@harperfast/harper 5.3.0 → 5.3.1

package/resources/auditStore.ts

@@ -1,3 +1,4 @@
+import { randomBytes } from 'node:crypto';
 import { readKey, writeKey } from 'ordered-binary';
 import { initSync, get as envGet } from '../utility/environment/environmentManager.ts';
 import { AUDIT_STORE_NAME } from '../utility/lmdb/terms.ts';
@@ -12,7 +13,8 @@ import { onStorageReclamation } from '../server/storageReclamation.ts';
 import { RocksDatabase } from '@harperfast/rocksdb-js';
 import { asBinary } from 'lmdb';
 import { RocksTransactionLogStore } from './RocksTransactionLogStore.ts';
-import { isReadOnlyMode } from './databases.ts';
+import { endSubscriptionsFromEarlierHandles } from './transactionBroadcast.ts';
+import { isReadOnlyMode, openRocksDatabase } from './databases.ts';
 
 /**
  * This module is responsible for the binary representation of audit records in an efficient form.
@@ -138,9 +140,10 @@ const FLOAT_BUFFER = new Uint8Array(FLOAT_TARGET.buffer);
 const AUDIT_FLOOR_KEY = Symbol.for('audit-floor');
 // The epoch `establishAuditFloor` stamped, never raised or removed. Its PRESENCE marks the floor as
 // unverified provenance — a guess bounded by what survived, blind to history a legacy prune removed
-// before tracking began — until a database generation (#2451) retires the mark. No comparison does:
-// a later prune certifies only what it removed, so `floor > bootstrap` says nothing about the older
-// gap. The value records how far the guess reached, for that repair. See `establishAuditFloor`.
+// before tracking began. No comparison retires the mark: a later prune certifies only what it
+// removed, so `floor > bootstrap` says nothing about the older gap. Resume never relies on it, since a
+// position is bound to a database generation and everything after its cursor was written after
+// tracking began. The value records how far the guess reached. See `establishAuditFloor`.
 const AUDIT_FLOOR_BOOTSTRAP_KEY = Symbol.for('audit-floor-bootstrap');
 /**
  * The floor's own eight bytes, deliberately NOT the FLOAT_TARGET/FLOAT_BUFFER pair the `last-removed`
@@ -152,6 +155,25 @@ const FLOOR_TARGET = new Float64Array(1);
 const FLOOR_BUFFER = new Uint8Array(FLOOR_TARGET.buffer);
 /** No trustworthy floor: the highest possible floor, so every cursor compares as stale. */
 const AUDIT_FLOOR_UNKNOWN = Infinity;
+/**
+ * Which copy of this database's history this is: a sixteen-byte random id, then the float64 time the
+ * generation began (0 for genesis). Every path that publishes a copy stamps a fresh one first
+ * (`stampDatabaseGeneration`), so a position naming an older id is refused (harper#2451). RocksDB only,
+ * like the resume floor: an LMDB database carries neither, so no position is resumable against it.
+ */
+const DATABASE_GENERATION_KEY = Symbol.for('database-generation');
+const GENERATION_ID_BYTES = 16;
+const GENERATION_RECORD_BYTES = GENERATION_ID_BYTES + 8;
+/**
+ * The highest prune cutoff within the current generation, encoded like the floor. Kept apart from the
+ * floor, which also steers `Table.commit`'s reconciliation: resume validity must restart at a copy and
+ * must not be absorbed by the floor's unknown sentinel, and doing either to the floor changes merges.
+ */
+const AUDIT_RESUME_FLOOR_KEY = Symbol.for('audit-resume-floor');
+
+function isRocksStore(store: any): boolean {
+	return store instanceof RocksTransactionLogStore || store instanceof RocksDatabase;
+}
 
 /** Last resort on a detached path: a failing log sink must not itself become an unhandled rejection. */
 function warnContained(message: string, error: unknown) {
@@ -191,6 +213,8 @@ export function openAuditStore(rootStore) {
 	rootStore.auditStore = auditStore;
 	auditStore.rootStore = rootStore;
 	establishAuditFloor(auditStore);
+	establishDatabaseGeneration(auditStore);
+	endSubscriptionsFromEarlierHandles(auditStore, isRocksStore(auditStore));
 	auditStore.tableStores = [];
 	const deleteCallbacks = [];
 	auditStore.addDeleteRemovalCallback = function (tableId, table, callback) {
@@ -508,13 +532,13 @@ function decodeAuditFloor(stored: any): number {
 }
 
 /**
- * Did the floor write land? A record has to be PRESENT, not merely decode to the value we wrote:
- * `decodeAuditFloor(undefined)` is the unknown sentinel too, so on a floorless store — where the
- * resolver writes exactly that sentinel — comparing decoded values alone reported a commit for a
- * write that never happened, and the caller pruned with nothing persisted.
+ * Did a metadata write land? A record has to be PRESENT with exactly the bytes written, not merely
+ * decode to the same value: `decodeAuditFloor(undefined)` is the unknown sentinel too, so on a
+ * floorless store — where the resolver writes exactly that sentinel — comparing decoded values alone
+ * reported a commit for a write that never happened, and the caller pruned with nothing persisted.
  */
-function floorWriteLanded(stored: any, floor: number): boolean {
-	return stored !== undefined && decodeAuditFloor(stored) === floor;
+function writeLanded(stored: any, written: Uint8Array): boolean {
+	return stored !== undefined && stored.byteLength === written.byteLength && Buffer.compare(stored, written) === 0;
 }
 
 /** Own eight bytes per write: the store must never be handed a live view of the reused module buffer. */
@@ -538,50 +562,66 @@ function updateAuditFloor(
 	resolve: (current: number, recorded: boolean) => number | undefined,
 	key: symbol = AUDIT_FLOOR_KEY
 ): void {
-	// A legacy `auditPath` layout is opened as its own standalone LMDB root (databases.ts) and has no
-	// `.rootStore`, so it owns the transaction itself.
-	const transactionOwner = auditStore?.rootStore ?? auditStore;
+	commitAuditMetadata(
+		auditStore,
+		(read) => {
+			const stored = read(key);
+			const floor = resolve(decodeAuditFloor(stored), stored !== undefined);
+			return floor === undefined ? undefined : [[key, encodeAuditFloor(floor)]];
+		},
+		'audit retention floor'
+	);
+}
+
+/**
+ * Write audit metadata records under one store transaction, all or none: every write is read back,
+ * and a mismatch throws inside the transaction so it aborts — returning false commits on LMDB.
+ */
+function commitAuditMetadata(
+	store: any,
+	plan: (read: (key: symbol) => any) => Array<[symbol, Uint8Array]> | undefined,
+	what: string
+): void {
+	// A copy being stamped passes its RocksDB root directly. A legacy `auditPath` layout is opened as its
+	// own standalone LMDB root (databases.ts) and has no `.rootStore`, so it owns the transaction itself.
+	const onRocksDB = isRocksStore(store);
+	const transactionOwner = store instanceof RocksDatabase ? store : (store?.rootStore ?? store);
 	if (!transactionOwner?.transactionSync)
-		throw new Error('Cannot record the audit retention floor: this database has no audit store');
-	// Both branches read their own write back and report `false` on mismatch, and the caller demands an
-	// explicit `true`. Both halves are load-bearing: a RocksDB transactionSync returns undefined for a
-	// swallowed abort rather than throwing (see RecordEncoder.saveStructures), and a write that fails
-	// without throwing is otherwise indistinguishable from one that landed — a caller pruning against a
-	// floor never recorded. Reads inside a write transaction see their own writes on both engines, so
-	// the read-back observes what commit will make durable.
-	const committed =
-		auditStore instanceof RocksTransactionLogStore
-			? transactionOwner.transactionSync(
-					(txn) => {
-						const stored = txn.getBinarySync(key);
-						const floor = resolve(decodeAuditFloor(stored), stored !== undefined);
-						if (floor !== undefined) {
-							txn.putSync(key, asBinary(encodeAuditFloor(floor)));
-							if (!floorWriteLanded(txn.getBinarySync(key), floor)) return false;
-						}
-						return true;
-					},
-					{ retryOnBusy: true }
-				)
-			: transactionOwner.transactionSync(() => {
-					const stored = auditStore.getBinary(key);
-					const floor = resolve(decodeAuditFloor(stored), stored !== undefined);
-					// `put` rather than `putSync`, and inside the transaction: lmdb's putSync is
-					// `put(...) === SYNC_PROMISE_SUCCESS`, so it drops whatever put returns, and a rejected put
-					// would leak with no owner. Within a write transaction put writes synchronously and returns
-					// an already-resolved sentinel, so the value is visible immediately either way and this
-					// only takes ownership of the failure case.
-					// asBinary: a legacy standalone audit root's encoder has no Uint8Array passthrough, so raw
-					// bytes would reach createAuditEntry and throw. This bypasses both encoders.
-					if (floor !== undefined) {
-						auditStore
-							.put(key, asBinary(encodeAuditFloor(floor)))
-							?.catch?.((error) => warnContained('Error writing the audit retention floor', error));
-						if (!floorWriteLanded(auditStore.getBinary(key), floor)) return false;
+		throw new Error(`Cannot record the ${what}: this database has no audit store`);
+	const notCommitted = () => new Error(`The ${what} transaction did not commit`);
+	// The caller still demands an explicit `true`: a RocksDB transactionSync returns undefined for a
+	// swallowed abort rather than throwing (see RecordEncoder.saveStructures). Reads inside a write
+	// transaction see their own writes on both engines, so the read-back observes what commit will make
+	// durable.
+	const committed = onRocksDB
+		? transactionOwner.transactionSync(
+				(txn) => {
+					const writes = plan((key) => txn.getBinarySync(key));
+					if (writes) {
+						for (const [key, bytes] of writes) txn.putSync(key, asBinary(bytes));
+						for (const [key, bytes] of writes) if (!writeLanded(txn.getBinarySync(key), bytes)) throw notCommitted();
 					}
 					return true;
-				});
-	if (committed !== true) throw new Error('The audit retention floor transaction did not commit');
+				},
+				{ retryOnBusy: true }
+			)
+		: transactionOwner.transactionSync(() => {
+				const writes = plan((key) => store.getBinary(key));
+				// `put` rather than `putSync`, and inside the transaction: lmdb's putSync is
+				// `put(...) === SYNC_PROMISE_SUCCESS`, so it drops whatever put returns, and a rejected put
+				// would leak with no owner. Within a write transaction put writes synchronously and returns
+				// an already-resolved sentinel, so the value is visible immediately either way and this
+				// only takes ownership of the failure case.
+				// asBinary: a legacy standalone audit root's encoder has no Uint8Array passthrough, so raw
+				// bytes would reach createAuditEntry and throw. This bypasses both encoders.
+				if (writes) {
+					for (const [key, bytes] of writes)
+						store.put(key, asBinary(bytes))?.catch?.((error) => warnContained(`Error writing the ${what}`, error));
+					for (const [key, bytes] of writes) if (!writeLanded(store.getBinary(key), bytes)) throw notCommitted();
+				}
+				return true;
+			});
+	if (committed !== true) throw notCommitted();
 }
 
 /**
@@ -666,25 +706,42 @@ export function raiseAuditFloor(auditStore: any, cutoff: number): void {
 	// database, a cutoff below one a wider prune already set), and taking the env write lock to
 	// discover that serializes every worker's boot and reclamation on it. The in-transaction guards
 	// below stay authoritative.
-	// Skips only the case it can prove is a no-op: a record that exists and already sits at or above
-	// the cutoff. An absent record is NOT decided here — the presence question is settled inside the
-	// transaction below, because another worker's establishAuditFloor can land between this read and
-	// that write.
+	// Skips only the case it can prove is a no-op: the floor, and on RocksDB the resume record, exist and
+	// already sit at or above the cutoff. An absent record is NOT decided here — the presence question is
+	// settled inside the transaction below, because another worker's establishAuditFloor can land between
+	// this read and that write.
+	const resumeTracked = isRocksStore(auditStore);
 	if (auditStore?.getBinary) {
 		const stored = auditStore.getBinary(AUDIT_FLOOR_KEY);
-		if (stored !== undefined && !(cutoff > decodeAuditFloor(stored))) return;
+		const resume = resumeTracked ? auditStore.getBinary(AUDIT_RESUME_FLOOR_KEY) : undefined;
+		if (
+			stored !== undefined &&
+			!(cutoff > decodeAuditFloor(stored)) &&
+			(!resumeTracked || (resume !== undefined && !(cutoff > decodeAuditFloor(resume))))
+		)
+			return;
 	}
-	updateAuditFloor(auditStore, (current, recorded) => {
-		// Still no record, and we are about to prune: persist the unknown sentinel. Leaving no marker
-		// lets the next open stamp a FINITE epoch, and a prune bound above that epoch (a future
-		// `endTime`, or a rolled-back clock) then certifies cursors whose history this prune deleted.
-		// Unknown is the honest value, because a store with no record may have been pruned before this
-		// run too.
-		if (!recorded) return AUDIT_FLOOR_UNKNOWN;
-		// A record appeared while we were getting here, so this is an ordinary monotonic raise: pruning
-		// to `cutoff` against a floor left below it is exactly the silent gap this function prevents.
-		return cutoff > current ? cutoff : undefined;
-	});
+	commitAuditMetadata(
+		auditStore,
+		(read) => {
+			const writes: Array<[symbol, Uint8Array]> = [];
+			const stored = read(AUDIT_FLOOR_KEY);
+			// Still no record, and we are about to prune: persist the unknown sentinel. Leaving no marker
+			// lets the next open stamp a FINITE epoch, and a prune bound above that epoch (a future
+			// `endTime`, or a rolled-back clock) then certifies cursors whose history this prune deleted.
+			// Unknown is the honest value, because a store with no record may have been pruned before this
+			// run too.
+			if (stored === undefined) writes.push([AUDIT_FLOOR_KEY, encodeAuditFloor(AUDIT_FLOOR_UNKNOWN)]);
+			else if (cutoff > decodeAuditFloor(stored)) writes.push([AUDIT_FLOOR_KEY, encodeAuditFloor(cutoff)]);
+			if (resumeTracked) {
+				const resume = read(AUDIT_RESUME_FLOOR_KEY);
+				if (resume === undefined || cutoff > decodeAuditFloor(resume))
+					writes.push([AUDIT_RESUME_FLOOR_KEY, encodeAuditFloor(cutoff)]);
+			}
+			return writes;
+		},
+		'audit retention floor'
+	);
 }
 
 /**
@@ -715,8 +772,9 @@ export function raiseAuditFloor(auditStore: any, cutoff: number): void {
  * unverified pre-tracking window for as long as it exists, however far the floor has since moved: a
  * prune raising the floor above the epoch certifies only what that prune removed, and says nothing
  * about history removed before tracking began — which may sit above the epoch, since that is exactly
- * the case the guess cannot see. Retiring the mark takes a database generation (harper#2451), not a
- * floor that has climbed past it.
+ * the case the guess cannot see. A floor that has climbed past it does not retire it. Resumable
+ * positions never depend on it: each is bound to a database generation minted after tracking began,
+ * so the pre-tracking window lies below every one of them (`getDatabaseGeneration`).
  *
  * Ordering. The provenance record is written **first**, so a crash between the two writes leaves a
  * record with no floor — which the next open retries, since the early return above tests the floor.
@@ -787,10 +845,10 @@ export function establishAuditFloor(auditStore: any): void {
 
 /**
  * The floor of this database's retained audit history: every audit entry at or after the returned
- * time is still retained, so a consumer whose last-processed audit-log cursor is `>=` it can resume
- * incrementally, and one below it must resync — it may have lost nothing, but the floor cannot
- * certify it either way. Returns `Infinity` when
- * the floor is unknown, which fails closed — no cursor compares as safe.
+ * time is still retained, and below it the log may have lost anything. Returns `Infinity` when the
+ * floor is unknown, which fails closed — no cursor compares as safe. A resume is decided by the
+ * database generation and its resume floor (`getDatabaseGeneration`) instead: this floor survives a
+ * restore, and an unknown one is absorbing.
  *
  * **One exception, and it is the only one: history removed before the floor existed.** Every prune
  * that runs with a floor recorded is covered — it raises the floor first, so it cannot remove an entry
@@ -798,7 +856,9 @@ export function establishAuditFloor(auditStore: any): void {
  * (`establishAuditFloor`), and a legacy `Table.deleteHistory` that removed a table's newest entries,
  * followed by a clock rollback, leaves that stamp below history that is gone; a cursor in the window
  * is then certified over the gap. So the guarantee is one-directional for tracked prunes and silent
- * about untracked ones. Only a database generation can close that (harper#2451).
+ * about untracked ones. Resumable positions do not face it: they are checked against the database
+ * generation and its resume floor (`getDatabaseGeneration`), not against this floor, which also steers
+ * `Table.commit`'s reconciliation and so cannot move in the direction a cursor check would want.
  *
  * The time domain is the audit-log key: what `subscribe`'s events carry as `localTime` and what MQTT
  * durable sessions persist as `startTime`, so those compare against the floor directly.
@@ -817,8 +877,9 @@ export function establishAuditFloor(auditStore: any): void {
  *
  * **A moment-in-time observation.** Retention can advance between this call and whatever the caller
  * does with the answer, so a check-then-resume sequence has a window where the floor moves under it.
- * Closing that requires validating the cursor inside the resume itself (harper#2448); until then a
- * lost race degrades to the truncation that happens today, never to anything worse.
+ * Closing that requires validating the position inside the resume itself (harper#2448), against the
+ * generation and the resume floor; until then a lost race degrades to the truncation that happens
+ * today, never to anything worse.
  *
  * **On RocksDB the floor tracks the configured horizon, not retained reality.** That branch purges at
  * whole-log-file granularity, so it cannot know before the fact which entries a purge will drop, and
@@ -828,18 +889,179 @@ export function establishAuditFloor(auditStore: any): void {
  * resync. Conservative in the one safe direction, and the reason the LMDB branch (which can see a
  * single eligible entry) instead raises off the first one it finds.
  *
- * **Not covered: copying a database's state without its history.** `restore_backup` replaces a
- * database with the backup's, floor and all, and a RocksDB checkpoint (a branch database) copies the
- * floor record but no transaction logs — so in both cases a cursor from after the copy point sits
- * above a floor that is present, and therefore trusted, for history that is not there. Nothing here
- * can detect that on its own, and it is not the audit log's problem alone: the same copy rolls back
- * record versions and per-node replication sequence state, so making this one field honest while
- * those stay stale would not give a consumer a coherent answer. It needs a database-level epoch —
- * harper#2451.
+ * **Copies of a database's state.** `restore_backup` copies this record with everything else, and it
+ * stays accurate for the log the restore carried; a branch checkpoint copies it with no log at all.
+ * What a copy changes is which history the database is, and that is the database generation's to
+ * answer: every copy path stamps a new one (`stampDatabaseGeneration`), and a copy that carried no log
+ * raises this floor to the generation's epoch.
  */
 export function getAuditFloor(auditStore: any): number {
 	return decodeAuditFloor(auditStore.getBinary(AUDIT_FLOOR_KEY));
 }
+
+export interface DatabaseGeneration {
+	/** Thirty-two lowercase hex characters. */
+	id: string;
+	/** When the generation began, in the audit-log key domain; 0 for a genesis generation. */
+	epoch: number;
+}
+
+/**
+ * The generation this handle's open established, or undefined when it could not establish one, in
+ * which case nothing may resume against it. Per database per node, never cluster-wide.
+ */
+export function getDatabaseGeneration(auditStore: any): DatabaseGeneration | undefined {
+	return auditStore?.databaseGeneration;
+}
+
+/**
+ * The bound a resumed position is checked against, or `Infinity` when unknown: the resume floor, but
+ * never below a finite audit floor, since a binary that predates the resume floor raised only the
+ * audit floor when it pruned. Such a binary pruning while the audit floor was unknown records nothing
+ * either record can show.
+ */
+export function getAuditResumeFloor(auditStore: any): number {
+	// The audit floor first: both records only rise, and an unknown audit floor stays unknown, so a prune
+	// committing between the two reads is still seen through the resume floor read second.
+	const auditFloor = getAuditFloor(auditStore);
+	const resumeFloor = decodeAuditFloor(auditStore.getBinary(AUDIT_RESUME_FLOOR_KEY));
+	return Number.isFinite(auditFloor) && auditFloor > resumeFloor ? auditFloor : resumeFloor;
+}
+
+/**
+ * Whether a position may resume here: it names this handle's generation and carries a finite cursor
+ * at or above the resume floor. The cursor must be progress-based — no lower than the log position
+ * observed when the position was established — or a quiet scope's snapshot key falls below a floor
+ * that retention keeps advancing and is refused on every resume. The answer holds as of the read: a
+ * prune that commits after it returns is the caller's to order against its replay. Always false on
+ * LMDB, which has no generation: a caller applies the check to RocksDB databases only.
+ */
+export function isResumablePosition(auditStore: any, generationId: string | undefined, cursor: number): boolean {
+	const generation = getDatabaseGeneration(auditStore);
+	return (
+		generation !== undefined &&
+		generationId === generation.id &&
+		typeof cursor === 'number' &&
+		Number.isFinite(cursor) &&
+		cursor >= getAuditResumeFloor(auditStore)
+	);
+}
+
+function newGeneration(epoch: number): DatabaseGeneration {
+	return { id: randomBytes(GENERATION_ID_BYTES).toString('hex'), epoch };
+}
+
+function encodeGeneration(generation: DatabaseGeneration): Uint8Array {
+	const bytes = new Uint8Array(GENERATION_RECORD_BYTES);
+	bytes.set(Buffer.from(generation.id, 'hex'));
+	new DataView(bytes.buffer).setFloat64(GENERATION_ID_BYTES, generation.epoch, true);
+	return bytes;
+}
+
+function decodeGeneration(stored: any): DatabaseGeneration | undefined {
+	if (stored?.byteLength !== GENERATION_RECORD_BYTES) return undefined;
+	const epoch = new DataView(stored.buffer, stored.byteOffset, GENERATION_RECORD_BYTES).getFloat64(
+		GENERATION_ID_BYTES,
+		true
+	);
+	if (!Number.isFinite(epoch) || epoch < 0 || Object.is(epoch, -0)) return undefined;
+	return { id: Buffer.from(stored.buffer, stored.byteOffset, GENERATION_ID_BYTES).toString('hex'), epoch };
+}
+
+/**
+ * Establish this handle's generation at open: adopt the recorded one, or mint genesis for a RocksDB
+ * store that has none (an LMDB store gets none). An undecodable record is never replaced here — an
+ * ordinary open cannot know whether another worker already serves it; only a copy's stamp replaces a
+ * generation. Never fails the open.
+ */
+export function establishDatabaseGeneration(auditStore: any): void {
+	auditStore.databaseGeneration = undefined;
+	if (!isRocksStore(auditStore)) return;
+	try {
+		let stored = auditStore.getBinary(DATABASE_GENERATION_KEY);
+		if (stored === undefined) {
+			if (isReadOnlyMode()) return;
+			const genesis = encodeGeneration(newGeneration(0));
+			commitAuditMetadata(
+				auditStore,
+				(read) => {
+					// compare-and-set on absence, so racing workers converge on the first one's id
+					if (read(DATABASE_GENERATION_KEY) !== undefined) return undefined;
+					const writes: Array<[symbol, Uint8Array]> = [[DATABASE_GENERATION_KEY, genesis]];
+					if (read(AUDIT_RESUME_FLOOR_KEY) === undefined) {
+						// starting at the audit floor rather than 0 keeps the next prune's lock-free skip effective
+						const auditFloor = decodeAuditFloor(read(AUDIT_FLOOR_KEY));
+						writes.push([AUDIT_RESUME_FLOOR_KEY, encodeAuditFloor(Number.isFinite(auditFloor) ? auditFloor : 0)]);
+					}
+					return writes;
+				},
+				'database generation'
+			);
+			stored = auditStore.getBinary(DATABASE_GENERATION_KEY);
+		}
+		auditStore.databaseGeneration = decodeGeneration(stored);
+		if (!auditStore.databaseGeneration)
+			warnContained(
+				'The database generation record is unreadable, so no subscription can resume against this database',
+				new Error(`database generation record of ${stored?.byteLength} bytes`)
+			);
+	} catch (error) {
+		warnContained('Error establishing the database generation', error);
+	}
+}
+
+/**
+ * Give a copy of a database a generation of its own, before anything can read it; the caller holds
+ * the copy exclusively and makes the stamp durable before publishing. A copy that kept no
+ * transaction log has nothing below its epoch, so a finite floor is raised to it — raising it over
+ * history a copy carried would change `Table.commit`'s reconciliation, and an unknown floor stays
+ * unknown. `generation` lets a caller replay a value it recorded durably first.
+ */
+export function stampDatabaseGeneration(
+	store: any,
+	{ carriesLog, generation = newGeneration(Date.now()) }: { carriesLog: boolean; generation?: DatabaseGeneration }
+): DatabaseGeneration {
+	if (
+		!/^[0-9a-f]{32}$/.test(generation?.id) ||
+		decodeGeneration(encodeGeneration(generation))?.epoch !== generation.epoch
+	)
+		throw new Error(`Invalid database generation: ${JSON.stringify(generation)}`);
+	const stamp: Array<[symbol, Uint8Array]> = [
+		[DATABASE_GENERATION_KEY, encodeGeneration(generation)],
+		[AUDIT_RESUME_FLOOR_KEY, encodeAuditFloor(0)],
+	];
+	commitAuditMetadata(
+		store,
+		(read) => {
+			if (carriesLog) return stamp;
+			const stored = read(AUDIT_FLOOR_KEY);
+			const floor = decodeAuditFloor(stored);
+			return stored !== undefined && Number.isFinite(floor) && floor < generation.epoch
+				? [...stamp, [AUDIT_FLOOR_KEY, encodeAuditFloor(generation.epoch)]]
+				: stamp;
+		},
+		'database generation'
+	);
+	return generation;
+}
+
+/**
+ * Stamp the RocksDB database at `path`, which nothing else may have open, and flush it: a root-store
+ * write alone is not power-loss durable, and directory fsync is best-effort where unsupported.
+ */
+export async function stampDatabaseDirectory(
+	path: string,
+	options: { carriesLog: boolean; generation?: DatabaseGeneration }
+): Promise<DatabaseGeneration> {
+	const database = openRocksDatabase(path, { disableWAL: false });
+	try {
+		const generation = stampDatabaseGeneration(database, options);
+		await database.flush({ allowWriteStall: true });
+		return generation;
+	} finally {
+		database.close();
+	}
+}
 export function setAuditRetention(retentionTime, defaultDelay = DEFAULT_AUDIT_CLEANUP_DELAY) {
 	auditRetention = retentionTime;
 	DEFAULT_AUDIT_CLEANUP_DELAY = defaultDelay;

package/resources/branchDatabase.ts

@@ -26,6 +26,7 @@ import {
 	retakeBranchIdentity,
 } from './databases.ts';
 import { replayLogs, replayTimeBudgetMs } from './replayLogs.ts';
+import { stampDatabaseDirectory } from './auditStore.ts';
 
 /**
  * Private per-application forks of a database, for running several variants of an application against
@@ -345,6 +346,7 @@ async function materializeBranch(
 		await base.createCheckpoint(staging);
 		report.progress();
 		await cloneBlobRoots(baseName, baseRoots, blobRoots, report.progress);
+		await stampDatabaseDirectory(staging, { carriesLog: false });
 		await writeFile(join(staging, COMPLETION_MARKER), JSON.stringify({ blobRoots } satisfies BranchCompletion));
 		await rename(staging, branchPath);
 		return blobRoots;

package/resources/databases.ts

@@ -1459,6 +1459,9 @@ function initStores(
 		const fullTextIndexRetirements = persistedFullTextIndexNames(primaryAttribute.fullTextIndexRetirements);
 		if (table && !recreateTable) {
 			if (primaryAttribute.audit === true && table.audit !== true) table.enableAuditing();
+			// Absent means unchanged, not `replicates`: NON_REPLICATING_SYSTEM_TABLES sets `replicate = false`
+			// on the live class and stores nothing, so an unconditional refresh would clear that override.
+			if (typeof primaryAttribute.replicate === 'boolean') table.replicate = primaryAttribute.replicate;
 			table.fullTextIndexes = fullTextIndexes;
 			table.fullTextFields = fullTextFields;
 			table.fullTextIndexGenerations = fullTextIndexGenerations;
@@ -4304,8 +4307,8 @@ function declareTable<TableResourceType>(target: TableTarget, tableDefinition: T
 					origin !== 'cluster' &&
 					(schemaDefinedMismatch ||
 						(typeof audit === 'boolean' && audit !== attributeDescriptor.audit) ||
-						(sealed !== undefined && sealed !== Table.sealed) ||
-						(replicate !== undefined && replicate !== Table.replicate) ||
+						(sealed !== undefined && sealed !== attributeDescriptor.sealed) ||
+						(replicate !== undefined && replicate !== attributeDescriptor.replicate) ||
 						(+expiration || undefined) !== (+attributeDescriptor.expiration || undefined) ||
 						(+eviction || undefined) !== (+attributeDescriptor.eviction || undefined) ||
 						attribute.type !== attributeDescriptor.type)
@@ -4695,6 +4698,7 @@ function declareTable<TableResourceType>(target: TableTarget, tableDefinition: T
 		);
 	refreshDerivedIndexes(Table);
 
+	if (typeof replicate === 'boolean' && origin !== 'cluster') Table.replicate = replicate;
 	Table.origin = origin;
 	// scope-private: replication and other global subscribers must not learn of a branch class
 	if ((hasChanges || refreshRelationshipAttributes) && !target.branch) {