@rocicorp/zero 1.11.0-canary.13 → 1.11.0-canary.15

package/out/replicache/src/replicache-impl.js

@@ -3,8 +3,9 @@ import { assertCookie } from "./cookies.js";
 import { isClientStateNotFoundResponse, isVersionNotSupportedResponse } from "./error-responses.js";
 import { getDefaultPuller, isDefaultPuller } from "./get-default-puller.js";
 import { getBrowserGlobal } from "../../shared/src/browser-env.js";
-import { promiseVoid } from "../../shared/src/resolved-promises.js";
+import { promiseFalse, promiseVoid } from "../../shared/src/resolved-promises.js";
 import { deepFreeze } from "./frozen-json.js";
+import { getStorageFailure } from "./storage-failure.js";
 import { makeIDBName } from "./make-idb-name.js";
 import { AbortError } from "../../shared/src/abort-error.js";
 import { InvalidRefCountError } from "./dag/invalid-ref-count-error.js";
@@ -19,6 +20,7 @@ import { disableClientGroup } from "./persist/client-groups.js";
 import { makeClientID } from "./persist/make-client-id.js";
 import { ClientStateNotFoundError, hasClientState, initClientV6 } from "./persist/clients.js";
 import { getKVStoreProvider } from "./get-kv-store-provider.js";
+import { MemFallbackStoreProvider } from "./kv/mem-fallback-store.js";
 import { createLogContext } from "./log-options.js";
 import { IDBDatabasesStore } from "./persist/idb-databases-store.js";
 import { COLLECT_IDB_INTERVAL, INITIAL_COLLECT_IDB_DELAY, dropDatabaseInternal, initCollectIDBDatabases } from "./persist/collect-idb-databases.js";
@@ -100,12 +102,12 @@ var ReplicacheImpl = class {
 		return makeIDBName(this.name, this.schemaVersion);
 	}
 	/**
-	* The KV store backing this instance. Its `kind` reflects the storage
-	* currently in use, e.g. `'mem'` after an IndexedDB store fell back to
-	* memory.
+	* The KV store backing this instance. It is a memory store once a storage
+	* failure while opening moved the instance onto memory (see
+	* `onStorageFailure`).
 	*/
 	get kvStore() {
-		return this.#perKVStore;
+		return this.#perKVStore.backing;
 	}
 	set auth(auth) {
 		if (this.#zero) this.#zero.auth = auth;
@@ -171,6 +173,16 @@ var ReplicacheImpl = class {
 	#lc;
 	#zero;
 	#closeAbortController = new AbortController();
+	/**
+	* Aborted on close AND on a storage failure after the open. The store-using
+	* background processes (heartbeat, client and client-group GC, database
+	* collection, mutation recovery, the new-client channel's store read) run
+	* on this signal, so the first detected failure stops them instead of
+	* letting each retry the fault at its interval and log `Error running.`
+	* every time. A failure during the open moves the stores onto memory
+	* instead, where they keep running.
+	*/
+	#storeProcessesAbortController = new AbortController();
 	#persistLock = new Lock();
 	#enableScheduledPersist;
 	#enableScheduledRefresh;
@@ -217,6 +229,51 @@ var ReplicacheImpl = class {
 	*/
 	onClientStateNotFound = reload;
 	/**
+	* `onStorageFailure` is called once, the first time the local kv store
+	* reports that its storage has failed rather than its data, by throwing a
+	* {@link StorageFailureError}: the device is out of space (`full`), the
+	* database cannot be opened (`cannot-open`), or a read or write to it failed
+	* (`io-error`). The SQLite stores and the IndexedDB store report these; a
+	* custom `StoreProvider` can throw a `StorageFailureError` itself.
+	*
+	* A failure while the instance is opening moves it onto memory for the
+	* session, where it runs as with `kvStore: 'mem'`: it persists and
+	* refreshes against the memory stores, and nothing reaches the disk.
+	*
+	* A failure after that stops persisting and refreshing: a rebuild would
+	* open the same failing storage, and a wipe would destroy an intact replica
+	* for a fault that is not in the data. Queries and mutations keep running
+	* against what the in-memory dag holds, and mutations keep pushing to the
+	* server, but nothing is persisted locally until the app creates a new
+	* instance. A read that needs a chunk the in-memory dag has not loaded from
+	* the store yet, or has since evicted from its cache, still goes to the
+	* store and fails the way any store read does; that failure reaches its
+	* caller and is not reclassified here. An invalid ref count reported while
+	* the store is failing with `cannot-open` or `io-error` is NOT treated as
+	* corruption (the database is not dropped and {@link onClientStateNotFound}
+	* is not called), because a store that cannot complete a read or write is
+	* what produces such readings. A `full` disk keeps the drop: a torn write
+	* there is real corruption, and deleting the database is also what frees
+	* the space.
+	*
+	* After the open, the failure is detected when a transaction on the store
+	* fails to begin or to commit, whichever caller ran it (`persist()`, the
+	* heartbeat, garbage collection, ...), and when `persist()` or `refresh()`
+	* fails. The background maintenance processes (heartbeat, client and
+	* client-group GC, database collection, mutation recovery) stop on it
+	* instead of retrying against the store at their interval.
+	*
+	* There is no default behavior other than logging. An app that shows a
+	* "free up space" or "restart" screen can do so from here.
+	*/
+	onStorageFailure = null;
+	/**
+	* Set the first time the kv store reports a storage failure. Read by every
+	* path that writes the perdag so none of them retries against storage
+	* that has failed.
+	*/
+	#storageFailure;
+	/**
 	* `onUpdateNeeded` is called when a code update is needed.
 	*
 	* A code update can be needed because:
@@ -268,8 +325,9 @@ var ReplicacheImpl = class {
 			name,
 			"replicache version": version
 		});
+		this.#closeAbortController.signal.addEventListener("abort", () => this.#storeProcessesAbortController.abort(), { once: true });
 		this.#subscriptions = new SubscriptionsManagerImpl(this.#queryInternal, this.#lc, this.#closeAbortController.signal);
-		const kvStoreProvider = getKVStoreProvider(this.#lc, options.kvStore);
+		const kvStoreProvider = new MemFallbackStoreProvider(getKVStoreProvider(this.#lc, options.kvStore), (failure) => this.#onMemFallBack(failure), (failure) => this.#handleStorageFailure(failure));
 		this.#kvStoreProvider = kvStoreProvider;
 		const perKVStore = kvStoreProvider.create(this.idbName);
 		this.#perKVStore = perKVStore;
@@ -294,12 +352,19 @@ var ReplicacheImpl = class {
 		this.#onPersist = initOnPersistChannel(this.name, this.#closeAbortController.signal, (persistInfo) => {
 			this.#handlePersist(persistInfo);
 		});
-		this.#open(indexes, enableClientGroupForking, enableMutationRecovery, clientMaxAgeMs, profileIDResolver.resolve, clientGroupIDResolver.resolve, readyResolver.resolve, onClientsDeleted).catch((e) => {
-			this.#openFailed.resolve();
+		const open = () => this.#open(indexes, enableClientGroupForking, enableMutationRecovery, clientMaxAgeMs, profileIDResolver.resolve, clientGroupIDResolver.resolve, readyResolver.resolve, onClientsDeleted);
+		open().catch((e) => {
 			if (e instanceof InvalidRefCountError) {
+				this.#openFailed.resolve();
 				this.#lc.debug?.("Open failed because the persistent store is corrupt");
 				return;
 			}
+			if (!this.#closed && this.#kvStoreProvider.fallBack(e)) return open().catch((e) => {
+				this.#openFailed.resolve();
+				if (this.#closed) this.#lc.debug?.("Open on memory ended by close", e);
+				else this.#lc.error?.("Open failed again on memory", e);
+			});
+			this.#openFailed.resolve();
 			throw e;
 		});
 	}
@@ -316,12 +381,13 @@ var ReplicacheImpl = class {
 		if (dbOpenMs > SLOW_DB_OPEN_THRESHOLD_MS) this.#lc.warn?.(`Opening the database took ${Math.round(dbOpenMs)}ms`, { name: this.idbName });
 		else this.#lc.debug?.("Opened the database in", Math.round(dbOpenMs), "ms");
 		await this.#zero?.init(headHash, this.memdag);
+		this.#kvStoreProvider.opened();
 		resolveReady();
 		if (this.#enablePullAndPushInOpen) {
 			this.pull().catch(noop);
 			this.push().catch(noop);
 		}
-		const { signal } = this.#closeAbortController;
+		const { signal } = this.#storeProcessesAbortController;
 		startHeartbeats(clientID, this.perdag, () => {
 			this.#clientStateNotFoundOnClient(clientID);
 		}, HEARTBEAT_INTERVAL, this.#lc, signal);
@@ -336,7 +402,7 @@ var ReplicacheImpl = class {
 		getBrowserGlobal("document")?.addEventListener("visibilitychange", this.#onVisibilityChange);
 	}
 	#onVisibilityChange = async () => {
-		if (this.#closed) return;
+		if (this.#closed || this.#storageFailure !== void 0) return;
 		if (getBrowserGlobal("document")?.visibilityState !== "visible") return;
 		await this.#checkForClientStateNotFoundAndCallHandler();
 	};
@@ -653,7 +719,7 @@ var ReplicacheImpl = class {
 		return this.#persistLock.withLock(async () => {
 			const { clientID } = this;
 			await this.#ready;
-			if (this.#closed) return;
+			if (this.#closed || this.#storageFailure) return;
 			try {
 				await persistDD31(this.#lc, clientID, this.memdag, this.perdag, this.#mutatorRegistry, () => this.#closed, 7, this.#zero?.getTxData);
 			} catch (e) {
@@ -662,7 +728,12 @@ var ReplicacheImpl = class {
 					await this.#handleInvalidRefCount(e);
 					return;
 				} else if (this.#closed) this.#lc.debug?.("Exception persisting during close", e);
-				else throw e;
+				else {
+					const failure = getStorageFailure(e);
+					if (failure === void 0) throw e;
+					this.#handleStorageFailure(failure);
+					return;
+				}
 			}
 			const clientGroupID = await this.#clientGroupIDPromise;
 			assert(clientGroupID, "Expected clientGroupID to be defined");
@@ -675,7 +746,7 @@ var ReplicacheImpl = class {
 	async refresh() {
 		await this.#ready;
 		const { clientID } = this;
-		if (this.#closed || !this.#enableRefresh()) return;
+		if (this.#closed || this.#storageFailure || !this.#enableRefresh()) return;
 		let refreshResult;
 		try {
 			refreshResult = await refresh(this.#lc, this.memdag, this.perdag, clientID, this.#mutatorRegistry, this.#subscriptions, () => this.closed, 7, this.#zero);
@@ -683,7 +754,12 @@ var ReplicacheImpl = class {
 			if (e instanceof ClientStateNotFoundError) this.#clientStateNotFoundOnClient(clientID);
 			else if (e instanceof InvalidRefCountError) await this.#handleInvalidRefCount(e);
 			else if (this.#closed) this.#lc.debug?.("Exception refreshing during close", e);
-			else throw e;
+			else {
+				const failure = getStorageFailure(e);
+				if (failure === void 0) throw e;
+				this.#handleStorageFailure(failure);
+				return;
+			}
 		}
 		if (refreshResult !== void 0) await this.#subscriptions.fire(refreshResult.diffs);
 	}
@@ -701,6 +777,42 @@ var ReplicacheImpl = class {
 	*/
 	#corruptDatabaseRecovery;
 	/**
+	* A storage failure after the open. Records the first one, stops the
+	* store-using background processes, logs it once and tells the app. Later
+	* failures of the same instance are the same fault and are not reported
+	* again; the scheduled persist and refresh stop asking.
+	*/
+	#handleStorageFailure(failure) {
+		if (this.#storageFailure !== void 0) return;
+		this.#storageFailure = failure;
+		this.#storeProcessesAbortController.abort();
+		this.#lc.warn?.(`Local store storage failed (${failure.kind}) for ${this.idbName}; persistence is stopped for this instance`, failure);
+		this.#fireOnStorageFailure(failure);
+	}
+	/**
+	* A storage failure during the open: the stores have moved onto memory, so
+	* the instance runs on them for the session, as with `kvStore: 'mem'`.
+	* Reported on a
+	* later microtask: the SQLite stores open in `create`, while the
+	* constructor is still running and before the app can attach the callback.
+	*/
+	#onMemFallBack(failure) {
+		this.#lc.warn?.(`Local store storage failed (${failure.kind}) opening ${this.idbName}; running this instance in memory`, failure);
+		queueMicrotask(() => this.#fireOnStorageFailure(failure));
+	}
+	/**
+	* The app's callback must not be able to stop the recovery around it: in
+	* the open path, a throw here would skip the open on memory and leave the
+	* instance never ready.
+	*/
+	#fireOnStorageFailure(failure) {
+		try {
+			this.onStorageFailure?.(failure);
+		} catch (e) {
+			this.#lc.error?.("onStorageFailure threw", e);
+		}
+	}
+	/**
 	* Installed on the dag store that writes to our database. The store calls
 	* it after the failing transaction has been released, so dropping the
 	* database from here does not race with the rollback.
@@ -728,6 +840,10 @@ var ReplicacheImpl = class {
 	* something we don't expect to ever happen.
 	*/
 	#handleInvalidRefCount(e) {
+		if (this.#storageFailure !== void 0 && this.#storageFailure.kind !== "full") {
+			this.#lc.warn?.(`Invalid ref count while the local store is failing (${this.#storageFailure.kind}); not dropping database ${this.idbName}`, e);
+			return promiseVoid;
+		}
 		if (this.#corruptDatabaseRecovery === void 0) {
 			this.#lc.error?.(`Client state is corrupt on client, clientID: ${this.clientID}. Dropping database ${this.idbName}`, e);
 			this.#corruptDatabaseRecovery = this.#dropDatabaseAndFireOnClientStateNotFound();
@@ -774,7 +890,7 @@ var ReplicacheImpl = class {
 		this.onUpdateNeeded?.(reason);
 	}
 	async #schedulePersist() {
-		if (!this.#enableScheduledPersist) return;
+		if (!this.#enableScheduledPersist || this.#storageFailure) return;
 		await this.#schedule("persist", this.#persistScheduler);
 	}
 	async #handlePersist(persistInfo) {
@@ -783,7 +899,7 @@ var ReplicacheImpl = class {
 		if (persistInfo.clientGroupID === clientGroupID) this.#scheduleRefresh();
 	}
 	async #scheduleRefresh() {
-		if (!this.#enableScheduledRefresh) return;
+		if (!this.#enableScheduledRefresh || this.#storageFailure) return;
 		await this.#schedule("refresh from storage", this.#refreshScheduler);
 	}
 	async #schedule(name, scheduler) {
@@ -941,7 +1057,9 @@ var ReplicacheImpl = class {
 		if (ex instanceof ChunkNotFoundError && await this.#checkForClientStateNotFoundAndCallHandler()) return new ClientStateNotFoundError(this.clientID);
 		return ex;
 	}
-	recoverMutations() {}
+	recoverMutations() {
+		if (this.#storeProcessesAbortController.signal.aborted) return promiseFalse;
+	}
 	/**
 	* List of pending mutations. The order of this is from oldest to newest.
 	*