@signaldb/core 2.0.0-beta.17 → 2.0.0-beta.19

package/dist/index39.mjs

@@ -0,0 +1,513 @@
+import isEqual from "./index2.mjs";
+import getIndexInfo from "./index17.mjs";
+import deepClone from "./index18.mjs";
+import match from "./index19.mjs";
+import modify from "./index20.mjs";
+import project from "./index22.mjs";
+import sortItems from "./index24.mjs";
+import queryId from "./index26.mjs";
+import idIndexQuery from "./index27.mjs";
+import storageIndexQuery from "./index33.mjs";
+//#region src/AutoFetchDataAdapter.ts
+/**
+* Default merge strategy: shallow spread (right wins)
+* @param a first item
+* @param b second item
+* @returns merged item
+*/
+function defaultMergeItems(a, b) {
+	return {
+		...a,
+		...b
+	};
+}
+/**
+* Generates a stable key for a selector
+* @param selector - the selector
+* @returns the key
+*/
+function selectorId(selector) {
+	return JSON.stringify(selector ?? {});
+}
+/**
+* AutoFetchDataAdapter
+*
+* A DataAdapter that:
+* - Mirrors the CollectionBackend surface (CRUD + query registry + lifecycle)
+* - Executes queries against a provided StorageAdapter (local cache)
+* - On first registration of a selector, auto-fetches from a remote source and
+*   ingests the result into storage (upsert), then pushes query result updates
+* - Optionally purges auto-fetched items for a selector once no observers remain
+* - Can subscribe to remote change notifications to re-fetch active selectors
+*
+* IMPORTANT: Purging only ever deletes items that were introduced via the
+* auto-fetch path and are no longer referenced by any active selector. Items
+* inserted through CRUD calls are never purged.
+*/
+var AutoFetchDataAdapter = class {
+	options;
+	id;
+	onError;
+	fetchQueryItems;
+	mergeItems;
+	purgeDelay;
+	storageAdapters = /* @__PURE__ */ new Map();
+	storageAdapterReady = /* @__PURE__ */ new Map();
+	collectionIndices = /* @__PURE__ */ new Map();
+	queries = /* @__PURE__ */ new Map();
+	activeObservers = /* @__PURE__ */ new Map();
+	observerTimeouts = /* @__PURE__ */ new Map();
+	selectorIds = /* @__PURE__ */ new Map();
+	idRefCounts = /* @__PURE__ */ new Map();
+	autoloadIds = /* @__PURE__ */ new Map();
+	constructor(options) {
+		this.options = options;
+		this.id = options.id || "autofetch-data-adapter";
+		this.onError = options.onError ?? ((error) => {
+			console.error(error);
+		});
+		this.fetchQueryItems = options.fetchQueryItems;
+		this.mergeItems = options.mergeItems ?? defaultMergeItems;
+		this.purgeDelay = options.purgeDelay ?? 1e4;
+		if (options.registerRemoteChange) options.registerRemoteChange(async () => {
+			await this.forceRefetchAll();
+		});
+	}
+	createCollectionBackend(collection, indices) {
+		this.collectionIndices.set(collection.name, indices);
+		this.queries.set(collection.name, /* @__PURE__ */ new Map());
+		this.ensureStorageAdapter(collection.name);
+		this.activeObservers.set(collection.name, /* @__PURE__ */ new Map());
+		this.observerTimeouts.set(collection.name, /* @__PURE__ */ new Map());
+		this.selectorIds.set(collection.name, /* @__PURE__ */ new Map());
+		this.idRefCounts.set(collection.name, /* @__PURE__ */ new Map());
+		this.autoloadIds.set(collection.name, /* @__PURE__ */ new Set());
+		const ready = this.setupStorage(collection.name, indices);
+		this.storageAdapterReady.set(collection.name, ready);
+		const registerQuery = (selector, options) => {
+			const qid = queryId(selector, options);
+			const registry = this.queries.get(collection.name);
+			if (!registry) throw new Error(`Collection ${collection.name} not initialized!`);
+			registry.set(qid, {
+				selector,
+				options,
+				items: [],
+				listeners: /* @__PURE__ */ new Set(),
+				...registry.get(qid),
+				state: "active",
+				error: null
+			});
+			const key = selectorId(selector);
+			const perColObservers = this.activeObservers.get(collection.name);
+			const current = perColObservers?.get(key)?.count ?? 0;
+			perColObservers?.set(key, {
+				selector,
+				count: current + 1
+			});
+			const t = this.observerTimeouts.get(collection.name)?.get(key);
+			if (t) clearTimeout(t);
+			if (current === 0) this.fetchAndIngest(collection.name, selector).catch(this.onError);
+			this.fulfillQuery(collection.name, selector, options).catch(this.onError);
+		};
+		const unregisterQuery = (selector, options) => {
+			const qid = queryId(selector, options);
+			this.queries.get(collection.name)?.delete(qid);
+			const key = selectorId(selector);
+			const perColObservers = this.activeObservers.get(collection.name);
+			const current = perColObservers?.get(key)?.count ?? 0;
+			const remaining = Math.max(0, current - 1);
+			if (remaining > 0) {
+				perColObservers?.set(key, {
+					selector,
+					count: remaining
+				});
+				return;
+			}
+			const doPurge = () => {
+				perColObservers?.delete(key);
+				this.purgeSelector(collection.name, selector).catch(this.onError);
+			};
+			if (this.purgeDelay === 0) doPurge();
+			else {
+				const timeouts = this.observerTimeouts.get(collection.name);
+				const t = timeouts?.get(key);
+				if (t) clearTimeout(t);
+				timeouts?.set(key, setTimeout(doPurge, this.purgeDelay));
+			}
+		};
+		const getQueryState = (selector, options) => {
+			return (this.queries.get(collection.name)?.get(queryId(selector, options)))?.state ?? "active";
+		};
+		const getQueryError = (selector, options) => {
+			return (this.queries.get(collection.name)?.get(queryId(selector, options)))?.error ?? null;
+		};
+		const getQueryResult = (selector, options) => {
+			return (this.queries.get(collection.name)?.get(queryId(selector, options)))?.items ?? [];
+		};
+		const onQueryStateChange = (selector, options, callback) => {
+			const qid = queryId(selector, options);
+			const registry = this.queries.get(collection.name);
+			if (!registry) throw new Error(`Collection ${collection.name} not initialized!`);
+			if (!registry.has(qid)) registry.set(qid, {
+				selector,
+				options,
+				state: "active",
+				error: null,
+				items: [],
+				listeners: /* @__PURE__ */ new Set()
+			});
+			registry.get(qid)?.listeners.add(callback);
+			return () => {
+				registry.get(qid)?.listeners.delete(callback);
+			};
+		};
+		return {
+			insert: async (item) => {
+				await ready;
+				return await this.insert(collection.name, item);
+			},
+			updateOne: async (selector, modifier) => {
+				await ready;
+				return this.updateOne(collection.name, selector, modifier);
+			},
+			updateMany: async (selector, modifier) => {
+				await ready;
+				return this.updateMany(collection.name, selector, modifier);
+			},
+			replaceOne: async (selector, replacement) => {
+				await ready;
+				return this.replaceOne(collection.name, selector, replacement);
+			},
+			removeOne: async (selector) => {
+				await ready;
+				return this.removeOne(collection.name, selector);
+			},
+			removeMany: async (selector) => {
+				await ready;
+				return this.removeMany(collection.name, selector);
+			},
+			registerQuery,
+			unregisterQuery,
+			getQueryState,
+			getQueryError,
+			getQueryResult,
+			onQueryStateChange,
+			executeQuery: async (selector, options) => {
+				await ready;
+				registerQuery(selector, options);
+				await new Promise((resolve) => {
+					let stop = () => {};
+					stop = onQueryStateChange(selector, options, (state) => {
+						if (state === "active") return;
+						resolve();
+						stop();
+					});
+				});
+				const result = getQueryResult(selector, options);
+				unregisterQuery(selector, options);
+				return result;
+			},
+			dispose: async () => {
+				this.storageAdapters.delete(collection.name);
+				this.queries.delete(collection.name);
+				this.collectionIndices.delete(collection.name);
+				this.storageAdapterReady.delete(collection.name);
+				this.activeObservers.delete(collection.name);
+				this.observerTimeouts.delete(collection.name);
+				this.selectorIds.delete(collection.name);
+				this.idRefCounts.delete(collection.name);
+				this.autoloadIds.delete(collection.name);
+			},
+			isReady: async () => {
+				await ready;
+			}
+		};
+	}
+	async forceRefetchAll() {
+		const tasks = [];
+		for (const [collectionName, observers] of this.activeObservers.entries()) for (const { selector, count } of observers.values()) if (count > 0) tasks.push(this.fetchAndIngest(collectionName, selector));
+		await Promise.all(tasks);
+	}
+	async fetchAndIngest(collectionName, selector) {
+		this.publishForSelector(collectionName, selector, "active", null);
+		try {
+			const items = await this.fetchQueryItems(collectionName, selector);
+			if (!items || !Array.isArray(items)) throw new Error("AutoFetchDataAdapter: fetchQueryItems must resolve to { items: T[] }");
+			const ids = items.map((i) => i.id);
+			const selectorKey = selectorId(selector);
+			const selMap = this.selectorIds.get(collectionName);
+			const previous = selMap?.get(selectorKey) ?? /* @__PURE__ */ new Set();
+			ids.forEach((id) => previous.add(id));
+			selMap?.set(selectorKey, previous);
+			await this.upsertMerged(collectionName, items);
+			await this.checkQueryUpdates(collectionName, items);
+		} catch (error) {
+			this.publishForSelector(collectionName, selector, "error", error);
+			this.onError(error);
+		}
+	}
+	async purgeSelector(collectionName, selector) {
+		const selectorKey = selectorId(selector);
+		const selMap = this.selectorIds.get(collectionName);
+		const ids = selMap?.get(selectorKey);
+		selMap?.delete(selectorKey);
+		if (!ids || ids.size === 0) return;
+		const referenceMap = this.idRefCounts.get(collectionName);
+		const autoload = this.autoloadIds.get(collectionName);
+		const toRemove = [];
+		for (const id of ids) {
+			const current = referenceMap?.get(id) ?? 0;
+			const next = Math.max(0, current - 1);
+			if (next === 0) {
+				referenceMap?.delete(id);
+				if (autoload?.has(id)) toRemove.push(id);
+			} else referenceMap?.set(id, next);
+		}
+		if (toRemove.length === 0) return;
+		const storage = this.storageAdapters.get(collectionName);
+		if (!storage) throw new Error(`No persistence adapter for collection ${collectionName}`);
+		const items = await storage.readIds(toRemove);
+		if (items.length > 0) {
+			await storage.remove(items);
+			await this.checkQueryUpdates(collectionName, items);
+		}
+		toRemove.forEach((id) => autoload?.delete(id));
+	}
+	async setupStorage(collectionName, indices) {
+		const storage = this.storageAdapters.get(collectionName);
+		if (!storage) throw new Error(`No persistence adapter for collection ${collectionName}`);
+		await storage.setup();
+		await Promise.all(indices.map((field) => storage.createIndex(field)));
+	}
+	ensureStorageAdapter(name) {
+		if (this.storageAdapters.has(name)) return;
+		const adapter = this.options.storage && this.options.storage(name);
+		if (!adapter) return;
+		this.storageAdapters.set(name, adapter);
+	}
+	publishForSelector(collectionName, selector, state, error) {
+		const registry = this.queries.get(collectionName);
+		if (!registry) return;
+		for (const query of registry.values()) {
+			if (!isEqual(query.selector, selector)) continue;
+			this.publishState(collectionName, queryId(query.selector, query.options), state, error);
+		}
+	}
+	publishState(collectionName, qid, state, error) {
+		const query = this.queries.get(collectionName)?.get(qid);
+		if (!query) return;
+		query.state = state;
+		query.error = error;
+		for (const callback of query.listeners) try {
+			callback(state);
+		} catch (error_) {
+			this.onError(error_);
+		}
+	}
+	publishResult(collectionName, qid, items) {
+		const query = this.queries.get(collectionName)?.get(qid);
+		if (!query) return;
+		query.items = items;
+		this.queries.get(collectionName)?.set(qid, query);
+	}
+	async fulfillQuery(collectionName, selector, options) {
+		const qid = queryId(selector, options);
+		const registry = this.queries.get(collectionName);
+		if (!registry) throw new Error(`Collection ${collectionName} not initialized!`);
+		if (!registry.get(qid)) return;
+		this.publishState(collectionName, qid, "active", null);
+		try {
+			const items = await this.executeQuery(collectionName, selector, options);
+			this.publishResult(collectionName, qid, items);
+			this.publishState(collectionName, qid, "complete", null);
+		} catch (error) {
+			this.publishState(collectionName, qid, "error", error);
+		}
+	}
+	async getIndexInfo(collectionName, selector) {
+		const storage = this.storageAdapters.get(collectionName);
+		if (!storage) throw new Error(`No persistence adapter for collection ${collectionName}`);
+		if (selector != null && Object.keys(selector).length === 1 && "id" in selector) {
+			const idResult = idIndexQuery(selector);
+			if (idResult.matched) return {
+				matched: true,
+				ids: idResult.ids,
+				optimizedSelector: {}
+			};
+		}
+		if (selector == null) return {
+			matched: false,
+			ids: [],
+			optimizedSelector: {}
+		};
+		return getIndexInfo((this.collectionIndices.get(collectionName) ?? []).map((field) => storageIndexQuery(storage, field)), selector);
+	}
+	async queryItems(collectionName, selector) {
+		const storage = this.storageAdapters.get(collectionName);
+		if (!storage) throw new Error(`No persistence adapter for collection ${collectionName}`);
+		const index = await this.getIndexInfo(collectionName, selector);
+		const matchItems = (item) => {
+			if (index.optimizedSelector == null) return true;
+			if (Object.keys(index.optimizedSelector).length <= 0) return true;
+			return match(item, index.optimizedSelector);
+		};
+		if (index.matched) {
+			const items = await storage.readIds(index.ids);
+			if (isEqual(index.optimizedSelector, {})) return items;
+			return items.filter(matchItems);
+		} else {
+			const allItems = await storage.readAll();
+			if (isEqual(selector, {})) return allItems;
+			return allItems.filter(matchItems);
+		}
+	}
+	async executeQuery(collectionName, selector, options) {
+		const items = await this.queryItems(collectionName, selector || {});
+		const { sort, skip, limit, fields } = options || {};
+		const sorted = sort ? sortItems(items, sort) : items;
+		const skipped = skip ? sorted.slice(skip) : sorted;
+		const limited = limit ? skipped.slice(0, limit) : skipped;
+		const idExcluded = fields && fields.id === 0;
+		return limited.map((item) => {
+			if (!fields) return item;
+			return {
+				...idExcluded ? {} : { id: item.id },
+				...project(item, fields)
+			};
+		});
+	}
+	async checkQueryUpdates(collectionName, changedItems) {
+		const registry = this.queries.get(collectionName);
+		if (!registry) throw new Error(`Collection ${collectionName} not initialized!`);
+		if (registry.size === 0) return;
+		const affected = [...registry.values()].filter(({ selector }) => changedItems.some((item) => match(item, selector)));
+		if (affected.length === 0) return;
+		for (const { selector, options } of affected) {
+			const qid = queryId(selector, options);
+			this.publishState(collectionName, qid, "active", null);
+		}
+		await Promise.all(affected.map(async ({ selector, options }) => {
+			const qid = queryId(selector, options);
+			try {
+				const items = await this.executeQuery(collectionName, selector, options);
+				this.publishResult(collectionName, qid, items);
+				this.publishState(collectionName, qid, "complete", null);
+			} catch (error) {
+				this.publishState(collectionName, qid, "error", error);
+			}
+		}));
+	}
+	async insert(collectionName, newItem) {
+		const storage = this.storageAdapters.get(collectionName);
+		if (!storage) throw new Error(`No persistence adapter for collection ${collectionName}`);
+		if ((await this.executeQuery(collectionName, { id: newItem.id }, { limit: 1 })).length > 0) throw new Error(`Item with id ${String(newItem.id)} already exists`);
+		await storage.insert([newItem]);
+		await this.checkQueryUpdates(collectionName, [newItem]);
+		return newItem;
+	}
+	async updateOne(collectionName, selector, modifier) {
+		const storage = this.storageAdapters.get(collectionName);
+		if (!storage) throw new Error(`No persistence adapter for collection ${collectionName}`);
+		const [item] = await this.executeQuery(collectionName, selector, { limit: 1 });
+		const { $setOnInsert, ...rest } = modifier;
+		if (item == null) return [];
+		const modified = modify(deepClone(item), rest);
+		if (item.id !== modified.id) {
+			if ((await this.executeQuery(collectionName, { id: modified.id }, { limit: 1 })).length > 0) throw new Error(`Item with id ${String(modified.id)} already exists`);
+		}
+		await storage.replace([modified]);
+		await this.checkQueryUpdates(collectionName, [item, modified]);
+		return [modified];
+	}
+	async updateMany(collectionName, selector, modifier) {
+		const storage = this.storageAdapters.get(collectionName);
+		if (!storage) throw new Error(`No persistence adapter for collection ${collectionName}`);
+		const items = await this.executeQuery(collectionName, selector);
+		if (items.length === 0) return [];
+		const { $setOnInsert, ...rest } = modifier;
+		const changed = await Promise.all(items.map(async (item) => {
+			const modified = modify(deepClone(item), rest);
+			if (item.id !== modified.id) {
+				if ((await this.executeQuery(collectionName, { id: modified.id }, { limit: 1 })).length > 0) throw new Error(`Item with id ${String(modified.id)} already exists`);
+			}
+			return modified;
+		}));
+		await storage.replace(changed);
+		await this.checkQueryUpdates(collectionName, [...items, ...changed]);
+		return changed;
+	}
+	async replaceOne(collectionName, selector, replacement) {
+		const storage = this.storageAdapters.get(collectionName);
+		if (!storage) throw new Error(`No persistence adapter for collection ${collectionName}`);
+		const [item] = await this.executeQuery(collectionName, selector, { limit: 1 });
+		if (item == null) return [];
+		const modified = {
+			...replacement,
+			id: replacement.id ?? item.id
+		};
+		if (item.id !== modified.id) {
+			if ((await this.executeQuery(collectionName, { id: modified.id }, { limit: 1 })).length > 0) throw new Error(`Item with id ${String(modified.id)} already exists`);
+		}
+		await storage.replace([modified]);
+		await this.checkQueryUpdates(collectionName, [item, modified]);
+		return [modified];
+	}
+	async removeOne(collectionName, selector) {
+		const storage = this.storageAdapters.get(collectionName);
+		if (!storage) throw new Error(`No persistence adapter for collection ${collectionName}`);
+		const [item] = await this.executeQuery(collectionName, selector, { limit: 1 });
+		if (item == null) return [];
+		await storage.remove([item]);
+		await this.checkQueryUpdates(collectionName, [item]);
+		return [item];
+	}
+	async removeMany(collectionName, selector) {
+		const storage = this.storageAdapters.get(collectionName);
+		if (!storage) throw new Error(`No persistence adapter for collection ${collectionName}`);
+		const items = await this.executeQuery(collectionName, selector);
+		if (items.length === 0) return [];
+		await storage.remove(items);
+		await this.checkQueryUpdates(collectionName, items);
+		return items;
+	}
+	async upsertMerged(collectionName, incoming) {
+		if (incoming.length === 0) return;
+		const storage = this.storageAdapters.get(collectionName);
+		if (!storage) throw new Error(`No persistence adapter for collection ${collectionName}`);
+		const ids = incoming.map((i) => i.id);
+		const existing = await storage.readIds(ids);
+		const existingById = new Map(existing.map((it) => [it.id, it]));
+		const toInsert = [];
+		const toReplace = [];
+		const referenceMap = this.idRefCounts.get(collectionName);
+		const autoload = this.autoloadIds.get(collectionName);
+		for (const item of incoming) {
+			const previous = existingById.get(item.id);
+			if (previous) toReplace.push(this.mergeItems(previous, item));
+			else toInsert.push(item);
+			autoload?.add(item.id);
+			referenceMap?.set(item.id, (referenceMap?.get(item.id) ?? 0) + 1);
+		}
+		if (toInsert.length > 0) await storage.insert(toInsert);
+		if (toReplace.length > 0) await storage.replace(toReplace);
+	}
+};
+/**
+* Usage (example):
+*
+* const adapter = new AutoFetchDataAdapter({
+*   storage: (name) => new IndexedDBStorage(name),
+*   fetchQueryItems: async (collectionName, selector) => {
+*     const res = await fetch(`/api/${collectionName}?q=${encodeURIComponent(JSON.stringify(selector||{}))}`)
+*     const items = await res.json()
+*     return { items }
+*   },
+*   registerRemoteChange: (onChange) => subscribeToWS(onChange),
+*   mergeItems: (a, b) => ({ ...a, ...b }),
+*   purgeDelay: 10_000,
+* })
+*
+* const backend = adapter.createCollectionBackend(myCollection, ['status', 'projectId'])
+*/
+//#endregion
+export { AutoFetchDataAdapter as default };

package/dist/types/StorageAdapter.d.ts

@@ -12,7 +12,15 @@ export default interface StorageAdapter<T extends {
     readIds(positions: I[]): Promise<T[]>;
     createIndex(field: string): Promise<void>;
     dropIndex(field: string): Promise<void>;
-    readIndex(field: string): Promise<Map<any, Set<I>>>;
+    /**
+     * The index, keyed by `serializeValue(value)` — not by the raw field value.
+     *
+     * SignalDB looks an index up with the serialized form, because that is what
+     * makes `3`, `'3'` and `new Date(...)` comparable as map keys at all. An
+     * adapter that stores its backend's own keys instead answers nothing for
+     * every non-string field, and everything for a `$ne` on one.
+     */
+    readIndex(field: string): Promise<Map<string | null, Set<I>>>;
     insert(items: T[]): Promise<void>;
     replace(items: T[]): Promise<void>;
     remove(items: T[]): Promise<void>;

package/dist/utils/idIndexQuery.d.ts

@@ -0,0 +1,20 @@
+import { BaseItem } from '../Collection/types';
+import { IndexResult } from '../types/IndexProvider';
+import { FlatSelector } from '../types/Selector';
+/**
+ * Resolves a selector on `id` into the ids it names, without consulting an index.
+ *
+ * `id` is the one field every storage adapter can look up directly — that is what
+ * `readIds` is — so a query on it never needs an index to be declared and never
+ * needs the whole collection to be read. This behaves like an index provider that
+ * happens to need no stored index, because the ids are already in the selector.
+ *
+ * Only inclusive forms can be answered this way. `$ne`/`$nin` describe everything
+ * except* something, which cannot be enumerated without knowing every id, so they
+ * report no match and take the ordinary path.
+ * @template T - The type of the items in the collection.
+ * @template I - The type of the unique identifier for the items.
+ * @param selector - The flat selector to resolve.
+ * @returns An index result naming the matched ids, or `{ matched: false }`.
+ */
+export default function idIndexQuery<T extends BaseItem<I> = BaseItem, I = any>(selector: FlatSelector<T>): IndexResult<I>;

package/dist/utils/storageIndexQuery.d.ts

@@ -0,0 +1,20 @@
+import { BaseItem } from '../Collection/types';
+import { default as StorageAdapter } from '../types/StorageAdapter';
+import { AsynchronousQueryFunction } from '../types/IndexProvider';
+/**
+ * Builds the index provider a data adapter uses to narrow a selector down through
+ * a storage adapter's index.
+ *
+ * Every adapter that keeps its data in a `StorageAdapter` needs exactly this, and
+ * each of them used to carry its own copy — three transcriptions of one set of
+ * rules about null, `$exists`, inclusion and exclusion, which is how they drift
+ * apart without anyone noticing. The index is keyed by `serializeValue(value)`,
+ * which is what `getMatchingKeys` produces, so the two only agree while they stay
+ * in one place.
+ * @template T - The type of the items in the collection.
+ * @template I - The type of the unique identifier for the items.
+ * @param storage - The storage adapter holding the index.
+ * @param field - The indexed field this provider answers for.
+ * @returns A query function for `getIndexInfo`.
+ */
+export default function storageIndexQuery<T extends BaseItem<I>, I = any>(storage: Pick<StorageAdapter<T, I>, 'readIndex'>, field: string): AsynchronousQueryFunction<T, I>;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@signaldb/core",
-  "version": "2.0.0-beta.17",
+  "version": "2.0.0-beta.19",
   "description": "SignalDB is a client-side database that provides a simple MongoDB-like interface to the data with first-class typescript support to achieve an optimistic UI. Data persistence can be achieved by using storage providers that store the data through a JSON interface to places such as localStorage.",
   "scripts": {
     "build": "rimraf dist && vite build",