@signaldb/svelte 2.0.0-beta.13 → 2.0.0-beta.15

package/dist/base/core/src/Collection/index.js

@@ -29,9 +29,32 @@ class Collection extends EventEmitter_1.default {
     static fieldTracking = false;
     static onCreationCallbacks = [];
     static onDisposeCallbacks = [];
+    // How many rows a live query may hold before it is reported as expensive.
+    // `null` disables the check, which is the default: it costs a stack capture
+    // per registered query, which is worth paying while developing and not in
+    // production. `enableDebugMode()` turns it on.
+    static largeQueryWarningThreshold = null;
+    static reportedLargeQueries = new Set();
     static getCollections() {
         return Collection.collections;
     }
+    /**
+     * Reports live queries whose result is larger than `rows`, once each, with
+     * the stack that registered them.
+     *
+     * A reactive query is re-evaluated whenever the data under it changes, and
+     * one registered from a long-lived place — a navigation bar, a provider
+     * near the root — keeps that cost for the lifetime of the application. There
+     * is otherwise nothing to see: the query works, and its price is only
+     * visible as an application that has grown slow. Finding one such query in a
+     * real app took a purpose-built profiler and the better part of a day.
+     * @param rows - Result size to report above, or `null` to switch the check off.
+     */
+    static reportLargeQueries(rows) {
+        Collection.largeQueryWarningThreshold = rows;
+        if (rows == null)
+            Collection.reportedLargeQueries.clear();
+    }
     static onCreation(callback) {
         Collection.onCreationCallbacks.push(callback);
     }
@@ -43,6 +66,12 @@ class Collection extends EventEmitter_1.default {
      */
     static enableDebugMode = () => {
         Collection.debugMode = true;
+        // A query large enough to matter is exactly the kind of thing debug mode
+        // exists to surface, and it is invisible otherwise. Call
+        // `reportLargeQueries()` afterwards to pick a different threshold or turn
+        // it off again.
+        if (Collection.largeQueryWarningThreshold == null)
+            Collection.reportLargeQueries(500);
         Collection.collections.forEach((collection) => {
             collection.setDebugMode(true);
         });
@@ -57,13 +86,23 @@ class Collection extends EventEmitter_1.default {
             collection.setFieldTracking(enable);
         });
     };
-    static batch(callback) {
-        Collection.batchOperationInProgress = true;
-        const execute = () => Collection.collections.reduce((memo, collection) => () => {
+    static batch(collectionsOrCallback, maybeCallback) {
+        const scoped = Array.isArray(collectionsOrCallback);
+        const callback = (scoped ? maybeCallback : collectionsOrCallback);
+        if (typeof callback !== 'function')
+            throw new TypeError('Collection.batch requires a callback');
+        const collections = scoped ? collectionsOrCallback : Collection.collections;
+        // Only a batch that really covers every collection may claim the global
+        // flag; a scoped one must not make unrelated collections report themselves
+        // as batching through `isBatchOperationInProgress()`.
+        if (!scoped)
+            Collection.batchOperationInProgress = true;
+        const execute = () => collections.reduce((memo, collection) => () => {
             return collection.batch(memo);
         }, callback)();
         const afterBatch = () => {
-            Collection.batchOperationInProgress = false;
+            if (!scoped)
+                Collection.batchOperationInProgress = false;
         };
         let maybePromise;
         try {
@@ -141,6 +180,34 @@ class Collection extends EventEmitter_1.default {
             .catch(() => { });
         Collection.onCreationCallbacks.forEach(callback => callback(this));
     }
+    /**
+     * Reports a live query the first time its result is found to be larger than
+     * the configured threshold. Once per query, because it re-runs on every
+     * write and a warning per write would be its own performance problem.
+     * @param selector - The query's selector.
+     * @param options - The query's options.
+     * @param registrationStack - Where the query was registered, if captured.
+     */
+    reportIfLargeQuery(selector, options, registrationStack) {
+        const threshold = Collection.largeQueryWarningThreshold;
+        if (threshold == null)
+            return;
+        const id = `${this.name}:${(0, queryId_1.default)(selector, options)}`;
+        if (Collection.reportedLargeQueries.has(id))
+            return;
+        const rows = this.backend.getQueryResult(selector, options || {}).length;
+        if (rows <= threshold)
+            return;
+        Collection.reportedLargeQueries.add(id);
+        // The selector's *keys*, never its values: the shape is what identifies the
+        // problem — an empty one means the query holds the whole collection — and
+        // the values would put user data into a log.
+        const keys = selector && typeof selector === 'object' ? Object.keys(selector) : [];
+        // eslint-disable-next-line no-console
+        console.warn(`[SignalDB] Live query on "${this.name}" holds ${rows} rows `
+            + `with selector {${keys.join(', ')}}. It is re-evaluated on every write to this `
+            + `collection, for as long as it stays registered. ${registrationStack ?? ''}`);
+    }
     isBatchOperationInProgress() {
         return Collection.batchOperationInProgress || this.batchOperationInProgress;
     }
@@ -338,7 +405,7 @@ class Collection extends EventEmitter_1.default {
                     callback();
                 }),
             },
-            bindEvents: (requery) => {
+            bindEvents: (requery, applyDelta) => {
                 const handleRequery = () => {
                     if (this.batchOperationInProgress) {
                         this.postBatchCallbacks.add(requery);
@@ -346,13 +413,28 @@ class Collection extends EventEmitter_1.default {
                     }
                     requery();
                 };
+                // A `transformAll` sits between the backend's result and what the cursor holds, and it
+                // is free to produce anything at all — so a delta describing the backend's result says
+                // nothing about the cursor's. Those collections keep comparing.
+                const canApplyDeltas = !this.options.transformAll && !options?.async;
                 // register query if not yet registered
                 const listeners = this.queryListeners({ selector, options });
                 const didRegister = listeners === 0;
                 if (didRegister)
                     this.backend.registerQuery(selector, options || {});
                 this.queryListeners({ selector, options }, listeners + 1);
-                const queryStateChangeCleanup = this.backend.onQueryStateChange(selector, options || {}, (state) => {
+                // Captured at registration, not at completion: by the time the result
+                // arrives the stack is the adapter's, and the only useful thing to
+                // report is where the query was asked for.
+                const registrationStack = didRegister && Collection.largeQueryWarningThreshold != null
+                    ? new Error('query registered here').stack
+                    : undefined;
+                // A synchronous adapter already holds the result here and never
+                // reports `'complete'`, so the check has to happen at both points.
+                // Reporting is once per query, which makes the overlap harmless.
+                if (didRegister)
+                    this.reportIfLargeQuery(selector, options, registrationStack);
+                const queryStateChangeCleanup = this.backend.onQueryStateChange(selector, options || {}, (state, delta) => {
                     // A failed query never reaches `'complete'`, so the cursor keeps
                     // serving its neutral empty value — indistinguishable from "no
                     // data" for anyone reading it. Surfacing the failure as an event
@@ -367,6 +449,14 @@ class Collection extends EventEmitter_1.default {
                     }
                     if (state !== 'complete')
                         return;
+                    this.reportIfLargeQuery(selector, options, registrationStack);
+                    // Inside a batch the update is deferred to the end of it, by which point this delta
+                    // is one of several and no longer describes the whole change — so the batch always
+                    // ends in a comparison.
+                    if (delta != null && canApplyDeltas && !this.batchOperationInProgress) {
+                        applyDelta(delta);
+                        return;
+                    }
                     handleRequery();
                 });
                 this.emit('observer.created', selector, options);
@@ -516,26 +606,30 @@ class Collection extends EventEmitter_1.default {
         if (!modifier)
             throw new Error('Invalid modifier');
         const { $setOnInsert, ...restModifier } = modifier;
-        const item = await this.getItem(selector, { async: true });
-        if (item == null) {
-            if (options?.upsert) {
-                // if upsert is enabled, insert a new item
-                const newItem = (0, modify_1.default)({}, {
-                    ...restModifier,
-                    $set: {
-                        ...$setOnInsert,
-                        ...restModifier.$set,
-                    },
-                });
-                await this.insert(newItem);
-                return 1;
-            }
-            return 0; // no item found, and upsert is not enabled
+        // Reading the item back before writing it is worth a round trip to the data layer only when
+        // something is waiting to inspect it: a validator gets to refuse the write, and it can only do
+        // that beforehand. Otherwise the backend's own answer says everything there is to know — what
+        // it returns is what changed, and an empty answer is what turns an upsert into an insert.
+        if (this.listenerCount('validate') > 0) {
+            const item = await this.getItem(selector, { async: true });
+            if (item != null)
+                this.emit('validate', (0, modify_1.default)((0, deepClone_1.default)(item), restModifier));
         }
-        const modifiedItem = (0, modify_1.default)((0, deepClone_1.default)(item), restModifier);
-        this.emit('validate', modifiedItem);
         const changes = await this.withPushState(() => this.backend.updateOne(selector, modifier));
-        this.emit('changed', modifiedItem, restModifier);
+        if (changes.length === 0) {
+            if (!options?.upsert)
+                return 0; // no item found, and upsert is not enabled
+            const newItem = (0, modify_1.default)({}, {
+                ...restModifier,
+                $set: {
+                    ...$setOnInsert,
+                    ...restModifier.$set,
+                },
+            });
+            await this.insert(newItem);
+            return 1;
+        }
+        changes.forEach(item => this.emit('changed', item, restModifier));
         this.emit('updateOne', selector, modifier);
         this.executeInDebugMode(callstack => this.emit('_debug.updateOne', callstack, selector, modifier));
         return changes.length;
@@ -557,27 +651,27 @@ class Collection extends EventEmitter_1.default {
         if (!modifier)
             throw new Error('Invalid modifier');
         const { $setOnInsert, ...restModifier } = modifier;
-        const items = await this.getItems(selector, { async: true });
-        if (items.length === 0) {
-            if (options?.upsert) {
-                // if upsert is enabled, insert a new item
-                const newItem = (0, modify_1.default)({}, {
-                    ...restModifier,
-                    $set: {
-                        ...$setOnInsert,
-                        ...restModifier.$set,
-                    },
-                });
-                await this.insert(newItem);
-                return 1;
-            }
-            return 0; // no items found, and upsert is not enabled
+        // See `updateOne`: the items are only fetched up front for the sake of a validator.
+        if (this.listenerCount('validate') > 0) {
+            const items = await this.getItems(selector, { async: true });
+            items.forEach((item) => {
+                this.emit('validate', (0, modify_1.default)((0, deepClone_1.default)(item), restModifier));
+            });
         }
-        items.forEach((item) => {
-            const modifiedItem = (0, modify_1.default)((0, deepClone_1.default)(item), restModifier);
-            this.emit('validate', modifiedItem);
-        });
         const changes = await this.withPushState(() => this.backend.updateMany(selector, modifier));
+        if (changes.length === 0) {
+            if (!options?.upsert)
+                return 0; // no items found, and upsert is not enabled
+            const newItem = (0, modify_1.default)({}, {
+                ...restModifier,
+                $set: {
+                    ...$setOnInsert,
+                    ...restModifier.$set,
+                },
+            });
+            await this.insert(newItem);
+            return 1;
+        }
         changes.forEach((item) => {
             this.emit('changed', item, restModifier);
         });
@@ -599,18 +693,20 @@ class Collection extends EventEmitter_1.default {
             throw new Error('Collection is disposed');
         if (!selector)
             throw new Error('Invalid selector');
-        const item = await this.getItem(selector, { async: true });
-        if (item == null) {
-            if (options?.upsert) {
-                await this.insert(replacement);
-                return 1;
-            }
-            return 0; // no item found, and upsert is not enabled
+        // See `updateOne`: the item is only fetched up front for the sake of a validator.
+        if (this.listenerCount('validate') > 0) {
+            const item = await this.getItem(selector, { async: true });
+            if (item != null)
+                this.emit('validate', { id: item.id, ...replacement });
         }
-        const modifiedItem = { id: item.id, ...replacement };
-        this.emit('validate', modifiedItem);
         const changes = await this.withPushState(() => this.backend.replaceOne(selector, replacement));
-        this.emit('changed', modifiedItem, replacement);
+        if (changes.length === 0) {
+            if (!options?.upsert)
+                return 0; // no item found, and upsert is not enabled
+            await this.insert(replacement);
+            return 1;
+        }
+        changes.forEach(item => this.emit('changed', item, replacement));
         this.emit('replaceOne', selector, replacement);
         this.executeInDebugMode(callstack => this.emit('_debug.replaceOne', callstack, selector, replacement));
         return changes.length;

package/dist/base/core/src/DataAdapter.d.ts

@@ -2,6 +2,7 @@ import type { BaseItem, FieldSpecifier, SortSpecifier } from './Collection';
 import type Collection from './Collection';
 import type Modifier from './types/Modifier';
 import type Selector from './types/Selector';
+import type { QueryDelta } from './utils/queryDelta';
 export interface QueryOptions<T extends BaseItem> {
     /** Sort order (default: natural order) */
     sort?: SortSpecifier<T> | undefined;
@@ -12,7 +13,19 @@ export interface QueryOptions<T extends BaseItem> {
     /** Dictionary of fields to return or exclude. */
     fields?: FieldSpecifier<T> | undefined;
 }
-export type StateChangeCallback = (state: 'active' | 'complete' | 'error') => void;
+/**
+ * Notified when a query's state changes.
+ *
+ * A `'complete'` notification may carry a delta describing how the result changed since the last
+ * one. An adapter that can produce one saves its listeners from rediscovering the change by
+ * comparing the whole result against the whole previous result; one that cannot simply omits it,
+ * and its listeners fall back to exactly that comparison.
+ *
+ * A delta is only ever passed when it is relative to what `getQueryResult` returned the last time
+ * it was asked. An adapter that layers anything on top of its stored result — an optimistic write
+ * still in flight, for instance — must omit the delta for as long as it does.
+ */
+export type StateChangeCallback<T extends BaseItem = BaseItem> = (state: 'active' | 'complete' | 'error', delta?: QueryDelta<T>) => void;
 export interface CollectionBackend<T extends BaseItem<I>, I> {
     insert(item: T): Promise<T>;
     updateOne(selector: Selector<T>, modifier: Modifier<T>): Promise<T[]>;
@@ -32,7 +45,7 @@ export interface CollectionBackend<T extends BaseItem<I>, I> {
     getQueryError<O extends QueryOptions<T>>(selector: Selector<T>, options: O): Error | null;
     getQueryResult<O extends QueryOptions<T>>(selector: Selector<T>, options: O): T[];
     executeQuery<O extends QueryOptions<T>>(selector: Selector<T>, options: O): Promise<T[]>;
-    onQueryStateChange<O extends QueryOptions<T>>(selector: Selector<T>, options: O, callback: StateChangeCallback): () => void;
+    onQueryStateChange<O extends QueryOptions<T>>(selector: Selector<T>, options: O, callback: StateChangeCallback<T>): () => void;
     dispose(): Promise<void>;
     isReady(): Promise<void>;
 }

package/dist/base/core/src/DefaultDataAdapter.js

@@ -10,7 +10,9 @@ const EventEmitter_1 = __importDefault(require("./utils/EventEmitter"));
 const isEqual_1 = __importDefault(require("./utils/isEqual"));
 const match_1 = __importDefault(require("./utils/match"));
 const modify_1 = __importDefault(require("./utils/modify"));
-const project_1 = __importDefault(require("./utils/project"));
+const projectItems_1 = __importDefault(require("./utils/projectItems"));
+const incrementalQueryUpdate_1 = __importDefault(require("./utils/incrementalQueryUpdate"));
+const queryDelta_1 = require("./utils/queryDelta");
 const queryId_1 = __importDefault(require("./utils/queryId"));
 const serializeValue_1 = __importDefault(require("./utils/serializeValue"));
 const sortItems_1 = __importDefault(require("./utils/sortItems"));
@@ -142,15 +144,7 @@ class DefaultDataAdapter {
         const sorted = sort ? (0, sortItems_1.default)(items, sort) : items;
         const skipped = skip ? sorted.slice(skip) : sorted;
         const limited = limit ? skipped.slice(0, limit) : skipped;
-        const idExcluded = fields && fields.id === 0;
-        return limited.map((item) => {
-            if (!fields)
-                return item;
-            return {
-                ...idExcluded ? {} : { id: item.id },
-                ...(0, project_1.default)(item, fields),
-            };
-        });
+        return (0, projectItems_1.default)(limited, fields);
     }
     flushQueuedQueryUpdates(collection) {
         if (!this.queuedQueryUpdates.get(collection.name))
@@ -159,6 +153,10 @@ class DefaultDataAdapter {
         if (!changes || !hasPendingUpdates(changes))
             return;
         this.queuedQueryUpdates.set(collection.name, { added: [], modified: [], removed: [] });
+        const changeset = {
+            upserts: [...changes.added, ...changes.modified],
+            deletes: changes.removed.map(item => item.id),
+        };
         const flatItems = [...changes.added, ...changes.modified, ...changes.removed];
         const itemIds = new Set(flatItems.map(i => i.id));
         const queries = [
@@ -173,17 +171,27 @@ class DefaultDataAdapter {
             return flatItems.some(item => (0, match_1.default)(item, selector));
         });
         queries.forEach(({ selector, options }) => {
-            this.executeAndCacheQuery(collection, selector, options);
+            this.executeAndCacheQuery(collection, selector, options, changeset);
         });
     }
-    executeAndCacheQuery(collection, selector, options) {
-        const result = this.executeQuery(collection, selector, options);
+    executeAndCacheQuery(collection, selector, options, changes) {
+        const cached = this.cachedQueryResults.get(collection.name)?.get((0, queryId_1.default)(selector, options));
+        // With the change in hand and the previous result cached, the query can be brought up to date
+        // without walking every item this collection holds — and the listeners can be told what
+        // changed instead of being left to compare the two results themselves.
+        const incremental = changes && cached
+            ? (0, incrementalQueryUpdate_1.default)(cached, selector, options, changes)
+            : null;
+        const result = incremental ?? this.executeQuery(collection, selector, options);
+        const delta = cached ? (0, queryDelta_1.diffQueryResults)(cached, result) : undefined;
         this.cachedQueryResults.set(collection.name, this.cachedQueryResults.get(collection.name) || new Map());
         this.cachedQueryResults.get(collection.name)?.set((0, queryId_1.default)(selector, options), result);
         const emitter = this.queryEmitters.get(collection.name);
         if (!emitter)
             return;
-        emitter.emit('change', selector, options, 'complete');
+        if (delta && (0, queryDelta_1.isEmptyQueryDelta)(delta))
+            return;
+        emitter.emit('change', selector, options, 'complete', delta);
     }
     updateQueries(collection, changes) {
         this.queuedQueryUpdates.set(collection.name, this.queuedQueryUpdates.get(collection.name) || { added: [], modified: [], removed: [] });
@@ -336,10 +344,10 @@ class DefaultDataAdapter {
                 const emitter = this.queryEmitters.get(collection.name);
                 if (!emitter)
                     throw new Error(`Query emitter not found for collection ${collection.name}`);
-                const handler = (querySelector, queryOptions, state) => {
+                const handler = (querySelector, queryOptions, state, delta) => {
                     if ((0, queryId_1.default)(querySelector, queryOptions) !== (0, queryId_1.default)(selector, options))
                         return;
-                    callback(state);
+                    (0, queryDelta_1.callWithDelta)(callback, state, delta);
                 };
                 emitter.on('change', handler);
                 return () => {

package/dist/base/core/src/WorkerDataAdapter.d.ts

@@ -22,12 +22,33 @@ export default class WorkerDataAdapter implements DataAdapter {
     private collectionReady;
     private batchExecutionHelpers;
     private queries;
+    private pendingRequests;
     private pendingWrites;
     private pendingWriteSeq;
+    private pendingWriteVersions;
+    private bumpPendingWriteVersion;
     constructor(worker: WorkerDataAdapterEndpoint, options: WorkerDataAdapterOptions);
+    private resolveWorkerReady;
+    private handleWorkerMessage;
+    private handleQueryUpdate;
     private exec;
+    /**
+     * Issues a call whose result nobody is waiting for, and makes sure a failure has somewhere to
+     * go. A bare rejection here would surface as an uncaught error — which is what a disposed
+     * collection produced every time a cursor was cleaned up after it.
+     * @param method - The method to call on the worker.
+     * @param collectionName - The collection it applies to.
+     * @param args - The remaining arguments.
+     * @param onError - Called when the call fails, in place of merely logging it.
+     */
+    private execInBackground;
+    private queryItemsById;
+    private flattenPendingWrites;
+    private static providesFullItems;
     private observableItems;
-    private mergePendingWrites;
+    private observableItemsByIds;
+    private servedResult;
+    private computeServedResult;
     /**
      * Registers a write's effect locally and notifies every active query it
      * touches, then returns a function that drops it again once the write
@@ -38,7 +59,9 @@ export default class WorkerDataAdapter implements DataAdapter {
      * @returns A function that drops the pending write and re-notifies.
      */
     private applyPendingWrite;
-    private notifyAffectedQueries;
+    private affectedQueries;
+    private servedResults;
+    private notifyWithDeltas;
     private matchObservableItems;
     private resolveUpdate;
     private resolveRemoval;