@minnowdb/core 0.7.7 → 0.7.10

package/dist/engine/live.js

@@ -4,52 +4,8 @@ const DEFAULT_LIVE_QUERY_MAX_SUBSCRIPTIONS = 1024;
 const MAX_LIVE_QUERY_GROUPS = 4096;
 const MAX_LIVE_QUERY_SUBSCRIPTIONS = 16384;
 const MAX_LIVE_QUERY_SETS_PER_DATABASE = 256;
+const LIVE_QUERY_EXECUTION_CONCURRENCY = 8;
 import { LiveQueryLimitError as LiveQueryLimitError2 } from "./errors.js";
-const digestScratch = new DataView(new ArrayBuffer(8));
-function digestResult(result) {
-  let hash = 2166136261;
-  const mixByte = (byte) => {
-    hash = Math.imul(hash ^ byte, 16777619) >>> 0;
-  };
-  const mixNumber = (value) => {
-    digestScratch.setFloat64(0, value);
-    for (let index = 0; index < 8; index += 1)
-      mixByte(digestScratch.getUint8(index));
-  };
-  const mixString = (value) => {
-    for (let index = 0; index < value.length; index += 1) {
-      const code = value.charCodeAt(index);
-      mixByte(code & 255);
-      mixByte(code >>> 8);
-    }
-    mixByte(255);
-  };
-  for (const column of result.columns)
-    mixString(column);
-  for (const domain of result.columnDomains)
-    mixString(JSON.stringify(domain));
-  for (const row of result.rows) {
-    for (const column of result.columns) {
-      const value = row[column] ?? null;
-      if (value === null)
-        mixByte(1);
-      else if (typeof value === "number") {
-        mixByte(2);
-        mixNumber(value);
-      } else if (typeof value === "string") {
-        mixByte(3);
-        mixString(value);
-      } else if (typeof value === "boolean")
-        mixByte(value ? 4 : 5);
-      else {
-        mixByte(6);
-        mixNumber(dateMilliseconds(value));
-      }
-    }
-    mixByte(254);
-  }
-  return hash;
-}
 function sameQueryValue(left, right) {
   if (left instanceof Date || right instanceof Date) {
     return left instanceof Date && right instanceof Date && Object.is(dateMilliseconds(left), dateMilliseconds(right));
@@ -67,12 +23,15 @@ function sameResult(left, right) {
       return false;
     }
   }
+  const columns = left.columns;
   for (let rowIndex = 0; rowIndex < left.rows.length; rowIndex += 1) {
     const leftRow = left.rows[rowIndex];
     const rightRow = right.rows[rowIndex];
     if (leftRow === void 0 || rightRow === void 0)
       return false;
-    for (const column of left.columns) {
+    if (leftRow === rightRow)
+      continue;
+    for (const column of columns) {
       if (!sameQueryValue(leftRow[column] ?? null, rightRow[column] ?? null))
         return false;
     }
@@ -148,6 +107,9 @@ function sameProbe(left, right) {
 function versionOrdinal(version) {
   return version ?? -1;
 }
+function newerProbe(left, right) {
+  return versionOrdinal(right.manifestVersion) > versionOrdinal(left.manifestVersion) ? right : left;
+}
 function boundedLiveLimit(value, maximum, label) {
   if (!Number.isSafeInteger(value) || value < 1 || value > maximum) {
     throw new RangeError(`Live query ${label} limit must be between 1 and ${String(maximum)}`);
@@ -157,6 +119,28 @@ function boundedLiveLimit(value, maximum, label) {
 function catalogChangedBetween(previous, current) {
   return versionOrdinal(current.manifestVersion) < versionOrdinal(previous.manifestVersion) || current.schemaEpoch !== previous.schemaEpoch;
 }
+function isResultSubscriber(subscriber) {
+  return subscriber.kind === "result" && !subscriber.closed;
+}
+function isObserver(subscriber) {
+  return subscriber.kind === "observer" && !subscriber.closed;
+}
+function groupExecutes(group) {
+  for (const subscriber of group.subscribers) {
+    if (subscriber.closed)
+      continue;
+    if (subscriber.kind === "result" || subscriber.options.suppressUnchanged === true)
+      return true;
+  }
+  return false;
+}
+function groupMemoizes(group) {
+  for (const subscriber of group.subscribers) {
+    if (!subscriber.closed && subscriber.kind === "observer")
+      return true;
+  }
+  return false;
+}
 class LiveQuerySet {
   #host;
   #channel;
@@ -166,8 +150,11 @@ class LiveQuerySet {
   #groups = /* @__PURE__ */ new Map();
   #opening = /* @__PURE__ */ new Map();
   #groupsByTable = /* @__PURE__ */ new Map();
+  #lagging = /* @__PURE__ */ new Set();
   #maxGroups;
   #maxSubscriptions;
+  #sharedResults;
+  #incremental;
   #stats = {
     hints: 0,
     versionChecks: 0,
@@ -177,10 +164,16 @@ class LiveQuerySet {
     zoneSkips: 0,
     notificationsSuppressed: 0,
     invalidations: 0,
+    maintained: 0,
+    groupsVisited: 0,
+    retainedRows: 0,
     sharedExecutions: 0,
     lastSweepMs: 0
   };
   #lastProbe;
+  #pendingProbe;
+  #executing = 0;
+  #executionWaiters = [];
   #sweepChain = Promise.resolve();
   #sweepQueued = false;
   #pollTimer;
@@ -194,6 +187,8 @@ class LiveQuerySet {
     }
     this.#maxGroups = boundedLiveLimit(options.maxGroups ?? DEFAULT_LIVE_QUERY_MAX_GROUPS, MAX_LIVE_QUERY_GROUPS, "group");
     this.#maxSubscriptions = boundedLiveLimit(options.maxSubscriptions ?? DEFAULT_LIVE_QUERY_MAX_SUBSCRIPTIONS, MAX_LIVE_QUERY_SUBSCRIPTIONS, "subscription");
+    this.#sharedResults = options.sharedResults === true;
+    this.#incremental = options.incremental !== false;
     this.#host = host;
     if (options.channel !== void 0) {
       this.#channel = options.channel;
@@ -214,7 +209,22 @@ class LiveQuerySet {
     }
   }
   get stats() {
-    return { ...this.#stats };
+    let retainedRows = 0;
+    for (const group of this.#groups.values())
+      retainedRows += group.result?.rows.length ?? 0;
+    return { ...this.#stats, retainedRows };
+  }
+  #freshProbe() {
+    let pending = this.#pendingProbe;
+    if (pending === void 0) {
+      pending = Promise.resolve().then(() => {
+        if (this.#pendingProbe === pending)
+          this.#pendingProbe = void 0;
+        return this.#host.currentProbe();
+      });
+      this.#pendingProbe = pending;
+    }
+    return pending;
   }
   #throwIfClosed() {
     if (this.#closed)
@@ -232,22 +242,25 @@ class LiveQuerySet {
     let group;
     let subscriber;
     try {
-      group = await this.#getOrOpenGroup(query);
+      const opened = await this.#getOrOpenGroup(query);
+      group = opened.group;
       this.#throwIfClosed();
       subscriber = {
         kind: "result",
         options,
         delivered: false,
-        closed: false
+        closed: false,
+        seenDelivery: -1
       };
       group.subscribers.add(subscriber);
-      await this.refresh();
+      await this.#settleGroup(group, opened.fresh, true);
       this.#throwIfClosed();
       if (!subscriber.delivered) {
         const result = group.result ?? (await this.#executeGroup(group)).result;
         this.#throwIfClosed();
-        if (!subscriber.closed)
-          this.#deliverResult(subscriber, result);
+        if (!subscriber.closed) {
+          this.#deliverResult(group, subscriber, result, { ...group.seenProbe, initial: true });
+        }
       }
     } catch (error) {
       if (group !== void 0 && subscriber !== void 0) {
@@ -267,7 +280,8 @@ class LiveQuerySet {
     let group;
     let subscriber;
     try {
-      group = await this.#getOrOpenGroup(query);
+      const opened = await this.#getOrOpenGroup(query);
+      group = opened.group;
       this.#throwIfClosed();
       subscriber = {
         kind: "observer",
@@ -276,7 +290,7 @@ class LiveQuerySet {
         closed: false
       };
       group.subscribers.add(subscriber);
-      await this.refresh();
+      await this.#settleGroup(group, opened.fresh, options.suppressUnchanged === true);
       this.#throwIfClosed();
       if (!subscriber.delivered) {
         this.#deliverInvalidation(subscriber, { ...group.seenProbe, initial: true });
@@ -293,6 +307,22 @@ class LiveQuerySet {
     }
     return this.#subscriptionHandle(group, subscriber);
   }
+  async #settleGroup(group, fresh, execute) {
+    if (execute && group.result === void 0) {
+      if (group.execution !== void 0) {
+        this.#stats.sharedExecutions += 1;
+        await group.execution;
+      } else {
+        await this.#executeGroup(group, {
+          probe: group.seenProbe,
+          memoize: groupMemoizes(group)
+        });
+      }
+      this.#throwIfClosed();
+    }
+    if (!fresh || this.#lagging.has(group))
+      await this.refresh();
+  }
   #subscriptionHandle(group, subscriber) {
     return {
       dependencyTableIds: [...group.dependencies],
@@ -317,6 +347,7 @@ class LiveQuerySet {
     if (this.#groups.get(group.key) !== group)
       return;
     this.#groups.delete(group.key);
+    this.#lagging.delete(group);
     this.#unindexGroup(group, group.dependencies);
   }
   async #getOrOpenGroup(query) {
@@ -324,12 +355,12 @@ class LiveQuerySet {
     const existing = this.#groups.get(key);
     if (existing !== void 0) {
       this.#stats.sharedExecutions += 1;
-      return existing;
+      return { group: existing, fresh: false };
     }
     const opening = this.#opening.get(key);
     if (opening !== void 0) {
       this.#stats.sharedExecutions += 1;
-      return opening;
+      return { group: await opening, fresh: false };
     }
     if (this.#groups.size + this.#opening.size >= this.#maxGroups) {
       throw new LiveQueryLimitError("group", this.#maxGroups);
@@ -337,7 +368,7 @@ class LiveQuerySet {
     const created = this.#openGroup(key, query);
     this.#opening.set(key, created);
     try {
-      return await created;
+      return { group: await created, fresh: true };
     } finally {
       this.#opening.delete(key);
     }
@@ -351,13 +382,17 @@ class LiveQuerySet {
       subscribers: /* @__PURE__ */ new Set(),
       seenProbe: after,
       result: void 0,
-      digest: void 0,
-      execution: void 0
+      execution: void 0,
+      deliveries: 0,
+      maintenance: void 0,
+      unmaintainable: false
     };
     this.#groups.set(key, group);
     this.#indexGroup(group, dependencies);
-    if (this.#lastProbe === void 0 || versionOrdinal(after.manifestVersion) < versionOrdinal(this.#lastProbe.manifestVersion)) {
+    if (this.#lastProbe === void 0)
       this.#lastProbe = after;
+    else if (versionOrdinal(after.manifestVersion) < versionOrdinal(this.#lastProbe.manifestVersion) || after.schemaEpoch !== this.#lastProbe.schemaEpoch) {
+      this.#lagging.add(group);
     }
     return group;
   }
@@ -388,12 +423,12 @@ class LiveQuerySet {
     this.#indexGroup(group, next);
   }
   async #stableDependencies(query) {
-    let before = await this.#host.currentProbe();
+    let before = await this.#freshProbe();
     this.#throwIfClosed();
     for (let attempt = 0; attempt < 8; attempt += 1) {
-      const dependencies = await this.#host.dependencyTableIds(query);
+      const dependencies = await this.#host.dependencyTableIds(query, before);
       this.#throwIfClosed();
-      const after = await this.#host.currentProbe();
+      const after = await this.#freshProbe();
       this.#throwIfClosed();
       if (!catalogChangedBetween(before, after))
         return { dependencies, probe: after };
@@ -401,20 +436,34 @@ class LiveQuerySet {
     }
     throw new Error("Catalog kept changing while live-query dependencies were resolved");
   }
-  async #executeGroup(group) {
+  async #executeGroup(group, context) {
     if (group.execution !== void 0) {
       this.#stats.sharedExecutions += 1;
       return group.execution;
     }
     const execution = (async () => {
-      const executed = await this.#host.execute(group.query);
-      const digest = digestResult(executed);
+      await this.#acquireExecutionSlot();
+      let executed;
+      let maintenance;
+      try {
+        if (this.#incremental && this.#host.executeMaintainable !== void 0 && !group.unmaintainable) {
+          const maintained = await this.#host.executeMaintainable(group.query, context);
+          if (maintained === void 0)
+            group.unmaintainable = true;
+          else {
+            executed = maintained.result;
+            maintenance = maintained.state;
+          }
+        }
+        executed ??= await this.#host.execute(group.query, context);
+      } finally {
+        this.#releaseExecutionSlot();
+      }
       const previous = group.result;
-      const changed = previous === void 0 || group.digest !== digest || !sameResult(previous, executed);
-      const retained = cloneResult(executed);
-      group.result = retained;
-      group.digest = digest;
-      return { result: retained, changed };
+      const changed = previous === void 0 || !sameResult(previous, executed);
+      group.result = executed;
+      group.maintenance = maintenance;
+      return { result: executed, changed };
     })();
     group.execution = execution;
     try {
@@ -424,11 +473,66 @@ class LiveQuerySet {
         group.execution = void 0;
     }
   }
-  #deliverResult(subscriber, result) {
+  async #acquireExecutionSlot() {
+    if (this.#executing < LIVE_QUERY_EXECUTION_CONCURRENCY) {
+      this.#executing += 1;
+      return;
+    }
+    await new Promise((resolve) => {
+      this.#executionWaiters.push(resolve);
+    });
+    this.#executing += 1;
+  }
+  #releaseExecutionSlot() {
+    this.#executing -= 1;
+    this.#executionWaiters.shift()?.();
+  }
+  async #maintainGroup(group, tableIds, after, until, probe) {
+    const host = this.#host;
+    if (host.maintain === void 0 || group.result === void 0 || group.maintenance === void 0 || group.execution !== void 0) {
+      return void 0;
+    }
+    const retained = group.result;
+    const state = group.maintenance;
+    const verdict = { declined: false };
+    const execution = (async () => {
+      await this.#acquireExecutionSlot();
+      let maintained;
+      try {
+        maintained = await host.maintain?.(group.query, retained, state, tableIds, after, until, probe);
+      } catch {
+        maintained = void 0;
+      } finally {
+        this.#releaseExecutionSlot();
+      }
+      if (maintained === void 0) {
+        verdict.declined = true;
+        return { result: retained, changed: false };
+      }
+      group.result = maintained.result;
+      group.maintenance = maintained.state;
+      return {
+        result: maintained.result,
+        changed: maintained.changed,
+        ...maintained.retained === void 0 ? {} : { retained: maintained.retained }
+      };
+    })();
+    group.execution = execution;
+    try {
+      const outcome = await execution;
+      return verdict.declined ? void 0 : outcome;
+    } finally {
+      if (group.execution === execution)
+        group.execution = void 0;
+    }
+  }
+  #deliverResult(group, subscriber, result, delivery, retained) {
     if (subscriber.closed)
       return;
     subscriber.delivered = true;
-    subscriber.options.onChange(cloneResult(result));
+    const consecutive = subscriber.seenDelivery === group.deliveries - 1;
+    subscriber.options.onChange(this.#sharedResults ? result : cloneResult(result), retained !== void 0 && consecutive && !delivery.initial ? { ...delivery, retained } : delivery);
+    subscriber.seenDelivery = group.deliveries;
   }
   #deliverInvalidation(subscriber, invalidation) {
     if (subscriber.closed)
@@ -456,6 +560,7 @@ class LiveQuerySet {
     const groups = [...this.#groups.values()];
     this.#groups.clear();
     this.#groupsByTable.clear();
+    this.#lagging.clear();
     for (const group of groups) {
       const subscribers = [...group.subscribers];
       group.subscribers.clear();
@@ -489,19 +594,39 @@ class LiveQuerySet {
   #stillOpen() {
     return !this.#closed;
   }
+  async #sweepCandidates(last, current, changedSince) {
+    if (catalogChangedBetween(last, current))
+      return [...this.#groups.values()];
+    if (sameProbe(last, current))
+      return [...this.#lagging];
+    const changed = await changedSince(last.manifestVersion);
+    if (!this.#stillOpen())
+      return void 0;
+    if (changed === "all")
+      return [...this.#groups.values()];
+    const candidates = new Set(this.#lagging);
+    for (const tableId of changed) {
+      const groups = this.#groupsByTable.get(tableId);
+      if (groups === void 0)
+        continue;
+      for (const group of groups)
+        candidates.add(group);
+    }
+    return [...candidates];
+  }
   async #sweep() {
     if (this.#closed || this.#groups.size === 0)
       return;
     this.#stats.versionChecks += 1;
-    const current = await this.#host.currentProbe();
+    const current = await this.#freshProbe();
     if (!this.#stillOpen())
       return;
     const last = this.#lastProbe ?? current;
-    const anyLagging = [...this.#groups.values()].some((group) => !sameProbe(group.seenProbe, current));
-    if (sameProbe(last, current) && !anyLagging)
+    if (sameProbe(last, current) && this.#lagging.size === 0)
       return;
     const started = performance.now();
     this.#stats.sweeps += 1;
+    const rerunsBefore = this.#stats.reruns;
     const windowCache = /* @__PURE__ */ new Map();
     const changedSince = (after) => {
       const key = versionOrdinal(after);
@@ -512,95 +637,133 @@ class LiveQuerySet {
       }
       return pending;
     };
-    for (const group of [...this.#groups.values()]) {
+    const candidates = await this.#sweepCandidates(last, current, changedSince);
+    if (candidates === void 0)
+      return;
+    for (const group of candidates) {
       if (!this.#stillOpen())
         return;
-      if (group.subscribers.size === 0 || sameProbe(group.seenProbe, current))
-        continue;
-      const prior = group.seenProbe;
-      const catalogChanged = catalogChangedBetween(prior, current);
-      if (catalogChanged) {
-        try {
-          await this.#refreshDependencies(group);
-        } catch (error) {
-          this.#notifyGroupError(group, error);
-          continue;
-        }
+      await this.#sweepGroup(group, last, current, changedSince);
+    }
+    this.#lastProbe = current;
+    this.#stats.rerunsAvoided += this.#groups.size - (this.#stats.reruns - rerunsBefore);
+    this.#stats.lastSweepMs = performance.now() - started;
+  }
+  async #sweepGroup(group, last, current, changedSince) {
+    if (!this.#stillOpen())
+      return;
+    if (group.subscribers.size === 0 || this.#groups.get(group.key) !== group)
+      return;
+    this.#stats.groupsVisited += 1;
+    if (sameProbe(group.seenProbe, current)) {
+      this.#lagging.delete(group);
+      return;
+    }
+    const prior = this.#lagging.has(group) ? group.seenProbe : newerProbe(group.seenProbe, last);
+    const catalogChanged = catalogChangedBetween(prior, current);
+    if (catalogChanged) {
+      group.maintenance = void 0;
+      group.unmaintainable = false;
+      try {
+        await this.#refreshDependencies(group);
+      } catch (error) {
+        this.#lagging.add(group);
+        this.#notifyGroupError(group, error);
+        return;
       }
-      let relevant = [];
-      let affected = catalogChanged;
-      if (!affected && prior.manifestVersion !== current.manifestVersion) {
-        const changed = await changedSince(prior.manifestVersion);
-        if (!this.#stillOpen())
-          return;
-        if (changed === "all")
-          affected = true;
-        else {
-          relevant = [...group.dependencies].filter((tableId) => changed.has(tableId));
-          affected = relevant.length > 0;
-        }
+    }
+    let relevant = [];
+    let affected = catalogChanged;
+    if (!affected && prior.manifestVersion !== current.manifestVersion) {
+      const changed = await changedSince(prior.manifestVersion);
+      if (!this.#stillOpen())
+        return;
+      if (changed === "all")
+        affected = true;
+      else {
+        relevant = [...group.dependencies].filter((tableId) => changed.has(tableId));
+        affected = relevant.length > 0;
       }
-      if (!affected) {
-        this.#stats.rerunsAvoided += 1;
-        group.seenProbe = current;
-        continue;
+    }
+    if (!affected) {
+      this.#settleProbe(group, current);
+      return;
+    }
+    if (!catalogChanged && relevant.length > 0 && current.manifestVersion !== null && this.#host.changeCanAffect !== void 0) {
+      let canAffect;
+      try {
+        canAffect = await this.#host.changeCanAffect(group.query, relevant, prior.manifestVersion, current.manifestVersion);
+      } catch {
+        canAffect = true;
       }
-      if (!catalogChanged && relevant.length > 0 && current.manifestVersion !== null && this.#host.changeCanAffect !== void 0) {
-        let canAffect;
-        try {
-          canAffect = await this.#host.changeCanAffect(group.query, relevant, prior.manifestVersion, current.manifestVersion);
-        } catch {
-          canAffect = true;
-        }
-        if (!this.#stillOpen())
+      if (!this.#stillOpen())
+        return;
+      if (!canAffect) {
+        this.#stats.zoneSkips += 1;
+        this.#settleProbe(group, current);
+        return;
+      }
+    }
+    try {
+      let execution;
+      if (groupExecutes(group)) {
+        if (group.result === void 0 && group.execution !== void 0) {
+          this.#lagging.add(group);
           return;
-        if (!canAffect) {
-          this.#stats.rerunsAvoided += 1;
-          this.#stats.zoneSkips += 1;
-          group.seenProbe = current;
-          continue;
         }
-      }
-      const resultSubscribers = [...group.subscribers].filter((subscriber) => subscriber.kind === "result" && !subscriber.closed);
-      const observers = [...group.subscribers].filter((subscriber) => subscriber.kind === "observer" && !subscriber.closed);
-      try {
-        let execution;
-        if (resultSubscribers.length > 0) {
-          if (group.result === void 0 && group.execution !== void 0)
-            continue;
+        if (!catalogChanged && relevant.length > 0 && current.manifestVersion !== null) {
+          execution = await this.#maintainGroup(group, relevant, prior.manifestVersion, current.manifestVersion, current);
+          if (!this.#stillOpen() || this.#groups.get(group.key) !== group)
+            return;
+        }
+        if (execution !== void 0)
+          this.#stats.maintained += 1;
+        else {
           this.#stats.reruns += 1;
-          execution = await this.#executeGroup(group);
-          if (!this.#stillOpen() || !this.#groups.has(group.key))
-            continue;
+          execution = await this.#executeGroup(group, {
+            probe: current,
+            memoize: groupMemoizes(group)
+          });
+          if (!this.#stillOpen() || this.#groups.get(group.key) !== group)
+            return;
         }
-        for (const observer of observers) {
+      }
+      const invalidation = { ...current, initial: false };
+      if (execution?.changed === true)
+        group.deliveries += 1;
+      for (const subscriber of [...group.subscribers]) {
+        if (isObserver(subscriber)) {
+          if (execution !== void 0 && !execution.changed && subscriber.options.suppressUnchanged === true) {
+            this.#stats.notificationsSuppressed += 1;
+            return;
+          }
           try {
-            this.#deliverInvalidation(observer, { ...current, initial: false });
+            this.#deliverInvalidation(subscriber, invalidation);
             this.#stats.invalidations += 1;
           } catch (error) {
-            observer.options.onError?.(error);
+            subscriber.options.onError?.(error);
           }
-        }
-        if (execution !== void 0) {
-          if (execution.changed) {
-            for (const subscriber of resultSubscribers) {
-              try {
-                this.#deliverResult(subscriber, execution.result);
-              } catch (error) {
-                subscriber.options.onError?.(error);
-              }
-            }
-          } else {
-            this.#stats.notificationsSuppressed += resultSubscribers.length;
+        } else if (isResultSubscriber(subscriber) && execution !== void 0) {
+          if (!execution.changed) {
+            this.#stats.notificationsSuppressed += 1;
+            return;
+          }
+          try {
+            this.#deliverResult(group, subscriber, execution.result, invalidation, execution.retained);
+          } catch (error) {
+            subscriber.options.onError?.(error);
           }
         }
-        group.seenProbe = current;
-      } catch (error) {
-        this.#notifyGroupError(group, error);
       }
+      this.#settleProbe(group, current);
+    } catch (error) {
+      this.#lagging.add(group);
+      this.#notifyGroupError(group, error);
     }
-    this.#lastProbe = current;
-    this.#stats.lastSweepMs = performance.now() - started;
+  }
+  #settleProbe(group, probe) {
+    group.seenProbe = probe;
+    this.#lagging.delete(group);
   }
   #notifyGroupError(group, error) {
     for (const subscriber of group.subscribers) {