@dxos/echo-pipeline 0.8.3-main.7f5a14c → 0.8.3-staging.0fa589b

package/dist/lib/browser/index.mjs

@@ -809,6 +809,10 @@ var AutomergeHost = class extends Resource3 {
       shouldSyncCollection: this._shouldSyncCollection.bind(this)
     });
     this.collectionStateUpdated = new Event2();
+    /**
+    * Fired after a batch of documents was saved to disk.
+    */
+    this.documentsSaved = new Event2();
     this._db = db;
     this._storage = new LevelDBStorageAdapter({
       db: db.sublevel("automerge"),
@@ -955,7 +959,7 @@ var AutomergeHost = class extends Resource3 {
       await Promise.all(headsToWait.map(async (entry, index) => {
         const handle = await this.loadDoc(Context.default(void 0, {
           F: __dxlog_file3,
-          L: 288
+          L: 293
         }), entry.documentId);
         await waitForHeads(handle, entry.heads);
       }));
@@ -968,7 +972,7 @@ var AutomergeHost = class extends Resource3 {
         documentId
       }, {
         F: __dxlog_file3,
-        L: 302,
+        L: 307,
         S: this,
         C: (f, a) => f(...a)
       });
@@ -978,7 +982,7 @@ var AutomergeHost = class extends Resource3 {
           documentId
         }, {
           F: __dxlog_file3,
-          L: 305,
+          L: 310,
           S: this,
           C: (f, a) => f(...a)
         });
@@ -991,7 +995,7 @@ var AutomergeHost = class extends Resource3 {
     }
     log3("done re-indexing heads", void 0, {
       F: __dxlog_file3,
-      L: 314,
+      L: 319,
       S: this,
       C: (f, a) => f(...a)
     });
@@ -1059,6 +1063,7 @@ var AutomergeHost = class extends Resource3 {
       const heads = getHeads(document);
       this._onHeadsChanged(documentId, heads);
     }
+    this.documentsSaved.emit();
   }
   _automergePeers() {
     return this._repo.peers;
@@ -1212,7 +1217,7 @@ var AutomergeHost = class extends Resource3 {
       count: toReplicate.length
     }, {
       F: __dxlog_file3,
-      L: 557,
+      L: 563,
       S: this,
       C: (f, a) => f(...a)
     });
@@ -1272,7 +1277,7 @@ var changeIsPresentInDoc = (doc, changeHash) => {
 var decodeCollectionState = (state) => {
   invariant2(typeof state === "object" && state !== null, "Invalid state", {
     F: __dxlog_file3,
-    L: 608,
+    L: 614,
     S: void 0,
     A: [
       "typeof state === 'object' && state !== null",
@@ -2451,7 +2456,7 @@ var createSelectedDocumentsIterator = (automergeHost) => (
 // packages/core/echo/echo-pipeline/src/db-host/query-service.ts
 import { getHeads as getHeads3 } from "@automerge/automerge";
 import { Schema } from "effect";
-import { DeferredTask } from "@dxos/async";
+import { DeferredTask, scheduleMicroTask, synchronized as synchronized2 } from "@dxos/async";
 import { Stream as Stream2 } from "@dxos/codec-protobuf/stream";
 import { Context as Context4, Resource as Resource7 } from "@dxos/context";
 import { raise } from "@dxos/debug";
@@ -2539,6 +2544,8 @@ var QueryPlanner = class {
         return this._generateReferenceTraversalClause(query, context);
       case "union":
         return this._generateUnionClause(query, context);
+      case "set-difference":
+        return this._generateSetDifferenceClause(query, context);
       default:
         throw new QueryError(`Unsupported query type: ${query.type}`, {
           context: {
@@ -2690,7 +2697,7 @@ var QueryPlanner = class {
           const typenames = filter.filters.map((f) => {
             invariant9(f.type === "object" && f.typename !== null, void 0, {
               F: __dxlog_file10,
-              L: 189,
+              L: 191,
               S: this,
               A: [
                 "f.type === 'object' && f.typename !== null",
@@ -2754,6 +2761,15 @@ var QueryPlanner = class {
       }
     ]);
   }
+  _generateSetDifferenceClause(query, context) {
+    return QueryPlan.Plan.make([
+      {
+        _tag: "SetDifferenceStep",
+        source: this._generate(query.source, context),
+        exclude: this._generate(query.exclude, context)
+      }
+    ]);
+  }
   _generateReferenceTraversalClause(query, context) {
     return QueryPlan.Plan.make([
       ...this._generate(query.anchor, context).steps,
@@ -3008,7 +3024,7 @@ var QueryExecutor = class extends Resource6 {
       ]
     });
     const prevResultSet = this._lastResultSet;
-    const { workingSet, trace: trace6 } = await this._execPlan(this._plan, this._lastResultSet);
+    const { workingSet, trace: trace6 } = await this._execPlan(this._plan, []);
     this._lastResultSet = workingSet;
     trace6.name = "Root";
     trace6.details = JSON.stringify({
@@ -3025,6 +3041,7 @@ var QueryExecutor = class extends Resource6 {
   }
   async _execPlan(plan, workingSet) {
     const trace6 = ExecutionTrace.makeEmpty();
+    const begin = performance.now();
     for (const step of plan.steps) {
       if (this._ctx.disposed) {
         throw new ContextDisposedError();
@@ -3033,6 +3050,8 @@ var QueryExecutor = class extends Resource6 {
       workingSet = result.workingSet;
       trace6.children.push(result.trace);
     }
+    trace6.objectCount = workingSet.length;
+    trace6.executionTime = performance.now() - begin;
     return {
       workingSet,
       trace: trace6
@@ -3064,6 +3083,9 @@ var QueryExecutor = class extends Resource6 {
       case "UnionStep":
         ({ workingSet: newWorkingSet, trace: trace6 } = await this._execUnionStep(step, workingSet));
         break;
+      case "SetDifferenceStep":
+        ({ workingSet: newWorkingSet, trace: trace6 } = await this._execSetDifferenceStep(step, workingSet));
+        break;
       case "TraverseStep":
         ({ workingSet: newWorkingSet, trace: trace6 } = await this._execTraverseStep(step, workingSet));
         break;
@@ -3190,6 +3212,17 @@ var QueryExecutor = class extends Resource6 {
     };
   }
   async _execFilterDeletedStep(step, workingSet) {
+    if (workingSet.length === 6) {
+      log9.info("FilterDeletedStep", {
+        step,
+        workingSet
+      }, {
+        F: __dxlog_file11,
+        L: 386,
+        S: this,
+        C: (f, a) => f(...a)
+      });
+    }
     const expected = step.mode === "only-deleted";
     const result = workingSet.filter((item) => ObjectStructure.isDeleted(item.doc) === expected);
     return {
@@ -3215,27 +3248,29 @@ var QueryExecutor = class extends Resource6 {
         switch (step.traversal.direction) {
           case "outgoing": {
             const property = EscapedPropPath.unescape(step.traversal.property);
-            const refs = workingSet.map((item) => {
+            const refs = workingSet.flatMap((item) => {
               const ref = getDeep(item.doc.data, property);
-              if (!isEncodedReference(ref)) {
-                return null;
-              }
-              try {
-                return {
-                  ref: DXN.parse(ref["/"]),
-                  spaceId: item.spaceId
-                };
-              } catch {
-                log9.warn("Invalid reference", {
-                  ref: ref["/"]
-                }, {
-                  F: __dxlog_file11,
-                  L: 421,
-                  S: this,
-                  C: (f, a) => f(...a)
-                });
-                return null;
-              }
+              const refs2 = Array.isArray(ref) ? ref : [
+                ref
+              ];
+              return refs2.map((ref2) => {
+                try {
+                  return isEncodedReference(ref2) ? {
+                    ref: DXN.parse(ref2["/"]),
+                    spaceId: item.spaceId
+                  } : null;
+                } catch {
+                  log9.warn("Invalid reference", {
+                    ref: ref2["/"]
+                  }, {
+                    F: __dxlog_file11,
+                    L: 431,
+                    S: this,
+                    C: (f, a) => f(...a)
+                  });
+                  return null;
+                }
+              });
             }).filter(isNonNullable2);
             const beginLoad = performance.now();
             const items = await Promise.all(refs.map(({ ref, spaceId }) => this._loadFromDXN(ref, {
@@ -3287,7 +3322,7 @@ var QueryExecutor = class extends Resource6 {
                   ref: ref["/"]
                 }, {
                   F: __dxlog_file11,
-                  L: 483,
+                  L: 494,
                   S: this,
                   C: (f, a) => f(...a)
                 });
@@ -3356,6 +3391,26 @@ var QueryExecutor = class extends Resource6 {
       trace: trace6
     };
   }
+  async _execSetDifferenceStep(step, workingSet) {
+    const trace6 = {
+      ...ExecutionTrace.makeEmpty(),
+      name: "SetDifference"
+    };
+    const sourceResult = await this._execPlan(step.source, [
+      ...workingSet
+    ]);
+    const excludeResult = await this._execPlan(step.exclude, [
+      ...workingSet
+    ]);
+    trace6.children.push(sourceResult.trace, excludeResult.trace);
+    return {
+      workingSet: sourceResult.workingSet.filter((item) => {
+        const index = excludeResult.workingSet.findIndex((i) => i.objectId === item.objectId);
+        return index === -1;
+      }),
+      trace: trace6
+    };
+  }
   async _loadDocumentsAfterIndexQuery(indexHits) {
     return Promise.all(indexHits.map(async (hit) => {
       return this._loadFromIndexHit(hit);
@@ -3365,7 +3420,7 @@ var QueryExecutor = class extends Resource6 {
     const { objectId, documentId, spaceKey: spaceKeyInIndex } = objectPointerCodec3.decode(hit.id);
     const handle = await this._automergeHost.loadDoc(Context3.default(void 0, {
       F: __dxlog_file11,
-      L: 571
+      L: 604
     }), documentId);
     const doc = handle.doc();
     if (!doc) {
@@ -3393,7 +3448,7 @@ var QueryExecutor = class extends Resource6 {
         dxn
       }, {
         F: __dxlog_file11,
-        L: 598,
+        L: 631,
         S: this,
         C: (f, a) => f(...a)
       });
@@ -3406,7 +3461,7 @@ var QueryExecutor = class extends Resource6 {
         spaceId
       }, {
         F: __dxlog_file11,
-        L: 606,
+        L: 639,
         S: this,
         C: (f, a) => f(...a)
       });
@@ -3418,7 +3473,7 @@ var QueryExecutor = class extends Resource6 {
         spaceId
       }, {
         F: __dxlog_file11,
-        L: 611,
+        L: 644,
         S: this,
         C: (f, a) => f(...a)
       });
@@ -3439,7 +3494,7 @@ var QueryExecutor = class extends Resource6 {
     }
     const handle = await this._automergeHost.loadDoc(Context3.default(void 0, {
       F: __dxlog_file11,
-      L: 630
+      L: 663
     }), link);
     const doc = handle.doc();
     if (!doc) {
@@ -3459,27 +3514,17 @@ var QueryExecutor = class extends Resource6 {
 };
 
 // packages/core/echo/echo-pipeline/src/db-host/query-service.ts
+function _ts_decorate5(decorators, target, key, desc) {
+  var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+  if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+  else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+  return c > 3 && r && Object.defineProperty(target, key, r), r;
+}
 var __dxlog_file12 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/db-host/query-service.ts";
 var QueryServiceImpl = class extends Resource7 {
   // TODO(burdon): OK for options, but not params. Pass separately and type readonly here.
   constructor(_params) {
-    super(), this._params = _params, this._queries = /* @__PURE__ */ new Set(), this._updateQueries = new DeferredTask(this._ctx, async () => {
-      await Promise.all(Array.from(this._queries).map(async (query) => {
-        try {
-          const { changed } = await query.executor.execQuery();
-          if (changed) {
-            query.sendResults(query.executor.getResults());
-          }
-        } catch (err) {
-          log10.catch(err, void 0, {
-            F: __dxlog_file12,
-            L: 57,
-            S: this,
-            C: (f, a) => f(...a)
-          });
-        }
-      }));
-    });
+    super(), this._params = _params, this._queries = /* @__PURE__ */ new Set();
     trace4.diagnostic({
       id: "active-queries",
       name: "Active Queries",
@@ -3495,9 +3540,11 @@ var QueryServiceImpl = class extends Resource7 {
     });
   }
   async _open() {
-    this._params.indexer.updated.on(this._ctx, () => this._updateQueries.schedule());
+    this._params.indexer.updated.on(this._ctx, () => this.invalidateQueries());
+    this._updateQueries = new DeferredTask(this._ctx, this._executeQueries.bind(this));
   }
   async _close() {
+    await this._updateQueries.join();
     await Promise.all(Array.from(this._queries).map((query) => query.close()));
   }
   async setConfig(config) {
@@ -3505,40 +3552,11 @@ var QueryServiceImpl = class extends Resource7 {
   }
   execQuery(request) {
     return new Stream2(({ next, close, ctx }) => {
-      const parsedQuery = QueryAST.Query.pipe(Schema.decodeUnknownSync)(JSON.parse(request.query));
-      const queryEntry = {
-        executor: new QueryExecutor({
-          indexer: this._params.indexer,
-          automergeHost: this._params.automergeHost,
-          queryId: request.queryId ?? raise(new Error("query id required")),
-          query: parsedQuery,
-          reactivity: request.reactivity,
-          spaceStateManager: this._params.spaceStateManager
-        }),
-        sendResults: (results) => {
-          if (ctx.disposed) {
-            return;
-          }
-          next({
-            queryId: request.queryId,
-            results
-          });
-        },
-        close: async () => {
-          close();
-          await queryEntry.executor.close();
-          this._queries.delete(queryEntry);
-        }
-      };
-      this._queries.add(queryEntry);
-      queueMicrotask(async () => {
+      const queryEntry = this._createQuery(ctx, request, next, close, close);
+      scheduleMicroTask(ctx, async () => {
         await queryEntry.executor.open();
-        try {
-          await queryEntry.executor.execQuery();
-          queryEntry.sendResults(queryEntry.executor.getResults());
-        } catch (error) {
-          close(error);
-        }
+        queryEntry.open = true;
+        this._updateQueries.schedule();
       });
       return queryEntry.close;
     });
@@ -3549,7 +3567,7 @@ var QueryServiceImpl = class extends Resource7 {
   async reindex() {
     log10("Reindexing all documents...", void 0, {
       F: __dxlog_file12,
-      L: 141,
+      L: 113,
       S: this,
       C: (f, a) => f(...a)
     });
@@ -3564,7 +3582,7 @@ var QueryServiceImpl = class extends Resource7 {
           count: ids.size
         }, {
           F: __dxlog_file12,
-          L: 149,
+          L: 121,
           S: this,
           C: (f, a) => f(...a)
         });
@@ -3574,13 +3592,100 @@ var QueryServiceImpl = class extends Resource7 {
       count: ids.size
     }, {
       F: __dxlog_file12,
-      L: 153,
+      L: 125,
       S: this,
       C: (f, a) => f(...a)
     });
     await this._params.indexer.reindex(ids);
   }
+  /**
+  * Schedule re-execution of all queries.
+  */
+  invalidateQueries() {
+    for (const query of this._queries) {
+      query.dirty = true;
+    }
+    this._updateQueries.schedule();
+  }
+  _createQuery(ctx, request, onResults, onError, onClose) {
+    const parsedQuery = QueryAST.Query.pipe(Schema.decodeUnknownSync)(JSON.parse(request.query));
+    const queryEntry = {
+      executor: new QueryExecutor({
+        indexer: this._params.indexer,
+        automergeHost: this._params.automergeHost,
+        queryId: request.queryId ?? raise(new Error("query id required")),
+        query: parsedQuery,
+        reactivity: request.reactivity,
+        spaceStateManager: this._params.spaceStateManager
+      }),
+      dirty: true,
+      open: false,
+      firstResult: true,
+      sendResults: (results) => {
+        if (ctx.disposed) {
+          return;
+        }
+        onResults({
+          queryId: request.queryId,
+          results
+        });
+      },
+      onError,
+      close: async () => {
+        onClose();
+        await queryEntry.executor.close();
+        this._queries.delete(queryEntry);
+      }
+    };
+    this._queries.add(queryEntry);
+    return queryEntry;
+  }
+  async _executeQueries() {
+    const begin = performance.now();
+    let count = 0;
+    await Promise.all(Array.from(this._queries).map(async (query) => {
+      if (!query.dirty || !query.open) {
+        return;
+      }
+      count++;
+      try {
+        const { changed } = await query.executor.execQuery();
+        query.dirty = false;
+        if (changed || query.firstResult) {
+          query.firstResult = false;
+          query.sendResults(query.executor.getResults());
+        }
+      } catch (err) {
+        log10.catch(err, void 0, {
+          F: __dxlog_file12,
+          L: 196,
+          S: this,
+          C: (f, a) => f(...a)
+        });
+      }
+    }));
+    log10.verbose("executed queries", {
+      count,
+      duration: performance.now() - begin
+    }, {
+      F: __dxlog_file12,
+      L: 200,
+      S: this,
+      C: (f, a) => f(...a)
+    });
+  }
 };
+_ts_decorate5([
+  synchronized2
+], QueryServiceImpl.prototype, "_close", null);
+_ts_decorate5([
+  trace4.span({
+    showInBrowserTimeline: true
+  })
+], QueryServiceImpl.prototype, "_executeQueries", null);
+QueryServiceImpl = _ts_decorate5([
+  trace4.resource()
+], QueryServiceImpl);
 var createDocumentsIterator = (automergeHost) => (
   /**
   * Recursively get all object data blobs from loaded documents from Automerge Repo.
@@ -3615,7 +3720,7 @@ var createDocumentsIterator = (automergeHost) => (
           }
           const linkHandle = await automergeHost.loadDoc(Context4.default(void 0, {
             F: __dxlog_file12,
-            L: 195
+            L: 240
           }), urlString);
           for await (const result of getObjectsFromHandle(linkHandle)) {
             yield result;
@@ -3989,6 +4094,9 @@ var EchoHost = class extends Resource9 {
       void this._automergeHost.updateLocalCollectionState(deriveCollectionIdFromSpaceId(e.spaceId), e.documentIds);
       void this._automergeHost.updateLocalCollectionState(deriveCollectionIdFromSpaceId(e.spaceId, e.spaceRootId), e.documentIds);
     });
+    this._automergeHost.documentsSaved.on(this._ctx, () => {
+      this._queryService.invalidateQueries();
+    });
   }
   async _close(ctx) {
     await this._queryService.close(ctx);
@@ -4029,7 +4137,7 @@ var EchoHost = class extends Resource9 {
   async createSpaceRoot(spaceKey) {
     invariant13(this._lifecycleState === LifecycleState5.OPEN, void 0, {
       F: __dxlog_file16,
-      L: 252,
+      L: 255,
       S: this,
       A: [
         "this._lifecycleState === LifecycleState.OPEN",
@@ -4057,7 +4165,7 @@ var EchoHost = class extends Resource9 {
   async openSpaceRoot(spaceId, automergeUrl) {
     invariant13(this._lifecycleState === LifecycleState5.OPEN, void 0, {
       F: __dxlog_file16,
-      L: 271,
+      L: 274,
       S: this,
       A: [
         "this._lifecycleState === LifecycleState.OPEN",
@@ -4088,7 +4196,7 @@ var EchoHost = class extends Resource9 {
 
 // packages/core/echo/echo-pipeline/src/edge/echo-edge-replicator.ts
 import { cbor as cbor2 } from "@automerge/automerge-repo";
-import { Mutex, scheduleTask as scheduleTask2, scheduleMicroTask } from "@dxos/async";
+import { Mutex, scheduleTask as scheduleTask2, scheduleMicroTask as scheduleMicroTask2 } from "@dxos/async";
 import { Context as Context6, Resource as Resource11 } from "@dxos/context";
 import { randomUUID } from "@dxos/crypto";
 import { invariant as invariant14 } from "@dxos/invariant";
@@ -4248,7 +4356,7 @@ var EchoEdgeReplicator = class {
       L: 63
     });
     this._ctx.onDispose(this._edgeConnection.onReconnected(() => {
-      this._ctx && scheduleMicroTask(this._ctx, () => this._handleReconnect());
+      this._ctx && scheduleMicroTask2(this._ctx, () => this._handleReconnect());
     }));
   }
   async _handleReconnect() {