@dxos/client-services 0.4.8-main.eb5ae94 → 0.4.8-main.fff1521

package/dist/types/src/version.d.ts

@@ -1,2 +1,2 @@
-export declare const DXOS_VERSION = "0.4.8-main.eb5ae94";
+export declare const DXOS_VERSION = "0.4.8-main.fff1521";
 //# sourceMappingURL=version.d.ts.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dxos/client-services",
-  "version": "0.4.8-main.eb5ae94",
+  "version": "0.4.8-main.fff1521",
   "description": "DXOS client services implementation",
   "homepage": "https://dxos.org",
   "bugs": "https://github.com/dxos/dxos/issues",
@@ -22,46 +22,46 @@
   ],
   "dependencies": {
     "platform": "^1.3.6",
-    "@dxos/async": "0.4.8-main.eb5ae94",
-    "@dxos/config": "0.4.8-main.eb5ae94",
-    "@dxos/automerge": "0.4.8-main.eb5ae94",
-    "@dxos/codec-protobuf": "0.4.8-main.eb5ae94",
-    "@dxos/client-protocol": "0.4.8-main.eb5ae94",
-    "@dxos/context": "0.4.8-main.eb5ae94",
-    "@dxos/crypto": "0.4.8-main.eb5ae94",
-    "@dxos/credentials": "0.4.8-main.eb5ae94",
-    "@dxos/debug": "0.4.8-main.eb5ae94",
-    "@dxos/echo-db": "0.4.8-main.eb5ae94",
-    "@dxos/document-model": "0.4.8-main.eb5ae94",
-    "@dxos/echo-schema": "0.4.8-main.eb5ae94",
-    "@dxos/echo-pipeline": "0.4.8-main.eb5ae94",
-    "@dxos/feed-store": "0.4.8-main.eb5ae94",
-    "@dxos/invariant": "0.4.8-main.eb5ae94",
-    "@dxos/indexing": "0.4.8-main.eb5ae94",
-    "@dxos/keys": "0.4.8-main.eb5ae94",
-    "@dxos/keyring": "0.4.8-main.eb5ae94",
-    "@dxos/messaging": "0.4.8-main.eb5ae94",
-    "@dxos/log": "0.4.8-main.eb5ae94",
-    "@dxos/model-factory": "0.4.8-main.eb5ae94",
-    "@dxos/lock-file": "0.4.8-main.eb5ae94",
-    "@dxos/network-manager": "0.4.8-main.eb5ae94",
-    "@dxos/random-access-storage": "0.4.8-main.eb5ae94",
-    "@dxos/protocols": "0.4.8-main.eb5ae94",
-    "@dxos/node-std": "0.4.8-main.eb5ae94",
-    "@dxos/rpc": "0.4.8-main.eb5ae94",
-    "@dxos/teleport": "0.4.8-main.eb5ae94",
-    "@dxos/teleport-extension-gossip": "0.4.8-main.eb5ae94",
-    "@dxos/text-model": "0.4.8-main.eb5ae94",
-    "@dxos/teleport-extension-object-sync": "0.4.8-main.eb5ae94",
-    "@dxos/timeframe": "0.4.8-main.eb5ae94",
-    "@dxos/tracing": "0.4.8-main.eb5ae94",
-    "@dxos/util": "0.4.8-main.eb5ae94",
-    "@dxos/websocket-rpc": "0.4.8-main.eb5ae94"
+    "@dxos/async": "0.4.8-main.fff1521",
+    "@dxos/automerge": "0.4.8-main.fff1521",
+    "@dxos/codec-protobuf": "0.4.8-main.fff1521",
+    "@dxos/config": "0.4.8-main.fff1521",
+    "@dxos/client-protocol": "0.4.8-main.fff1521",
+    "@dxos/credentials": "0.4.8-main.fff1521",
+    "@dxos/context": "0.4.8-main.fff1521",
+    "@dxos/crypto": "0.4.8-main.fff1521",
+    "@dxos/debug": "0.4.8-main.fff1521",
+    "@dxos/document-model": "0.4.8-main.fff1521",
+    "@dxos/echo-db": "0.4.8-main.fff1521",
+    "@dxos/echo-schema": "0.4.8-main.fff1521",
+    "@dxos/echo-pipeline": "0.4.8-main.fff1521",
+    "@dxos/feed-store": "0.4.8-main.fff1521",
+    "@dxos/indexing": "0.4.8-main.fff1521",
+    "@dxos/keyring": "0.4.8-main.fff1521",
+    "@dxos/invariant": "0.4.8-main.fff1521",
+    "@dxos/keys": "0.4.8-main.fff1521",
+    "@dxos/lock-file": "0.4.8-main.fff1521",
+    "@dxos/log": "0.4.8-main.fff1521",
+    "@dxos/messaging": "0.4.8-main.fff1521",
+    "@dxos/network-manager": "0.4.8-main.fff1521",
+    "@dxos/node-std": "0.4.8-main.fff1521",
+    "@dxos/model-factory": "0.4.8-main.fff1521",
+    "@dxos/protocols": "0.4.8-main.fff1521",
+    "@dxos/rpc": "0.4.8-main.fff1521",
+    "@dxos/teleport": "0.4.8-main.fff1521",
+    "@dxos/random-access-storage": "0.4.8-main.fff1521",
+    "@dxos/teleport-extension-gossip": "0.4.8-main.fff1521",
+    "@dxos/teleport-extension-object-sync": "0.4.8-main.fff1521",
+    "@dxos/text-model": "0.4.8-main.fff1521",
+    "@dxos/timeframe": "0.4.8-main.fff1521",
+    "@dxos/websocket-rpc": "0.4.8-main.fff1521",
+    "@dxos/tracing": "0.4.8-main.fff1521",
+    "@dxos/util": "0.4.8-main.fff1521"
   },
   "devDependencies": {
     "@types/platform": "^1.3.4",
     "@types/readable-stream": "^2.3.9",
-    "@dxos/signal": "0.4.8-main.eb5ae94"
+    "@dxos/signal": "0.4.8-main.fff1521"
   },
   "publishConfig": {
     "access": "public"

package/src/packlets/indexing/index.ts

@@ -0,0 +1,5 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+export * from './util';

package/src/packlets/indexing/util.ts

@@ -0,0 +1,89 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { getHeads } from '@dxos/automerge/automerge';
+import { type DocHandle } from '@dxos/automerge/automerge-repo';
+import { warnAfterTimeout } from '@dxos/debug';
+import { type AutomergeHost } from '@dxos/echo-pipeline';
+import { type ObjectSnapshot } from '@dxos/indexing';
+import { idCodec } from '@dxos/protocols';
+
+/**
+ * Factory for `loadDocuments` iterator.
+ */
+export const createLoadDocuments = (automergeHost: AutomergeHost) =>
+  /**
+   * Get object data blobs from Automerge Repo by ids.
+   * @param ids
+   */
+  // TODO(mykola): Unload automerge handles after usage.
+  async function* loadDocuments(ids: string[]) {
+    for (const id of ids) {
+      const { documentId, objectId } = idCodec.decode(id);
+      const handle = automergeHost.repo.find(documentId as any);
+      await warnAfterTimeout(5000, 'to long to load doc', () => handle.whenReady());
+      const doc = handle.docSync();
+      const heads = getHeads(doc);
+      yield [{ id, object: doc.objects[objectId], currentHash: heads.at(-1)! }];
+    }
+  };
+
+/**
+ * Factory for `getAllDocuments` iterator.
+ */
+export const createGetAllDocuments = (automergeHost: AutomergeHost) =>
+  /**
+   * Recursively get all object data blobs from Automerge Repo.
+   * @param ids
+   */
+  // TODO(mykola): Unload automerge handles after usage.
+  async function* getAllDocuments(): AsyncGenerator<ObjectSnapshot[], void, 'done'> {
+    /** visited automerge handles */
+    const visited = new Set<string>();
+
+    async function* getObjectsFromHandle(handle: DocHandle<any>): AsyncGenerator<ObjectSnapshot[]> {
+      if (visited.has(handle.documentId)) {
+        return;
+      }
+
+      await warnAfterTimeout(5000, 'to long to load doc', () => handle.whenReady());
+      const doc = handle.docSync();
+
+      const heads = getHeads(doc);
+
+      if (doc.objects) {
+        yield Object.entries(doc.objects as { [key: string]: any }).map(([objectId, object]) => {
+          return {
+            id: idCodec.encode({ documentId: handle.documentId, objectId }),
+            object,
+            currentHash: heads.at(-1)!,
+          };
+        });
+      }
+
+      if (doc.links) {
+        for (const id of Object.values(doc.links as { [echoId: string]: string })) {
+          if (visited.has(id)) {
+            continue;
+          }
+          const linkHandle = automergeHost.repo.find(id as any);
+          for await (const result of getObjectsFromHandle(linkHandle)) {
+            yield result;
+          }
+        }
+      }
+
+      visited.add(handle.documentId);
+    }
+
+    for (const handle of Object.values(automergeHost.repo.handles)) {
+      if (visited.has(handle.documentId)) {
+        continue;
+      }
+      for await (const result of getObjectsFromHandle(handle)) {
+        yield result;
+      }
+      visited.add(handle.documentId);
+    }
+  };

package/src/packlets/services/service-context.ts

@@ -3,10 +3,9 @@
 //
 
 import { Trigger } from '@dxos/async';
-import { getHeads } from '@dxos/automerge/automerge';
 import { Context } from '@dxos/context';
 import { type CredentialProcessor, getCredentialAssertion } from '@dxos/credentials';
-import { failUndefined, warnAfterTimeout } from '@dxos/debug';
+import { failUndefined } from '@dxos/debug';
 import {
   valueEncoding,
   MetadataStore,
@@ -24,7 +23,7 @@ import { log } from '@dxos/log';
 import { type SignalManager } from '@dxos/messaging';
 import { type ModelFactory } from '@dxos/model-factory';
 import { type NetworkManager } from '@dxos/network-manager';
-import { InvalidStorageVersionError, STORAGE_VERSION, idCodec, trace } from '@dxos/protocols';
+import { InvalidStorageVersionError, STORAGE_VERSION, trace } from '@dxos/protocols';
 import { Invitation } from '@dxos/protocols/proto/dxos/client/services';
 import type { FeedMessage } from '@dxos/protocols/proto/dxos/echo/feed';
 import { type ProfileDocument, type Credential } from '@dxos/protocols/proto/dxos/halo/credentials';
@@ -39,6 +38,7 @@ import {
   type IdentityManagerRuntimeParams,
   type JoinIdentityParams,
 } from '../identity';
+import { createGetAllDocuments, createLoadDocuments } from '../indexing';
 import {
   DeviceInvitationProtocol,
   InvitationsHandler,
@@ -136,19 +136,8 @@ export class ServiceContext {
     this.indexer = new Indexer({
       indexStore: new IndexStore({ directory: storage.createDirectory('index-store') }),
       metadataStore: this.indexMetadata,
-      loadDocuments: async (ids: string[]) => {
-        const snapshots = await Promise.all(
-          ids.map(async (id) => {
-            const { documentId, objectId } = idCodec.decode(id);
-            const handle = this.automergeHost.repo.find(documentId as any);
-            await warnAfterTimeout(1000, 'to long to load doc', () => handle.whenReady());
-            const doc = handle.docSync();
-            const heads = getHeads(doc);
-            return { id, object: doc.objects[objectId], currentHash: heads.at(-1)! };
-          }),
-        );
-        return snapshots.filter((snapshot) => snapshot.object);
-      },
+      loadDocuments: createLoadDocuments(this.automergeHost),
+      getAllDocuments: createGetAllDocuments(this.automergeHost),
     });
 
     this.invitations = new InvitationsHandler(this.networkManager);

package/src/version.ts

@@ -1 +1 @@
-export const DXOS_VERSION = "0.4.8-main.eb5ae94";
+export const DXOS_VERSION = "0.4.8-main.fff1521";