@dxos/echo-pipeline 0.4.10-main.ef6fbc2 → 0.4.10-main.f635fb0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. package/dist/lib/browser/{chunk-RA6MLCZM.mjs → chunk-SYE4EK33.mjs} +17 -9
  2. package/dist/lib/browser/chunk-SYE4EK33.mjs.map +7 -0
  3. package/dist/lib/browser/index.mjs +304 -230
  4. package/dist/lib/browser/index.mjs.map +4 -4
  5. package/dist/lib/browser/meta.json +1 -1
  6. package/dist/lib/browser/testing/index.mjs +8 -2
  7. package/dist/lib/browser/testing/index.mjs.map +4 -4
  8. package/dist/lib/node/{chunk-KGIYLJBT.cjs → chunk-WCTX6RNS.cjs} +19 -11
  9. package/dist/lib/node/chunk-WCTX6RNS.cjs.map +7 -0
  10. package/dist/lib/node/index.cjs +325 -254
  11. package/dist/lib/node/index.cjs.map +4 -4
  12. package/dist/lib/node/meta.json +1 -1
  13. package/dist/lib/node/testing/index.cjs +18 -13
  14. package/dist/lib/node/testing/index.cjs.map +4 -4
  15. package/dist/types/src/automerge/automerge-host.d.ts +19 -17
  16. package/dist/types/src/automerge/automerge-host.d.ts.map +1 -1
  17. package/dist/types/src/automerge/automerge-repo.test.d.ts +2 -0
  18. package/dist/types/src/automerge/automerge-repo.test.d.ts.map +1 -0
  19. package/dist/types/src/automerge/index.d.ts +1 -0
  20. package/dist/types/src/automerge/index.d.ts.map +1 -1
  21. package/dist/types/src/automerge/level.test.d.ts +2 -0
  22. package/dist/types/src/automerge/level.test.d.ts.map +1 -0
  23. package/dist/types/src/automerge/leveldb-storage-adapter.d.ts +30 -0
  24. package/dist/types/src/automerge/leveldb-storage-adapter.d.ts.map +1 -0
  25. package/dist/types/src/automerge/local-host-network-adapter.d.ts +8 -1
  26. package/dist/types/src/automerge/local-host-network-adapter.d.ts.map +1 -1
  27. package/dist/types/src/automerge/migrations.d.ts +7 -0
  28. package/dist/types/src/automerge/migrations.d.ts.map +1 -0
  29. package/dist/types/src/automerge/storage-adapter.test.d.ts +2 -0
  30. package/dist/types/src/automerge/storage-adapter.test.d.ts.map +1 -0
  31. package/dist/types/src/automerge/types.d.ts +6 -0
  32. package/dist/types/src/automerge/types.d.ts.map +1 -1
  33. package/dist/types/src/metadata/metadata-store.d.ts +2 -1
  34. package/dist/types/src/metadata/metadata-store.d.ts.map +1 -1
  35. package/dist/types/src/testing/index.d.ts +1 -0
  36. package/dist/types/src/testing/index.d.ts.map +1 -1
  37. package/dist/types/src/testing/level.d.ts +3 -0
  38. package/dist/types/src/testing/level.d.ts.map +1 -0
  39. package/package.json +33 -30
  40. package/src/automerge/automerge-host.test.ts +22 -8
  41. package/src/automerge/automerge-host.ts +52 -114
  42. package/src/automerge/automerge-repo.test.ts +29 -0
  43. package/src/automerge/index.ts +1 -0
  44. package/src/automerge/level.test.ts +64 -0
  45. package/src/automerge/leveldb-storage-adapter.ts +117 -0
  46. package/src/automerge/local-host-network-adapter.ts +19 -13
  47. package/src/automerge/migrations.ts +41 -0
  48. package/src/automerge/storage-adapter.test.ts +90 -0
  49. package/src/automerge/types.ts +6 -3
  50. package/src/metadata/metadata-store.ts +17 -8
  51. package/src/testing/index.ts +1 -0
  52. package/src/testing/level.ts +11 -0
  53. package/dist/lib/browser/chunk-RA6MLCZM.mjs.map +0 -7
  54. package/dist/lib/node/chunk-KGIYLJBT.cjs.map +0 -7
@@ -2,83 +2,81 @@
2
2
  // Copyright 2023 DXOS.org
3
3
  //
4
4
 
5
- import { next as automerge, getHeads } from '@dxos/automerge/automerge';
6
- import {
7
- Repo,
8
- type PeerId,
9
- type DocumentId,
10
- type StorageKey,
11
- type DocHandle,
12
- type DocHandleChangePayload,
13
- } from '@dxos/automerge/automerge-repo';
14
- import { IndexedDBStorageAdapter } from '@dxos/automerge/automerge-repo-storage-indexeddb';
5
+ import { asyncTimeout } from '@dxos/async';
6
+ import { next as automerge } from '@dxos/automerge/automerge';
7
+ import { Repo, type DocumentId, type PeerId, type StorageAdapterInterface } from '@dxos/automerge/automerge-repo';
15
8
  import { type Stream } from '@dxos/codec-protobuf';
16
- import { Context } from '@dxos/context';
9
+ import { Context, type Lifecycle } from '@dxos/context';
17
10
  import { PublicKey } from '@dxos/keys';
18
11
  import { log } from '@dxos/log';
19
- import { idCodec } from '@dxos/protocols';
20
12
  import {
21
13
  type FlushRequest,
22
14
  type HostInfo,
23
15
  type SyncRepoRequest,
24
16
  type SyncRepoResponse,
25
17
  } from '@dxos/protocols/proto/dxos/echo/service';
26
- import { StorageType, type Directory } from '@dxos/random-access-storage';
18
+ import { type Directory } from '@dxos/random-access-storage';
27
19
  import { type AutomergeReplicator } from '@dxos/teleport-extension-automerge-replicator';
28
20
  import { trace } from '@dxos/tracing';
29
21
  import { ComplexMap, ComplexSet, defaultMap, mapValues } from '@dxos/util';
30
22
 
31
- import { AutomergeStorageAdapter } from './automerge-storage-adapter';
32
- import { AutomergeStorageWrapper } from './automerge-storage–wrapper';
23
+ import { LevelDBStorageAdapter, type StorageCallbacks } from './leveldb-storage-adapter';
33
24
  import { LocalHostNetworkAdapter } from './local-host-network-adapter';
34
25
  import { MeshNetworkAdapter } from './mesh-network-adapter';
26
+ import { levelMigration } from './migrations';
27
+ import { type SubLevelDB } from './types';
35
28
 
36
29
  export type { DocumentId };
37
30
 
38
- export interface MetadataMethods {
39
- markDirty(idToLastHash: Map<string, string>): Promise<void>;
40
- }
41
-
42
31
  export type AutomergeHostParams = {
43
- directory: Directory;
44
- metadata?: MetadataMethods;
32
+ db: SubLevelDB;
33
+ /**
34
+ * For migration purposes.
35
+ */
36
+ directory?: Directory;
37
+ storageCallbacks?: StorageCallbacks;
45
38
  };
46
39
 
47
40
  @trace.resource()
48
41
  export class AutomergeHost {
49
42
  private readonly _ctx = new Context();
50
- private readonly _repo: Repo;
51
- private readonly _meshNetwork: MeshNetworkAdapter;
52
- private readonly _clientNetwork: LocalHostNetworkAdapter;
53
- private readonly _storage: AutomergeStorageWrapper;
43
+ private readonly _directory?: Directory;
44
+ private readonly _db: SubLevelDB;
45
+ private readonly _storageCallbacks?: StorageCallbacks;
46
+
47
+ private _repo!: Repo;
48
+ private _meshNetwork!: MeshNetworkAdapter;
49
+ private _clientNetwork!: LocalHostNetworkAdapter;
50
+ private _storage!: StorageAdapterInterface & Lifecycle;
54
51
 
55
52
  @trace.info()
56
- private readonly _peerId: string;
53
+ private _peerId!: string;
57
54
 
58
55
  /**
59
56
  * spaceKey -> deviceKey[]
60
57
  */
61
58
  private readonly _authorizedDevices = new ComplexMap<PublicKey, ComplexSet<PublicKey>>(PublicKey.hash);
62
59
 
63
- private readonly _updatingMetadata = new Map<string, Promise<void>>();
64
- private readonly _metadata?: MetadataMethods;
65
-
66
60
  public _requestedDocs = new Set<string>();
67
61
 
68
- constructor({ directory, metadata }: AutomergeHostParams) {
69
- this._metadata = metadata;
70
- this._meshNetwork = new MeshNetworkAdapter();
71
- this._clientNetwork = new LocalHostNetworkAdapter();
62
+ constructor({ directory, db, storageCallbacks }: AutomergeHostParams) {
63
+ this._directory = directory;
64
+ this._db = db;
65
+ this._storageCallbacks = storageCallbacks;
66
+ }
72
67
 
73
- this._storage = new AutomergeStorageWrapper({
74
- storage:
75
- // TODO(mykola): Delete specific handling of IDB storage.
76
- directory.type === StorageType.IDB
77
- ? new IndexedDBStorageAdapter(directory.path, 'data')
78
- : new AutomergeStorageAdapter(directory),
79
- callbacks: { beforeSave: (params) => this._beforeSave(params) },
68
+ async open() {
69
+ // TODO(mykola): remove this before 0.6 release.
70
+ this._directory && (await levelMigration({ db: this._db, directory: this._directory }));
71
+ this._storage = new LevelDBStorageAdapter({
72
+ db: this._db,
73
+ callbacks: this._storageCallbacks,
80
74
  });
75
+ await this._storage.open?.();
81
76
  this._peerId = `host-${PublicKey.random().toHex()}` as PeerId;
77
+
78
+ this._meshNetwork = new MeshNetworkAdapter();
79
+ this._clientNetwork = new LocalHostNetworkAdapter();
82
80
  this._repo = new Repo({
83
81
  peerId: this._peerId as PeerId,
84
82
  network: [this._clientNetwork, this._meshNetwork],
@@ -138,59 +136,17 @@ export class AutomergeHost {
138
136
  this._clientNetwork.ready();
139
137
  this._meshNetwork.ready();
140
138
 
141
- {
142
- const listener = ({ handle }: { handle: DocHandle<any> }) => this._onDocument(handle);
143
- this._repo.on('document', listener);
144
- this._ctx.onDispose(() => {
145
- this._repo.off('document', listener);
146
- Object.values(this._repo.handles).forEach((handle) => handle.off('change'));
147
- });
148
- }
149
- }
150
-
151
- get repo(): Repo {
152
- return this._repo;
139
+ await this._clientNetwork.whenConnected();
153
140
  }
154
141
 
155
- private async _beforeSave(path: StorageKey) {
156
- const id = path[0];
157
- if (this._updatingMetadata.has(id)) {
158
- return this._updatingMetadata.get(id);
159
- }
160
- }
161
-
162
- private _onDocument(handle: DocHandle<any>) {
163
- const listener = (event: DocHandleChangePayload<any>) => this._onUpdate(event);
164
- handle.on('change', listener);
142
+ async close() {
143
+ await this._storage.close?.();
144
+ await this._clientNetwork.close();
145
+ await this._ctx.dispose();
165
146
  }
166
147
 
167
- private _onUpdate(event: DocHandleChangePayload<any>) {
168
- if (this._metadata == null) {
169
- return;
170
- }
171
-
172
- const objectIds = getInlineChanges(event);
173
- if (objectIds.length === 0) {
174
- return;
175
- }
176
-
177
- const heads = getHeads(event.doc);
178
- const lastAvailableHash = heads.join('');
179
- if (!lastAvailableHash) {
180
- return;
181
- }
182
-
183
- const encodedIds = objectIds.map((objectId) => idCodec.encode({ documentId: event.handle.documentId, objectId }));
184
- const idToLastHash = new Map(encodedIds.map((id) => [id, lastAvailableHash]));
185
- const markingDirtyPromise = this._metadata
186
- .markDirty(idToLastHash)
187
- .then(() => {
188
- this._updatingMetadata.delete(event.handle.documentId);
189
- })
190
- .catch((err: Error) => {
191
- this._ctx.disposed && log.catch(err);
192
- });
193
- this._updatingMetadata.set(event.handle.documentId, markingDirtyPromise);
148
+ get repo(): Repo {
149
+ return this._repo;
194
150
  }
195
151
 
196
152
  @trace.info({ depth: null })
@@ -224,12 +180,6 @@ export class AutomergeHost {
224
180
  return this._repo.peers;
225
181
  }
226
182
 
227
- async close() {
228
- await this._storage.close();
229
- await this._clientNetwork.close();
230
- await this._ctx.dispose();
231
- }
232
-
233
183
  //
234
184
  // Methods for client-services.
235
185
  //
@@ -237,7 +187,13 @@ export class AutomergeHost {
237
187
  async flush({ documentIds }: FlushRequest): Promise<void> {
238
188
  // Note: Wait for all requested documents to be loaded/synced from thin-client.
239
189
  await Promise.all(documentIds?.map((id) => this._repo.find(id as DocumentId).whenReady()) ?? []);
240
- await this._repo.flush(documentIds as DocumentId[]);
190
+
191
+ // TODO(dmaretskyi): Workaround until the flush issue gets resolved.
192
+ try {
193
+ await asyncTimeout(this._repo.flush(documentIds as DocumentId[]), 500);
194
+ } catch (err) {
195
+ log.warn('flush error', { documentIds, err });
196
+ }
241
197
  }
242
198
 
243
199
  syncRepo(request: SyncRepoRequest): Stream<SyncRepoResponse> {
@@ -266,24 +222,6 @@ export class AutomergeHost {
266
222
  }
267
223
  }
268
224
 
269
- // TODO(mykola): Reconcile with `getInlineAndLinkChanges` in AutomergeDB.
270
- const getInlineChanges = (event: DocHandleChangePayload<any>) => {
271
- const inlineChangedObjectIds = new Set<string>();
272
- for (const { path } of event.patches) {
273
- if (path.length < 2) {
274
- continue;
275
- }
276
- switch (path[0]) {
277
- case 'objects':
278
- if (path.length >= 2) {
279
- inlineChangedObjectIds.add(path[1]);
280
- }
281
- break;
282
- }
283
- }
284
- return [...inlineChangedObjectIds];
285
- };
286
-
287
225
  export const getSpaceKeyFromDoc = (doc: any): string | null => {
288
226
  // experimental_spaceKey is set on old documents, new ones are created with doc.access.spaceKey
289
227
  const rawSpaceKey = doc.access?.spaceKey ?? doc.experimental_spaceKey;
@@ -0,0 +1,29 @@
1
+ //
2
+ // Copyright 2024 DXOS.org
3
+ //
4
+
5
+ import { Repo } from '@dxos/automerge/automerge-repo';
6
+ import { randomBytes } from '@dxos/crypto';
7
+ import { StorageType, createStorage } from '@dxos/random-access-storage';
8
+ import { describe, test } from '@dxos/test';
9
+
10
+ import { AutomergeStorageAdapter } from './automerge-storage-adapter';
11
+
12
+ describe('AutomergeRepo', () => {
13
+ // Currently failing
14
+ test.skip('flush', async () => {
15
+ const repo = new Repo({
16
+ network: [],
17
+ storage: new AutomergeStorageAdapter(createStorage({ type: StorageType.NODE }).createDirectory()),
18
+ });
19
+ const handle = repo.create<{ field?: string }>();
20
+
21
+ for (let i = 0; i < 10; i++) {
22
+ const p = repo.flush([handle.documentId]);
23
+ handle.change((doc: any) => {
24
+ doc.field += randomBytes(1024).toString('hex');
25
+ });
26
+ await p;
27
+ }
28
+ });
29
+ });
@@ -5,6 +5,7 @@
5
5
  export * from './automerge-host';
6
6
  export * from './automerge-storage-adapter';
7
7
  export * from './automerge-doc-loader';
8
+ export * from './leveldb-storage-adapter';
8
9
  export * from './local-host-network-adapter';
9
10
  export * from './mesh-network-adapter';
10
11
  export * from './types';
@@ -0,0 +1,64 @@
1
+ //
2
+ // Copyright 2024 DXOS.org
3
+ //
4
+
5
+ import { expect } from 'chai';
6
+ import { Level } from 'level';
7
+
8
+ import { PublicKey } from '@dxos/keys';
9
+ import { describe, openAndClose, test } from '@dxos/test';
10
+
11
+ import { type SubLevelDB } from './types';
12
+ import { createTestLevel } from '../testing';
13
+
14
+ describe('Level', () => {
15
+ test('missing keys', async () => {
16
+ const level = createTestLevel();
17
+ await openAndClose(level);
18
+
19
+ expect(() => level.get('missing')).to.throw;
20
+ });
21
+
22
+ test('data persistance after reload', async () => {
23
+ const path = `/tmp/dxos-${PublicKey.random().toHex()}`;
24
+ const level = new Level<string, string>(path);
25
+ await level.open();
26
+
27
+ const key = 'name';
28
+ const value = 'Rich';
29
+ {
30
+ await level.put(key, value);
31
+ expect(await level.get(key)).to.equal(value);
32
+ }
33
+
34
+ await level.close();
35
+
36
+ {
37
+ const level = new Level<string, string>(path);
38
+ await level.open();
39
+ expect(await level.get(key)).to.equal(value);
40
+ await level.clear();
41
+ expect(() => level.get(key)).to.throw;
42
+ }
43
+ });
44
+
45
+ test('batch different sublevels', async () => {
46
+ const level = createTestLevel();
47
+ await openAndClose(level);
48
+
49
+ const first: SubLevelDB = level.sublevel('first');
50
+ const second: SubLevelDB = level.sublevel('second');
51
+
52
+ const batch = first.batch();
53
+
54
+ const key = 'key';
55
+ const value = 'first-level-value';
56
+ batch.put(key, value, { sublevel: second });
57
+ await batch.write();
58
+
59
+ expect(() => first.get(key)).to.throw;
60
+ expect(await level.sublevel('second').get(key)).to.equal(value);
61
+ expect(await second.get(key)).to.equal(value);
62
+ expect(await second.iterator().all()).to.deep.equal([[key, value]]);
63
+ });
64
+ });
@@ -0,0 +1,117 @@
1
+ //
2
+ // Copyright 2024 DXOS.org
3
+ // s
4
+
5
+ import { type MixedEncoding } from 'level-transcoder';
6
+
7
+ import { type StorageAdapterInterface, type Chunk, type StorageKey } from '@dxos/automerge/automerge-repo';
8
+ import { LifecycleState, Resource } from '@dxos/context';
9
+ import { type MaybePromise } from '@dxos/util';
10
+
11
+ import { type BatchLevel, type SubLevelDB } from './types';
12
+
13
+ export type LevelDBStorageAdapterParams = {
14
+ db: SubLevelDB;
15
+ callbacks?: StorageCallbacks;
16
+ };
17
+
18
+ export type BeforeSaveParams = { path: StorageKey; batch: BatchLevel };
19
+
20
+ export interface StorageCallbacks {
21
+ beforeSave(params: BeforeSaveParams): MaybePromise<void>;
22
+ afterSave(path: StorageKey): MaybePromise<void>;
23
+ }
24
+
25
+ export class LevelDBStorageAdapter extends Resource implements StorageAdapterInterface {
26
+ constructor(private readonly _params: LevelDBStorageAdapterParams) {
27
+ super();
28
+ }
29
+
30
+ async load(keyArray: StorageKey): Promise<Uint8Array | undefined> {
31
+ try {
32
+ if (this._lifecycleState !== LifecycleState.OPEN) {
33
+ // TODO(mykola): this should be an error.
34
+ return undefined;
35
+ }
36
+ return await this._params.db.get<StorageKey, Uint8Array>(keyArray, { ...encodingOptions });
37
+ } catch (err: any) {
38
+ if (isLevelDbNotFoundError(err)) {
39
+ return undefined;
40
+ }
41
+ throw err;
42
+ }
43
+ }
44
+
45
+ async save(keyArray: StorageKey, binary: Uint8Array): Promise<void> {
46
+ if (this._lifecycleState !== LifecycleState.OPEN) {
47
+ return undefined;
48
+ }
49
+ const batch = this._params.db.batch();
50
+
51
+ await this._params.callbacks?.beforeSave?.({ path: keyArray, batch });
52
+ batch.put<StorageKey, Uint8Array>(keyArray, Buffer.from(binary), {
53
+ ...encodingOptions,
54
+ });
55
+ await batch.write();
56
+
57
+ await this._params.callbacks?.afterSave?.(keyArray);
58
+ }
59
+
60
+ async remove(keyArray: StorageKey): Promise<void> {
61
+ if (this._lifecycleState !== LifecycleState.OPEN) {
62
+ return undefined;
63
+ }
64
+ await this._params.db.del<StorageKey>(keyArray, { ...encodingOptions });
65
+ }
66
+
67
+ async loadRange(keyPrefix: StorageKey): Promise<Chunk[]> {
68
+ if (this._lifecycleState !== LifecycleState.OPEN) {
69
+ return [];
70
+ }
71
+ const result: Chunk[] = [];
72
+ for await (const [key, value] of this._params.db.iterator<StorageKey, Uint8Array>({
73
+ gte: keyPrefix,
74
+ lte: [...keyPrefix, '\uffff'],
75
+ ...encodingOptions,
76
+ })) {
77
+ result.push({
78
+ key,
79
+ data: value,
80
+ });
81
+ }
82
+ return result;
83
+ }
84
+
85
+ async removeRange(keyPrefix: StorageKey): Promise<void> {
86
+ if (this._lifecycleState !== LifecycleState.OPEN) {
87
+ return undefined;
88
+ }
89
+ const batch = this._params.db.batch();
90
+
91
+ for await (const [key] of this._params.db.iterator<StorageKey, Uint8Array>({
92
+ gte: keyPrefix,
93
+ lte: [...keyPrefix, '\uffff'],
94
+ ...encodingOptions,
95
+ })) {
96
+ batch.del<StorageKey>(key, { ...encodingOptions });
97
+ }
98
+ await batch.write();
99
+ }
100
+ }
101
+
102
+ const keyEncoder: MixedEncoding<StorageKey, Uint8Array, StorageKey> = {
103
+ encode: (key: StorageKey): Uint8Array =>
104
+ Buffer.from(key.map((k) => k.replaceAll('%', '%25').replaceAll('-', '%2D')).join('-')),
105
+ decode: (key: Uint8Array): StorageKey =>
106
+ Buffer.from(key)
107
+ .toString()
108
+ .split('-')
109
+ .map((k) => k.replaceAll('%2D', '-').replaceAll('%25', '%')),
110
+ };
111
+
112
+ export const encodingOptions = {
113
+ keyEncoding: keyEncoder,
114
+ valueEncoding: 'buffer',
115
+ };
116
+
117
+ const isLevelDbNotFoundError = (err: any): boolean => err.code === 'LEVEL_NOT_FOUND';
@@ -6,7 +6,6 @@ import { Trigger } from '@dxos/async';
6
6
  import { NetworkAdapter, type Message, type PeerId, cbor } from '@dxos/automerge/automerge-repo';
7
7
  import { Stream } from '@dxos/codec-protobuf';
8
8
  import { invariant } from '@dxos/invariant';
9
- import { log } from '@dxos/log';
10
9
  import { type HostInfo, type SyncRepoRequest, type SyncRepoResponse } from '@dxos/protocols/proto/dxos/echo/service';
11
10
 
12
11
  type ClientSyncState = {
@@ -32,10 +31,17 @@ export class LocalHostNetworkAdapter extends NetworkAdapter {
32
31
  });
33
32
  }
34
33
 
35
- private _connected = new Trigger();
34
+ private readonly _connected = new Trigger();
35
+ private _isConnected: boolean = false;
36
36
 
37
+ /**
38
+ * Called by `Repo` to connect to the network.
39
+ *
40
+ * @param peerId Our peer Id.
41
+ */
37
42
  override connect(peerId: PeerId): void {
38
43
  this.peerId = peerId;
44
+ this._isConnected = true;
39
45
  this._connected.wake();
40
46
  // No-op. Client always connects first
41
47
  }
@@ -56,6 +62,10 @@ export class LocalHostNetworkAdapter extends NetworkAdapter {
56
62
  // No-op
57
63
  }
58
64
 
65
+ async whenConnected(): Promise<void> {
66
+ await this._connected.wait({ timeout: 10_000 });
67
+ }
68
+
59
69
  syncRepo({ id, syncMessage }: SyncRepoRequest): Stream<SyncRepoResponse> {
60
70
  const peerId = this._getPeerId(id);
61
71
 
@@ -77,26 +87,22 @@ export class LocalHostNetworkAdapter extends NetworkAdapter {
77
87
  },
78
88
  });
79
89
 
80
- this._connected
81
- .wait({ timeout: 1_000 })
82
- .then(() => {
83
- this.emit('peer-candidate', {
84
- peerMetadata: {},
85
- peerId,
86
- });
87
- })
88
- .catch((err) => log.catch(err));
90
+ invariant(this._isConnected);
91
+ this.emit('peer-candidate', {
92
+ peerMetadata: {},
93
+ peerId,
94
+ });
89
95
  });
90
96
  }
91
97
 
92
98
  async sendSyncMessage({ id, syncMessage }: SyncRepoRequest): Promise<void> {
93
- await this._connected.wait({ timeout: 1_000 });
99
+ invariant(this._isConnected);
94
100
  const message = cbor.decode(syncMessage!) as Message;
95
101
  this.emit('message', message);
96
102
  }
97
103
 
98
104
  async getHostInfo(): Promise<HostInfo> {
99
- await this._connected.wait({ timeout: 1_000 });
105
+ invariant(this._isConnected);
100
106
  invariant(this.peerId, 'Peer id not set.');
101
107
  return {
102
108
  peerId: this.peerId,
@@ -0,0 +1,41 @@
1
+ //
2
+ // Copyright 2024 DXOS.org
3
+ //
4
+
5
+ import { type StorageKey } from '@dxos/automerge/automerge-repo';
6
+ import { IndexedDBStorageAdapter } from '@dxos/automerge/automerge-repo-storage-indexeddb';
7
+ import { log } from '@dxos/log';
8
+ import { StorageType, type Directory } from '@dxos/random-access-storage';
9
+
10
+ import { AutomergeStorageAdapter } from './automerge-storage-adapter';
11
+ import { encodingOptions } from './leveldb-storage-adapter';
12
+ import { type SubLevelDB } from './types';
13
+
14
+ export const levelMigration = async ({ db, directory }: { db: SubLevelDB; directory: Directory }) => {
15
+ // Note: Make automigration from previous storage to leveldb here.
16
+ const isNewLevel = !(await db
17
+ .iterator<StorageKey, Uint8Array>({
18
+ ...encodingOptions,
19
+ })
20
+ .next());
21
+
22
+ if (!isNewLevel) {
23
+ return;
24
+ }
25
+
26
+ const oldStorageAdapter =
27
+ directory.type === StorageType.IDB
28
+ ? new IndexedDBStorageAdapter(directory.path, 'data')
29
+ : new AutomergeStorageAdapter(directory);
30
+
31
+ const chunks = await oldStorageAdapter.loadRange([]);
32
+ if (chunks.length === 0) {
33
+ return;
34
+ }
35
+ const batch = db.batch();
36
+ log.info('found chunks on old storage adapter', { chunks: chunks.length });
37
+ for (const { key, data } of await oldStorageAdapter.loadRange([])) {
38
+ data && batch.put<StorageKey, Uint8Array>(key, data, { ...encodingOptions });
39
+ }
40
+ await batch.write();
41
+ };
@@ -0,0 +1,90 @@
1
+ //
2
+ // Copyright 2024 DXOS.org
3
+ //
4
+
5
+ import { expect } from 'chai';
6
+
7
+ import { type StorageAdapterInterface } from '@dxos/automerge/automerge-repo';
8
+ import { PublicKey } from '@dxos/keys';
9
+ import { StorageType, createStorage } from '@dxos/random-access-storage';
10
+ import { afterTest, describe, openAndClose, test } from '@dxos/test';
11
+ import { type MaybePromise } from '@dxos/util';
12
+
13
+ import { AutomergeStorageAdapter } from './automerge-storage-adapter';
14
+ import { LevelDBStorageAdapter } from './leveldb-storage-adapter';
15
+ import { createTestLevel } from '../testing';
16
+
17
+ const runTests = (
18
+ testNamespace: string,
19
+ /** Run per test. Expects automatic clean-up with `afterTest`. */ createAdapter: () => MaybePromise<StorageAdapterInterface>,
20
+ ) => {
21
+ describe(testNamespace, () => {
22
+ const chunks = [
23
+ { key: ['a', 'b', 'c', '1'], data: PublicKey.random().asUint8Array() },
24
+ { key: ['a', 'b', 'c', '2'], data: PublicKey.random().asUint8Array() },
25
+ { key: ['a', 'b', 'd', '3'], data: PublicKey.random().asUint8Array() },
26
+ { key: ['a', 'b', 'd', '4'], data: PublicKey.random().asUint8Array() },
27
+ ];
28
+
29
+ test('should store and retrieve data', async () => {
30
+ const adapter = await createAdapter();
31
+
32
+ await adapter.save(chunks[0].key, chunks[0].data);
33
+ expect(await adapter.load(chunks[0].key)).to.deep.equal(chunks[0].data);
34
+ });
35
+
36
+ test('loadRange return inputs with correct prefixes', async () => {
37
+ const adapter = await createAdapter();
38
+
39
+ for (const chunk of chunks) {
40
+ await adapter.save(chunk.key, chunk.data);
41
+ }
42
+
43
+ expect((await adapter.loadRange(['a', 'b'])).length).to.equal(4);
44
+ expect((await adapter.loadRange(['a', 'b', 'c']))[0]).to.deep.equal(chunks[0]);
45
+ expect((await adapter.loadRange(['a', 'b', 'c'])).length).to.equal(2);
46
+ });
47
+
48
+ test('deletion works', async () => {
49
+ const adapter = await createAdapter();
50
+
51
+ for (const chunk of chunks) {
52
+ await adapter.save(chunk.key, chunk.data);
53
+ }
54
+
55
+ await adapter.remove(['a', 'b', 'c', '1']);
56
+
57
+ expect((await adapter.loadRange(['a', 'b'])).length).to.equal(3);
58
+ expect((await adapter.loadRange(['a', 'b', 'c'])).length).to.equal(1);
59
+
60
+ await adapter.removeRange(['a', 'b', 'd']);
61
+
62
+ expect((await adapter.loadRange(['a', 'b'])).length).to.equal(1);
63
+ expect((await adapter.loadRange(['a', 'b']))[0]).to.deep.equal(chunks[1]);
64
+ expect(await adapter.load(['a', 'b', 'c', '2'])).to.deep.equal(chunks[1].data);
65
+ expect(await adapter.load(['a', 'b', 'd', '3'])).to.be.undefined;
66
+ });
67
+ });
68
+ };
69
+
70
+ /**
71
+ * Run tests for AutomergeStorageAdapter.
72
+ */
73
+ runTests('AutomergeStorageAdapter', () => {
74
+ const storage = createStorage({ type: StorageType.RAM });
75
+ afterTest(() => storage.close());
76
+ const dir = storage.createDirectory('automerge');
77
+ const adapter = new AutomergeStorageAdapter(dir);
78
+ afterTest(() => adapter.close());
79
+ return adapter;
80
+ });
81
+
82
+ /**
83
+ * Run tests for LevelDBStorageAdapter.
84
+ */
85
+ runTests('LevelDBStorageAdapter', async () => {
86
+ const level = createTestLevel();
87
+ const adapter = new LevelDBStorageAdapter({ db: level.sublevel('automerge') });
88
+ await openAndClose(level, adapter as any);
89
+ return adapter;
90
+ });
@@ -2,9 +2,8 @@
2
2
  // Copyright 2024 DXOS.org
3
3
  //
4
4
 
5
- //
6
- // Copyright 2023 DXOS.org
7
- //
5
+ import { type AbstractChainedBatch, type AbstractSublevel } from 'abstract-level';
6
+ import { type Level } from 'level';
8
7
 
9
8
  import { type EncodedReferenceObject } from './reference';
10
9
 
@@ -81,3 +80,7 @@ export type ObjectSystem = {
81
80
  */
82
81
  type?: EncodedReferenceObject;
83
82
  };
83
+
84
+ export type LevelDB = Level<string, string>;
85
+ export type SubLevelDB = AbstractSublevel<any, string | Buffer | Uint8Array, string, string>;
86
+ export type BatchLevel = AbstractChainedBatch<any, string, string>;