@dxos/echo-pipeline 0.4.10-main.ef6fbc2 → 0.4.10-main.f635fb0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/browser/{chunk-RA6MLCZM.mjs → chunk-SYE4EK33.mjs} +17 -9
- package/dist/lib/browser/chunk-SYE4EK33.mjs.map +7 -0
- package/dist/lib/browser/index.mjs +304 -230
- package/dist/lib/browser/index.mjs.map +4 -4
- package/dist/lib/browser/meta.json +1 -1
- package/dist/lib/browser/testing/index.mjs +8 -2
- package/dist/lib/browser/testing/index.mjs.map +4 -4
- package/dist/lib/node/{chunk-KGIYLJBT.cjs → chunk-WCTX6RNS.cjs} +19 -11
- package/dist/lib/node/chunk-WCTX6RNS.cjs.map +7 -0
- package/dist/lib/node/index.cjs +325 -254
- package/dist/lib/node/index.cjs.map +4 -4
- package/dist/lib/node/meta.json +1 -1
- package/dist/lib/node/testing/index.cjs +18 -13
- package/dist/lib/node/testing/index.cjs.map +4 -4
- package/dist/types/src/automerge/automerge-host.d.ts +19 -17
- package/dist/types/src/automerge/automerge-host.d.ts.map +1 -1
- package/dist/types/src/automerge/automerge-repo.test.d.ts +2 -0
- package/dist/types/src/automerge/automerge-repo.test.d.ts.map +1 -0
- package/dist/types/src/automerge/index.d.ts +1 -0
- package/dist/types/src/automerge/index.d.ts.map +1 -1
- package/dist/types/src/automerge/level.test.d.ts +2 -0
- package/dist/types/src/automerge/level.test.d.ts.map +1 -0
- package/dist/types/src/automerge/leveldb-storage-adapter.d.ts +30 -0
- package/dist/types/src/automerge/leveldb-storage-adapter.d.ts.map +1 -0
- package/dist/types/src/automerge/local-host-network-adapter.d.ts +8 -1
- package/dist/types/src/automerge/local-host-network-adapter.d.ts.map +1 -1
- package/dist/types/src/automerge/migrations.d.ts +7 -0
- package/dist/types/src/automerge/migrations.d.ts.map +1 -0
- package/dist/types/src/automerge/storage-adapter.test.d.ts +2 -0
- package/dist/types/src/automerge/storage-adapter.test.d.ts.map +1 -0
- package/dist/types/src/automerge/types.d.ts +6 -0
- package/dist/types/src/automerge/types.d.ts.map +1 -1
- package/dist/types/src/metadata/metadata-store.d.ts +2 -1
- package/dist/types/src/metadata/metadata-store.d.ts.map +1 -1
- package/dist/types/src/testing/index.d.ts +1 -0
- package/dist/types/src/testing/index.d.ts.map +1 -1
- package/dist/types/src/testing/level.d.ts +3 -0
- package/dist/types/src/testing/level.d.ts.map +1 -0
- package/package.json +33 -30
- package/src/automerge/automerge-host.test.ts +22 -8
- package/src/automerge/automerge-host.ts +52 -114
- package/src/automerge/automerge-repo.test.ts +29 -0
- package/src/automerge/index.ts +1 -0
- package/src/automerge/level.test.ts +64 -0
- package/src/automerge/leveldb-storage-adapter.ts +117 -0
- package/src/automerge/local-host-network-adapter.ts +19 -13
- package/src/automerge/migrations.ts +41 -0
- package/src/automerge/storage-adapter.test.ts +90 -0
- package/src/automerge/types.ts +6 -3
- package/src/metadata/metadata-store.ts +17 -8
- package/src/testing/index.ts +1 -0
- package/src/testing/level.ts +11 -0
- package/dist/lib/browser/chunk-RA6MLCZM.mjs.map +0 -7
- package/dist/lib/node/chunk-KGIYLJBT.cjs.map +0 -7
|
@@ -2,83 +2,81 @@
|
|
|
2
2
|
// Copyright 2023 DXOS.org
|
|
3
3
|
//
|
|
4
4
|
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
7
|
-
|
|
8
|
-
type PeerId,
|
|
9
|
-
type DocumentId,
|
|
10
|
-
type StorageKey,
|
|
11
|
-
type DocHandle,
|
|
12
|
-
type DocHandleChangePayload,
|
|
13
|
-
} from '@dxos/automerge/automerge-repo';
|
|
14
|
-
import { IndexedDBStorageAdapter } from '@dxos/automerge/automerge-repo-storage-indexeddb';
|
|
5
|
+
import { asyncTimeout } from '@dxos/async';
|
|
6
|
+
import { next as automerge } from '@dxos/automerge/automerge';
|
|
7
|
+
import { Repo, type DocumentId, type PeerId, type StorageAdapterInterface } from '@dxos/automerge/automerge-repo';
|
|
15
8
|
import { type Stream } from '@dxos/codec-protobuf';
|
|
16
|
-
import { Context } from '@dxos/context';
|
|
9
|
+
import { Context, type Lifecycle } from '@dxos/context';
|
|
17
10
|
import { PublicKey } from '@dxos/keys';
|
|
18
11
|
import { log } from '@dxos/log';
|
|
19
|
-
import { idCodec } from '@dxos/protocols';
|
|
20
12
|
import {
|
|
21
13
|
type FlushRequest,
|
|
22
14
|
type HostInfo,
|
|
23
15
|
type SyncRepoRequest,
|
|
24
16
|
type SyncRepoResponse,
|
|
25
17
|
} from '@dxos/protocols/proto/dxos/echo/service';
|
|
26
|
-
import {
|
|
18
|
+
import { type Directory } from '@dxos/random-access-storage';
|
|
27
19
|
import { type AutomergeReplicator } from '@dxos/teleport-extension-automerge-replicator';
|
|
28
20
|
import { trace } from '@dxos/tracing';
|
|
29
21
|
import { ComplexMap, ComplexSet, defaultMap, mapValues } from '@dxos/util';
|
|
30
22
|
|
|
31
|
-
import {
|
|
32
|
-
import { AutomergeStorageWrapper } from './automerge-storage–wrapper';
|
|
23
|
+
import { LevelDBStorageAdapter, type StorageCallbacks } from './leveldb-storage-adapter';
|
|
33
24
|
import { LocalHostNetworkAdapter } from './local-host-network-adapter';
|
|
34
25
|
import { MeshNetworkAdapter } from './mesh-network-adapter';
|
|
26
|
+
import { levelMigration } from './migrations';
|
|
27
|
+
import { type SubLevelDB } from './types';
|
|
35
28
|
|
|
36
29
|
export type { DocumentId };
|
|
37
30
|
|
|
38
|
-
export interface MetadataMethods {
|
|
39
|
-
markDirty(idToLastHash: Map<string, string>): Promise<void>;
|
|
40
|
-
}
|
|
41
|
-
|
|
42
31
|
export type AutomergeHostParams = {
|
|
43
|
-
|
|
44
|
-
|
|
32
|
+
db: SubLevelDB;
|
|
33
|
+
/**
|
|
34
|
+
* For migration purposes.
|
|
35
|
+
*/
|
|
36
|
+
directory?: Directory;
|
|
37
|
+
storageCallbacks?: StorageCallbacks;
|
|
45
38
|
};
|
|
46
39
|
|
|
47
40
|
@trace.resource()
|
|
48
41
|
export class AutomergeHost {
|
|
49
42
|
private readonly _ctx = new Context();
|
|
50
|
-
private readonly
|
|
51
|
-
private readonly
|
|
52
|
-
private readonly
|
|
53
|
-
|
|
43
|
+
private readonly _directory?: Directory;
|
|
44
|
+
private readonly _db: SubLevelDB;
|
|
45
|
+
private readonly _storageCallbacks?: StorageCallbacks;
|
|
46
|
+
|
|
47
|
+
private _repo!: Repo;
|
|
48
|
+
private _meshNetwork!: MeshNetworkAdapter;
|
|
49
|
+
private _clientNetwork!: LocalHostNetworkAdapter;
|
|
50
|
+
private _storage!: StorageAdapterInterface & Lifecycle;
|
|
54
51
|
|
|
55
52
|
@trace.info()
|
|
56
|
-
private
|
|
53
|
+
private _peerId!: string;
|
|
57
54
|
|
|
58
55
|
/**
|
|
59
56
|
* spaceKey -> deviceKey[]
|
|
60
57
|
*/
|
|
61
58
|
private readonly _authorizedDevices = new ComplexMap<PublicKey, ComplexSet<PublicKey>>(PublicKey.hash);
|
|
62
59
|
|
|
63
|
-
private readonly _updatingMetadata = new Map<string, Promise<void>>();
|
|
64
|
-
private readonly _metadata?: MetadataMethods;
|
|
65
|
-
|
|
66
60
|
public _requestedDocs = new Set<string>();
|
|
67
61
|
|
|
68
|
-
constructor({ directory,
|
|
69
|
-
this.
|
|
70
|
-
this.
|
|
71
|
-
this.
|
|
62
|
+
constructor({ directory, db, storageCallbacks }: AutomergeHostParams) {
|
|
63
|
+
this._directory = directory;
|
|
64
|
+
this._db = db;
|
|
65
|
+
this._storageCallbacks = storageCallbacks;
|
|
66
|
+
}
|
|
72
67
|
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
callbacks: { beforeSave: (params) => this._beforeSave(params) },
|
|
68
|
+
async open() {
|
|
69
|
+
// TODO(mykola): remove this before 0.6 release.
|
|
70
|
+
this._directory && (await levelMigration({ db: this._db, directory: this._directory }));
|
|
71
|
+
this._storage = new LevelDBStorageAdapter({
|
|
72
|
+
db: this._db,
|
|
73
|
+
callbacks: this._storageCallbacks,
|
|
80
74
|
});
|
|
75
|
+
await this._storage.open?.();
|
|
81
76
|
this._peerId = `host-${PublicKey.random().toHex()}` as PeerId;
|
|
77
|
+
|
|
78
|
+
this._meshNetwork = new MeshNetworkAdapter();
|
|
79
|
+
this._clientNetwork = new LocalHostNetworkAdapter();
|
|
82
80
|
this._repo = new Repo({
|
|
83
81
|
peerId: this._peerId as PeerId,
|
|
84
82
|
network: [this._clientNetwork, this._meshNetwork],
|
|
@@ -138,59 +136,17 @@ export class AutomergeHost {
|
|
|
138
136
|
this._clientNetwork.ready();
|
|
139
137
|
this._meshNetwork.ready();
|
|
140
138
|
|
|
141
|
-
|
|
142
|
-
const listener = ({ handle }: { handle: DocHandle<any> }) => this._onDocument(handle);
|
|
143
|
-
this._repo.on('document', listener);
|
|
144
|
-
this._ctx.onDispose(() => {
|
|
145
|
-
this._repo.off('document', listener);
|
|
146
|
-
Object.values(this._repo.handles).forEach((handle) => handle.off('change'));
|
|
147
|
-
});
|
|
148
|
-
}
|
|
149
|
-
}
|
|
150
|
-
|
|
151
|
-
get repo(): Repo {
|
|
152
|
-
return this._repo;
|
|
139
|
+
await this._clientNetwork.whenConnected();
|
|
153
140
|
}
|
|
154
141
|
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
}
|
|
160
|
-
}
|
|
161
|
-
|
|
162
|
-
private _onDocument(handle: DocHandle<any>) {
|
|
163
|
-
const listener = (event: DocHandleChangePayload<any>) => this._onUpdate(event);
|
|
164
|
-
handle.on('change', listener);
|
|
142
|
+
async close() {
|
|
143
|
+
await this._storage.close?.();
|
|
144
|
+
await this._clientNetwork.close();
|
|
145
|
+
await this._ctx.dispose();
|
|
165
146
|
}
|
|
166
147
|
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
return;
|
|
170
|
-
}
|
|
171
|
-
|
|
172
|
-
const objectIds = getInlineChanges(event);
|
|
173
|
-
if (objectIds.length === 0) {
|
|
174
|
-
return;
|
|
175
|
-
}
|
|
176
|
-
|
|
177
|
-
const heads = getHeads(event.doc);
|
|
178
|
-
const lastAvailableHash = heads.join('');
|
|
179
|
-
if (!lastAvailableHash) {
|
|
180
|
-
return;
|
|
181
|
-
}
|
|
182
|
-
|
|
183
|
-
const encodedIds = objectIds.map((objectId) => idCodec.encode({ documentId: event.handle.documentId, objectId }));
|
|
184
|
-
const idToLastHash = new Map(encodedIds.map((id) => [id, lastAvailableHash]));
|
|
185
|
-
const markingDirtyPromise = this._metadata
|
|
186
|
-
.markDirty(idToLastHash)
|
|
187
|
-
.then(() => {
|
|
188
|
-
this._updatingMetadata.delete(event.handle.documentId);
|
|
189
|
-
})
|
|
190
|
-
.catch((err: Error) => {
|
|
191
|
-
this._ctx.disposed && log.catch(err);
|
|
192
|
-
});
|
|
193
|
-
this._updatingMetadata.set(event.handle.documentId, markingDirtyPromise);
|
|
148
|
+
get repo(): Repo {
|
|
149
|
+
return this._repo;
|
|
194
150
|
}
|
|
195
151
|
|
|
196
152
|
@trace.info({ depth: null })
|
|
@@ -224,12 +180,6 @@ export class AutomergeHost {
|
|
|
224
180
|
return this._repo.peers;
|
|
225
181
|
}
|
|
226
182
|
|
|
227
|
-
async close() {
|
|
228
|
-
await this._storage.close();
|
|
229
|
-
await this._clientNetwork.close();
|
|
230
|
-
await this._ctx.dispose();
|
|
231
|
-
}
|
|
232
|
-
|
|
233
183
|
//
|
|
234
184
|
// Methods for client-services.
|
|
235
185
|
//
|
|
@@ -237,7 +187,13 @@ export class AutomergeHost {
|
|
|
237
187
|
async flush({ documentIds }: FlushRequest): Promise<void> {
|
|
238
188
|
// Note: Wait for all requested documents to be loaded/synced from thin-client.
|
|
239
189
|
await Promise.all(documentIds?.map((id) => this._repo.find(id as DocumentId).whenReady()) ?? []);
|
|
240
|
-
|
|
190
|
+
|
|
191
|
+
// TODO(dmaretskyi): Workaround until the flush issue gets resolved.
|
|
192
|
+
try {
|
|
193
|
+
await asyncTimeout(this._repo.flush(documentIds as DocumentId[]), 500);
|
|
194
|
+
} catch (err) {
|
|
195
|
+
log.warn('flush error', { documentIds, err });
|
|
196
|
+
}
|
|
241
197
|
}
|
|
242
198
|
|
|
243
199
|
syncRepo(request: SyncRepoRequest): Stream<SyncRepoResponse> {
|
|
@@ -266,24 +222,6 @@ export class AutomergeHost {
|
|
|
266
222
|
}
|
|
267
223
|
}
|
|
268
224
|
|
|
269
|
-
// TODO(mykola): Reconcile with `getInlineAndLinkChanges` in AutomergeDB.
|
|
270
|
-
const getInlineChanges = (event: DocHandleChangePayload<any>) => {
|
|
271
|
-
const inlineChangedObjectIds = new Set<string>();
|
|
272
|
-
for (const { path } of event.patches) {
|
|
273
|
-
if (path.length < 2) {
|
|
274
|
-
continue;
|
|
275
|
-
}
|
|
276
|
-
switch (path[0]) {
|
|
277
|
-
case 'objects':
|
|
278
|
-
if (path.length >= 2) {
|
|
279
|
-
inlineChangedObjectIds.add(path[1]);
|
|
280
|
-
}
|
|
281
|
-
break;
|
|
282
|
-
}
|
|
283
|
-
}
|
|
284
|
-
return [...inlineChangedObjectIds];
|
|
285
|
-
};
|
|
286
|
-
|
|
287
225
|
export const getSpaceKeyFromDoc = (doc: any): string | null => {
|
|
288
226
|
// experimental_spaceKey is set on old documents, new ones are created with doc.access.spaceKey
|
|
289
227
|
const rawSpaceKey = doc.access?.spaceKey ?? doc.experimental_spaceKey;
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
//
|
|
2
|
+
// Copyright 2024 DXOS.org
|
|
3
|
+
//
|
|
4
|
+
|
|
5
|
+
import { Repo } from '@dxos/automerge/automerge-repo';
|
|
6
|
+
import { randomBytes } from '@dxos/crypto';
|
|
7
|
+
import { StorageType, createStorage } from '@dxos/random-access-storage';
|
|
8
|
+
import { describe, test } from '@dxos/test';
|
|
9
|
+
|
|
10
|
+
import { AutomergeStorageAdapter } from './automerge-storage-adapter';
|
|
11
|
+
|
|
12
|
+
describe('AutomergeRepo', () => {
|
|
13
|
+
// Currently failing
|
|
14
|
+
test.skip('flush', async () => {
|
|
15
|
+
const repo = new Repo({
|
|
16
|
+
network: [],
|
|
17
|
+
storage: new AutomergeStorageAdapter(createStorage({ type: StorageType.NODE }).createDirectory()),
|
|
18
|
+
});
|
|
19
|
+
const handle = repo.create<{ field?: string }>();
|
|
20
|
+
|
|
21
|
+
for (let i = 0; i < 10; i++) {
|
|
22
|
+
const p = repo.flush([handle.documentId]);
|
|
23
|
+
handle.change((doc: any) => {
|
|
24
|
+
doc.field += randomBytes(1024).toString('hex');
|
|
25
|
+
});
|
|
26
|
+
await p;
|
|
27
|
+
}
|
|
28
|
+
});
|
|
29
|
+
});
|
package/src/automerge/index.ts
CHANGED
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
export * from './automerge-host';
|
|
6
6
|
export * from './automerge-storage-adapter';
|
|
7
7
|
export * from './automerge-doc-loader';
|
|
8
|
+
export * from './leveldb-storage-adapter';
|
|
8
9
|
export * from './local-host-network-adapter';
|
|
9
10
|
export * from './mesh-network-adapter';
|
|
10
11
|
export * from './types';
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
//
|
|
2
|
+
// Copyright 2024 DXOS.org
|
|
3
|
+
//
|
|
4
|
+
|
|
5
|
+
import { expect } from 'chai';
|
|
6
|
+
import { Level } from 'level';
|
|
7
|
+
|
|
8
|
+
import { PublicKey } from '@dxos/keys';
|
|
9
|
+
import { describe, openAndClose, test } from '@dxos/test';
|
|
10
|
+
|
|
11
|
+
import { type SubLevelDB } from './types';
|
|
12
|
+
import { createTestLevel } from '../testing';
|
|
13
|
+
|
|
14
|
+
describe('Level', () => {
|
|
15
|
+
test('missing keys', async () => {
|
|
16
|
+
const level = createTestLevel();
|
|
17
|
+
await openAndClose(level);
|
|
18
|
+
|
|
19
|
+
expect(() => level.get('missing')).to.throw;
|
|
20
|
+
});
|
|
21
|
+
|
|
22
|
+
test('data persistance after reload', async () => {
|
|
23
|
+
const path = `/tmp/dxos-${PublicKey.random().toHex()}`;
|
|
24
|
+
const level = new Level<string, string>(path);
|
|
25
|
+
await level.open();
|
|
26
|
+
|
|
27
|
+
const key = 'name';
|
|
28
|
+
const value = 'Rich';
|
|
29
|
+
{
|
|
30
|
+
await level.put(key, value);
|
|
31
|
+
expect(await level.get(key)).to.equal(value);
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
await level.close();
|
|
35
|
+
|
|
36
|
+
{
|
|
37
|
+
const level = new Level<string, string>(path);
|
|
38
|
+
await level.open();
|
|
39
|
+
expect(await level.get(key)).to.equal(value);
|
|
40
|
+
await level.clear();
|
|
41
|
+
expect(() => level.get(key)).to.throw;
|
|
42
|
+
}
|
|
43
|
+
});
|
|
44
|
+
|
|
45
|
+
test('batch different sublevels', async () => {
|
|
46
|
+
const level = createTestLevel();
|
|
47
|
+
await openAndClose(level);
|
|
48
|
+
|
|
49
|
+
const first: SubLevelDB = level.sublevel('first');
|
|
50
|
+
const second: SubLevelDB = level.sublevel('second');
|
|
51
|
+
|
|
52
|
+
const batch = first.batch();
|
|
53
|
+
|
|
54
|
+
const key = 'key';
|
|
55
|
+
const value = 'first-level-value';
|
|
56
|
+
batch.put(key, value, { sublevel: second });
|
|
57
|
+
await batch.write();
|
|
58
|
+
|
|
59
|
+
expect(() => first.get(key)).to.throw;
|
|
60
|
+
expect(await level.sublevel('second').get(key)).to.equal(value);
|
|
61
|
+
expect(await second.get(key)).to.equal(value);
|
|
62
|
+
expect(await second.iterator().all()).to.deep.equal([[key, value]]);
|
|
63
|
+
});
|
|
64
|
+
});
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
//
|
|
2
|
+
// Copyright 2024 DXOS.org
|
|
3
|
+
// s
|
|
4
|
+
|
|
5
|
+
import { type MixedEncoding } from 'level-transcoder';
|
|
6
|
+
|
|
7
|
+
import { type StorageAdapterInterface, type Chunk, type StorageKey } from '@dxos/automerge/automerge-repo';
|
|
8
|
+
import { LifecycleState, Resource } from '@dxos/context';
|
|
9
|
+
import { type MaybePromise } from '@dxos/util';
|
|
10
|
+
|
|
11
|
+
import { type BatchLevel, type SubLevelDB } from './types';
|
|
12
|
+
|
|
13
|
+
export type LevelDBStorageAdapterParams = {
|
|
14
|
+
db: SubLevelDB;
|
|
15
|
+
callbacks?: StorageCallbacks;
|
|
16
|
+
};
|
|
17
|
+
|
|
18
|
+
export type BeforeSaveParams = { path: StorageKey; batch: BatchLevel };
|
|
19
|
+
|
|
20
|
+
export interface StorageCallbacks {
|
|
21
|
+
beforeSave(params: BeforeSaveParams): MaybePromise<void>;
|
|
22
|
+
afterSave(path: StorageKey): MaybePromise<void>;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export class LevelDBStorageAdapter extends Resource implements StorageAdapterInterface {
|
|
26
|
+
constructor(private readonly _params: LevelDBStorageAdapterParams) {
|
|
27
|
+
super();
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
async load(keyArray: StorageKey): Promise<Uint8Array | undefined> {
|
|
31
|
+
try {
|
|
32
|
+
if (this._lifecycleState !== LifecycleState.OPEN) {
|
|
33
|
+
// TODO(mykola): this should be an error.
|
|
34
|
+
return undefined;
|
|
35
|
+
}
|
|
36
|
+
return await this._params.db.get<StorageKey, Uint8Array>(keyArray, { ...encodingOptions });
|
|
37
|
+
} catch (err: any) {
|
|
38
|
+
if (isLevelDbNotFoundError(err)) {
|
|
39
|
+
return undefined;
|
|
40
|
+
}
|
|
41
|
+
throw err;
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
async save(keyArray: StorageKey, binary: Uint8Array): Promise<void> {
|
|
46
|
+
if (this._lifecycleState !== LifecycleState.OPEN) {
|
|
47
|
+
return undefined;
|
|
48
|
+
}
|
|
49
|
+
const batch = this._params.db.batch();
|
|
50
|
+
|
|
51
|
+
await this._params.callbacks?.beforeSave?.({ path: keyArray, batch });
|
|
52
|
+
batch.put<StorageKey, Uint8Array>(keyArray, Buffer.from(binary), {
|
|
53
|
+
...encodingOptions,
|
|
54
|
+
});
|
|
55
|
+
await batch.write();
|
|
56
|
+
|
|
57
|
+
await this._params.callbacks?.afterSave?.(keyArray);
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
async remove(keyArray: StorageKey): Promise<void> {
|
|
61
|
+
if (this._lifecycleState !== LifecycleState.OPEN) {
|
|
62
|
+
return undefined;
|
|
63
|
+
}
|
|
64
|
+
await this._params.db.del<StorageKey>(keyArray, { ...encodingOptions });
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
async loadRange(keyPrefix: StorageKey): Promise<Chunk[]> {
|
|
68
|
+
if (this._lifecycleState !== LifecycleState.OPEN) {
|
|
69
|
+
return [];
|
|
70
|
+
}
|
|
71
|
+
const result: Chunk[] = [];
|
|
72
|
+
for await (const [key, value] of this._params.db.iterator<StorageKey, Uint8Array>({
|
|
73
|
+
gte: keyPrefix,
|
|
74
|
+
lte: [...keyPrefix, '\uffff'],
|
|
75
|
+
...encodingOptions,
|
|
76
|
+
})) {
|
|
77
|
+
result.push({
|
|
78
|
+
key,
|
|
79
|
+
data: value,
|
|
80
|
+
});
|
|
81
|
+
}
|
|
82
|
+
return result;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
async removeRange(keyPrefix: StorageKey): Promise<void> {
|
|
86
|
+
if (this._lifecycleState !== LifecycleState.OPEN) {
|
|
87
|
+
return undefined;
|
|
88
|
+
}
|
|
89
|
+
const batch = this._params.db.batch();
|
|
90
|
+
|
|
91
|
+
for await (const [key] of this._params.db.iterator<StorageKey, Uint8Array>({
|
|
92
|
+
gte: keyPrefix,
|
|
93
|
+
lte: [...keyPrefix, '\uffff'],
|
|
94
|
+
...encodingOptions,
|
|
95
|
+
})) {
|
|
96
|
+
batch.del<StorageKey>(key, { ...encodingOptions });
|
|
97
|
+
}
|
|
98
|
+
await batch.write();
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
const keyEncoder: MixedEncoding<StorageKey, Uint8Array, StorageKey> = {
|
|
103
|
+
encode: (key: StorageKey): Uint8Array =>
|
|
104
|
+
Buffer.from(key.map((k) => k.replaceAll('%', '%25').replaceAll('-', '%2D')).join('-')),
|
|
105
|
+
decode: (key: Uint8Array): StorageKey =>
|
|
106
|
+
Buffer.from(key)
|
|
107
|
+
.toString()
|
|
108
|
+
.split('-')
|
|
109
|
+
.map((k) => k.replaceAll('%2D', '-').replaceAll('%25', '%')),
|
|
110
|
+
};
|
|
111
|
+
|
|
112
|
+
export const encodingOptions = {
|
|
113
|
+
keyEncoding: keyEncoder,
|
|
114
|
+
valueEncoding: 'buffer',
|
|
115
|
+
};
|
|
116
|
+
|
|
117
|
+
const isLevelDbNotFoundError = (err: any): boolean => err.code === 'LEVEL_NOT_FOUND';
|
|
@@ -6,7 +6,6 @@ import { Trigger } from '@dxos/async';
|
|
|
6
6
|
import { NetworkAdapter, type Message, type PeerId, cbor } from '@dxos/automerge/automerge-repo';
|
|
7
7
|
import { Stream } from '@dxos/codec-protobuf';
|
|
8
8
|
import { invariant } from '@dxos/invariant';
|
|
9
|
-
import { log } from '@dxos/log';
|
|
10
9
|
import { type HostInfo, type SyncRepoRequest, type SyncRepoResponse } from '@dxos/protocols/proto/dxos/echo/service';
|
|
11
10
|
|
|
12
11
|
type ClientSyncState = {
|
|
@@ -32,10 +31,17 @@ export class LocalHostNetworkAdapter extends NetworkAdapter {
|
|
|
32
31
|
});
|
|
33
32
|
}
|
|
34
33
|
|
|
35
|
-
private _connected = new Trigger();
|
|
34
|
+
private readonly _connected = new Trigger();
|
|
35
|
+
private _isConnected: boolean = false;
|
|
36
36
|
|
|
37
|
+
/**
|
|
38
|
+
* Called by `Repo` to connect to the network.
|
|
39
|
+
*
|
|
40
|
+
* @param peerId Our peer Id.
|
|
41
|
+
*/
|
|
37
42
|
override connect(peerId: PeerId): void {
|
|
38
43
|
this.peerId = peerId;
|
|
44
|
+
this._isConnected = true;
|
|
39
45
|
this._connected.wake();
|
|
40
46
|
// No-op. Client always connects first
|
|
41
47
|
}
|
|
@@ -56,6 +62,10 @@ export class LocalHostNetworkAdapter extends NetworkAdapter {
|
|
|
56
62
|
// No-op
|
|
57
63
|
}
|
|
58
64
|
|
|
65
|
+
async whenConnected(): Promise<void> {
|
|
66
|
+
await this._connected.wait({ timeout: 10_000 });
|
|
67
|
+
}
|
|
68
|
+
|
|
59
69
|
syncRepo({ id, syncMessage }: SyncRepoRequest): Stream<SyncRepoResponse> {
|
|
60
70
|
const peerId = this._getPeerId(id);
|
|
61
71
|
|
|
@@ -77,26 +87,22 @@ export class LocalHostNetworkAdapter extends NetworkAdapter {
|
|
|
77
87
|
},
|
|
78
88
|
});
|
|
79
89
|
|
|
80
|
-
this.
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
peerId,
|
|
86
|
-
});
|
|
87
|
-
})
|
|
88
|
-
.catch((err) => log.catch(err));
|
|
90
|
+
invariant(this._isConnected);
|
|
91
|
+
this.emit('peer-candidate', {
|
|
92
|
+
peerMetadata: {},
|
|
93
|
+
peerId,
|
|
94
|
+
});
|
|
89
95
|
});
|
|
90
96
|
}
|
|
91
97
|
|
|
92
98
|
async sendSyncMessage({ id, syncMessage }: SyncRepoRequest): Promise<void> {
|
|
93
|
-
|
|
99
|
+
invariant(this._isConnected);
|
|
94
100
|
const message = cbor.decode(syncMessage!) as Message;
|
|
95
101
|
this.emit('message', message);
|
|
96
102
|
}
|
|
97
103
|
|
|
98
104
|
async getHostInfo(): Promise<HostInfo> {
|
|
99
|
-
|
|
105
|
+
invariant(this._isConnected);
|
|
100
106
|
invariant(this.peerId, 'Peer id not set.');
|
|
101
107
|
return {
|
|
102
108
|
peerId: this.peerId,
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
//
|
|
2
|
+
// Copyright 2024 DXOS.org
|
|
3
|
+
//
|
|
4
|
+
|
|
5
|
+
import { type StorageKey } from '@dxos/automerge/automerge-repo';
|
|
6
|
+
import { IndexedDBStorageAdapter } from '@dxos/automerge/automerge-repo-storage-indexeddb';
|
|
7
|
+
import { log } from '@dxos/log';
|
|
8
|
+
import { StorageType, type Directory } from '@dxos/random-access-storage';
|
|
9
|
+
|
|
10
|
+
import { AutomergeStorageAdapter } from './automerge-storage-adapter';
|
|
11
|
+
import { encodingOptions } from './leveldb-storage-adapter';
|
|
12
|
+
import { type SubLevelDB } from './types';
|
|
13
|
+
|
|
14
|
+
export const levelMigration = async ({ db, directory }: { db: SubLevelDB; directory: Directory }) => {
|
|
15
|
+
// Note: Make automigration from previous storage to leveldb here.
|
|
16
|
+
const isNewLevel = !(await db
|
|
17
|
+
.iterator<StorageKey, Uint8Array>({
|
|
18
|
+
...encodingOptions,
|
|
19
|
+
})
|
|
20
|
+
.next());
|
|
21
|
+
|
|
22
|
+
if (!isNewLevel) {
|
|
23
|
+
return;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
const oldStorageAdapter =
|
|
27
|
+
directory.type === StorageType.IDB
|
|
28
|
+
? new IndexedDBStorageAdapter(directory.path, 'data')
|
|
29
|
+
: new AutomergeStorageAdapter(directory);
|
|
30
|
+
|
|
31
|
+
const chunks = await oldStorageAdapter.loadRange([]);
|
|
32
|
+
if (chunks.length === 0) {
|
|
33
|
+
return;
|
|
34
|
+
}
|
|
35
|
+
const batch = db.batch();
|
|
36
|
+
log.info('found chunks on old storage adapter', { chunks: chunks.length });
|
|
37
|
+
for (const { key, data } of await oldStorageAdapter.loadRange([])) {
|
|
38
|
+
data && batch.put<StorageKey, Uint8Array>(key, data, { ...encodingOptions });
|
|
39
|
+
}
|
|
40
|
+
await batch.write();
|
|
41
|
+
};
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
//
|
|
2
|
+
// Copyright 2024 DXOS.org
|
|
3
|
+
//
|
|
4
|
+
|
|
5
|
+
import { expect } from 'chai';
|
|
6
|
+
|
|
7
|
+
import { type StorageAdapterInterface } from '@dxos/automerge/automerge-repo';
|
|
8
|
+
import { PublicKey } from '@dxos/keys';
|
|
9
|
+
import { StorageType, createStorage } from '@dxos/random-access-storage';
|
|
10
|
+
import { afterTest, describe, openAndClose, test } from '@dxos/test';
|
|
11
|
+
import { type MaybePromise } from '@dxos/util';
|
|
12
|
+
|
|
13
|
+
import { AutomergeStorageAdapter } from './automerge-storage-adapter';
|
|
14
|
+
import { LevelDBStorageAdapter } from './leveldb-storage-adapter';
|
|
15
|
+
import { createTestLevel } from '../testing';
|
|
16
|
+
|
|
17
|
+
const runTests = (
|
|
18
|
+
testNamespace: string,
|
|
19
|
+
/** Run per test. Expects automatic clean-up with `afterTest`. */ createAdapter: () => MaybePromise<StorageAdapterInterface>,
|
|
20
|
+
) => {
|
|
21
|
+
describe(testNamespace, () => {
|
|
22
|
+
const chunks = [
|
|
23
|
+
{ key: ['a', 'b', 'c', '1'], data: PublicKey.random().asUint8Array() },
|
|
24
|
+
{ key: ['a', 'b', 'c', '2'], data: PublicKey.random().asUint8Array() },
|
|
25
|
+
{ key: ['a', 'b', 'd', '3'], data: PublicKey.random().asUint8Array() },
|
|
26
|
+
{ key: ['a', 'b', 'd', '4'], data: PublicKey.random().asUint8Array() },
|
|
27
|
+
];
|
|
28
|
+
|
|
29
|
+
test('should store and retrieve data', async () => {
|
|
30
|
+
const adapter = await createAdapter();
|
|
31
|
+
|
|
32
|
+
await adapter.save(chunks[0].key, chunks[0].data);
|
|
33
|
+
expect(await adapter.load(chunks[0].key)).to.deep.equal(chunks[0].data);
|
|
34
|
+
});
|
|
35
|
+
|
|
36
|
+
test('loadRange return inputs with correct prefixes', async () => {
|
|
37
|
+
const adapter = await createAdapter();
|
|
38
|
+
|
|
39
|
+
for (const chunk of chunks) {
|
|
40
|
+
await adapter.save(chunk.key, chunk.data);
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
expect((await adapter.loadRange(['a', 'b'])).length).to.equal(4);
|
|
44
|
+
expect((await adapter.loadRange(['a', 'b', 'c']))[0]).to.deep.equal(chunks[0]);
|
|
45
|
+
expect((await adapter.loadRange(['a', 'b', 'c'])).length).to.equal(2);
|
|
46
|
+
});
|
|
47
|
+
|
|
48
|
+
test('deletion works', async () => {
|
|
49
|
+
const adapter = await createAdapter();
|
|
50
|
+
|
|
51
|
+
for (const chunk of chunks) {
|
|
52
|
+
await adapter.save(chunk.key, chunk.data);
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
await adapter.remove(['a', 'b', 'c', '1']);
|
|
56
|
+
|
|
57
|
+
expect((await adapter.loadRange(['a', 'b'])).length).to.equal(3);
|
|
58
|
+
expect((await adapter.loadRange(['a', 'b', 'c'])).length).to.equal(1);
|
|
59
|
+
|
|
60
|
+
await adapter.removeRange(['a', 'b', 'd']);
|
|
61
|
+
|
|
62
|
+
expect((await adapter.loadRange(['a', 'b'])).length).to.equal(1);
|
|
63
|
+
expect((await adapter.loadRange(['a', 'b']))[0]).to.deep.equal(chunks[1]);
|
|
64
|
+
expect(await adapter.load(['a', 'b', 'c', '2'])).to.deep.equal(chunks[1].data);
|
|
65
|
+
expect(await adapter.load(['a', 'b', 'd', '3'])).to.be.undefined;
|
|
66
|
+
});
|
|
67
|
+
});
|
|
68
|
+
};
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* Run tests for AutomergeStorageAdapter.
|
|
72
|
+
*/
|
|
73
|
+
runTests('AutomergeStorageAdapter', () => {
|
|
74
|
+
const storage = createStorage({ type: StorageType.RAM });
|
|
75
|
+
afterTest(() => storage.close());
|
|
76
|
+
const dir = storage.createDirectory('automerge');
|
|
77
|
+
const adapter = new AutomergeStorageAdapter(dir);
|
|
78
|
+
afterTest(() => adapter.close());
|
|
79
|
+
return adapter;
|
|
80
|
+
});
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* Run tests for LevelDBStorageAdapter.
|
|
84
|
+
*/
|
|
85
|
+
runTests('LevelDBStorageAdapter', async () => {
|
|
86
|
+
const level = createTestLevel();
|
|
87
|
+
const adapter = new LevelDBStorageAdapter({ db: level.sublevel('automerge') });
|
|
88
|
+
await openAndClose(level, adapter as any);
|
|
89
|
+
return adapter;
|
|
90
|
+
});
|
package/src/automerge/types.ts
CHANGED
|
@@ -2,9 +2,8 @@
|
|
|
2
2
|
// Copyright 2024 DXOS.org
|
|
3
3
|
//
|
|
4
4
|
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
//
|
|
5
|
+
import { type AbstractChainedBatch, type AbstractSublevel } from 'abstract-level';
|
|
6
|
+
import { type Level } from 'level';
|
|
8
7
|
|
|
9
8
|
import { type EncodedReferenceObject } from './reference';
|
|
10
9
|
|
|
@@ -81,3 +80,7 @@ export type ObjectSystem = {
|
|
|
81
80
|
*/
|
|
82
81
|
type?: EncodedReferenceObject;
|
|
83
82
|
};
|
|
83
|
+
|
|
84
|
+
export type LevelDB = Level<string, string>;
|
|
85
|
+
export type SubLevelDB = AbstractSublevel<any, string | Buffer | Uint8Array, string, string>;
|
|
86
|
+
export type BatchLevel = AbstractChainedBatch<any, string, string>;
|