@weaveprotocol/core 0.1.1 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +264 -42
- package/dist/identity/contact-key.d.ts +42 -0
- package/dist/identity/contact-key.d.ts.map +1 -0
- package/dist/identity/contact-key.js +133 -0
- package/dist/identity/contact-key.js.map +1 -0
- package/dist/index.d.ts +16 -7
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +10 -5
- package/dist/index.js.map +1 -1
- package/dist/network/mesh.d.ts +6 -0
- package/dist/network/mesh.d.ts.map +1 -1
- package/dist/network/mesh.js +11 -1
- package/dist/network/mesh.js.map +1 -1
- package/dist/network/multi-signaling.d.ts +12 -2
- package/dist/network/multi-signaling.d.ts.map +1 -1
- package/dist/network/multi-signaling.js +86 -14
- package/dist/network/multi-signaling.js.map +1 -1
- package/dist/network/peer-auth.d.ts +50 -16
- package/dist/network/peer-auth.d.ts.map +1 -1
- package/dist/network/peer-auth.js +52 -15
- package/dist/network/peer-auth.js.map +1 -1
- package/dist/node/carrier.d.ts +87 -0
- package/dist/node/carrier.d.ts.map +1 -1
- package/dist/node/carrier.js +180 -59
- package/dist/node/carrier.js.map +1 -1
- package/dist/node/copy.d.ts.map +1 -1
- package/dist/node/copy.js +6 -3
- package/dist/node/copy.js.map +1 -1
- package/dist/node/host.d.ts +117 -0
- package/dist/node/host.d.ts.map +1 -0
- package/dist/node/host.js +192 -0
- package/dist/node/host.js.map +1 -0
- package/dist/node/index.d.ts +6 -2
- package/dist/node/index.d.ts.map +1 -1
- package/dist/node/index.js +1 -0
- package/dist/node/index.js.map +1 -1
- package/dist/node/node.d.ts.map +1 -1
- package/dist/node/node.js +640 -25
- package/dist/node/node.js.map +1 -1
- package/dist/node/space-runtime.d.ts +42 -25
- package/dist/node/space-runtime.d.ts.map +1 -1
- package/dist/node/space-runtime.js +734 -35
- package/dist/node/space-runtime.js.map +1 -1
- package/dist/node/types.d.ts +269 -3
- package/dist/node/types.d.ts.map +1 -1
- package/dist/privacy/space-encryption.d.ts +16 -0
- package/dist/privacy/space-encryption.d.ts.map +1 -1
- package/dist/privacy/space-encryption.js +40 -0
- package/dist/privacy/space-encryption.js.map +1 -1
- package/dist/query/engine.js +1 -1
- package/dist/query/engine.js.map +1 -1
- package/dist/query/types.d.ts +7 -0
- package/dist/query/types.d.ts.map +1 -1
- package/dist/query/types.js.map +1 -1
- package/dist/react/use-query.d.ts +1 -1
- package/dist/react/use-query.d.ts.map +1 -1
- package/dist/records/topics.d.ts +21 -0
- package/dist/records/topics.d.ts.map +1 -0
- package/dist/records/topics.js +86 -0
- package/dist/records/topics.js.map +1 -0
- package/dist/schema/collection-def.d.ts +7 -0
- package/dist/schema/collection-def.d.ts.map +1 -1
- package/dist/schema/collection-def.js +4 -0
- package/dist/schema/collection-def.js.map +1 -1
- package/dist/schema/expression.d.ts +2 -0
- package/dist/schema/expression.d.ts.map +1 -1
- package/dist/schema/expression.js +1 -0
- package/dist/schema/expression.js.map +1 -1
- package/dist/schemas/contacts.d.ts +77 -0
- package/dist/schemas/contacts.d.ts.map +1 -0
- package/dist/schemas/contacts.js +35 -0
- package/dist/schemas/contacts.js.map +1 -0
- package/dist/schemas/index.d.ts +2 -0
- package/dist/schemas/index.d.ts.map +1 -1
- package/dist/schemas/index.js +1 -0
- package/dist/schemas/index.js.map +1 -1
- package/dist/session/auth.d.ts.map +1 -1
- package/dist/session/auth.js +14 -3
- package/dist/session/auth.js.map +1 -1
- package/dist/session/connect.d.ts +25 -1
- package/dist/session/connect.d.ts.map +1 -1
- package/dist/session/connect.js +9 -2
- package/dist/session/connect.js.map +1 -1
- package/dist/session/hosting.d.ts +155 -0
- package/dist/session/hosting.d.ts.map +1 -0
- package/dist/session/hosting.js +160 -0
- package/dist/session/hosting.js.map +1 -0
- package/dist/space/account-registry.d.ts +6 -0
- package/dist/space/account-registry.d.ts.map +1 -1
- package/dist/space/account-registry.js +16 -4
- package/dist/space/account-registry.js.map +1 -1
- package/dist/space/notify.d.ts +88 -0
- package/dist/space/notify.d.ts.map +1 -0
- package/dist/space/notify.js +127 -0
- package/dist/space/notify.js.map +1 -0
- package/dist/space/pass.d.ts +6 -0
- package/dist/space/pass.d.ts.map +1 -1
- package/dist/space/pass.js +5 -2
- package/dist/space/pass.js.map +1 -1
- package/dist/space/roles.d.ts +71 -0
- package/dist/space/roles.d.ts.map +1 -1
- package/dist/space/roles.js +114 -0
- package/dist/space/roles.js.map +1 -1
- package/dist/space/space-access.d.ts +24 -0
- package/dist/space/space-access.d.ts.map +1 -1
- package/dist/space/space-access.js +24 -0
- package/dist/space/space-access.js.map +1 -1
- package/dist/space/space-manager.d.ts +35 -1
- package/dist/space/space-manager.d.ts.map +1 -1
- package/dist/space/space-manager.js +72 -25
- package/dist/space/space-manager.js.map +1 -1
- package/dist/storage/blob/memory.d.ts +9 -0
- package/dist/storage/blob/memory.d.ts.map +1 -0
- package/dist/storage/blob/memory.js +20 -0
- package/dist/storage/blob/memory.js.map +1 -0
- package/dist/storage/blob/s3.d.ts +16 -0
- package/dist/storage/blob/s3.d.ts.map +1 -0
- package/dist/storage/blob/s3.js +86 -0
- package/dist/storage/blob/s3.js.map +1 -0
- package/dist/storage/blob-store.d.ts +26 -0
- package/dist/storage/blob-store.d.ts.map +1 -0
- package/dist/storage/blob-store.js +2 -0
- package/dist/storage/blob-store.js.map +1 -0
- package/dist/storage/encrypted-adapter.d.ts +3 -3
- package/dist/storage/encrypted-adapter.js +3 -3
- package/dist/storage/folder-adapter.d.ts +8 -8
- package/dist/storage/folder-adapter.d.ts.map +1 -1
- package/dist/storage/folder-adapter.js +7 -7
- package/dist/storage/folder-reconcile.d.ts +12 -14
- package/dist/storage/folder-reconcile.d.ts.map +1 -1
- package/dist/storage/folder-reconcile.js +20 -14
- package/dist/storage/folder-reconcile.js.map +1 -1
- package/dist/storage/index.d.ts +0 -1
- package/dist/storage/index.d.ts.map +1 -1
- package/dist/storage/index.js +0 -1
- package/dist/storage/index.js.map +1 -1
- package/dist/storage/indexeddb-adapter.d.ts.map +1 -1
- package/dist/storage/indexeddb-adapter.js +8 -9
- package/dist/storage/indexeddb-adapter.js.map +1 -1
- package/dist/storage/mirror.d.ts +68 -0
- package/dist/storage/mirror.d.ts.map +1 -0
- package/dist/storage/mirror.js +174 -0
- package/dist/storage/mirror.js.map +1 -0
- package/dist/storage/segment.d.ts +20 -0
- package/dist/storage/segment.d.ts.map +1 -0
- package/dist/storage/segment.js +27 -0
- package/dist/storage/segment.js.map +1 -0
- package/dist/storage/storage-provider.d.ts +36 -40
- package/dist/storage/storage-provider.d.ts.map +1 -1
- package/dist/storage/storage-provider.js +208 -117
- package/dist/storage/storage-provider.js.map +1 -1
- package/dist/sync/index.d.ts +1 -1
- package/dist/sync/index.d.ts.map +1 -1
- package/dist/sync/index.js +1 -1
- package/dist/sync/index.js.map +1 -1
- package/dist/sync/negentropy.d.ts +68 -0
- package/dist/sync/negentropy.d.ts.map +1 -0
- package/dist/sync/negentropy.js +379 -0
- package/dist/sync/negentropy.js.map +1 -0
- package/dist/sync/sync-engine.d.ts +45 -13
- package/dist/sync/sync-engine.d.ts.map +1 -1
- package/dist/sync/sync-engine.js +290 -207
- package/dist/sync/sync-engine.js.map +1 -1
- package/dist/sync/sync-messages.d.ts +42 -32
- package/dist/sync/sync-messages.d.ts.map +1 -1
- package/dist/sync/sync-messages.js +1 -1
- package/dist/sync/sync-messages.js.map +1 -1
- package/dist/types.d.ts +9 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js.map +1 -1
- package/dist/utils/hash.d.ts +9 -0
- package/dist/utils/hash.d.ts.map +1 -1
- package/dist/utils/hash.js +33 -0
- package/dist/utils/hash.js.map +1 -1
- package/package.json +9 -3
- package/dist/storage/mst.d.ts +0 -121
- package/dist/storage/mst.d.ts.map +0 -1
- package/dist/storage/mst.js +0 -402
- package/dist/storage/mst.js.map +0 -1
- package/dist/sync/anti-entropy.d.ts +0 -49
- package/dist/sync/anti-entropy.d.ts.map +0 -1
- package/dist/sync/anti-entropy.js +0 -62
- package/dist/sync/anti-entropy.js.map +0 -1
|
@@ -5,21 +5,22 @@
|
|
|
5
5
|
* Two origins — or two devices behind Dropbox, iCloud or Syncthing — can write
|
|
6
6
|
* into one folder at the same time. Almost nothing there can actually conflict:
|
|
7
7
|
* expression files are named by their own content hash, so two writers either
|
|
8
|
-
* produce different files or produce byte-identical ones. The
|
|
9
|
-
*
|
|
10
|
-
*
|
|
8
|
+
* produce different files or produce byte-identical ones. The entries saying
|
|
9
|
+
* which version is current can be written by both, and one write can be lost;
|
|
10
|
+
* that costs nothing, because they are derived from the files.
|
|
11
11
|
*
|
|
12
12
|
* So the rule is: **the set of expression files wins.** Reconciling means
|
|
13
|
-
*
|
|
14
|
-
*
|
|
13
|
+
* placing every file that turned up — which also puts right a current-version
|
|
14
|
+
* entry the other writer overwrote — and dropping entries whose file went
|
|
15
|
+
* away. That is the whole concurrency story.
|
|
15
16
|
*/
|
|
16
17
|
/**
|
|
17
|
-
* Re-reads the folder and brings the
|
|
18
|
+
* Re-reads the folder and brings the entries back in line with it.
|
|
18
19
|
*
|
|
19
20
|
* Safe to call as often as you like: with nothing new on disk it is two
|
|
20
21
|
* directory listings and no writes.
|
|
21
22
|
*
|
|
22
|
-
* @param storage The provider whose
|
|
23
|
+
* @param storage The provider whose entries should be brought up to date
|
|
23
24
|
* @param adapter The folder adapter underneath it
|
|
24
25
|
* @param accept Whether a version found on disk may be placed. Anyone who can
|
|
25
26
|
* write the folder can drop a file in it — another site given the folder, a
|
|
@@ -29,13 +30,18 @@
|
|
|
29
30
|
*/
|
|
30
31
|
export async function reconcileFolder(storage, adapter, accept = async () => true) {
|
|
31
32
|
const disk = await adapter.reload();
|
|
33
|
+
// Another writer's entries are on disk now too: what sync compares is read again.
|
|
34
|
+
if (disk.added.length > 0 || disk.removed.length > 0)
|
|
35
|
+
storage.invalidate();
|
|
32
36
|
const onDisk = new Set(await adapter.listExpressionIds());
|
|
33
|
-
// Versions another writer put on disk
|
|
34
|
-
//
|
|
35
|
-
// is
|
|
36
|
-
//
|
|
37
|
-
|
|
38
|
-
const
|
|
37
|
+
// Versions another writer put on disk. Each goes through the ordering rule
|
|
38
|
+
// like anything else: it becomes current, is kept as history, or loses and
|
|
39
|
+
// is dropped — the same decision the other writer reached, so both converge.
|
|
40
|
+
// Files that appeared since the last pass are placed again even if indexed:
|
|
41
|
+
// placing is idempotent, and it corrects a current entry a race left wrong.
|
|
42
|
+
const indexed = new Set(await storage.versionIds());
|
|
43
|
+
const appeared = new Set(disk.added);
|
|
44
|
+
const missing = [...onDisk].filter((id) => !indexed.has(id) || appeared.has(id));
|
|
39
45
|
const placed = [];
|
|
40
46
|
for (const id of missing) {
|
|
41
47
|
const expression = await adapter.getExpression(id);
|
|
@@ -48,7 +54,7 @@ export async function reconcileFolder(storage, adapter, accept = async () => tru
|
|
|
48
54
|
}
|
|
49
55
|
// Entries whose file is gone: another writer superseded and dropped it, and
|
|
50
56
|
// its replacement was placed above.
|
|
51
|
-
const stale =
|
|
57
|
+
const stale = (await storage.versionIds()).filter((id) => !onDisk.has(id) && !missing.includes(id));
|
|
52
58
|
for (const id of stale) {
|
|
53
59
|
if (!(await adapter.getExpression(id)))
|
|
54
60
|
await storage.removeExpression(id);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"folder-reconcile.js","sourceRoot":"","sources":["../../src/storage/folder-reconcile.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"folder-reconcile.js","sourceRoot":"","sources":["../../src/storage/folder-reconcile.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAuBH;;;;;;;;;;;;;GAaG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,OAAwB,EACxB,OAAsB,EACtB,SAAuD,KAAK,IAAI,EAAE,CAAC,IAAI;IAEvE,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,MAAM,EAAE,CAAC;IACpC,kFAAkF;IAClF,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,CAAC,UAAU,EAAE,CAAC;IAC3E,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,MAAM,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAAC;IAE1D,2EAA2E;IAC3E,2EAA2E;IAC3E,6EAA6E;IAC7E,4EAA4E;IAC5E,4EAA4E;IAC5E,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC;IACpD,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACrC,MAAM,OAAO,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;IACjF,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,KAAK,MAAM,EAAE,IAAI,OAAO,EAAE,CAAC;QACzB,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;QACnD,sEAAsE;QACtE,2EAA2E;QAC3E,IAAI,UAAU,IAAI,CAAC,MAAM,MAAM,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC;YAC7C,MAAM,OAAO,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC;YACxC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAClB,CAAC;IACH,CAAC;IAED,4EAA4E;IAC5E,oCAAoC;IACpC,MAAM,KAAK,GAAG,CAAC,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;IACpG,KAAK,MAAM,EAAE,IAAI,KAAK,EAAE,CAAC;QACvB,IAAI,CAAC,CAAC,MAAM,OAAO,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;YAAE,MAAM,OAAO,CAAC,gBAAgB,CAAC,EAAE,CAAC,CAAC;IAC7E,CAAC;IAED,wEAAwE;IACxE,MAAM,QAAQ,GAAG,CAAC,GAAG,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC;IACvC,OAAO;QACL,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,OAAO,EAAE,IAAI,CAAC,OAAO;QACrB,QAAQ;QACR,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC;KACjF,CAAC;AACJ,CAAC"}
|
package/dist/storage/index.d.ts
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/storage/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,cAAc,wBAAwB,CAAC;AACvC,cAAc,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/storage/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,cAAc,wBAAwB,CAAC;AACvC,cAAc,uBAAuB,CAAC;AACtC,cAAc,uBAAuB,CAAC;AACtC,YAAY,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC"}
|
package/dist/storage/index.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/storage/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,cAAc,wBAAwB,CAAC;AACvC,cAAc,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/storage/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,cAAc,wBAAwB,CAAC;AACvC,cAAc,uBAAuB,CAAC;AACtC,cAAc,uBAAuB,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"indexeddb-adapter.d.ts","sourceRoot":"","sources":["../../src/storage/indexeddb-adapter.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,cAAc,EAAuB,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"indexeddb-adapter.d.ts","sourceRoot":"","sources":["../../src/storage/indexeddb-adapter.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,cAAc,EAAuB,MAAM,aAAa,CAAC;AA0CvE;;;;GAIG;AACH,wBAAsB,sBAAsB,CAAC,MAAM,GAAE,MAAwB,GAAG,OAAO,CAAC,cAAc,CAAC,CAqKtG"}
|
|
@@ -3,10 +3,11 @@
|
|
|
3
3
|
* IndexedDB implementation of StorageAdapter for P2P network.
|
|
4
4
|
*/
|
|
5
5
|
/**
|
|
6
|
-
* Bumped to
|
|
7
|
-
*
|
|
6
|
+
* Bumped to 3 when the Merkle tree gave way to plain entries and sync by
|
|
7
|
+
* reconciliation. Opening an older database wipes it: it is a local copy,
|
|
8
|
+
* rebuilt by syncing with peers.
|
|
8
9
|
*/
|
|
9
|
-
const DB_VERSION =
|
|
10
|
+
const DB_VERSION = 3;
|
|
10
11
|
/**
|
|
11
12
|
* Promisify an IDBRequest.
|
|
12
13
|
* @param request The IDBRequest to promisify
|
|
@@ -45,17 +46,15 @@ export async function createIndexedDBAdapter(dbName = 'weave-storage') {
|
|
|
45
46
|
request.onupgradeneeded = (event) => {
|
|
46
47
|
const db = request.result;
|
|
47
48
|
const previousVersion = event.oldVersion;
|
|
48
|
-
//
|
|
49
|
-
//
|
|
50
|
-
|
|
51
|
-
// rebuilt by syncing with peers.
|
|
52
|
-
if (previousVersion > 0 && previousVersion < 2) {
|
|
49
|
+
// Older versions indexed records in a Merkle tree this store no longer
|
|
50
|
+
// has. The local copy is dropped and rebuilt by syncing with peers.
|
|
51
|
+
if (previousVersion > 0 && previousVersion < 3) {
|
|
53
52
|
for (const name of ['kv', 'expressions']) {
|
|
54
53
|
if (db.objectStoreNames.contains(name))
|
|
55
54
|
db.deleteObjectStore(name);
|
|
56
55
|
}
|
|
57
56
|
}
|
|
58
|
-
// Store for key-value
|
|
57
|
+
// Store for key-value: which version is current, what sync compares, space records
|
|
59
58
|
if (!db.objectStoreNames.contains('kv')) {
|
|
60
59
|
db.createObjectStore('kv');
|
|
61
60
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"indexeddb-adapter.js","sourceRoot":"","sources":["../../src/storage/indexeddb-adapter.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAIH
|
|
1
|
+
{"version":3,"file":"indexeddb-adapter.js","sourceRoot":"","sources":["../../src/storage/indexeddb-adapter.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAIH;;;;GAIG;AACH,MAAM,UAAU,GAAG,CAAC,CAAC;AAErB;;;;GAIG;AACH,SAAS,UAAU,CAAI,OAAsB;IAC3C,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrC,OAAO,CAAC,SAAS,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAClD,OAAO,CAAC,OAAO,GAAG,GAAG,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAChD,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;;;;GAMG;AACH,SAAS,cAAc,CACrB,EAAe,EACf,MAAgB,EAChB,IAAwB;IAExB,MAAM,EAAE,GAAG,EAAE,CAAC,WAAW,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IACxC,MAAM,QAAQ,GAAG,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrD,EAAE,CAAC,UAAU,GAAG,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC;QAChC,EAAE,CAAC,OAAO,GAAG,GAAG,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC;QACpC,EAAE,CAAC,OAAO,GAAG,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC,CAAC;IAC9D,CAAC,CAAC,CAAC;IACH,OAAO,EAAE,EAAE,EAAE,QAAQ,EAAE,CAAC;AAC1B,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAAC,SAAiB,eAAe;IAC3E,MAAM,EAAE,GAAG,MAAM,IAAI,OAAO,CAAc,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QAC5D,MAAM,OAAO,GAAG,UAAU,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;QAE9D,OAAO,CAAC,eAAe,GAAG,CAAC,KAAK,EAAE,EAAE;YAClC,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC;YAC1B,MAAM,eAAe,GAAI,KAA+B,CAAC,UAAU,CAAC;YAEpE,uEAAuE;YACvE,oEAAoE;YACpE,IAAI,eAAe,GAAG,CAAC,IAAI,eAAe,GAAG,CAAC,EAAE,CAAC;gBAC/C,KAAK,MAAM,IAAI,IAAI,CAAC,IAAI,EAAE,aAAa,CAAC,EAAE,CAAC;oBACzC,IAAI,EAAE,CAAC,gBAAgB,CAAC,QAAQ,CAAC,IAAI,CAAC;wBAAE,EAAE,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC;gBACrE,CAAC;YACH,CAAC;YAED,mFAAmF;YACnF,IAAI,CAAC,EAAE,CAAC,gBAAgB,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;gBACxC,EAAE,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC;YAC7B,CAAC;YAED,wBAAwB;YACxB,IAAI,CAAC,EAAE,CAAC,gBAAgB,CAAC,QAAQ,CAAC,aAAa,CAAC,EAAE,CAAC;gBACjD,MAAM,QAAQ,GAAG,EAAE,CAAC,iBAAiB,CAAC,aAAa,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;gBACxE,QAAQ,CAAC,WAAW,CAAC,YAAY,EAAE,YAAY,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;gBACpE,QAAQ,CAAC,WAAW,CAAC,QAAQ,EAAE,QAAQ,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;gBAC5D,QAAQ,CAAC,WAAW,CAAC,WAAW,EAAE,WAAW,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;YACpE,CAAC;QACH,CAAC,CAAC;QAEF,OAAO,CAAC,SAAS,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAClD,OAAO,CAAC,OAAO,GAAG,GAAG,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAChD,CAAC,CAAC,CAAC;IAEH,OAAO,MAAM,CAAC,MAAM,CAAC;QACnB,KAAK,CAAC,GAAG,CAAC,GAAW;YACnB,MAAM,EAAE,EAAE,EAAE,GAAG,cAAc,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,UAAU,CAAC,CAAC;YACtD,MAAM,KAAK,GAAG,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;YACnC,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;YAChD,OAAO,MAAM,CAAC,CAAC,CAAC,IAAI,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAChD,CAAC;QAED,KAAK,CAAC,GAAG,CAAC,GAAW,EAAE,KAAiB;YACtC,MAAM,EAAE,EAAE,EAAE,QAAQ,EAAE,GAAG,cAAc,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,WAAW,CAAC,CAAC;YACjE,MAAM,KAAK,GAAG,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;YACnC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;YAC7B,MAAM,QAAQ,CAAC;QACjB,CAAC;QAED,KAAK,CAAC,MAAM,CAAC,GAAW;YACtB,MAAM,EAAE,EAAE,EAAE,QAAQ,EAAE,GAAG,cAAc,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,WAAW,CAAC,CAAC;YACjE,MAAM,KAAK,GAAG,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;YACnC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAClB,MAAM,QAAQ,CAAC;QACjB,CAAC;QAED,KAAK,CAAC,GAAG,CAAC,GAAW;YACnB,MAAM,EAAE,EAAE,EAAE,GAAG,cAAc,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,UAAU,CAAC,CAAC;YACtD,MAAM,KAAK,GAAG,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;YACnC,MAAM,KAAK,GAAG,MAAM,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;YACjD,OAAO,KAAK,GAAG,CAAC,CAAC;QACnB,CAAC;QAED,KAAK,CAAC,IAAI,CAAC,SAAiB,EAAE;YAC5B,MAAM,EAAE,EAAE,EAAE,GAAG,cAAc,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,UAAU,CAAC,CAAC;YACtD,MAAM,KAAK,GAAG,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;YACnC,MAAM,IAAI,GAAa,EAAE,CAAC;YAE1B,IAAI,OAAO,CAAC;YACZ,IAAI,MAAM,EAAE,CAAC;gBACX,MAAM,KAAK,GAAG,MAAM,GAAG,QAAQ,CAAC;gBAChC,OAAO,GAAG,KAAK,CAAC,UAAU,CAAC,WAAW,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC;YAC7E,CAAC;iBAAM,CAAC;gBACN,OAAO,GAAG,KAAK,CAAC,UAAU,EAAE,CAAC;YAC/B,CAAC;YAED,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;gBAC1C,OAAO,CAAC,SAAS,GAAG,CAAC,KAAK,EAAE,EAAE;oBAC5B,MAAM,MAAM,GAAI,KAAK,CAAC,MAAyC,CAAC,MAAM,CAAC;oBACvE,IAAI,MAAM,EAAE,CAAC;wBACX,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,GAAa,CAAC,CAAC;wBAChC,MAAM,CAAC,QAAQ,EAAE,CAAC;oBACpB,CAAC;yBAAM,CAAC;wBACN,OAAO,EAAE,CAAC;oBACZ,CAAC;gBACH,CAAC,CAAC;gBACF,OAAO,CAAC,OAAO,GAAG,GAAG,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YAChD,CAAC,CAAC,CAAC;YACH,OAAO,IAAI,CAAC;QACd,CAAC;QAED,KAAK,CAAC,gBAAgB,CAAC,UAAkB,EAAE,QAAgB,EAAE,EAAE,MAAe;YAC5E,MAAM,EAAE,EAAE,EAAE,GAAG,cAAc,CAAC,EAAE,EAAE,CAAC,aAAa,CAAC,EAAE,UAAU,CAAC,CAAC;YAC/D,MAAM,KAAK,GAAG,EAAE,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC;YAC5C,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;YAExC,MAAM,KAAK,GAAG,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAC3C,MAAM,OAAO,GAAG,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;YAExC,MAAM,OAAO,GAAiB,EAAE,CAAC;YACjC,IAAI,QAAQ,GAAG,CAAC,MAAM,CAAC;YAEvB,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;gBAC1C,OAAO,CAAC,SAAS,GAAG,CAAC,KAAK,EAAE,EAAE;oBAC5B,MAAM,SAAS,GAAI,KAAK,CAAC,MAAyC,CAAC,MAAM,CAAC;oBAC1E,IAAI,CAAC,SAAS,IAAI,OAAO,CAAC,MAAM,IAAI,KAAK,EAAE,CAAC;wBAC1C,OAAO,EAAE,CAAC;wBACV,OAAO;oBACT,CAAC;oBAED,IAAI,CAAC,QAAQ,IAAI,MAAM,IAAI,SAAS,CAAC,UAAU,KAAK,MAAM,EAAE,CAAC;wBAC3D,QAAQ,GAAG,IAAI,CAAC;wBAChB,SAAS,CAAC,QAAQ,EAAE,CAAC;wBACrB,OAAO;oBACT,CAAC;oBAED,IAAI,QAAQ,EAAE,CAAC;wBACb,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;oBAChC,CAAC;oBACD,SAAS,CAAC,QAAQ,EAAE,CAAC;gBACvB,CAAC,CAAC;gBACF,OAAO,CAAC,OAAO,GAAG,GAAG,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YAChD,CAAC,CAAC,CAAC;YAEH,OAAO,OAAO,CAAC;QACjB,CAAC;QAED,KAAK,CAAC,aAAa,CAAC,UAAsB;YACxC,MAAM,EAAE,EAAE,EAAE,QAAQ,EAAE,GAAG,cAAc,CAAC,EAAE,EAAE,CAAC,aAAa,CAAC,EAAE,WAAW,CAAC,CAAC;YAC1E,MAAM,KAAK,GAAG,EAAE,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC;YAC5C,KAAK,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;YACtB,MAAM,QAAQ,CAAC;QACjB,CAAC;QAED,KAAK,CAAC,aAAa,CAAC,EAAU;YAC5B,MAAM,EAAE,EAAE,EAAE,GAAG,cAAc,CAAC,EAAE,EAAE,CAAC,aAAa,CAAC,EAAE,UAAU,CAAC,CAAC;YAC/D,MAAM,KAAK,GAAG,EAAE,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC;YAC5C,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;YAC/C,OAAO,MAAM,IAAI,IAAI,CAAC;QACxB,CAAC;QAED,KAAK,CAAC,gBAAgB,CAAC,EAAU;YAC/B,MAAM,EAAE,EAAE,EAAE,QAAQ,EAAE,GAAG,cAAc,CAAC,EAAE,EAAE,CAAC,aAAa,CAAC,EAAE,WAAW,CAAC,CAAC;YAC1E,MAAM,KAAK,GAAG,EAAE,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC;YAC5C,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YACjB,MAAM,QAAQ,CAAC;QACjB,CAAC;QAED,KAAK,CAAC,KAAK,CAAC,GAA2B;YACrC,MAAM,EAAE,EAAE,EAAE,QAAQ,EAAE,GAAG,cAAc,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,WAAW,CAAC,CAAC;YACjE,MAAM,KAAK,GAAG,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;YACnC,KAAK,MAAM,EAAE,IAAI,GAAG,EAAE,CAAC;gBACrB,IAAI,EAAE,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;oBACtB,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC;gBACrC,CAAC;qBAAM,IAAI,EAAE,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;oBAChC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC;gBACvB,CAAC;YACH,CAAC;YACD,MAAM,QAAQ,CAAC;QACjB,CAAC;QAED,KAAK,CAAC,KAAK;YACT,EAAE,CAAC,KAAK,EAAE,CAAC;QACb,CAAC;KACF,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @module storage/mirror
|
|
3
|
+
* A space kept in a dumb file store, synced like a peer that never runs code.
|
|
4
|
+
*
|
|
5
|
+
* ```
|
|
6
|
+
* <space-id>/
|
|
7
|
+
* <writer-id>/000001-<hash>.seg immutable: written once, never changed
|
|
8
|
+
* <writer-id>/000002-<hash>.seg
|
|
9
|
+
* <other writer>/…
|
|
10
|
+
* ```
|
|
11
|
+
*
|
|
12
|
+
* A **writer** is one store on one device — a browser's database, a host —
|
|
13
|
+
* with a random id of its own. It only ever creates files in its own folder,
|
|
14
|
+
* so nothing is written twice, nothing is overwritten, and nothing needs a
|
|
15
|
+
* lock. A segment holds versions exactly as they travel; each reader builds
|
|
16
|
+
* its own tree from them. Merging is the protocol's own: the set of versions
|
|
17
|
+
* only grows, and the version rule picks the same current one everywhere.
|
|
18
|
+
*
|
|
19
|
+
* **Push**: versions this store holds that the store is not known to hold go
|
|
20
|
+
* out in a new segment. "Known" is everything this writer uploaded or read,
|
|
21
|
+
* so nobody re-uploads everyone else's records.
|
|
22
|
+
*
|
|
23
|
+
* **Pull**: segments not read yet are fetched once, and every version in them
|
|
24
|
+
* goes through the same gates as one from a peer — the store is untrusted.
|
|
25
|
+
* It can hide things (the mesh fills the gap) but not forge them.
|
|
26
|
+
*
|
|
27
|
+
* **Compaction**: a writer may rewrite its own segments into fewer, keeping
|
|
28
|
+
* only what its store still keeps, then delete the old ones. New first,
|
|
29
|
+
* delete after: a reader in between sees duplicates, which are harmless.
|
|
30
|
+
*/
|
|
31
|
+
import type { Expression, StorageAdapter } from '../types.js';
|
|
32
|
+
import type { BlobStore } from './blob-store.js';
|
|
33
|
+
import type { StorageProvider } from './storage-provider.js';
|
|
34
|
+
/** What became of a version handed over from the store */
|
|
35
|
+
export type Taken = 'stored' | 'later' | 'refused';
|
|
36
|
+
export interface MirrorConfig {
|
|
37
|
+
readonly store: BlobStore;
|
|
38
|
+
readonly space: string;
|
|
39
|
+
readonly storage: StorageProvider;
|
|
40
|
+
/** The gates sync uses, and storing what passes: nothing from the store is trusted more than a peer */
|
|
41
|
+
readonly accept: (version: Expression) => Promise<Taken>;
|
|
42
|
+
/** Where this writer keeps its id, what it read and what it knows the store holds */
|
|
43
|
+
readonly state: StorageAdapter;
|
|
44
|
+
/** Pack a segment once this much is waiting. Default 256 KiB. */
|
|
45
|
+
readonly flushBytes?: number;
|
|
46
|
+
/** Or this long after the first change. Default 5 s. */
|
|
47
|
+
readonly flushMs?: number;
|
|
48
|
+
/** Rewrite this writer's segments once it has this many. Default 32. */
|
|
49
|
+
readonly compactAt?: number;
|
|
50
|
+
}
|
|
51
|
+
export interface Mirror {
|
|
52
|
+
/** Takes in what others left in the store. How many versions were stored. */
|
|
53
|
+
pull(): Promise<{
|
|
54
|
+
readonly added: number;
|
|
55
|
+
}>;
|
|
56
|
+
/** Uploads what the store is missing, now */
|
|
57
|
+
flush(): Promise<void>;
|
|
58
|
+
/** Something changed here: flush soon */
|
|
59
|
+
changed(): void;
|
|
60
|
+
/** Rewrites this writer's segments into fewer */
|
|
61
|
+
compact(): Promise<void>;
|
|
62
|
+
/** Flushes, and stops the timer */
|
|
63
|
+
close(): Promise<void>;
|
|
64
|
+
}
|
|
65
|
+
/** Deletes everything a space keeps in a store — every writer's segments */
|
|
66
|
+
export declare function deleteMirrored(store: BlobStore, space: string): Promise<void>;
|
|
67
|
+
export declare function createMirror(config: MirrorConfig): Promise<Mirror>;
|
|
68
|
+
//# sourceMappingURL=mirror.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"mirror.d.ts","sourceRoot":"","sources":["../../src/storage/mirror.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,OAAO,KAAK,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC9D,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAI7D,0DAA0D;AAC1D,MAAM,MAAM,KAAK,GAAG,QAAQ,GAAG,OAAO,GAAG,SAAS,CAAC;AAEnD,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,KAAK,EAAE,SAAS,CAAC;IAC1B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,OAAO,EAAE,eAAe,CAAC;IAClC,uGAAuG;IACvG,QAAQ,CAAC,MAAM,EAAE,CAAC,OAAO,EAAE,UAAU,KAAK,OAAO,CAAC,KAAK,CAAC,CAAC;IACzD,qFAAqF;IACrF,QAAQ,CAAC,KAAK,EAAE,cAAc,CAAC;IAC/B,iEAAiE;IACjE,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAC7B,wDAAwD;IACxD,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAC1B,wEAAwE;IACxE,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;CAC7B;AAED,MAAM,WAAW,MAAM;IACrB,6EAA6E;IAC7E,IAAI,IAAI,OAAO,CAAC;QAAE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC5C,6CAA6C;IAC7C,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACvB,yCAAyC;IACzC,OAAO,IAAI,IAAI,CAAC;IAChB,iDAAiD;IACjD,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACzB,mCAAmC;IACnC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACxB;AAQD,4EAA4E;AAC5E,wBAAsB,cAAc,CAAC,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAEnF;AAED,wBAAsB,YAAY,CAAC,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CA2JxE"}
|
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
import { packSegment, segmentName, SEGMENT_SUFFIX, unpackSegment, versionSize } from './segment.js';
|
|
2
|
+
import { utf8Decode, utf8Encode } from '../utils/encoding.js';
|
|
3
|
+
const WRITER_KEY = 'mirror:writer';
|
|
4
|
+
const COUNTER_KEY = 'mirror:counter';
|
|
5
|
+
const readKey = (name) => `mirror:read:${name}`;
|
|
6
|
+
const knownKey = (id) => `mirror:known:${id}`;
|
|
7
|
+
const MARK = utf8Encode('1');
|
|
8
|
+
/** Deletes everything a space keeps in a store — every writer's segments */
|
|
9
|
+
export async function deleteMirrored(store, space) {
|
|
10
|
+
for (const key of await store.list(`${space}/`))
|
|
11
|
+
await store.delete(key);
|
|
12
|
+
}
|
|
13
|
+
export async function createMirror(config) {
|
|
14
|
+
const { store, space, storage, state } = config;
|
|
15
|
+
const flushBytes = config.flushBytes ?? 256 * 1024;
|
|
16
|
+
const flushMs = config.flushMs ?? 5000;
|
|
17
|
+
const compactAt = config.compactAt ?? 32;
|
|
18
|
+
// 128 random bits, never a DID: a folder listing must not say whose device wrote it.
|
|
19
|
+
let writerBytes = await state.get(WRITER_KEY);
|
|
20
|
+
if (!writerBytes) {
|
|
21
|
+
writerBytes = utf8Encode(Array.from(globalThis.crypto.getRandomValues(new Uint8Array(16)), (b) => b.toString(16).padStart(2, '0')).join(''));
|
|
22
|
+
await state.put(WRITER_KEY, writerBytes);
|
|
23
|
+
}
|
|
24
|
+
const writer = utf8Decode(writerBytes);
|
|
25
|
+
const mine = `${space}/${writer}/`;
|
|
26
|
+
const counterBytes = await state.get(COUNTER_KEY);
|
|
27
|
+
let counter = counterBytes ? Number(utf8Decode(counterBytes)) : 0;
|
|
28
|
+
const isKnown = async (id) => (await state.get(knownKey(id))) !== null;
|
|
29
|
+
const markKnown = (id) => state.put(knownKey(id), MARK);
|
|
30
|
+
let timer = null;
|
|
31
|
+
let closed = false;
|
|
32
|
+
/** One upload or compaction at a time: two at once would pack the same versions twice */
|
|
33
|
+
let busy = Promise.resolve();
|
|
34
|
+
const serial = (work) => {
|
|
35
|
+
const next = busy.then(work, work);
|
|
36
|
+
busy = next.catch(() => { });
|
|
37
|
+
return next;
|
|
38
|
+
};
|
|
39
|
+
async function writeSegment(versions) {
|
|
40
|
+
const bytes = packSegment(versions);
|
|
41
|
+
counter += 1;
|
|
42
|
+
const name = `${mine}${await segmentName(counter, bytes)}`;
|
|
43
|
+
await store.put(name, bytes);
|
|
44
|
+
await state.put(COUNTER_KEY, utf8Encode(String(counter)));
|
|
45
|
+
await state.put(readKey(name), MARK);
|
|
46
|
+
return name;
|
|
47
|
+
}
|
|
48
|
+
/** Every version this store keeps: current, first and retained */
|
|
49
|
+
async function held() {
|
|
50
|
+
return storage.versionIds();
|
|
51
|
+
}
|
|
52
|
+
async function flushNow() {
|
|
53
|
+
const waiting = [];
|
|
54
|
+
for (const id of await held()) {
|
|
55
|
+
if (await isKnown(id))
|
|
56
|
+
continue;
|
|
57
|
+
const version = await storage.getExpression(id);
|
|
58
|
+
if (version)
|
|
59
|
+
waiting.push(version);
|
|
60
|
+
}
|
|
61
|
+
let batch = [];
|
|
62
|
+
let size = 0;
|
|
63
|
+
const send = async () => {
|
|
64
|
+
if (batch.length === 0)
|
|
65
|
+
return;
|
|
66
|
+
await writeSegment(batch);
|
|
67
|
+
for (const version of batch)
|
|
68
|
+
await markKnown(version.id);
|
|
69
|
+
batch = [];
|
|
70
|
+
size = 0;
|
|
71
|
+
};
|
|
72
|
+
for (const version of waiting) {
|
|
73
|
+
batch.push(version);
|
|
74
|
+
size += versionSize(version);
|
|
75
|
+
if (size >= flushBytes)
|
|
76
|
+
await send();
|
|
77
|
+
}
|
|
78
|
+
await send();
|
|
79
|
+
if ((await store.list(mine)).filter((key) => key.endsWith(SEGMENT_SUFFIX)).length >= compactAt)
|
|
80
|
+
await compactNow();
|
|
81
|
+
}
|
|
82
|
+
async function compactNow() {
|
|
83
|
+
const old = (await store.list(mine)).filter((key) => key.endsWith(SEGMENT_SUFFIX));
|
|
84
|
+
if (old.length < 2)
|
|
85
|
+
return;
|
|
86
|
+
const wanted = new Set(await held());
|
|
87
|
+
const keep = new Map();
|
|
88
|
+
for (const key of old) {
|
|
89
|
+
const bytes = await store.get(key);
|
|
90
|
+
for (const version of bytes ? unpackSegment(bytes) : [])
|
|
91
|
+
if (wanted.has(version.id))
|
|
92
|
+
keep.set(version.id, version);
|
|
93
|
+
}
|
|
94
|
+
// New first, then delete: a reader in between sees each version twice, which is harmless.
|
|
95
|
+
let batch = [];
|
|
96
|
+
let size = 0;
|
|
97
|
+
for (const version of keep.values()) {
|
|
98
|
+
batch.push(version);
|
|
99
|
+
size += versionSize(version);
|
|
100
|
+
if (size >= flushBytes) {
|
|
101
|
+
await writeSegment(batch);
|
|
102
|
+
batch = [];
|
|
103
|
+
size = 0;
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
if (batch.length)
|
|
107
|
+
await writeSegment(batch);
|
|
108
|
+
for (const key of old)
|
|
109
|
+
await store.delete(key);
|
|
110
|
+
}
|
|
111
|
+
async function pullNow() {
|
|
112
|
+
const fresh = [];
|
|
113
|
+
for (const key of await store.list(`${space}/`)) {
|
|
114
|
+
// Our own segments hold what we uploaded; a listing that lags is trusted less than our own writes.
|
|
115
|
+
if (!key.endsWith(SEGMENT_SUFFIX) || key.startsWith(mine) || (await state.get(readKey(key))))
|
|
116
|
+
continue;
|
|
117
|
+
fresh.push(key);
|
|
118
|
+
}
|
|
119
|
+
let added = 0;
|
|
120
|
+
for (const key of fresh.sort()) {
|
|
121
|
+
const bytes = await store.get(key);
|
|
122
|
+
// Deleted between listing and fetching — compacted by its writer. Its versions are in a newer segment.
|
|
123
|
+
if (!bytes)
|
|
124
|
+
continue;
|
|
125
|
+
let pending = unpackSegment(bytes);
|
|
126
|
+
// Versions can depend on each other — a record on its first version, on the access changes it saw.
|
|
127
|
+
// Pass over the rest until nothing more goes in.
|
|
128
|
+
for (let progress = true; pending.length > 0 && progress;) {
|
|
129
|
+
progress = false;
|
|
130
|
+
const later = [];
|
|
131
|
+
for (const version of pending) {
|
|
132
|
+
const taken = await config.accept(version);
|
|
133
|
+
if (taken === 'later') {
|
|
134
|
+
later.push(version);
|
|
135
|
+
continue;
|
|
136
|
+
}
|
|
137
|
+
if (taken === 'stored')
|
|
138
|
+
added += 1;
|
|
139
|
+
await markKnown(version.id);
|
|
140
|
+
progress = true;
|
|
141
|
+
}
|
|
142
|
+
pending = later;
|
|
143
|
+
}
|
|
144
|
+
// Some still waiting on something no segment here has: read again next time.
|
|
145
|
+
if (pending.length === 0)
|
|
146
|
+
await state.put(readKey(key), MARK);
|
|
147
|
+
}
|
|
148
|
+
return { added };
|
|
149
|
+
}
|
|
150
|
+
return Object.freeze({
|
|
151
|
+
pull: () => serial(pullNow),
|
|
152
|
+
flush: () => serial(flushNow),
|
|
153
|
+
compact: () => serial(compactNow),
|
|
154
|
+
changed() {
|
|
155
|
+
if (closed || timer)
|
|
156
|
+
return;
|
|
157
|
+
timer = setTimeout(() => {
|
|
158
|
+
timer = null;
|
|
159
|
+
void serial(flushNow).catch(() => {
|
|
160
|
+
// The store is out of reach; the next change tries again, and nothing is lost meanwhile.
|
|
161
|
+
});
|
|
162
|
+
}, flushMs);
|
|
163
|
+
timer.unref?.();
|
|
164
|
+
},
|
|
165
|
+
async close() {
|
|
166
|
+
closed = true;
|
|
167
|
+
if (timer)
|
|
168
|
+
clearTimeout(timer);
|
|
169
|
+
timer = null;
|
|
170
|
+
await serial(flushNow).catch(() => { });
|
|
171
|
+
},
|
|
172
|
+
});
|
|
173
|
+
}
|
|
174
|
+
//# sourceMappingURL=mirror.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"mirror.js","sourceRoot":"","sources":["../../src/storage/mirror.ts"],"names":[],"mappings":"AAiCA,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,cAAc,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AACpG,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAkC9D,MAAM,UAAU,GAAG,eAAe,CAAC;AACnC,MAAM,WAAW,GAAG,gBAAgB,CAAC;AACrC,MAAM,OAAO,GAAG,CAAC,IAAY,EAAE,EAAE,CAAC,eAAe,IAAI,EAAE,CAAC;AACxD,MAAM,QAAQ,GAAG,CAAC,EAAU,EAAE,EAAE,CAAC,gBAAgB,EAAE,EAAE,CAAC;AACtD,MAAM,IAAI,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC;AAE7B,4EAA4E;AAC5E,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,KAAgB,EAAE,KAAa;IAClE,KAAK,MAAM,GAAG,IAAI,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,KAAK,GAAG,CAAC;QAAE,MAAM,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;AAC3E,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,MAAoB;IACrD,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,MAAM,CAAC;IAChD,MAAM,UAAU,GAAG,MAAM,CAAC,UAAU,IAAI,GAAG,GAAG,IAAI,CAAC;IACnD,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,IAAI,CAAC;IACvC,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,IAAI,EAAE,CAAC;IAEzC,qFAAqF;IACrF,IAAI,WAAW,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;IAC9C,IAAI,CAAC,WAAW,EAAE,CAAC;QACjB,WAAW,GAAG,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,eAAe,CAAC,IAAI,UAAU,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;QAC7I,MAAM,KAAK,CAAC,GAAG,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC;IAC3C,CAAC;IACD,MAAM,MAAM,GAAG,UAAU,CAAC,WAAW,CAAC,CAAC;IACvC,MAAM,IAAI,GAAG,GAAG,KAAK,IAAI,MAAM,GAAG,CAAC;IAEnC,MAAM,YAAY,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;IAClD,IAAI,OAAO,GAAG,YAAY,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAElE,MAAM,OAAO,GAAG,KAAK,EAAE,EAAU,EAAE,EAAE,CAAC,CAAC,MAAM,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC;IAC/E,MAAM,SAAS,GAAG,CAAC,EAAU,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,IAAI,CAAC,CAAC;IAEhE,IAAI,KAAK,GAAyC,IAAI,CAAC;IACvD,IAAI,MAAM,GAAG,KAAK,CAAC;IACnB,yFAAyF;IACzF,IAAI,IAAI,GAAqB,OAAO,CAAC,OAAO,EAAE,CAAC;IAC/C,MAAM,MAAM,GAAG,CAAI,IAAsB,EAAc,EAAE;QACvD,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QACnC,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAC5B,OAAO,IAAI,CAAC;IACd,CAAC,CAAC;IAEF,KAAK,UAAU,YAAY,CAAC,QAAmC;QAC7D,MAAM,KAAK,GAAG,WAAW,CAAC,QAAQ,CAAC,CAAC;QACpC,OAAO,IAAI,CAAC,CAAC;QACb,MAAM,IAAI,GAAG,GAAG,IAAI,GAAG,MAAM,WAAW,CAAC,OAAO,EAAE,KAAK,CAAC,EAAE,CAAC;QAC3D,MAAM,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QAC7B,MAAM,KAAK,CAAC,GAAG,CAAC,WAAW,EAAE,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAC1D,MAAM,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,CAAC;QACrC,OAAO,IAAI,CAAC;IACd,CAAC;IAED,kEAAkE;IAClE,KAAK,UAAU,IAAI;QACjB,OAAO,OAAO,CAAC,UAAU,EAAE,CAAC;IAC9B,CAAC;IAED,KAAK,UAAU,QAAQ;QACrB,MAAM,OAAO,GAAiB,EAAE,CAAC;QACjC,KAAK,MAAM,EAAE,IAAI,MAAM,IAAI,EAAE,EAAE,CAAC;YAC9B,IAAI,MAAM,OAAO,CAAC,EAAE,CAAC;gBAAE,SAAS;YAChC,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;YAChD,IAAI,OAAO;gBAAE,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACrC,CAAC;QACD,IAAI,KAAK,GAAiB,EAAE,CAAC;QAC7B,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,MAAM,IAAI,GAAG,KAAK,IAAI,EAAE;YACtB,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO;YAC/B,MAAM,YAAY,CAAC,KAAK,CAAC,CAAC;YAC1B,KAAK,MAAM,OAAO,IAAI,KAAK;gBAAE,MAAM,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;YACzD,KAAK,GAAG,EAAE,CAAC;YACX,IAAI,GAAG,CAAC,CAAC;QACX,CAAC,CAAC;QACF,KAAK,MAAM,OAAO,IAAI,OAAO,EAAE,CAAC;YAC9B,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACpB,IAAI,IAAI,WAAW,CAAC,OAAO,CAAC,CAAC;YAC7B,IAAI,IAAI,IAAI,UAAU;gBAAE,MAAM,IAAI,EAAE,CAAC;QACvC,CAAC;QACD,MAAM,IAAI,EAAE,CAAC;QACb,IAAI,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC,CAAC,MAAM,IAAI,SAAS;YAAE,MAAM,UAAU,EAAE,CAAC;IACrH,CAAC;IAED,KAAK,UAAU,UAAU;QACvB,MAAM,GAAG,GAAG,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC,CAAC;QACnF,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC;YAAE,OAAO;QAC3B,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC;QACrC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAsB,CAAC;QAC3C,KAAK,MAAM,GAAG,IAAI,GAAG,EAAE,CAAC;YACtB,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACnC,KAAK,MAAM,OAAO,IAAI,KAAK,CAAC,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE;gBAAE,IAAI,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;oBAAE,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;QACrH,CAAC;QACD,0FAA0F;QAC1F,IAAI,KAAK,GAAiB,EAAE,CAAC;QAC7B,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;YACpC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACpB,IAAI,IAAI,WAAW,CAAC,OAAO,CAAC,CAAC;YAC7B,IAAI,IAAI,IAAI,UAAU,EAAE,CAAC;gBACvB,MAAM,YAAY,CAAC,KAAK,CAAC,CAAC;gBAC1B,KAAK,GAAG,EAAE,CAAC;gBACX,IAAI,GAAG,CAAC,CAAC;YACX,CAAC;QACH,CAAC;QACD,IAAI,KAAK,CAAC,MAAM;YAAE,MAAM,YAAY,CAAC,KAAK,CAAC,CAAC;QAC5C,KAAK,MAAM,GAAG,IAAI,GAAG;YAAE,MAAM,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IACjD,CAAC;IAED,KAAK,UAAU,OAAO;QACpB,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,KAAK,MAAM,GAAG,IAAI,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,KAAK,GAAG,CAAC,EAAE,CAAC;YAChD,mGAAmG;YACnG,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,cAAc,CAAC,IAAI,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC;gBAAE,SAAS;YACvG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAClB,CAAC;QACD,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC;YAC/B,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACnC,uGAAuG;YACvG,IAAI,CAAC,KAAK;gBAAE,SAAS;YACrB,IAAI,OAAO,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;YACnC,mGAAmG;YACnG,iDAAiD;YACjD,KAAK,IAAI,QAAQ,GAAG,IAAI,EAAE,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,QAAQ,GAAI,CAAC;gBAC3D,QAAQ,GAAG,KAAK,CAAC;gBACjB,MAAM,KAAK,GAAiB,EAAE,CAAC;gBAC/B,KAAK,MAAM,OAAO,IAAI,OAAO,EAAE,CAAC;oBAC9B,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;oBAC3C,IAAI,KAAK,KAAK,OAAO,EAAE,CAAC;wBACtB,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;wBACpB,SAAS;oBACX,CAAC;oBACD,IAAI,KAAK,KAAK,QAAQ;wBAAE,KAAK,IAAI,CAAC,CAAC;oBACnC,MAAM,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;oBAC5B,QAAQ,GAAG,IAAI,CAAC;gBAClB,CAAC;gBACD,OAAO,GAAG,KAAK,CAAC;YAClB,CAAC;YACD,6EAA6E;YAC7E,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;gBAAE,MAAM,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC;QAChE,CAAC;QACD,OAAO,EAAE,KAAK,EAAE,CAAC;IACnB,CAAC;IAED,OAAO,MAAM,CAAC,MAAM,CAAC;QACnB,IAAI,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC;QAC3B,KAAK,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC;QAC7B,OAAO,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QAEjC,OAAO;YACL,IAAI,MAAM,IAAI,KAAK;gBAAE,OAAO;YAC5B,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;gBACtB,KAAK,GAAG,IAAI,CAAC;gBACb,KAAK,MAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE;oBAC/B,yFAAyF;gBAC3F,CAAC,CAAC,CAAC;YACL,CAAC,EAAE,OAAO,CAAC,CAAC;YACX,KAAgC,CAAC,KAAK,EAAE,EAAE,CAAC;QAC9C,CAAC;QAED,KAAK,CAAC,KAAK;YACT,MAAM,GAAG,IAAI,CAAC;YACd,IAAI,KAAK;gBAAE,YAAY,CAAC,KAAK,CAAC,CAAC;YAC/B,KAAK,GAAG,IAAI,CAAC;YACb,MAAM,MAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QACzC,CAAC;KACF,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @module storage/segment
|
|
3
|
+
* A batch of record versions, as a mirror keeps them: exactly as they travel
|
|
4
|
+
* on the wire — private bodies still sealed — packed into one immutable file.
|
|
5
|
+
*
|
|
6
|
+
* A segment's name sorts (a counter, for people reading a listing) and ends
|
|
7
|
+
* in the hash of its bytes, so an upload retried after a timeout lands on the
|
|
8
|
+
* same name, and two writers never write the same file differently.
|
|
9
|
+
*/
|
|
10
|
+
import type { Expression } from '../types.js';
|
|
11
|
+
export declare const SEGMENT_SUFFIX = ".seg";
|
|
12
|
+
/** Packs versions into a segment's bytes */
|
|
13
|
+
export declare function packSegment(versions: ReadonlyArray<Expression>): Uint8Array;
|
|
14
|
+
/** The versions in a segment; none for bytes that are not one */
|
|
15
|
+
export declare function unpackSegment(bytes: Uint8Array): Expression[];
|
|
16
|
+
/** A segment's file name: `000042-<first 16 hex of its hash>.seg` */
|
|
17
|
+
export declare function segmentName(counter: number, bytes: Uint8Array): Promise<string>;
|
|
18
|
+
/** Roughly how many bytes a version takes in a segment */
|
|
19
|
+
export declare const versionSize: (version: Expression) => number;
|
|
20
|
+
//# sourceMappingURL=segment.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"segment.d.ts","sourceRoot":"","sources":["../../src/storage/segment.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAI9C,eAAO,MAAM,cAAc,SAAS,CAAC;AAErC,4CAA4C;AAC5C,wBAAgB,WAAW,CAAC,QAAQ,EAAE,aAAa,CAAC,UAAU,CAAC,GAAG,UAAU,CAE3E;AAED,iEAAiE;AACjE,wBAAgB,aAAa,CAAC,KAAK,EAAE,UAAU,GAAG,UAAU,EAAE,CAQ7D;AAED,qEAAqE;AACrE,wBAAsB,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC,CAGrF;AAED,0DAA0D;AAC1D,eAAO,MAAM,WAAW,GAAI,SAAS,UAAU,WAA+C,CAAC"}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import { sha256 } from '../utils/hash.js';
|
|
2
|
+
import { utf8Decode, utf8Encode } from '../utils/encoding.js';
|
|
3
|
+
export const SEGMENT_SUFFIX = '.seg';
|
|
4
|
+
/** Packs versions into a segment's bytes */
|
|
5
|
+
export function packSegment(versions) {
|
|
6
|
+
return utf8Encode(JSON.stringify({ v: 1, versions }));
|
|
7
|
+
}
|
|
8
|
+
/** The versions in a segment; none for bytes that are not one */
|
|
9
|
+
export function unpackSegment(bytes) {
|
|
10
|
+
try {
|
|
11
|
+
const parsed = JSON.parse(utf8Decode(bytes));
|
|
12
|
+
if (parsed.v !== 1 || !Array.isArray(parsed.versions))
|
|
13
|
+
return [];
|
|
14
|
+
return parsed.versions.filter((version) => typeof version === 'object' && version !== null && typeof version.id === 'string');
|
|
15
|
+
}
|
|
16
|
+
catch {
|
|
17
|
+
return [];
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
/** A segment's file name: `000042-<first 16 hex of its hash>.seg` */
|
|
21
|
+
export async function segmentName(counter, bytes) {
|
|
22
|
+
const hash = Array.from((await sha256(bytes)).subarray(0, 8), (b) => b.toString(16).padStart(2, '0')).join('');
|
|
23
|
+
return `${String(counter).padStart(6, '0')}-${hash}${SEGMENT_SUFFIX}`;
|
|
24
|
+
}
|
|
25
|
+
/** Roughly how many bytes a version takes in a segment */
|
|
26
|
+
export const versionSize = (version) => utf8Encode(JSON.stringify(version)).length;
|
|
27
|
+
//# sourceMappingURL=segment.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"segment.js","sourceRoot":"","sources":["../../src/storage/segment.ts"],"names":[],"mappings":"AAUA,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAE9D,MAAM,CAAC,MAAM,cAAc,GAAG,MAAM,CAAC;AAErC,4CAA4C;AAC5C,MAAM,UAAU,WAAW,CAAC,QAAmC;IAC7D,OAAO,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC;AACxD,CAAC;AAED,iEAAiE;AACjE,MAAM,UAAU,aAAa,CAAC,KAAiB;IAC7C,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,CAAwC,CAAC;QACpF,IAAI,MAAM,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC;YAAE,OAAO,EAAE,CAAC;QACjE,OAAO,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,OAAO,EAAyB,EAAE,CAAC,OAAO,OAAO,KAAK,QAAQ,IAAI,OAAO,KAAK,IAAI,IAAI,OAAQ,OAAsB,CAAC,EAAE,KAAK,QAAQ,CAAC,CAAC;IACvK,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,qEAAqE;AACrE,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,OAAe,EAAE,KAAiB;IAClE,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC/G,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,IAAI,IAAI,GAAG,cAAc,EAAE,CAAC;AACxE,CAAC;AAED,0DAA0D;AAC1D,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,OAAmB,EAAE,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC"}
|
|
@@ -1,32 +1,37 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* @module storage-provider
|
|
3
|
-
* A space's store: record versions, and
|
|
4
|
-
*
|
|
5
|
-
* The tree maps **record keys to version ids**:
|
|
3
|
+
* A space's store: record versions, and plain entries saying which is which.
|
|
6
4
|
*
|
|
7
5
|
* ```
|
|
8
|
-
* r/<key>
|
|
9
|
-
* g/<key>
|
|
10
|
-
* h/<key>/<seq>/<id>
|
|
6
|
+
* r/<key> → the current version (every record)
|
|
7
|
+
* g/<key> → the record's first version (once it has been edited)
|
|
8
|
+
* h/<key>/<seq>/<id> → a superseded version (only if marked `retain`)
|
|
9
|
+
* i/<collection>/<time>/<id> → every version kept, by collection — what sync compares
|
|
11
10
|
* ```
|
|
12
11
|
*
|
|
13
12
|
* A version that arrives is compared with the one at `r/<key>` by the ordering
|
|
14
13
|
* rule (`records/version.ts`). The winner takes `r/<key>`; the loser's body is
|
|
15
14
|
* dropped — unless it is the record's first version, kept as proof of who
|
|
16
15
|
* created it, or its writer marked it `retain`. The same set of versions gives
|
|
17
|
-
* the same
|
|
18
|
-
* converge.
|
|
16
|
+
* the same entries whatever order they arrive in, which is what lets two
|
|
17
|
+
* peers converge.
|
|
18
|
+
*
|
|
19
|
+
* Sync compares the `i/` entries: the set of versions kept, one collection at
|
|
20
|
+
* a time (`sync/negentropy.ts`). Nothing else is stored for it. The sets are
|
|
21
|
+
* read once and kept in memory, updated by every change made here; a change
|
|
22
|
+
* made by another writer of the same store (a tab, a folder's other device)
|
|
23
|
+
* reaches them through {@link StorageProvider.invalidate}.
|
|
19
24
|
*/
|
|
20
25
|
import type { StorageAdapter, Expression } from '../types.js';
|
|
26
|
+
import { ItemSet, type Sum } from '../sync/negentropy.js';
|
|
21
27
|
export interface StorageProvider {
|
|
22
28
|
/**
|
|
23
29
|
* Takes in a version of a record: it becomes current if it supersedes the
|
|
24
30
|
* current one, and is kept or dropped otherwise. Idempotent.
|
|
25
|
-
* @returns The new tree root
|
|
26
31
|
*/
|
|
27
|
-
addExpression(expression: Expression): Promise<
|
|
28
|
-
/** Drops a version from the
|
|
29
|
-
removeExpression(id: string): Promise<
|
|
32
|
+
addExpression(expression: Expression): Promise<void>;
|
|
33
|
+
/** Drops a version from the store and wherever it is indexed. */
|
|
34
|
+
removeExpression(id: string): Promise<void>;
|
|
30
35
|
/** A version by its id, current or not, if this store holds it. */
|
|
31
36
|
getExpression(id: string): Promise<Expression | null>;
|
|
32
37
|
/** The current version of a record — possibly a delete. */
|
|
@@ -35,51 +40,42 @@ export interface StorageProvider {
|
|
|
35
40
|
getGenesis(key: string): Promise<Expression | null>;
|
|
36
41
|
/** Every version of a record this store keeps: current, first, and retained. Newest first. */
|
|
37
42
|
history(key: string): Promise<Expression[]>;
|
|
38
|
-
/** {@link history} for every record whose key starts with `prefix`, reading only
|
|
43
|
+
/** {@link history} for every record whose key starts with `prefix`, reading only those entries. */
|
|
39
44
|
histories(prefix: string): Promise<Map<string, Expression[]>>;
|
|
40
45
|
/** Current versions of every record, deletes included. */
|
|
41
46
|
listCurrent(): Promise<Expression[]>;
|
|
42
47
|
/** Current versions in one collection, deletes included — found through the store's collection index. */
|
|
43
48
|
queryExpressions(collection: string): Promise<Expression[]>;
|
|
44
|
-
/**
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
getRootCid(): Promise<string | null>;
|
|
49
|
+
/** The id of every version this store keeps — current, first and retained. */
|
|
50
|
+
versionIds(): Promise<string[]>;
|
|
51
|
+
/** The versions kept in one collection, as sync compares them: sorted, with sums. */
|
|
52
|
+
items(collection: string): Promise<ItemSet>;
|
|
53
|
+
/** Every collection this store keeps anything in, and the sum of its version ids. */
|
|
54
|
+
sums(): Promise<ReadonlyMap<string, Sum>>;
|
|
51
55
|
/**
|
|
52
|
-
*
|
|
53
|
-
*
|
|
54
|
-
* store grows with every edit ever made rather than with its records.
|
|
55
|
-
*
|
|
56
|
-
* Tabs share a browser's store, and a change another tab has in flight may
|
|
57
|
-
* build on nodes this one's root has left. So a node is deleted only once it
|
|
58
|
-
* was already unreachable at a compaction at least `graceMs` earlier —
|
|
59
|
-
* remembered in the store, so it holds across sessions. A store whose other
|
|
60
|
-
* writers can lag by more than that — a folder behind a sync service — must
|
|
61
|
-
* not be compacted at all.
|
|
62
|
-
* @returns How many nodes were deleted
|
|
56
|
+
* One fingerprint of everything kept, hex. Equal fingerprints mean the
|
|
57
|
+
* same versions kept — for status, and for tests.
|
|
63
58
|
*/
|
|
64
|
-
|
|
59
|
+
fingerprint(): Promise<string>;
|
|
60
|
+
/** Forgets what was read into memory: another writer changed the store underneath. */
|
|
61
|
+
invalidate(): void;
|
|
65
62
|
/** Get the underlying storage adapter. */
|
|
66
63
|
getAdapter(): StorageAdapter;
|
|
67
64
|
/** Close the storage adapter. */
|
|
68
65
|
close(): Promise<void>;
|
|
69
66
|
}
|
|
70
|
-
export interface StorageProviderOptions {
|
|
71
|
-
/** Compact on its own after this many changes (see `compact`). Off by default. */
|
|
72
|
-
readonly compactEvery?: number;
|
|
73
|
-
/** How long a node stays unreachable before `compact` deletes it. Default a minute. */
|
|
74
|
-
readonly graceMs?: number;
|
|
75
|
-
}
|
|
76
67
|
export declare const CURRENT_PREFIX = "r/";
|
|
77
68
|
export declare const GENESIS_PREFIX = "g/";
|
|
78
69
|
export declare const HISTORY_PREFIX = "h/";
|
|
70
|
+
export declare const ITEM_PREFIX = "i/";
|
|
71
|
+
/**
|
|
72
|
+
* Where a version sorts for sync: its own time, in whole seconds. Only an
|
|
73
|
+
* order — a wrong clock makes a version slower to find, never lost.
|
|
74
|
+
*/
|
|
75
|
+
export declare function syncTime(version: Pick<Expression, 'createdAt'>): number;
|
|
79
76
|
/**
|
|
80
77
|
* Creates a StorageProvider wrapping an adapter.
|
|
81
78
|
* @param adapter The initialized storage adapter.
|
|
82
|
-
* @returns The StorageProvider orchestrator.
|
|
83
79
|
*/
|
|
84
|
-
export declare function createStorageProvider(adapter: StorageAdapter
|
|
80
|
+
export declare function createStorageProvider(adapter: StorageAdapter): StorageProvider;
|
|
85
81
|
//# sourceMappingURL=storage-provider.d.ts.map
|