@automerge/automerge-repo 2.5.3 → 2.5.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/storage/SavedHeads.d.ts +31 -0
- package/dist/storage/SavedHeads.d.ts.map +1 -0
- package/dist/storage/SavedHeads.js +54 -0
- package/dist/storage/StorageSubsystem.d.ts +2 -0
- package/dist/storage/StorageSubsystem.d.ts.map +1 -1
- package/dist/storage/StorageSubsystem.js +22 -7
- package/package.json +2 -2
- package/src/storage/SavedHeads.ts +70 -0
- package/src/storage/StorageSubsystem.ts +28 -8
- package/test/StorageSubsystem.test.ts +213 -2
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { next as A } from "@automerge/automerge/slim";
|
|
2
|
+
import { DocumentId } from "../types.js";
|
|
3
|
+
/**
|
|
4
|
+
* A cache of the last saved heads for each document
|
|
5
|
+
*
|
|
6
|
+
* The reason for using this class, rather than just a Map<DocumentId, Heads>,
|
|
7
|
+
* is that we need to handle concurrent updates of the saved heads. This will
|
|
8
|
+
* occur when for example you have a compaction running whilst a new incremental
|
|
9
|
+
* save is begun. The incremental save can finish before the compaction and so
|
|
10
|
+
* we need to express the fact that the update to the saved heads made by the
|
|
11
|
+
* compaction should be ignored. We achieve this by maintaining a counter
|
|
12
|
+
* representing the time that the update was begin, and only applying updates
|
|
13
|
+
* to the saved heads if they are newer than the last update that was applied.
|
|
14
|
+
*/
|
|
15
|
+
export declare class SavedHeads {
|
|
16
|
+
#private;
|
|
17
|
+
/**
|
|
18
|
+
* Get the last saved heads for a document
|
|
19
|
+
*/
|
|
20
|
+
lastSavedHeads(documentId: DocumentId): HeadsHandle;
|
|
21
|
+
}
|
|
22
|
+
export declare class HeadsHandle {
|
|
23
|
+
#private;
|
|
24
|
+
constructor(documentId: DocumentId, seq: number, storedHeads: Map<DocumentId, {
|
|
25
|
+
heads: A.Heads;
|
|
26
|
+
seq: number;
|
|
27
|
+
}>);
|
|
28
|
+
get value(): A.Heads | null;
|
|
29
|
+
update(newHeads: A.Heads): void;
|
|
30
|
+
}
|
|
31
|
+
//# sourceMappingURL=SavedHeads.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"SavedHeads.d.ts","sourceRoot":"","sources":["../../src/storage/SavedHeads.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,IAAI,CAAC,EAAE,MAAM,2BAA2B,CAAA;AACrD,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AAIxC;;;;;;;;;;;GAWG;AACH,qBAAa,UAAU;;IAIrB;;OAEG;IACH,cAAc,CAAC,UAAU,EAAE,UAAU,GAAG,WAAW;CAGpD;AAID,qBAAa,WAAW;;gBAOpB,UAAU,EAAE,UAAU,EACtB,GAAG,EAAE,MAAM,EACX,WAAW,EAAE,GAAG,CAAC,UAAU,EAAE;QAAE,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,CAAC;IAO/D,IAAI,KAAK,IAAI,CAAC,CAAC,KAAK,GAAG,IAAI,CAE1B;IAED,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,KAAK;CAkBzB"}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import { headsAreSame } from "../helpers/headsAreSame.js";
|
|
2
|
+
import { encodeHeads } from "../index.js";
|
|
3
|
+
/**
|
|
4
|
+
* A cache of the last saved heads for each document
|
|
5
|
+
*
|
|
6
|
+
* The reason for using this class, rather than just a Map<DocumentId, Heads>,
|
|
7
|
+
* is that we need to handle concurrent updates of the saved heads. This will
|
|
8
|
+
* occur when for example you have a compaction running whilst a new incremental
|
|
9
|
+
* save is begun. The incremental save can finish before the compaction and so
|
|
10
|
+
* we need to express the fact that the update to the saved heads made by the
|
|
11
|
+
* compaction should be ignored. We achieve this by maintaining a counter
|
|
12
|
+
* representing the time that the update was begin, and only applying updates
|
|
13
|
+
* to the saved heads if they are newer than the last update that was applied.
|
|
14
|
+
*/
|
|
15
|
+
export class SavedHeads {
|
|
16
|
+
#seq = 0;
|
|
17
|
+
#data = new Map();
|
|
18
|
+
/**
|
|
19
|
+
* Get the last saved heads for a document
|
|
20
|
+
*/
|
|
21
|
+
lastSavedHeads(documentId) {
|
|
22
|
+
return new HeadsHandle(documentId, ++this.#seq, this.#data);
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
// Helpr class to manage applying heads updates in the correct order when there
|
|
26
|
+
// are concurrent saves
|
|
27
|
+
export class HeadsHandle {
|
|
28
|
+
#documentId;
|
|
29
|
+
#seq;
|
|
30
|
+
#storedHeads;
|
|
31
|
+
#appliedHeads = null;
|
|
32
|
+
constructor(documentId, seq, storedHeads) {
|
|
33
|
+
this.#documentId = documentId;
|
|
34
|
+
this.#seq = seq;
|
|
35
|
+
this.#storedHeads = storedHeads;
|
|
36
|
+
}
|
|
37
|
+
get value() {
|
|
38
|
+
return this.#storedHeads.get(this.#documentId)?.heads ?? null;
|
|
39
|
+
}
|
|
40
|
+
update(newHeads) {
|
|
41
|
+
if (this.#appliedHeads &&
|
|
42
|
+
!headsAreSame(encodeHeads(newHeads), encodeHeads(this.#appliedHeads))) {
|
|
43
|
+
throw new Error("attempting to reuase a heads update with different heads");
|
|
44
|
+
}
|
|
45
|
+
this.#appliedHeads = newHeads;
|
|
46
|
+
const currentSeq = this.#storedHeads.get(this.#documentId)?.seq ?? 0;
|
|
47
|
+
if (this.#seq >= currentSeq) {
|
|
48
|
+
this.#storedHeads.set(this.#documentId, {
|
|
49
|
+
heads: newHeads,
|
|
50
|
+
seq: this.#seq,
|
|
51
|
+
});
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
}
|
|
@@ -13,11 +13,13 @@ type StorageSubsystemEvents = {
|
|
|
13
13
|
"doc-compacted": (arg: {
|
|
14
14
|
documentId: DocumentId;
|
|
15
15
|
durationMillis: number;
|
|
16
|
+
savedHeads: A.Heads;
|
|
16
17
|
}) => void;
|
|
17
18
|
"doc-saved": (arg: {
|
|
18
19
|
documentId: DocumentId;
|
|
19
20
|
durationMillis: number;
|
|
20
21
|
sinceHeads: A.Heads;
|
|
22
|
+
savedHeads: A.Heads;
|
|
21
23
|
}) => void;
|
|
22
24
|
};
|
|
23
25
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"StorageSubsystem.d.ts","sourceRoot":"","sources":["../../src/storage/StorageSubsystem.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,IAAI,CAAC,EAAE,MAAM,2BAA2B,CAAA;AAIrD,OAAO,EAAE,KAAK,UAAU,EAAE,MAAM,aAAa,CAAA;AAC7C,OAAO,EAAE,uBAAuB,EAAE,MAAM,8BAA8B,CAAA;AACtE,OAAO,EAAyB,SAAS,EAAE,MAAM,YAAY,CAAA;AAG7D,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAA;
|
|
1
|
+
{"version":3,"file":"StorageSubsystem.d.ts","sourceRoot":"","sources":["../../src/storage/StorageSubsystem.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,IAAI,CAAC,EAAE,MAAM,2BAA2B,CAAA;AAIrD,OAAO,EAAE,KAAK,UAAU,EAAE,MAAM,aAAa,CAAA;AAC7C,OAAO,EAAE,uBAAuB,EAAE,MAAM,8BAA8B,CAAA;AACtE,OAAO,EAAyB,SAAS,EAAE,MAAM,YAAY,CAAA;AAG7D,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAA;AAI5C,KAAK,sBAAsB,GAAG;IAC5B,iBAAiB,EAAE,CAAC,GAAG,EAAE;QACvB,UAAU,EAAE,UAAU,CAAA;QACtB,cAAc,EAAE,MAAM,CAAA;QACtB,MAAM,EAAE,MAAM,CAAA;QACd,UAAU,EAAE,MAAM,CAAA;KACnB,KAAK,IAAI,CAAA;IACV,eAAe,EAAE,CAAC,GAAG,EAAE;QACrB,UAAU,EAAE,UAAU,CAAA;QACtB,cAAc,EAAE,MAAM,CAAA;QACtB,UAAU,EAAE,CAAC,CAAC,KAAK,CAAA;KACpB,KAAK,IAAI,CAAA;IACV,WAAW,EAAE,CAAC,GAAG,EAAE;QACjB,UAAU,EAAE,UAAU,CAAA;QACtB,cAAc,EAAE,MAAM,CAAA;QACtB,UAAU,EAAE,CAAC,CAAC,KAAK,CAAA;QACnB,UAAU,EAAE,CAAC,CAAC,KAAK,CAAA;KACpB,KAAK,IAAI,CAAA;CACX,CAAA;AAED;;;GAGG;AACH,qBAAa,gBAAiB,SAAQ,YAAY,CAAC,sBAAsB,CAAC;;gBAe5D,cAAc,EAAE,uBAAuB;IAK7C,EAAE,IAAI,OAAO,CAAC,SAAS,CAAC;IA2B9B,kCAAkC;IAC5B,IAAI;IACR,iFAAiF;IACjF,SAAS,EAAE,MAAM;IAEjB,yFAAyF;IACzF,GAAG,EAAE,MAAM,GACV,OAAO,CAAC,UAAU,GAAG,SAAS,CAAC;IAKlC,gCAAgC;IAC1B,IAAI;IACR,iFAAiF;IACjF,SAAS,EAAE,MAAM;IAEjB,yFAAyF;IACzF,GAAG,EAAE,MAAM;IAEX,sCAAsC;IACtC,IAAI,EAAE,UAAU,GACf,OAAO,CAAC,IAAI,CAAC;IAKhB,oCAAoC;IAC9B,MAAM;IACV,iFAAiF;IACjF,SAAS,EAAE,MAAM;IAEjB,2FAA2F;IAC3F,GAAG,EAAE,MAAM,GACV,OAAO,CAAC,IAAI,CAAC;IAOhB;;OAEG;IACG,WAAW,CAAC,UAAU,EAAE,UAAU,GAAG,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC;IAgDrE;;OAEG;IACG,OAAO,CAAC,CAAC,EAAE,UAAU,EAAE,UAAU,GAAG,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;IAsBlE;;;;;;OAMG;IACG,OAAO,CAAC,UAAU,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAazE;;OAEG;IACG,SAAS,CAAC,UAAU,EAAE,UAAU;IAiGhC,aAAa,CACjB,UAAU,EAAE,UAAU,EACtB,SAAS,EAAE,SAAS,GACnB,OAAO,CAAC,CAAC,CAAC,SAAS,GAAG,SAAS,CAAC;IAW7B,aAAa,CACjB,UAAU,EAAE,UAAU,EACtB,SAAS,EAAE,SAAS,EACpB,SAAS,EAAE,CAAC,CAAC,SAAS,GACrB,OAAO,CAAC,IAAI,CAAC;CA8CjB"}
|
|
@@ -6,6 +6,7 @@ import { keyHash, headsHash } from "./keyHash.js";
|
|
|
6
6
|
import * as Uuid from "uuid";
|
|
7
7
|
import { EventEmitter } from "eventemitter3";
|
|
8
8
|
import { encodeHeads } from "../AutomergeUrl.js";
|
|
9
|
+
import { SavedHeads } from "./SavedHeads.js";
|
|
9
10
|
/**
|
|
10
11
|
* The storage subsystem is responsible for saving and loading Automerge documents to and from
|
|
11
12
|
* storage adapter. It also provides a generic key/value storage interface for other uses.
|
|
@@ -14,7 +15,7 @@ export class StorageSubsystem extends EventEmitter {
|
|
|
14
15
|
/** The storage adapter to use for saving and loading documents */
|
|
15
16
|
#storageAdapter;
|
|
16
17
|
/** Record of the latest heads we've loaded or saved for each document */
|
|
17
|
-
#storedHeads = new
|
|
18
|
+
#storedHeads = new SavedHeads();
|
|
18
19
|
/** Metadata on the chunks we've already loaded for each document */
|
|
19
20
|
#chunkInfos = new Map();
|
|
20
21
|
/** Flag to avoid compacting when a compaction is already underway */
|
|
@@ -124,6 +125,7 @@ export class StorageSubsystem extends EventEmitter {
|
|
|
124
125
|
* Loads the Automerge document with the given ID from storage.
|
|
125
126
|
*/
|
|
126
127
|
async loadDoc(documentId) {
|
|
128
|
+
const headsHandle = this.#storedHeads.lastSavedHeads(documentId);
|
|
127
129
|
// Load and combine chunks
|
|
128
130
|
const binary = await this.loadDocData(documentId);
|
|
129
131
|
if (!binary)
|
|
@@ -138,7 +140,7 @@ export class StorageSubsystem extends EventEmitter {
|
|
|
138
140
|
...A.stats(newDoc),
|
|
139
141
|
});
|
|
140
142
|
// Record the latest heads for the document
|
|
141
|
-
|
|
143
|
+
headsHandle.update(A.getHeads(newDoc));
|
|
142
144
|
return newDoc;
|
|
143
145
|
}
|
|
144
146
|
/**
|
|
@@ -159,7 +161,6 @@ export class StorageSubsystem extends EventEmitter {
|
|
|
159
161
|
else {
|
|
160
162
|
await this.#saveIncremental(documentId, doc);
|
|
161
163
|
}
|
|
162
|
-
this.#storedHeads.set(documentId, A.getHeads(doc));
|
|
163
164
|
}
|
|
164
165
|
/**
|
|
165
166
|
* Removes the Automerge document with the given ID from storage
|
|
@@ -173,7 +174,14 @@ export class StorageSubsystem extends EventEmitter {
|
|
|
173
174
|
* Saves just the incremental changes since the last save.
|
|
174
175
|
*/
|
|
175
176
|
async #saveIncremental(documentId, doc) {
|
|
176
|
-
const
|
|
177
|
+
const headsHandle = this.#storedHeads.lastSavedHeads(documentId);
|
|
178
|
+
const sinceHeads = headsHandle.value;
|
|
179
|
+
if (!sinceHeads || sinceHeads.length === 0) {
|
|
180
|
+
// No prior save recorded — save a full snapshot instead of calling
|
|
181
|
+
// saveSince with empty heads (which would save the entire history).
|
|
182
|
+
await this.#saveTotal(documentId, doc, this.#chunkInfos.get(documentId) ?? []);
|
|
183
|
+
return;
|
|
184
|
+
}
|
|
177
185
|
const start = performance.now();
|
|
178
186
|
const binary = A.saveSince(doc, sinceHeads);
|
|
179
187
|
const end = performance.now();
|
|
@@ -181,6 +189,7 @@ export class StorageSubsystem extends EventEmitter {
|
|
|
181
189
|
documentId,
|
|
182
190
|
durationMillis: end - start,
|
|
183
191
|
sinceHeads,
|
|
192
|
+
savedHeads: A.getHeads(doc),
|
|
184
193
|
});
|
|
185
194
|
if (binary && binary.length > 0) {
|
|
186
195
|
const key = [documentId, "incremental", keyHash(binary)];
|
|
@@ -194,7 +203,7 @@ export class StorageSubsystem extends EventEmitter {
|
|
|
194
203
|
type: "incremental",
|
|
195
204
|
size: binary.length,
|
|
196
205
|
});
|
|
197
|
-
|
|
206
|
+
headsHandle.update(A.getHeads(doc));
|
|
198
207
|
}
|
|
199
208
|
else {
|
|
200
209
|
return Promise.resolve();
|
|
@@ -205,10 +214,15 @@ export class StorageSubsystem extends EventEmitter {
|
|
|
205
214
|
*/
|
|
206
215
|
async #saveTotal(documentId, doc, sourceChunks) {
|
|
207
216
|
this.#compacting = true;
|
|
217
|
+
const headsHandle = this.#storedHeads.lastSavedHeads(documentId);
|
|
208
218
|
const start = performance.now();
|
|
209
219
|
const binary = A.save(doc);
|
|
210
220
|
const end = performance.now();
|
|
211
|
-
this.emit("doc-compacted", {
|
|
221
|
+
this.emit("doc-compacted", {
|
|
222
|
+
documentId,
|
|
223
|
+
durationMillis: end - start,
|
|
224
|
+
savedHeads: A.getHeads(doc),
|
|
225
|
+
});
|
|
212
226
|
const snapshotHash = headsHash(A.getHeads(doc));
|
|
213
227
|
const key = [documentId, "snapshot", snapshotHash];
|
|
214
228
|
const oldKeys = new Set(sourceChunks.map(c => c.key).filter(k => k[2] !== snapshotHash));
|
|
@@ -221,6 +235,7 @@ export class StorageSubsystem extends EventEmitter {
|
|
|
221
235
|
const newChunkInfos = this.#chunkInfos.get(documentId)?.filter(c => !oldKeys.has(c.key)) ?? [];
|
|
222
236
|
newChunkInfos.push({ key, type: "snapshot", size: binary.length });
|
|
223
237
|
this.#chunkInfos.set(documentId, newChunkInfos);
|
|
238
|
+
headsHandle.update(A.getHeads(doc));
|
|
224
239
|
this.#compacting = false;
|
|
225
240
|
}
|
|
226
241
|
async loadSyncState(documentId, storageId) {
|
|
@@ -242,7 +257,7 @@ export class StorageSubsystem extends EventEmitter {
|
|
|
242
257
|
* Returns true if the document has changed since the last time it was saved.
|
|
243
258
|
*/
|
|
244
259
|
#shouldSave(documentId, doc) {
|
|
245
|
-
const oldHeads = this.#storedHeads.
|
|
260
|
+
const oldHeads = this.#storedHeads.lastSavedHeads(documentId).value;
|
|
246
261
|
if (!oldHeads) {
|
|
247
262
|
// we haven't saved this document before
|
|
248
263
|
return true;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@automerge/automerge-repo",
|
|
3
|
-
"version": "2.5.
|
|
3
|
+
"version": "2.5.4",
|
|
4
4
|
"description": "A repository object to manage a collection of automerge documents",
|
|
5
5
|
"repository": "https://github.com/automerge/automerge-repo/tree/master/packages/automerge-repo",
|
|
6
6
|
"author": "Peter van Hardenberg <pvh@pvh.ca>",
|
|
@@ -59,5 +59,5 @@
|
|
|
59
59
|
"publishConfig": {
|
|
60
60
|
"access": "public"
|
|
61
61
|
},
|
|
62
|
-
"gitHead": "
|
|
62
|
+
"gitHead": "70e3703e39f7151dbc446e865d9f9753f132ab3a"
|
|
63
63
|
}
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import { next as A } from "@automerge/automerge/slim"
|
|
2
|
+
import { DocumentId } from "../types.js"
|
|
3
|
+
import { headsAreSame } from "../helpers/headsAreSame.js"
|
|
4
|
+
import { encodeHeads } from "../index.js"
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* A cache of the last saved heads for each document
|
|
8
|
+
*
|
|
9
|
+
* The reason for using this class, rather than just a Map<DocumentId, Heads>,
|
|
10
|
+
* is that we need to handle concurrent updates of the saved heads. This will
|
|
11
|
+
* occur when for example you have a compaction running whilst a new incremental
|
|
12
|
+
* save is begun. The incremental save can finish before the compaction and so
|
|
13
|
+
* we need to express the fact that the update to the saved heads made by the
|
|
14
|
+
* compaction should be ignored. We achieve this by maintaining a counter
|
|
15
|
+
* representing the time that the update was begin, and only applying updates
|
|
16
|
+
* to the saved heads if they are newer than the last update that was applied.
|
|
17
|
+
*/
|
|
18
|
+
export class SavedHeads {
|
|
19
|
+
#seq: number = 0
|
|
20
|
+
#data: Map<DocumentId, { heads: A.Heads; seq: number }> = new Map()
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* Get the last saved heads for a document
|
|
24
|
+
*/
|
|
25
|
+
lastSavedHeads(documentId: DocumentId): HeadsHandle {
|
|
26
|
+
return new HeadsHandle(documentId, ++this.#seq, this.#data)
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
// Helpr class to manage applying heads updates in the correct order when there
|
|
31
|
+
// are concurrent saves
|
|
32
|
+
export class HeadsHandle {
|
|
33
|
+
#documentId: DocumentId
|
|
34
|
+
#seq: number
|
|
35
|
+
#storedHeads: Map<DocumentId, { heads: A.Heads; seq: number }>
|
|
36
|
+
#appliedHeads: A.Heads | null = null
|
|
37
|
+
|
|
38
|
+
constructor(
|
|
39
|
+
documentId: DocumentId,
|
|
40
|
+
seq: number,
|
|
41
|
+
storedHeads: Map<DocumentId, { heads: A.Heads; seq: number }>
|
|
42
|
+
) {
|
|
43
|
+
this.#documentId = documentId
|
|
44
|
+
this.#seq = seq
|
|
45
|
+
this.#storedHeads = storedHeads
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
get value(): A.Heads | null {
|
|
49
|
+
return this.#storedHeads.get(this.#documentId)?.heads ?? null
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
update(newHeads: A.Heads) {
|
|
53
|
+
if (
|
|
54
|
+
this.#appliedHeads &&
|
|
55
|
+
!headsAreSame(encodeHeads(newHeads), encodeHeads(this.#appliedHeads))
|
|
56
|
+
) {
|
|
57
|
+
throw new Error(
|
|
58
|
+
"attempting to reuase a heads update with different heads"
|
|
59
|
+
)
|
|
60
|
+
}
|
|
61
|
+
this.#appliedHeads = newHeads
|
|
62
|
+
const currentSeq = this.#storedHeads.get(this.#documentId)?.seq ?? 0
|
|
63
|
+
if (this.#seq >= currentSeq) {
|
|
64
|
+
this.#storedHeads.set(this.#documentId, {
|
|
65
|
+
heads: newHeads,
|
|
66
|
+
seq: this.#seq,
|
|
67
|
+
})
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
}
|
|
@@ -9,6 +9,7 @@ import { keyHash, headsHash } from "./keyHash.js"
|
|
|
9
9
|
import * as Uuid from "uuid"
|
|
10
10
|
import { EventEmitter } from "eventemitter3"
|
|
11
11
|
import { encodeHeads } from "../AutomergeUrl.js"
|
|
12
|
+
import { SavedHeads } from "./SavedHeads.js"
|
|
12
13
|
|
|
13
14
|
type StorageSubsystemEvents = {
|
|
14
15
|
"document-loaded": (arg: {
|
|
@@ -20,11 +21,13 @@ type StorageSubsystemEvents = {
|
|
|
20
21
|
"doc-compacted": (arg: {
|
|
21
22
|
documentId: DocumentId
|
|
22
23
|
durationMillis: number
|
|
24
|
+
savedHeads: A.Heads
|
|
23
25
|
}) => void
|
|
24
26
|
"doc-saved": (arg: {
|
|
25
27
|
documentId: DocumentId
|
|
26
28
|
durationMillis: number
|
|
27
29
|
sinceHeads: A.Heads
|
|
30
|
+
savedHeads: A.Heads
|
|
28
31
|
}) => void
|
|
29
32
|
}
|
|
30
33
|
|
|
@@ -37,7 +40,7 @@ export class StorageSubsystem extends EventEmitter<StorageSubsystemEvents> {
|
|
|
37
40
|
#storageAdapter: StorageAdapterInterface
|
|
38
41
|
|
|
39
42
|
/** Record of the latest heads we've loaded or saved for each document */
|
|
40
|
-
#storedHeads:
|
|
43
|
+
#storedHeads: SavedHeads = new SavedHeads()
|
|
41
44
|
|
|
42
45
|
/** Metadata on the chunks we've already loaded for each document */
|
|
43
46
|
#chunkInfos: Map<DocumentId, ChunkInfo[]> = new Map()
|
|
@@ -175,6 +178,7 @@ export class StorageSubsystem extends EventEmitter<StorageSubsystemEvents> {
|
|
|
175
178
|
* Loads the Automerge document with the given ID from storage.
|
|
176
179
|
*/
|
|
177
180
|
async loadDoc<T>(documentId: DocumentId): Promise<A.Doc<T> | null> {
|
|
181
|
+
const headsHandle = this.#storedHeads.lastSavedHeads(documentId)
|
|
178
182
|
// Load and combine chunks
|
|
179
183
|
const binary = await this.loadDocData(documentId)
|
|
180
184
|
if (!binary) return null
|
|
@@ -190,7 +194,7 @@ export class StorageSubsystem extends EventEmitter<StorageSubsystemEvents> {
|
|
|
190
194
|
})
|
|
191
195
|
|
|
192
196
|
// Record the latest heads for the document
|
|
193
|
-
|
|
197
|
+
headsHandle.update(A.getHeads(newDoc))
|
|
194
198
|
|
|
195
199
|
return newDoc
|
|
196
200
|
}
|
|
@@ -213,8 +217,6 @@ export class StorageSubsystem extends EventEmitter<StorageSubsystemEvents> {
|
|
|
213
217
|
} else {
|
|
214
218
|
await this.#saveIncremental(documentId, doc)
|
|
215
219
|
}
|
|
216
|
-
|
|
217
|
-
this.#storedHeads.set(documentId, A.getHeads(doc))
|
|
218
220
|
}
|
|
219
221
|
|
|
220
222
|
/**
|
|
@@ -233,7 +235,18 @@ export class StorageSubsystem extends EventEmitter<StorageSubsystemEvents> {
|
|
|
233
235
|
documentId: DocumentId,
|
|
234
236
|
doc: A.Doc<unknown>
|
|
235
237
|
): Promise<void> {
|
|
236
|
-
const
|
|
238
|
+
const headsHandle = this.#storedHeads.lastSavedHeads(documentId)
|
|
239
|
+
const sinceHeads = headsHandle.value
|
|
240
|
+
if (!sinceHeads || sinceHeads.length === 0) {
|
|
241
|
+
// No prior save recorded — save a full snapshot instead of calling
|
|
242
|
+
// saveSince with empty heads (which would save the entire history).
|
|
243
|
+
await this.#saveTotal(
|
|
244
|
+
documentId,
|
|
245
|
+
doc,
|
|
246
|
+
this.#chunkInfos.get(documentId) ?? []
|
|
247
|
+
)
|
|
248
|
+
return
|
|
249
|
+
}
|
|
237
250
|
const start = performance.now()
|
|
238
251
|
const binary = A.saveSince(doc, sinceHeads)
|
|
239
252
|
const end = performance.now()
|
|
@@ -241,6 +254,7 @@ export class StorageSubsystem extends EventEmitter<StorageSubsystemEvents> {
|
|
|
241
254
|
documentId,
|
|
242
255
|
durationMillis: end - start,
|
|
243
256
|
sinceHeads,
|
|
257
|
+
savedHeads: A.getHeads(doc),
|
|
244
258
|
})
|
|
245
259
|
|
|
246
260
|
if (binary && binary.length > 0) {
|
|
@@ -255,7 +269,7 @@ export class StorageSubsystem extends EventEmitter<StorageSubsystemEvents> {
|
|
|
255
269
|
type: "incremental",
|
|
256
270
|
size: binary.length,
|
|
257
271
|
})
|
|
258
|
-
|
|
272
|
+
headsHandle.update(A.getHeads(doc))
|
|
259
273
|
} else {
|
|
260
274
|
return Promise.resolve()
|
|
261
275
|
}
|
|
@@ -270,11 +284,16 @@ export class StorageSubsystem extends EventEmitter<StorageSubsystemEvents> {
|
|
|
270
284
|
sourceChunks: ChunkInfo[]
|
|
271
285
|
): Promise<void> {
|
|
272
286
|
this.#compacting = true
|
|
287
|
+
const headsHandle = this.#storedHeads.lastSavedHeads(documentId)
|
|
273
288
|
|
|
274
289
|
const start = performance.now()
|
|
275
290
|
const binary = A.save(doc)
|
|
276
291
|
const end = performance.now()
|
|
277
|
-
this.emit("doc-compacted", {
|
|
292
|
+
this.emit("doc-compacted", {
|
|
293
|
+
documentId,
|
|
294
|
+
durationMillis: end - start,
|
|
295
|
+
savedHeads: A.getHeads(doc),
|
|
296
|
+
})
|
|
278
297
|
|
|
279
298
|
const snapshotHash = headsHash(A.getHeads(doc))
|
|
280
299
|
const key = [documentId, "snapshot", snapshotHash]
|
|
@@ -296,6 +315,7 @@ export class StorageSubsystem extends EventEmitter<StorageSubsystemEvents> {
|
|
|
296
315
|
newChunkInfos.push({ key, type: "snapshot", size: binary.length })
|
|
297
316
|
|
|
298
317
|
this.#chunkInfos.set(documentId, newChunkInfos)
|
|
318
|
+
headsHandle.update(A.getHeads(doc))
|
|
299
319
|
this.#compacting = false
|
|
300
320
|
}
|
|
301
321
|
|
|
@@ -326,7 +346,7 @@ export class StorageSubsystem extends EventEmitter<StorageSubsystemEvents> {
|
|
|
326
346
|
* Returns true if the document has changed since the last time it was saved.
|
|
327
347
|
*/
|
|
328
348
|
#shouldSave(documentId: DocumentId, doc: A.Doc<unknown>): boolean {
|
|
329
|
-
const oldHeads = this.#storedHeads.
|
|
349
|
+
const oldHeads = this.#storedHeads.lastSavedHeads(documentId).value
|
|
330
350
|
if (!oldHeads) {
|
|
331
351
|
// we haven't saved this document before
|
|
332
352
|
return true
|
|
@@ -6,9 +6,10 @@ import os from "os"
|
|
|
6
6
|
import path from "path"
|
|
7
7
|
import { describe, it, expect } from "vitest"
|
|
8
8
|
import { generateAutomergeUrl, parseAutomergeUrl } from "../src/AutomergeUrl.js"
|
|
9
|
-
import { PeerId, cbor } from "../src/index.js"
|
|
9
|
+
import { PeerId, cbor, Chunk } from "../src/index.js"
|
|
10
10
|
import { StorageSubsystem } from "../src/storage/StorageSubsystem.js"
|
|
11
|
-
import { StorageId } from "../src/storage/types.js"
|
|
11
|
+
import { StorageId, StorageKey } from "../src/storage/types.js"
|
|
12
|
+
import { StorageAdapterInterface } from "../src/storage/StorageAdapterInterface.js"
|
|
12
13
|
import { DummyStorageAdapter } from "../src/helpers/DummyStorageAdapter.js"
|
|
13
14
|
import * as Uuid from "uuid"
|
|
14
15
|
import { chunkTypeFromKey } from "../src/storage/chunkTypeFromKey.js"
|
|
@@ -324,4 +325,214 @@ describe("StorageSubsystem", () => {
|
|
|
324
325
|
})
|
|
325
326
|
})
|
|
326
327
|
}
|
|
328
|
+
|
|
329
|
+
describe("concurrent save race condition", () => {
|
|
330
|
+
// A storage adapter that delays save() calls, simulating slow I/O.
|
|
331
|
+
// This widens the race window between concurrent saveDoc calls.
|
|
332
|
+
class SlowSaveAdapter implements StorageAdapterInterface {
|
|
333
|
+
#inner = new DummyStorageAdapter()
|
|
334
|
+
#saveDelayMs: number
|
|
335
|
+
|
|
336
|
+
constructor(saveDelayMs: number) {
|
|
337
|
+
this.#saveDelayMs = saveDelayMs
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
async load(key: StorageKey) {
|
|
341
|
+
return this.#inner.load(key)
|
|
342
|
+
}
|
|
343
|
+
async save(key: StorageKey, data: Uint8Array) {
|
|
344
|
+
await new Promise(resolve => setTimeout(resolve, this.#saveDelayMs))
|
|
345
|
+
return this.#inner.save(key, data)
|
|
346
|
+
}
|
|
347
|
+
async remove(key: StorageKey) {
|
|
348
|
+
return this.#inner.remove(key)
|
|
349
|
+
}
|
|
350
|
+
async loadRange(keyPrefix: StorageKey) {
|
|
351
|
+
return this.#inner.loadRange(keyPrefix)
|
|
352
|
+
}
|
|
353
|
+
async removeRange(keyPrefix: StorageKey) {
|
|
354
|
+
return this.#inner.removeRange(keyPrefix)
|
|
355
|
+
}
|
|
356
|
+
keys() {
|
|
357
|
+
return this.#inner.keys()
|
|
358
|
+
}
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
it("concurrent saveDoc calls should not save full history as an incremental chunk", async () => {
|
|
362
|
+
const adapter = new SlowSaveAdapter(50)
|
|
363
|
+
const storage = new StorageSubsystem(adapter)
|
|
364
|
+
const documentId = parseAutomergeUrl(generateAutomergeUrl()).documentId
|
|
365
|
+
|
|
366
|
+
// Create a document with enough data that the snapshot exceeds 1024 bytes,
|
|
367
|
+
// so that the second save won't trivially re-compact.
|
|
368
|
+
let doc = A.init<{ items: string[] }>()
|
|
369
|
+
doc = A.change(doc, d => {
|
|
370
|
+
d.items = Array(200)
|
|
371
|
+
.fill(0)
|
|
372
|
+
.map((_, i) => `item-${i}-${"x".repeat(20)}`)
|
|
373
|
+
})
|
|
374
|
+
|
|
375
|
+
// Compute the size of a full save for reference
|
|
376
|
+
const fullSaveSize = A.save(doc).length
|
|
377
|
+
|
|
378
|
+
// First saveDoc: no storedHeads, enters #saveTotal, sets #compacting = true,
|
|
379
|
+
// then awaits the slow adapter.save(). Don't await — let it be in-flight.
|
|
380
|
+
const save1 = storage.saveDoc(documentId, doc)
|
|
381
|
+
|
|
382
|
+
// Make a small change while the first save is still in-flight
|
|
383
|
+
const doc2 = A.change(doc, d => {
|
|
384
|
+
d.items.push("one-more-item")
|
|
385
|
+
})
|
|
386
|
+
|
|
387
|
+
// Second saveDoc: #compacting is true, so #shouldCompact returns false,
|
|
388
|
+
// falls through to #saveIncremental with sinceHeads = [] (empty).
|
|
389
|
+
const save2 = storage.saveDoc(documentId, doc2)
|
|
390
|
+
|
|
391
|
+
// Wait for both to complete
|
|
392
|
+
await Promise.all([save1, save2])
|
|
393
|
+
|
|
394
|
+
// Now inspect what was stored. Look at all incremental chunks.
|
|
395
|
+
const incrementalChunks = await adapter.loadRange([
|
|
396
|
+
documentId,
|
|
397
|
+
"incremental",
|
|
398
|
+
])
|
|
399
|
+
|
|
400
|
+
// If the bug is present, the incremental chunk will contain the full
|
|
401
|
+
// document history (roughly fullSaveSize). A correct incremental should
|
|
402
|
+
// only contain the delta — which is much smaller.
|
|
403
|
+
for (const chunk of incrementalChunks) {
|
|
404
|
+
expect(
|
|
405
|
+
chunk.data.length,
|
|
406
|
+
`incremental chunk should be much smaller than a full save ` +
|
|
407
|
+
`(${chunk.data.length} vs ${fullSaveSize}), ` +
|
|
408
|
+
`indicating saveSince was called with empty heads`
|
|
409
|
+
).toBeLessThan(fullSaveSize * 0.5)
|
|
410
|
+
}
|
|
411
|
+
})
|
|
412
|
+
|
|
413
|
+
it("compaction should never roll back storedHeads regardless of save timing", async () => {
|
|
414
|
+
// This test reproduces an issue where a the storedHeads of the storage
|
|
415
|
+
// subsystem would be rolled back to an old value. The scenario is
|
|
416
|
+
// roughly that a compaction starts, but it takes a long time to
|
|
417
|
+
// complete, during that time some incremental changes arrive and
|
|
418
|
+
// are saved before the compaction completes. This means that the
|
|
419
|
+
// storedheads are updated _after_ the compactions save call completes
|
|
420
|
+
// which means that the storedHeads roll back to before the incremental
|
|
421
|
+
// changes. This means that the next saveSince call will include
|
|
422
|
+
// all the incremental changes.
|
|
423
|
+
|
|
424
|
+
// An adapter where snapshot saves are slow but incremental saves are
|
|
425
|
+
// instant. This guarantees that when a compaction and an incremental
|
|
426
|
+
// save overlap, the compaction's adapter.save() completes *after* the
|
|
427
|
+
// incremental's — exactly the interleaving that triggers the heads
|
|
428
|
+
// rollback bug.
|
|
429
|
+
class SlowSnapshotAdapter implements StorageAdapterInterface {
|
|
430
|
+
#inner = new DummyStorageAdapter()
|
|
431
|
+
#snapshotDelayMs: number
|
|
432
|
+
|
|
433
|
+
constructor(snapshotDelayMs: number) {
|
|
434
|
+
this.#snapshotDelayMs = snapshotDelayMs
|
|
435
|
+
}
|
|
436
|
+
|
|
437
|
+
async load(key: StorageKey) {
|
|
438
|
+
return this.#inner.load(key)
|
|
439
|
+
}
|
|
440
|
+
async save(key: StorageKey, data: Uint8Array) {
|
|
441
|
+
if (key[1] === "snapshot") {
|
|
442
|
+
await new Promise(r => setTimeout(r, this.#snapshotDelayMs))
|
|
443
|
+
}
|
|
444
|
+
return this.#inner.save(key, data)
|
|
445
|
+
}
|
|
446
|
+
async remove(key: StorageKey) {
|
|
447
|
+
return this.#inner.remove(key)
|
|
448
|
+
}
|
|
449
|
+
async loadRange(keyPrefix: StorageKey) {
|
|
450
|
+
return this.#inner.loadRange(keyPrefix)
|
|
451
|
+
}
|
|
452
|
+
async removeRange(keyPrefix: StorageKey) {
|
|
453
|
+
return this.#inner.removeRange(keyPrefix)
|
|
454
|
+
}
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
const adapter = new SlowSnapshotAdapter(50)
|
|
458
|
+
const storage = new StorageSubsystem(adapter)
|
|
459
|
+
const documentId = parseAutomergeUrl(generateAutomergeUrl()).documentId
|
|
460
|
+
|
|
461
|
+
// Build a document large enough to trigger compaction
|
|
462
|
+
let doc = A.init<{ items: string[] }>()
|
|
463
|
+
doc = A.change(doc, d => {
|
|
464
|
+
d.items = Array(200)
|
|
465
|
+
.fill(0)
|
|
466
|
+
.map((_, i) => `item-${i}-${"x".repeat(20)}`)
|
|
467
|
+
})
|
|
468
|
+
await storage.saveDoc(documentId, doc)
|
|
469
|
+
|
|
470
|
+
// Add a large incremental so the next save triggers compaction
|
|
471
|
+
doc = A.change(doc, d => {
|
|
472
|
+
for (let i = 0; i < 200; i++) {
|
|
473
|
+
d.items.push(`extra-${i}-${"y".repeat(20)}`)
|
|
474
|
+
}
|
|
475
|
+
})
|
|
476
|
+
await storage.saveDoc(documentId, doc)
|
|
477
|
+
|
|
478
|
+
// Track events to verify the scenario we want actually happened
|
|
479
|
+
let sawCompaction = false
|
|
480
|
+
let sawIncrementalAfterCompaction = false
|
|
481
|
+
storage.on("doc-compacted", () => {
|
|
482
|
+
sawCompaction = true
|
|
483
|
+
})
|
|
484
|
+
storage.on("doc-saved", () => {
|
|
485
|
+
if (sawCompaction) {
|
|
486
|
+
sawIncrementalAfterCompaction = true
|
|
487
|
+
}
|
|
488
|
+
})
|
|
489
|
+
|
|
490
|
+
// Fire concurrent saves. The first will trigger compaction (slow
|
|
491
|
+
// snapshot save). The second will see #compacting=true and go
|
|
492
|
+
// through #saveIncremental (fast), completing before the compaction.
|
|
493
|
+
doc = A.change(doc, d => {
|
|
494
|
+
d.items.push("change-triggering-compaction")
|
|
495
|
+
})
|
|
496
|
+
const save1 = storage.saveDoc(documentId, doc)
|
|
497
|
+
|
|
498
|
+
doc = A.change(doc, d => {
|
|
499
|
+
d.items.push("change-during-compaction")
|
|
500
|
+
})
|
|
501
|
+
const save2 = storage.saveDoc(documentId, doc)
|
|
502
|
+
|
|
503
|
+
const lastConcurrentHeads = A.getHeads(doc)
|
|
504
|
+
await Promise.all([save1, save2])
|
|
505
|
+
|
|
506
|
+
// Verify we actually exercised the code path: a compaction happened,
|
|
507
|
+
// and an incremental save occurred while it was in flight.
|
|
508
|
+
expect(sawCompaction, "expected a compaction to have occurred").toBe(true)
|
|
509
|
+
expect(
|
|
510
|
+
sawIncrementalAfterCompaction,
|
|
511
|
+
"expected an incremental save after the compaction started"
|
|
512
|
+
).toBe(true)
|
|
513
|
+
|
|
514
|
+
// Now do a final sequential save. Its sinceHeads should match the
|
|
515
|
+
// heads from the last concurrent save. If the compaction's slower
|
|
516
|
+
// completion rolled back storedHeads, sinceHeads will be stale.
|
|
517
|
+
doc = A.change(doc, d => {
|
|
518
|
+
d.items.push("final-change")
|
|
519
|
+
})
|
|
520
|
+
|
|
521
|
+
let finalSinceHeads: A.Heads | undefined
|
|
522
|
+
storage.on("doc-saved", ({ sinceHeads }) => {
|
|
523
|
+
finalSinceHeads = sinceHeads
|
|
524
|
+
})
|
|
525
|
+
|
|
526
|
+
await storage.saveDoc(documentId, doc)
|
|
527
|
+
|
|
528
|
+
expect(
|
|
529
|
+
finalSinceHeads,
|
|
530
|
+
"final save should have been incremental (not compaction)"
|
|
531
|
+
).toBeDefined()
|
|
532
|
+
expect(
|
|
533
|
+
finalSinceHeads,
|
|
534
|
+
"sinceHeads was rolled back — expected heads from the latest concurrent save"
|
|
535
|
+
).toEqual(lastConcurrentHeads)
|
|
536
|
+
})
|
|
537
|
+
})
|
|
327
538
|
})
|