@agoric/swing-store 0.9.2-dev-2f092c3.0 → 0.9.2-dev-aa10ecd.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/docs/data-export.md +20 -4
- package/docs/swingstore.md +52 -0
- package/package.json +8 -5
- package/src/assertComplete.js +22 -0
- package/src/bundleStore.js +136 -47
- package/src/exporter.js +175 -0
- package/src/importer.js +121 -0
- package/src/index.js +11 -0
- package/src/internal.js +14 -0
- package/src/kvStore.js +172 -0
- package/src/repairMetadata.js +65 -0
- package/src/snapStore.js +160 -33
- package/src/snapStoreIO.js +8 -0
- package/src/swingStore.js +52 -587
- package/src/transcriptStore.js +170 -37
- package/src/types.d.ts +14 -0
- package/src/types.js +6 -0
- package/src/util.js +7 -1
- package/test/test-bundles.js +9 -7
- package/test/test-export.js +308 -0
- package/test/test-exportImport.js +36 -54
- package/test/test-import.js +476 -0
- package/test/test-repair-metadata.js +131 -0
- package/test/util.js +26 -0
package/src/swingStore.js
CHANGED
|
@@ -2,74 +2,34 @@
|
|
|
2
2
|
/* global Buffer */
|
|
3
3
|
import fs from 'fs';
|
|
4
4
|
import path from 'path';
|
|
5
|
-
import { performance } from 'perf_hooks';
|
|
6
5
|
|
|
7
6
|
import sqlite3 from 'better-sqlite3';
|
|
8
7
|
|
|
9
|
-
import {
|
|
10
|
-
import { makeMeasureSeconds } from '@agoric/internal';
|
|
8
|
+
import { Fail, q } from '@agoric/assert';
|
|
11
9
|
|
|
10
|
+
import { dbFileInDirectory } from './util.js';
|
|
11
|
+
import { makeKVStore, getKeyType } from './kvStore.js';
|
|
12
12
|
import { makeTranscriptStore } from './transcriptStore.js';
|
|
13
13
|
import { makeSnapStore } from './snapStore.js';
|
|
14
14
|
import { makeBundleStore } from './bundleStore.js';
|
|
15
15
|
import { createSHA256 } from './hasher.js';
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
/**
|
|
20
|
-
* This is a polyfill for the `buffer` function from Node's
|
|
21
|
-
* 'stream/consumers' package, which unfortunately only exists in newer versions
|
|
22
|
-
* of Node.
|
|
23
|
-
*
|
|
24
|
-
* @param {AsyncIterable<Buffer>} inStream
|
|
25
|
-
*/
|
|
26
|
-
export const buffer = async inStream => {
|
|
27
|
-
const chunks = [];
|
|
28
|
-
for await (const chunk of inStream) {
|
|
29
|
-
chunks.push(chunk);
|
|
30
|
-
}
|
|
31
|
-
return Buffer.concat(chunks);
|
|
32
|
-
};
|
|
33
|
-
|
|
34
|
-
export function makeSnapStoreIO() {
|
|
35
|
-
return {
|
|
36
|
-
measureSeconds: makeMeasureSeconds(performance.now),
|
|
37
|
-
};
|
|
38
|
-
}
|
|
39
|
-
|
|
40
|
-
/**
|
|
41
|
-
* @param {string} key
|
|
42
|
-
*/
|
|
43
|
-
function getKeyType(key) {
|
|
44
|
-
if (key.startsWith('local.')) {
|
|
45
|
-
return 'local';
|
|
46
|
-
} else if (key.startsWith('host.')) {
|
|
47
|
-
return 'host';
|
|
48
|
-
}
|
|
49
|
-
return 'consensus';
|
|
50
|
-
}
|
|
16
|
+
import { makeSnapStoreIO } from './snapStoreIO.js';
|
|
17
|
+
import { doRepairMetadata } from './repairMetadata.js';
|
|
51
18
|
|
|
52
19
|
/**
|
|
53
|
-
* @typedef {
|
|
54
|
-
* has: (key: string) => boolean,
|
|
55
|
-
* get: (key: string) => string | undefined,
|
|
56
|
-
* getNextKey: (previousKey: string) => string | undefined,
|
|
57
|
-
* set: (key: string, value: string, bypassHash?: boolean ) => void,
|
|
58
|
-
* delete: (key: string) => void,
|
|
59
|
-
* }} KVStore
|
|
20
|
+
* @typedef { import('./kvStore').KVStore } KVStore
|
|
60
21
|
*
|
|
61
22
|
* @typedef { import('./snapStore').SnapStore } SnapStore
|
|
62
|
-
* @typedef { import('./snapStore').SnapStoreInternal } SnapStoreInternal
|
|
63
23
|
* @typedef { import('./snapStore').SnapshotResult } SnapshotResult
|
|
64
24
|
*
|
|
65
25
|
* @typedef { import('./transcriptStore').TranscriptStore } TranscriptStore
|
|
66
|
-
* @typedef { import('./transcriptStore').TranscriptStoreInternal } TranscriptStoreInternal
|
|
67
26
|
* @typedef { import('./transcriptStore').TranscriptStoreDebug } TranscriptStoreDebug
|
|
68
27
|
*
|
|
69
28
|
* @typedef { import('./bundleStore').BundleStore } BundleStore
|
|
70
|
-
* @typedef { import('./bundleStore').BundleStoreInternal } BundleStoreInternal
|
|
71
29
|
* @typedef { import('./bundleStore').BundleStoreDebug } BundleStoreDebug
|
|
72
30
|
*
|
|
31
|
+
* @typedef { import('./exporter').KVPair } KVPair
|
|
32
|
+
*
|
|
73
33
|
* @typedef {{
|
|
74
34
|
* kvStore: KVStore, // a key-value API object to load and store data on behalf of the kernel
|
|
75
35
|
* transcriptStore: TranscriptStore, // a stream-oriented API object to append and read transcript entries
|
|
@@ -89,6 +49,7 @@ function getKeyType(key) {
|
|
|
89
49
|
* close: () => Promise<void>, // shutdown the store, abandoning any uncommitted changes
|
|
90
50
|
* diskUsage?: () => number, // optional stats method
|
|
91
51
|
* setExportCallback: (cb: (updates: KVPair[]) => void) => void, // Set a callback invoked by swingStore when new serializable data is available for export
|
|
52
|
+
* repairMetadata: (exporter: import('./exporter').SwingStoreExporter) => Promise<void>,
|
|
92
53
|
* }} SwingStoreHostStorage
|
|
93
54
|
*/
|
|
94
55
|
|
|
@@ -105,184 +66,13 @@ function getKeyType(key) {
|
|
|
105
66
|
* }} SwingStoreDebugTools
|
|
106
67
|
*
|
|
107
68
|
* @typedef {{
|
|
108
|
-
* transcriptStore: TranscriptStoreInternal,
|
|
109
|
-
* snapStore: SnapStoreInternal,
|
|
110
|
-
* bundleStore: BundleStoreInternal,
|
|
111
|
-
* }} SwingStoreInternal
|
|
112
|
-
*
|
|
113
|
-
* @typedef {{
|
|
114
69
|
* kernelStorage: SwingStoreKernelStorage,
|
|
115
70
|
* hostStorage: SwingStoreHostStorage,
|
|
116
71
|
* debug: SwingStoreDebugTools,
|
|
117
|
-
* internal: SwingStoreInternal,
|
|
72
|
+
* internal: import('./internal.js').SwingStoreInternal,
|
|
118
73
|
* }} SwingStore
|
|
119
74
|
*/
|
|
120
75
|
|
|
121
|
-
/**
|
|
122
|
-
* @template T
|
|
123
|
-
* @typedef { Iterable<T> | AsyncIterable<T> } AnyIterable<T>
|
|
124
|
-
*/
|
|
125
|
-
/**
|
|
126
|
-
* @template T
|
|
127
|
-
* @typedef { IterableIterator<T> | AsyncIterableIterator<T> } AnyIterableIterator<T>
|
|
128
|
-
*/
|
|
129
|
-
|
|
130
|
-
/**
|
|
131
|
-
* @typedef {readonly [
|
|
132
|
-
* key: string,
|
|
133
|
-
* value?: string | null | undefined,
|
|
134
|
-
* ]} KVPair
|
|
135
|
-
*
|
|
136
|
-
* @typedef {object} SwingStoreExporter
|
|
137
|
-
*
|
|
138
|
-
* Allows export of data from a swingStore as a fixed view onto the content as
|
|
139
|
-
* of the most recent commit point at the time the exporter was created. The
|
|
140
|
-
* exporter may be used while another SwingStore instance is active for the same
|
|
141
|
-
* DB, possibly in another thread or process. It guarantees that regardless of
|
|
142
|
-
* the concurrent activity of other swingStore instances, the data representing
|
|
143
|
-
* the commit point will stay consistent and available.
|
|
144
|
-
*
|
|
145
|
-
* @property {() => AnyIterableIterator<KVPair>} getExportData
|
|
146
|
-
*
|
|
147
|
-
* Get a full copy of the first-stage export data (key-value pairs) from the
|
|
148
|
-
* swingStore. This represents both the contents of the KVStore (excluding host
|
|
149
|
-
* and local prefixes), as well as any data needed to validate all artifacts,
|
|
150
|
-
* both current and historical. As such it represents the root of trust for the
|
|
151
|
-
* application.
|
|
152
|
-
*
|
|
153
|
-
* Content of validation data (with supporting entries for indexing):
|
|
154
|
-
* - kv.${key} = ${value} // ordinary kvStore data entry
|
|
155
|
-
* - snapshot.${vatID}.${snapPos} = ${{ vatID, snapPos, hash });
|
|
156
|
-
* - snapshot.${vatID}.current = `snapshot.${vatID}.${snapPos}`
|
|
157
|
-
* - transcript.${vatID}.${startPos} = ${{ vatID, startPos, endPos, hash }}
|
|
158
|
-
* - transcript.${vatID}.current = ${{ vatID, startPos, endPos, hash }}
|
|
159
|
-
*
|
|
160
|
-
* @property {() => AnyIterableIterator<string>} getArtifactNames
|
|
161
|
-
*
|
|
162
|
-
* Get a list of name of artifacts available from the swingStore. A name returned
|
|
163
|
-
* by this method guarantees that a call to `getArtifact` on the same exporter
|
|
164
|
-
* instance will succeed. Options control the filtering of the artifact names
|
|
165
|
-
* yielded.
|
|
166
|
-
*
|
|
167
|
-
* Artifact names:
|
|
168
|
-
* - transcript.${vatID}.${startPos}.${endPos}
|
|
169
|
-
* - snapshot.${vatID}.${snapPos}
|
|
170
|
-
*
|
|
171
|
-
* @property {(name: string) => AnyIterableIterator<Uint8Array>} getArtifact
|
|
172
|
-
*
|
|
173
|
-
* Retrieve an artifact by name. May throw if the artifact is not available,
|
|
174
|
-
* which can occur if the artifact is historical and wasn't been preserved.
|
|
175
|
-
*
|
|
176
|
-
* @property {() => Promise<void>} close
|
|
177
|
-
*
|
|
178
|
-
* Dispose of all resources held by this exporter. Any further operation on this
|
|
179
|
-
* exporter or its outstanding iterators will fail.
|
|
180
|
-
*/
|
|
181
|
-
|
|
182
|
-
/**
|
|
183
|
-
* @param {string} dirPath
|
|
184
|
-
* @param {string} exportMode
|
|
185
|
-
* @returns {SwingStoreExporter}
|
|
186
|
-
*/
|
|
187
|
-
export function makeSwingStoreExporter(dirPath, exportMode = 'current') {
|
|
188
|
-
typeof dirPath === 'string' || Fail`dirPath must be a string`;
|
|
189
|
-
exportMode === 'current' ||
|
|
190
|
-
exportMode === 'archival' ||
|
|
191
|
-
exportMode === 'debug' ||
|
|
192
|
-
Fail`invalid exportMode ${q(exportMode)}`;
|
|
193
|
-
const exportHistoricalSnapshots = exportMode === 'debug';
|
|
194
|
-
const exportHistoricalTranscripts = exportMode !== 'current';
|
|
195
|
-
const filePath = path.join(dirPath, 'swingstore.sqlite');
|
|
196
|
-
const db = sqlite3(filePath);
|
|
197
|
-
|
|
198
|
-
// Execute the data export in a (read) transaction, to ensure that we are
|
|
199
|
-
// capturing the state of the database at a single point in time.
|
|
200
|
-
const sqlBeginTransaction = db.prepare('BEGIN TRANSACTION');
|
|
201
|
-
sqlBeginTransaction.run();
|
|
202
|
-
|
|
203
|
-
// ensureTxn can be a dummy, we just started one
|
|
204
|
-
const ensureTxn = () => {};
|
|
205
|
-
const snapStore = makeSnapStore(db, ensureTxn, makeSnapStoreIO());
|
|
206
|
-
const bundleStore = makeBundleStore(db, ensureTxn);
|
|
207
|
-
const transcriptStore = makeTranscriptStore(db, ensureTxn, () => {});
|
|
208
|
-
|
|
209
|
-
const sqlGetAllKVData = db.prepare(`
|
|
210
|
-
SELECT key, value
|
|
211
|
-
FROM kvStore
|
|
212
|
-
ORDER BY key
|
|
213
|
-
`);
|
|
214
|
-
|
|
215
|
-
/**
|
|
216
|
-
* @returns {AsyncIterableIterator<KVPair>}
|
|
217
|
-
* @yields {KVPair}
|
|
218
|
-
*/
|
|
219
|
-
async function* getExportData() {
|
|
220
|
-
const kvPairs = sqlGetAllKVData.iterate();
|
|
221
|
-
for (const kv of kvPairs) {
|
|
222
|
-
if (getKeyType(kv.key) === 'consensus') {
|
|
223
|
-
yield [`kv.${kv.key}`, kv.value];
|
|
224
|
-
}
|
|
225
|
-
}
|
|
226
|
-
yield* snapStore.getExportRecords(true);
|
|
227
|
-
yield* transcriptStore.getExportRecords(true);
|
|
228
|
-
yield* bundleStore.getExportRecords();
|
|
229
|
-
}
|
|
230
|
-
|
|
231
|
-
/**
|
|
232
|
-
* @returns {AsyncIterableIterator<string>}
|
|
233
|
-
* @yields {string}
|
|
234
|
-
*/
|
|
235
|
-
async function* getArtifactNames() {
|
|
236
|
-
yield* snapStore.getArtifactNames(exportHistoricalSnapshots);
|
|
237
|
-
yield* transcriptStore.getArtifactNames(exportHistoricalTranscripts);
|
|
238
|
-
yield* bundleStore.getArtifactNames();
|
|
239
|
-
}
|
|
240
|
-
|
|
241
|
-
/**
|
|
242
|
-
* @param {string} name
|
|
243
|
-
* @returns {AsyncIterableIterator<Uint8Array>}
|
|
244
|
-
*/
|
|
245
|
-
function getArtifact(name) {
|
|
246
|
-
typeof name === 'string' || Fail`artifact name must be a string`;
|
|
247
|
-
const [type] = name.split('.', 1);
|
|
248
|
-
|
|
249
|
-
if (type === 'snapshot') {
|
|
250
|
-
return snapStore.exportSnapshot(name, exportHistoricalSnapshots);
|
|
251
|
-
} else if (type === 'transcript') {
|
|
252
|
-
return transcriptStore.exportSpan(name, exportHistoricalTranscripts);
|
|
253
|
-
} else if (type === 'bundle') {
|
|
254
|
-
return bundleStore.exportBundle(name);
|
|
255
|
-
} else {
|
|
256
|
-
throw Fail`invalid artifact type ${q(type)}`;
|
|
257
|
-
}
|
|
258
|
-
}
|
|
259
|
-
|
|
260
|
-
const sqlAbort = db.prepare('ROLLBACK');
|
|
261
|
-
|
|
262
|
-
async function close() {
|
|
263
|
-
// After all the data has been extracted, always abort the export
|
|
264
|
-
// transaction to ensure that the export was read-only (i.e., that no bugs
|
|
265
|
-
// inadvertantly modified the database).
|
|
266
|
-
sqlAbort.run();
|
|
267
|
-
db.close();
|
|
268
|
-
}
|
|
269
|
-
|
|
270
|
-
return harden({
|
|
271
|
-
getExportData,
|
|
272
|
-
getArtifactNames,
|
|
273
|
-
getArtifact,
|
|
274
|
-
close,
|
|
275
|
-
});
|
|
276
|
-
}
|
|
277
|
-
|
|
278
|
-
/**
|
|
279
|
-
* Function used to create a new swingStore from an object implementing the
|
|
280
|
-
* exporter API. The exporter API may be provided by a swingStore instance, or
|
|
281
|
-
* implemented by a host to restore data that was previously exported.
|
|
282
|
-
*
|
|
283
|
-
* @typedef {(exporter: SwingStoreExporter) => Promise<SwingStore>} ImportSwingStore
|
|
284
|
-
*/
|
|
285
|
-
|
|
286
76
|
/**
|
|
287
77
|
* A swing store holds the state of a swingset instance. This "store" is
|
|
288
78
|
* actually several different stores of different types that travel as a flock
|
|
@@ -342,7 +132,7 @@ export function makeSwingStoreExporter(dirPath, exportMode = 'current') {
|
|
|
342
132
|
*
|
|
343
133
|
* @returns {SwingStore}
|
|
344
134
|
*/
|
|
345
|
-
function makeSwingStore(dirPath, forceReset, options = {}) {
|
|
135
|
+
export function makeSwingStore(dirPath, forceReset, options = {}) {
|
|
346
136
|
const { serialized } = options;
|
|
347
137
|
if (serialized) {
|
|
348
138
|
Buffer.isBuffer(serialized) || Fail`options.serialized must be Buffer`;
|
|
@@ -374,7 +164,7 @@ function makeSwingStore(dirPath, forceReset, options = {}) {
|
|
|
374
164
|
}
|
|
375
165
|
}
|
|
376
166
|
fs.mkdirSync(dirPath, { recursive: true });
|
|
377
|
-
filePath =
|
|
167
|
+
filePath = dbFileInDirectory(dirPath);
|
|
378
168
|
} else {
|
|
379
169
|
filePath = ':memory:';
|
|
380
170
|
}
|
|
@@ -449,13 +239,6 @@ function makeSwingStore(dirPath, forceReset, options = {}) {
|
|
|
449
239
|
|
|
450
240
|
// Perform all database initialization in a single transaction
|
|
451
241
|
sqlBeginTransaction.run();
|
|
452
|
-
db.exec(`
|
|
453
|
-
CREATE TABLE IF NOT EXISTS kvStore (
|
|
454
|
-
key TEXT,
|
|
455
|
-
value TEXT,
|
|
456
|
-
PRIMARY KEY (key)
|
|
457
|
-
)
|
|
458
|
-
`);
|
|
459
242
|
|
|
460
243
|
db.exec(`
|
|
461
244
|
CREATE TABLE IF NOT EXISTS pendingExports (
|
|
@@ -465,10 +248,32 @@ function makeSwingStore(dirPath, forceReset, options = {}) {
|
|
|
465
248
|
)
|
|
466
249
|
`);
|
|
467
250
|
|
|
251
|
+
let exportCallback;
|
|
252
|
+
function setExportCallback(cb) {
|
|
253
|
+
typeof cb === 'function' || Fail`callback must be a function`;
|
|
254
|
+
exportCallback = cb;
|
|
255
|
+
}
|
|
256
|
+
if (options.exportCallback) {
|
|
257
|
+
setExportCallback(options.exportCallback);
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
const sqlAddPendingExport = db.prepare(`
|
|
261
|
+
INSERT INTO pendingExports (key, value)
|
|
262
|
+
VALUES (?, ?)
|
|
263
|
+
ON CONFLICT DO UPDATE SET value = excluded.value
|
|
264
|
+
`);
|
|
265
|
+
|
|
266
|
+
function noteExport(key, value) {
|
|
267
|
+
if (exportCallback) {
|
|
268
|
+
sqlAddPendingExport.run(key, value);
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
const kvStore = makeKVStore(db, ensureTxn, trace);
|
|
273
|
+
|
|
468
274
|
const { dumpTranscripts, ...transcriptStore } = makeTranscriptStore(
|
|
469
275
|
db,
|
|
470
276
|
ensureTxn,
|
|
471
|
-
// eslint-disable-next-line no-use-before-define
|
|
472
277
|
noteExport,
|
|
473
278
|
{
|
|
474
279
|
keepTranscripts,
|
|
@@ -478,7 +283,6 @@ function makeSwingStore(dirPath, forceReset, options = {}) {
|
|
|
478
283
|
db,
|
|
479
284
|
ensureTxn,
|
|
480
285
|
makeSnapStoreIO(),
|
|
481
|
-
// eslint-disable-next-line no-use-before-define
|
|
482
286
|
noteExport,
|
|
483
287
|
{
|
|
484
288
|
keepSnapshots,
|
|
@@ -487,7 +291,6 @@ function makeSwingStore(dirPath, forceReset, options = {}) {
|
|
|
487
291
|
const { dumpBundles, ...bundleStore } = makeBundleStore(
|
|
488
292
|
db,
|
|
489
293
|
ensureTxn,
|
|
490
|
-
// eslint-disable-next-line no-use-before-define
|
|
491
294
|
noteExport,
|
|
492
295
|
);
|
|
493
296
|
|
|
@@ -496,20 +299,11 @@ function makeSwingStore(dirPath, forceReset, options = {}) {
|
|
|
496
299
|
// At this point, all database initialization should be complete, so commit now.
|
|
497
300
|
sqlCommit.run();
|
|
498
301
|
|
|
499
|
-
let exportCallback;
|
|
500
|
-
function setExportCallback(cb) {
|
|
501
|
-
typeof cb === 'function' || Fail`callback must be a function`;
|
|
502
|
-
exportCallback = cb;
|
|
503
|
-
}
|
|
504
|
-
if (options.exportCallback) {
|
|
505
|
-
setExportCallback(options.exportCallback);
|
|
506
|
-
}
|
|
507
|
-
|
|
508
302
|
let inCrank = false;
|
|
509
303
|
|
|
510
304
|
function diskUsage() {
|
|
511
305
|
if (dirPath) {
|
|
512
|
-
const dataFilePath =
|
|
306
|
+
const dataFilePath = dbFileInDirectory(dirPath);
|
|
513
307
|
const stat = fs.statSync(dataFilePath);
|
|
514
308
|
return stat.size;
|
|
515
309
|
} else {
|
|
@@ -517,154 +311,13 @@ function makeSwingStore(dirPath, forceReset, options = {}) {
|
|
|
517
311
|
}
|
|
518
312
|
}
|
|
519
313
|
|
|
520
|
-
const sqlKVGet = db.prepare(`
|
|
521
|
-
SELECT value
|
|
522
|
-
FROM kvStore
|
|
523
|
-
WHERE key = ?
|
|
524
|
-
`);
|
|
525
|
-
sqlKVGet.pluck(true);
|
|
526
|
-
|
|
527
|
-
/**
|
|
528
|
-
* Obtain the value stored for a given key.
|
|
529
|
-
*
|
|
530
|
-
* @param {string} key The key whose value is sought.
|
|
531
|
-
*
|
|
532
|
-
* @returns {string | undefined} the (string) value for the given key, or
|
|
533
|
-
* undefined if there is no such value.
|
|
534
|
-
*
|
|
535
|
-
* @throws if key is not a string.
|
|
536
|
-
*/
|
|
537
|
-
function get(key) {
|
|
538
|
-
typeof key === 'string' || Fail`key must be a string`;
|
|
539
|
-
return sqlKVGet.get(key);
|
|
540
|
-
}
|
|
541
|
-
|
|
542
|
-
const sqlKVGetNextKey = db.prepare(`
|
|
543
|
-
SELECT key
|
|
544
|
-
FROM kvStore
|
|
545
|
-
WHERE key > ?
|
|
546
|
-
LIMIT 1
|
|
547
|
-
`);
|
|
548
|
-
sqlKVGetNextKey.pluck(true);
|
|
549
|
-
|
|
550
|
-
/**
|
|
551
|
-
* getNextKey enables callers to iterate over all keys within a
|
|
552
|
-
* given range. To build an iterator of all keys from start
|
|
553
|
-
* (inclusive) to end (exclusive), do:
|
|
554
|
-
*
|
|
555
|
-
* function* iterate(start, end) {
|
|
556
|
-
* if (kvStore.has(start)) {
|
|
557
|
-
* yield start;
|
|
558
|
-
* }
|
|
559
|
-
* let prev = start;
|
|
560
|
-
* while (true) {
|
|
561
|
-
* let next = kvStore.getNextKey(prev);
|
|
562
|
-
* if (!next || next >= end) {
|
|
563
|
-
* break;
|
|
564
|
-
* }
|
|
565
|
-
* yield next;
|
|
566
|
-
* prev = next;
|
|
567
|
-
* }
|
|
568
|
-
* }
|
|
569
|
-
*
|
|
570
|
-
* @param {string} previousKey The key returned will always be later than this one.
|
|
571
|
-
*
|
|
572
|
-
* @returns {string | undefined} a key string, or undefined if we reach the end of the store
|
|
573
|
-
*
|
|
574
|
-
* @throws if previousKey is not a string
|
|
575
|
-
*/
|
|
576
|
-
|
|
577
|
-
function getNextKey(previousKey) {
|
|
578
|
-
typeof previousKey === 'string' || Fail`previousKey must be a string`;
|
|
579
|
-
return sqlKVGetNextKey.get(previousKey);
|
|
580
|
-
}
|
|
581
|
-
|
|
582
|
-
/**
|
|
583
|
-
* Test if the state contains a value for a given key.
|
|
584
|
-
*
|
|
585
|
-
* @param {string} key The key that is of interest.
|
|
586
|
-
*
|
|
587
|
-
* @returns {boolean} true if a value is stored for the key, false if not.
|
|
588
|
-
*
|
|
589
|
-
* @throws if key is not a string.
|
|
590
|
-
*/
|
|
591
|
-
function has(key) {
|
|
592
|
-
typeof key === 'string' || Fail`key must be a string`;
|
|
593
|
-
return get(key) !== undefined;
|
|
594
|
-
}
|
|
595
|
-
|
|
596
|
-
const sqlKVSet = db.prepare(`
|
|
597
|
-
INSERT INTO kvStore (key, value)
|
|
598
|
-
VALUES (?, ?)
|
|
599
|
-
ON CONFLICT DO UPDATE SET value = excluded.value
|
|
600
|
-
`);
|
|
601
|
-
|
|
602
|
-
/**
|
|
603
|
-
* Store a value for a given key. The value will replace any prior value if
|
|
604
|
-
* there was one.
|
|
605
|
-
*
|
|
606
|
-
* @param {string} key The key whose value is being set.
|
|
607
|
-
* @param {string} value The value to set the key to.
|
|
608
|
-
*
|
|
609
|
-
* @throws if either parameter is not a string.
|
|
610
|
-
*/
|
|
611
|
-
function set(key, value) {
|
|
612
|
-
typeof key === 'string' || Fail`key must be a string`;
|
|
613
|
-
typeof value === 'string' || Fail`value must be a string`;
|
|
614
|
-
// synchronous read after write within a transaction is safe
|
|
615
|
-
// The transaction's overall success will be awaited during commit
|
|
616
|
-
ensureTxn();
|
|
617
|
-
sqlKVSet.run(key, value);
|
|
618
|
-
trace('set', key, value);
|
|
619
|
-
}
|
|
620
|
-
|
|
621
|
-
const sqlKVDel = db.prepare(`
|
|
622
|
-
DELETE FROM kvStore
|
|
623
|
-
WHERE key = ?
|
|
624
|
-
`);
|
|
625
|
-
|
|
626
|
-
/**
|
|
627
|
-
* Remove any stored value for a given key. It is permissible for there to
|
|
628
|
-
* be no existing stored value for the key.
|
|
629
|
-
*
|
|
630
|
-
* @param {string} key The key whose value is to be deleted
|
|
631
|
-
*
|
|
632
|
-
* @throws if key is not a string.
|
|
633
|
-
*/
|
|
634
|
-
function del(key) {
|
|
635
|
-
typeof key === 'string' || Fail`key must be a string`;
|
|
636
|
-
ensureTxn();
|
|
637
|
-
sqlKVDel.run(key);
|
|
638
|
-
trace('del', key);
|
|
639
|
-
}
|
|
640
|
-
|
|
641
|
-
const kvStore = {
|
|
642
|
-
has,
|
|
643
|
-
get,
|
|
644
|
-
getNextKey,
|
|
645
|
-
set,
|
|
646
|
-
delete: del,
|
|
647
|
-
};
|
|
648
|
-
|
|
649
|
-
const sqlAddPendingExport = db.prepare(`
|
|
650
|
-
INSERT INTO pendingExports (key, value)
|
|
651
|
-
VALUES (?, ?)
|
|
652
|
-
ON CONFLICT DO UPDATE SET value = excluded.value
|
|
653
|
-
`);
|
|
654
|
-
|
|
655
|
-
function noteExport(key, value) {
|
|
656
|
-
if (exportCallback) {
|
|
657
|
-
sqlAddPendingExport.run(key, value);
|
|
658
|
-
}
|
|
659
|
-
}
|
|
660
|
-
|
|
661
314
|
const kernelKVStore = {
|
|
662
315
|
...kvStore,
|
|
663
316
|
set(key, value) {
|
|
664
317
|
typeof key === 'string' || Fail`key must be a string`;
|
|
665
318
|
const keyType = getKeyType(key);
|
|
666
319
|
keyType !== 'host' || Fail`kernelKVStore refuses host keys`;
|
|
667
|
-
set(key, value);
|
|
320
|
+
kvStore.set(key, value);
|
|
668
321
|
if (keyType === 'consensus') {
|
|
669
322
|
noteExport(`kv.${key}`, value);
|
|
670
323
|
crankhasher.add('add');
|
|
@@ -679,7 +332,7 @@ function makeSwingStore(dirPath, forceReset, options = {}) {
|
|
|
679
332
|
typeof key === 'string' || Fail`key must be a string`;
|
|
680
333
|
const keyType = getKeyType(key);
|
|
681
334
|
keyType !== 'host' || Fail`kernelKVStore refuses host keys`;
|
|
682
|
-
|
|
335
|
+
kvStore.delete(key);
|
|
683
336
|
if (keyType === 'consensus') {
|
|
684
337
|
noteExport(`kv.${key}`, undefined);
|
|
685
338
|
crankhasher.add('delete');
|
|
@@ -695,12 +348,12 @@ function makeSwingStore(dirPath, forceReset, options = {}) {
|
|
|
695
348
|
set(key, value) {
|
|
696
349
|
const keyType = getKeyType(key);
|
|
697
350
|
keyType === 'host' || Fail`hostKVStore requires host keys`;
|
|
698
|
-
set(key, value);
|
|
351
|
+
kvStore.set(key, value);
|
|
699
352
|
},
|
|
700
353
|
delete(key) {
|
|
701
354
|
const keyType = getKeyType(key);
|
|
702
355
|
keyType === 'host' || Fail`hostKVStore requires host keys`;
|
|
703
|
-
|
|
356
|
+
kvStore.delete(key);
|
|
704
357
|
},
|
|
705
358
|
};
|
|
706
359
|
|
|
@@ -740,7 +393,7 @@ function makeSwingStore(dirPath, forceReset, options = {}) {
|
|
|
740
393
|
resetCrankhash();
|
|
741
394
|
|
|
742
395
|
// Get the old activityhash
|
|
743
|
-
let oldActivityhash = get('activityhash');
|
|
396
|
+
let oldActivityhash = kvStore.get('activityhash');
|
|
744
397
|
if (oldActivityhash === undefined) {
|
|
745
398
|
oldActivityhash = '';
|
|
746
399
|
}
|
|
@@ -756,7 +409,7 @@ function makeSwingStore(dirPath, forceReset, options = {}) {
|
|
|
756
409
|
|
|
757
410
|
// Store the new activityhash
|
|
758
411
|
const activityhash = hasher.finish();
|
|
759
|
-
set('activityhash', activityhash);
|
|
412
|
+
kvStore.set('activityhash', activityhash);
|
|
760
413
|
// Need to explicitly call noteExport here because activityhash is written
|
|
761
414
|
// directly to the low-level store to avoid recursive hashing, which
|
|
762
415
|
// bypasses the normal notification mechanism
|
|
@@ -766,7 +419,7 @@ function makeSwingStore(dirPath, forceReset, options = {}) {
|
|
|
766
419
|
}
|
|
767
420
|
|
|
768
421
|
function getActivityhash() {
|
|
769
|
-
return get('activityhash') || '';
|
|
422
|
+
return kvStore.get('activityhash') || '';
|
|
770
423
|
}
|
|
771
424
|
|
|
772
425
|
const sqlExportsGet = db.prepare(`
|
|
@@ -823,6 +476,13 @@ function makeSwingStore(dirPath, forceReset, options = {}) {
|
|
|
823
476
|
stopTrace();
|
|
824
477
|
}
|
|
825
478
|
|
|
479
|
+
/** @type {import('./internal.js').SwingStoreInternal} */
|
|
480
|
+
const internal = harden({ snapStore, transcriptStore, bundleStore });
|
|
481
|
+
|
|
482
|
+
async function repairMetadata(exporter) {
|
|
483
|
+
return doRepairMetadata(internal, exporter);
|
|
484
|
+
}
|
|
485
|
+
|
|
826
486
|
/**
|
|
827
487
|
* Return a Buffer with the entire DB state, useful for cloning a
|
|
828
488
|
* small swingstore in unit tests.
|
|
@@ -898,6 +558,7 @@ function makeSwingStore(dirPath, forceReset, options = {}) {
|
|
|
898
558
|
getActivityhash,
|
|
899
559
|
};
|
|
900
560
|
const hostStorage = {
|
|
561
|
+
repairMetadata,
|
|
901
562
|
kvStore: hostKVStore,
|
|
902
563
|
commit,
|
|
903
564
|
close,
|
|
@@ -909,11 +570,6 @@ function makeSwingStore(dirPath, forceReset, options = {}) {
|
|
|
909
570
|
dump,
|
|
910
571
|
getDatabase,
|
|
911
572
|
};
|
|
912
|
-
const internal = {
|
|
913
|
-
snapStore,
|
|
914
|
-
transcriptStore,
|
|
915
|
-
bundleStore,
|
|
916
|
-
};
|
|
917
573
|
|
|
918
574
|
return harden({
|
|
919
575
|
kernelStorage,
|
|
@@ -946,197 +602,6 @@ export function initSwingStore(dirPath = null, options = {}) {
|
|
|
946
602
|
return makeSwingStore(dirPath, true, options);
|
|
947
603
|
}
|
|
948
604
|
|
|
949
|
-
function parseVatArtifactExportKey(key) {
|
|
950
|
-
const parts = key.split('.');
|
|
951
|
-
const [_type, vatID, rawPos] = parts;
|
|
952
|
-
// prettier-ignore
|
|
953
|
-
parts.length === 3 ||
|
|
954
|
-
Fail`expected artifact name of the form '{type}.{vatID}.{pos}', saw ${q(key)}`;
|
|
955
|
-
const isCurrent = rawPos === 'current';
|
|
956
|
-
let pos;
|
|
957
|
-
if (isCurrent) {
|
|
958
|
-
pos = -1;
|
|
959
|
-
} else {
|
|
960
|
-
pos = Number(rawPos);
|
|
961
|
-
}
|
|
962
|
-
|
|
963
|
-
return { vatID, isCurrent, pos };
|
|
964
|
-
}
|
|
965
|
-
|
|
966
|
-
function artifactKey(type, vatID, pos) {
|
|
967
|
-
return `${type}.${vatID}.${pos}`;
|
|
968
|
-
}
|
|
969
|
-
|
|
970
|
-
/**
|
|
971
|
-
* @param {SwingStoreExporter} exporter
|
|
972
|
-
* @param {string | null} [dirPath]
|
|
973
|
-
* @param {object} options
|
|
974
|
-
* @returns {Promise<SwingStore>}
|
|
975
|
-
*/
|
|
976
|
-
export async function importSwingStore(exporter, dirPath = null, options = {}) {
|
|
977
|
-
if (dirPath) {
|
|
978
|
-
typeof dirPath === 'string' || Fail`dirPath must be a string`;
|
|
979
|
-
}
|
|
980
|
-
const { includeHistorical = false } = options;
|
|
981
|
-
const store = makeSwingStore(dirPath, true, options);
|
|
982
|
-
const { kernelStorage, internal } = store;
|
|
983
|
-
|
|
984
|
-
// Artifact metadata, keyed as `${type}.${vatID}.${pos}`
|
|
985
|
-
//
|
|
986
|
-
// Note that this key is almost but not quite the artifact name, since the
|
|
987
|
-
// names of transcript span artifacts also include the endPos, but the endPos
|
|
988
|
-
// value is in flux until the span is complete.
|
|
989
|
-
const artifactMetadata = new Map();
|
|
990
|
-
|
|
991
|
-
// Each vat requires a transcript span and (usually) a snapshot. This table
|
|
992
|
-
// tracks which of these we've seen, keyed by vatID.
|
|
993
|
-
// vatID -> { snapshotKey: metadataKey, transcriptKey: metatdataKey }
|
|
994
|
-
const vatArtifacts = new Map();
|
|
995
|
-
const bundleArtifacts = new Map();
|
|
996
|
-
|
|
997
|
-
for await (const [key, value] of exporter.getExportData()) {
|
|
998
|
-
const [tag] = key.split('.', 1);
|
|
999
|
-
const subKey = key.substring(tag.length + 1);
|
|
1000
|
-
if (tag === 'kv') {
|
|
1001
|
-
// 'kv' keys contain individual kvStore entries
|
|
1002
|
-
if (value == null) {
|
|
1003
|
-
// Note '==' rather than '===': any nullish value implies deletion
|
|
1004
|
-
kernelStorage.kvStore.delete(subKey);
|
|
1005
|
-
} else {
|
|
1006
|
-
kernelStorage.kvStore.set(subKey, value);
|
|
1007
|
-
}
|
|
1008
|
-
} else if (tag === 'bundle') {
|
|
1009
|
-
// 'bundle' keys contain bundle IDs
|
|
1010
|
-
if (value == null) {
|
|
1011
|
-
bundleArtifacts.delete(key);
|
|
1012
|
-
} else {
|
|
1013
|
-
bundleArtifacts.set(key, value);
|
|
1014
|
-
}
|
|
1015
|
-
} else if (tag === 'transcript' || tag === 'snapshot') {
|
|
1016
|
-
// 'transcript' and 'snapshot' keys contain artifact description info.
|
|
1017
|
-
assert(value); // make TypeScript shut up
|
|
1018
|
-
const { vatID, isCurrent, pos } = parseVatArtifactExportKey(key);
|
|
1019
|
-
if (isCurrent) {
|
|
1020
|
-
const vatInfo = vatArtifacts.get(vatID) || {};
|
|
1021
|
-
if (tag === 'snapshot') {
|
|
1022
|
-
// `export.snapshot.{vatID}.current` directly identifies the current snapshot artifact
|
|
1023
|
-
vatInfo.snapshotKey = value;
|
|
1024
|
-
} else if (tag === 'transcript') {
|
|
1025
|
-
// `export.transcript.${vatID}.current` contains a metadata record for the current
|
|
1026
|
-
// state of the current transcript span as of the time of export
|
|
1027
|
-
const metadata = JSON.parse(value);
|
|
1028
|
-
vatInfo.transcriptKey = artifactKey(tag, vatID, metadata.startPos);
|
|
1029
|
-
artifactMetadata.set(vatInfo.transcriptKey, metadata);
|
|
1030
|
-
}
|
|
1031
|
-
vatArtifacts.set(vatID, vatInfo);
|
|
1032
|
-
} else {
|
|
1033
|
-
artifactMetadata.set(artifactKey(tag, vatID, pos), JSON.parse(value));
|
|
1034
|
-
}
|
|
1035
|
-
} else {
|
|
1036
|
-
Fail`unknown artifact type tag ${q(tag)} on import`;
|
|
1037
|
-
}
|
|
1038
|
-
}
|
|
1039
|
-
|
|
1040
|
-
// At this point we should have acquired the entire KV store state, plus
|
|
1041
|
-
// sufficient metadata to identify the complete set of artifacts we'll need to
|
|
1042
|
-
// fetch along with the information required to validate each of them after
|
|
1043
|
-
// fetching.
|
|
1044
|
-
//
|
|
1045
|
-
// Depending on how the export was parameterized, the metadata may also include
|
|
1046
|
-
// information about historical artifacts that we might or might not actually
|
|
1047
|
-
// fetch depending on how this import was parameterized
|
|
1048
|
-
|
|
1049
|
-
// Fetch the set of current artifacts.
|
|
1050
|
-
|
|
1051
|
-
// Keep track of fetched artifacts in this set so we don't fetch them a second
|
|
1052
|
-
// time if we are trying for historical artifacts also.
|
|
1053
|
-
const fetchedArtifacts = new Set();
|
|
1054
|
-
|
|
1055
|
-
for await (const [vatID, vatInfo] of vatArtifacts.entries()) {
|
|
1056
|
-
// For each vat, we *must* have a transcript span. If this is not the very
|
|
1057
|
-
// first transcript span in the history of that vat, then we also must have
|
|
1058
|
-
// a snapshot for the state of the vat immediately prior to when the
|
|
1059
|
-
// transcript span begins.
|
|
1060
|
-
vatInfo.transcriptKey ||
|
|
1061
|
-
Fail`missing current transcript key for vat ${q(vatID)}`;
|
|
1062
|
-
const transcriptInfo = artifactMetadata.get(vatInfo.transcriptKey);
|
|
1063
|
-
transcriptInfo || Fail`missing transcript metadata for vat ${q(vatID)}`;
|
|
1064
|
-
let snapshotInfo;
|
|
1065
|
-
if (vatInfo.snapshotKey) {
|
|
1066
|
-
snapshotInfo = artifactMetadata.get(vatInfo.snapshotKey);
|
|
1067
|
-
snapshotInfo || Fail`missing snapshot metadata for vat ${q(vatID)}`;
|
|
1068
|
-
}
|
|
1069
|
-
if (!snapshotInfo) {
|
|
1070
|
-
transcriptInfo.startPos === 0 ||
|
|
1071
|
-
Fail`missing current snapshot for vat ${q(vatID)}`;
|
|
1072
|
-
} else {
|
|
1073
|
-
snapshotInfo.snapPos + 1 === transcriptInfo.startPos ||
|
|
1074
|
-
Fail`current transcript for vat ${q(vatID)} doesn't go with snapshot`;
|
|
1075
|
-
fetchedArtifacts.add(vatInfo.snapshotKey);
|
|
1076
|
-
}
|
|
1077
|
-
await (!snapshotInfo ||
|
|
1078
|
-
internal.snapStore.importSnapshot(
|
|
1079
|
-
vatInfo.snapshotKey,
|
|
1080
|
-
exporter,
|
|
1081
|
-
snapshotInfo,
|
|
1082
|
-
));
|
|
1083
|
-
|
|
1084
|
-
const transcriptArtifactName = `${vatInfo.transcriptKey}.${transcriptInfo.endPos}`;
|
|
1085
|
-
await internal.transcriptStore.importSpan(
|
|
1086
|
-
transcriptArtifactName,
|
|
1087
|
-
exporter,
|
|
1088
|
-
transcriptInfo,
|
|
1089
|
-
);
|
|
1090
|
-
fetchedArtifacts.add(transcriptArtifactName);
|
|
1091
|
-
}
|
|
1092
|
-
const bundleArtifactNames = Array.from(bundleArtifacts.keys()).sort();
|
|
1093
|
-
for await (const bundleArtifactName of bundleArtifactNames) {
|
|
1094
|
-
await internal.bundleStore.importBundle(
|
|
1095
|
-
bundleArtifactName,
|
|
1096
|
-
exporter,
|
|
1097
|
-
bundleArtifacts.get(bundleArtifactName),
|
|
1098
|
-
);
|
|
1099
|
-
}
|
|
1100
|
-
|
|
1101
|
-
if (!includeHistorical) {
|
|
1102
|
-
await exporter.close();
|
|
1103
|
-
return store;
|
|
1104
|
-
}
|
|
1105
|
-
|
|
1106
|
-
// If we're also importing historical artifacts, have the exporter enumerate
|
|
1107
|
-
// the complete set of artifacts it has and fetch all of them except for the
|
|
1108
|
-
// ones we've already fetched.
|
|
1109
|
-
for await (const artifactName of exporter.getArtifactNames()) {
|
|
1110
|
-
if (fetchedArtifacts.has(artifactName)) {
|
|
1111
|
-
continue;
|
|
1112
|
-
}
|
|
1113
|
-
let fetchedP;
|
|
1114
|
-
if (artifactName.startsWith('snapshot.')) {
|
|
1115
|
-
fetchedP = internal.snapStore.importSnapshot(
|
|
1116
|
-
artifactName,
|
|
1117
|
-
exporter,
|
|
1118
|
-
artifactMetadata.get(artifactName),
|
|
1119
|
-
);
|
|
1120
|
-
} else if (artifactName.startsWith('transcript.')) {
|
|
1121
|
-
// strip endPos off artifact name
|
|
1122
|
-
const metadataKey = artifactName.split('.').slice(0, 3).join('.');
|
|
1123
|
-
fetchedP = internal.transcriptStore.importSpan(
|
|
1124
|
-
artifactName,
|
|
1125
|
-
exporter,
|
|
1126
|
-
artifactMetadata.get(metadataKey),
|
|
1127
|
-
);
|
|
1128
|
-
} else if (artifactName.startsWith('bundle.')) {
|
|
1129
|
-
// already taken care of
|
|
1130
|
-
continue;
|
|
1131
|
-
} else {
|
|
1132
|
-
Fail`unknown artifact type: ${artifactName}`;
|
|
1133
|
-
}
|
|
1134
|
-
await fetchedP;
|
|
1135
|
-
}
|
|
1136
|
-
await exporter.close();
|
|
1137
|
-
return store;
|
|
1138
|
-
}
|
|
1139
|
-
|
|
1140
605
|
/**
|
|
1141
606
|
* Open a persistent swingset store. If there is no existing store at the given
|
|
1142
607
|
* `dirPath`, a new, empty store will be created.
|
|
@@ -1167,7 +632,7 @@ export function openSwingStore(dirPath, options = {}) {
|
|
|
1167
632
|
export function isSwingStore(dirPath) {
|
|
1168
633
|
typeof dirPath === 'string' || Fail`dirPath must be a string`;
|
|
1169
634
|
if (fs.existsSync(dirPath)) {
|
|
1170
|
-
const storeFile =
|
|
635
|
+
const storeFile = dbFileInDirectory(dirPath);
|
|
1171
636
|
if (fs.existsSync(storeFile)) {
|
|
1172
637
|
return true;
|
|
1173
638
|
}
|