@agoric/swing-store 0.9.1 → 0.9.2-calypso-dev-84eb287.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/swingStore.js CHANGED
@@ -1,74 +1,34 @@
1
1
  // @ts-check
2
2
  /* global Buffer */
3
- import fs from 'fs';
4
- import path from 'path';
5
- import { performance } from 'perf_hooks';
3
+ import * as fs from 'fs';
4
+ import * as path from 'path';
6
5
 
7
6
  import sqlite3 from 'better-sqlite3';
8
7
 
9
- import { assert, Fail, q } from '@agoric/assert';
10
- import { makeMeasureSeconds } from '@agoric/internal';
8
+ import { Fail, q } from '@agoric/assert';
11
9
 
10
+ import { dbFileInDirectory } from './util.js';
11
+ import { makeKVStore, getKeyType } from './kvStore.js';
12
12
  import { makeTranscriptStore } from './transcriptStore.js';
13
13
  import { makeSnapStore } from './snapStore.js';
14
14
  import { makeBundleStore } from './bundleStore.js';
15
15
  import { createSHA256 } from './hasher.js';
16
-
17
- export { makeSnapStore, makeBundleStore };
16
+ import { makeSnapStoreIO } from './snapStoreIO.js';
17
+ import { doRepairMetadata } from './repairMetadata.js';
18
18
 
19
19
  /**
20
- * This is a polyfill for the `buffer` function from Node's
21
- * 'stream/consumers' package, which unfortunately only exists in newer versions
22
- * of Node.
20
+ * @typedef { import('./kvStore.js').KVStore } KVStore
23
21
  *
24
- * @param {AsyncIterable<Buffer>} inStream
25
- */
26
- export const buffer = async inStream => {
27
- const chunks = [];
28
- for await (const chunk of inStream) {
29
- chunks.push(chunk);
30
- }
31
- return Buffer.concat(chunks);
32
- };
33
-
34
- export function makeSnapStoreIO() {
35
- return {
36
- measureSeconds: makeMeasureSeconds(performance.now),
37
- };
38
- }
39
-
40
- /**
41
- * @param {string} key
42
- */
43
- function getKeyType(key) {
44
- if (key.startsWith('local.')) {
45
- return 'local';
46
- } else if (key.startsWith('host.')) {
47
- return 'host';
48
- }
49
- return 'consensus';
50
- }
51
-
52
- /**
53
- * @typedef {{
54
- * has: (key: string) => boolean,
55
- * get: (key: string) => string | undefined,
56
- * getNextKey: (previousKey: string) => string | undefined,
57
- * set: (key: string, value: string, bypassHash?: boolean ) => void,
58
- * delete: (key: string) => void,
59
- * }} KVStore
22
+ * @typedef { import('./snapStore.js').SnapStore } SnapStore
23
+ * @typedef { import('./snapStore.js').SnapshotResult } SnapshotResult
60
24
  *
61
- * @typedef { import('./snapStore').SnapStore } SnapStore
62
- * @typedef { import('./snapStore').SnapStoreInternal } SnapStoreInternal
63
- * @typedef { import('./snapStore').SnapshotResult } SnapshotResult
25
+ * @typedef { import('./transcriptStore.js').TranscriptStore } TranscriptStore
26
+ * @typedef { import('./transcriptStore.js').TranscriptStoreDebug } TranscriptStoreDebug
64
27
  *
65
- * @typedef { import('./transcriptStore').TranscriptStore } TranscriptStore
66
- * @typedef { import('./transcriptStore').TranscriptStoreInternal } TranscriptStoreInternal
67
- * @typedef { import('./transcriptStore').TranscriptStoreDebug } TranscriptStoreDebug
28
+ * @typedef { import('./bundleStore.js').BundleStore } BundleStore
29
+ * @typedef { import('./bundleStore.js').BundleStoreDebug } BundleStoreDebug
68
30
  *
69
- * @typedef { import('./bundleStore').BundleStore } BundleStore
70
- * @typedef { import('./bundleStore').BundleStoreInternal } BundleStoreInternal
71
- * @typedef { import('./bundleStore').BundleStoreDebug } BundleStoreDebug
31
+ * @typedef { import('./exporter.js').KVPair } KVPair
72
32
  *
73
33
  * @typedef {{
74
34
  * kvStore: KVStore, // a key-value API object to load and store data on behalf of the kernel
@@ -89,6 +49,7 @@ function getKeyType(key) {
89
49
  * close: () => Promise<void>, // shutdown the store, abandoning any uncommitted changes
90
50
  * diskUsage?: () => number, // optional stats method
91
51
  * setExportCallback: (cb: (updates: KVPair[]) => void) => void, // Set a callback invoked by swingStore when new serializable data is available for export
52
+ * repairMetadata: (exporter: import('./exporter.js').SwingStoreExporter) => Promise<void>,
92
53
  * }} SwingStoreHostStorage
93
54
  */
94
55
 
@@ -105,184 +66,13 @@ function getKeyType(key) {
105
66
  * }} SwingStoreDebugTools
106
67
  *
107
68
  * @typedef {{
108
- * transcriptStore: TranscriptStoreInternal,
109
- * snapStore: SnapStoreInternal,
110
- * bundleStore: BundleStoreInternal,
111
- * }} SwingStoreInternal
112
- *
113
- * @typedef {{
114
69
  * kernelStorage: SwingStoreKernelStorage,
115
70
  * hostStorage: SwingStoreHostStorage,
116
71
  * debug: SwingStoreDebugTools,
117
- * internal: SwingStoreInternal,
72
+ * internal: import('./internal.js').SwingStoreInternal,
118
73
  * }} SwingStore
119
74
  */
120
75
 
121
- /**
122
- * @template T
123
- * @typedef { Iterable<T> | AsyncIterable<T> } AnyIterable<T>
124
- */
125
- /**
126
- * @template T
127
- * @typedef { IterableIterator<T> | AsyncIterableIterator<T> } AnyIterableIterator<T>
128
- */
129
-
130
- /**
131
- * @typedef {readonly [
132
- * key: string,
133
- * value?: string | null | undefined,
134
- * ]} KVPair
135
- *
136
- * @typedef {object} SwingStoreExporter
137
- *
138
- * Allows export of data from a swingStore as a fixed view onto the content as
139
- * of the most recent commit point at the time the exporter was created. The
140
- * exporter may be used while another SwingStore instance is active for the same
141
- * DB, possibly in another thread or process. It guarantees that regardless of
142
- * the concurrent activity of other swingStore instances, the data representing
143
- * the commit point will stay consistent and available.
144
- *
145
- * @property {() => AnyIterableIterator<KVPair>} getExportData
146
- *
147
- * Get a full copy of the first-stage export data (key-value pairs) from the
148
- * swingStore. This represents both the contents of the KVStore (excluding host
149
- * and local prefixes), as well as any data needed to validate all artifacts,
150
- * both current and historical. As such it represents the root of trust for the
151
- * application.
152
- *
153
- * Content of validation data (with supporting entries for indexing):
154
- * - kv.${key} = ${value} // ordinary kvStore data entry
155
- * - snapshot.${vatID}.${snapPos} = ${{ vatID, snapPos, hash });
156
- * - snapshot.${vatID}.current = `snapshot.${vatID}.${snapPos}`
157
- * - transcript.${vatID}.${startPos} = ${{ vatID, startPos, endPos, hash }}
158
- * - transcript.${vatID}.current = ${{ vatID, startPos, endPos, hash }}
159
- *
160
- * @property {() => AnyIterableIterator<string>} getArtifactNames
161
- *
162
- * Get a list of name of artifacts available from the swingStore. A name returned
163
- * by this method guarantees that a call to `getArtifact` on the same exporter
164
- * instance will succeed. Options control the filtering of the artifact names
165
- * yielded.
166
- *
167
- * Artifact names:
168
- * - transcript.${vatID}.${startPos}.${endPos}
169
- * - snapshot.${vatID}.${snapPos}
170
- *
171
- * @property {(name: string) => AnyIterableIterator<Uint8Array>} getArtifact
172
- *
173
- * Retrieve an artifact by name. May throw if the artifact is not available,
174
- * which can occur if the artifact is historical and wasn't been preserved.
175
- *
176
- * @property {() => Promise<void>} close
177
- *
178
- * Dispose of all resources held by this exporter. Any further operation on this
179
- * exporter or its outstanding iterators will fail.
180
- */
181
-
182
- /**
183
- * @param {string} dirPath
184
- * @param {string} exportMode
185
- * @returns {SwingStoreExporter}
186
- */
187
- export function makeSwingStoreExporter(dirPath, exportMode = 'current') {
188
- typeof dirPath === 'string' || Fail`dirPath must be a string`;
189
- exportMode === 'current' ||
190
- exportMode === 'archival' ||
191
- exportMode === 'debug' ||
192
- Fail`invalid exportMode ${q(exportMode)}`;
193
- const exportHistoricalSnapshots = exportMode === 'debug';
194
- const exportHistoricalTranscripts = exportMode !== 'current';
195
- const filePath = path.join(dirPath, 'swingstore.sqlite');
196
- const db = sqlite3(filePath);
197
-
198
- // Execute the data export in a (read) transaction, to ensure that we are
199
- // capturing the state of the database at a single point in time.
200
- const sqlBeginTransaction = db.prepare('BEGIN TRANSACTION');
201
- sqlBeginTransaction.run();
202
-
203
- // ensureTxn can be a dummy, we just started one
204
- const ensureTxn = () => {};
205
- const snapStore = makeSnapStore(db, ensureTxn, makeSnapStoreIO());
206
- const bundleStore = makeBundleStore(db, ensureTxn);
207
- const transcriptStore = makeTranscriptStore(db, ensureTxn, () => {});
208
-
209
- const sqlGetAllKVData = db.prepare(`
210
- SELECT key, value
211
- FROM kvStore
212
- ORDER BY key
213
- `);
214
-
215
- /**
216
- * @returns {AsyncIterableIterator<KVPair>}
217
- * @yields {KVPair}
218
- */
219
- async function* getExportData() {
220
- const kvPairs = sqlGetAllKVData.iterate();
221
- for (const kv of kvPairs) {
222
- if (getKeyType(kv.key) === 'consensus') {
223
- yield [`kv.${kv.key}`, kv.value];
224
- }
225
- }
226
- yield* snapStore.getExportRecords(true);
227
- yield* transcriptStore.getExportRecords(true);
228
- yield* bundleStore.getExportRecords();
229
- }
230
-
231
- /**
232
- * @returns {AsyncIterableIterator<string>}
233
- * @yields {string}
234
- */
235
- async function* getArtifactNames() {
236
- yield* snapStore.getArtifactNames(exportHistoricalSnapshots);
237
- yield* transcriptStore.getArtifactNames(exportHistoricalTranscripts);
238
- yield* bundleStore.getArtifactNames();
239
- }
240
-
241
- /**
242
- * @param {string} name
243
- * @returns {AsyncIterableIterator<Uint8Array>}
244
- */
245
- function getArtifact(name) {
246
- typeof name === 'string' || Fail`artifact name must be a string`;
247
- const [type] = name.split('.', 1);
248
-
249
- if (type === 'snapshot') {
250
- return snapStore.exportSnapshot(name, exportHistoricalSnapshots);
251
- } else if (type === 'transcript') {
252
- return transcriptStore.exportSpan(name, exportHistoricalTranscripts);
253
- } else if (type === 'bundle') {
254
- return bundleStore.exportBundle(name);
255
- } else {
256
- throw Fail`invalid artifact type ${q(type)}`;
257
- }
258
- }
259
-
260
- const sqlAbort = db.prepare('ROLLBACK');
261
-
262
- async function close() {
263
- // After all the data has been extracted, always abort the export
264
- // transaction to ensure that the export was read-only (i.e., that no bugs
265
- // inadvertantly modified the database).
266
- sqlAbort.run();
267
- db.close();
268
- }
269
-
270
- return harden({
271
- getExportData,
272
- getArtifactNames,
273
- getArtifact,
274
- close,
275
- });
276
- }
277
-
278
- /**
279
- * Function used to create a new swingStore from an object implementing the
280
- * exporter API. The exporter API may be provided by a swingStore instance, or
281
- * implemented by a host to restore data that was previously exported.
282
- *
283
- * @typedef {(exporter: SwingStoreExporter) => Promise<SwingStore>} ImportSwingStore
284
- */
285
-
286
76
  /**
287
77
  * A swing store holds the state of a swingset instance. This "store" is
288
78
  * actually several different stores of different types that travel as a flock
@@ -342,7 +132,7 @@ export function makeSwingStoreExporter(dirPath, exportMode = 'current') {
342
132
  *
343
133
  * @returns {SwingStore}
344
134
  */
345
- function makeSwingStore(dirPath, forceReset, options = {}) {
135
+ export function makeSwingStore(dirPath, forceReset, options = {}) {
346
136
  const { serialized } = options;
347
137
  if (serialized) {
348
138
  Buffer.isBuffer(serialized) || Fail`options.serialized must be Buffer`;
@@ -374,7 +164,7 @@ function makeSwingStore(dirPath, forceReset, options = {}) {
374
164
  }
375
165
  }
376
166
  fs.mkdirSync(dirPath, { recursive: true });
377
- filePath = path.join(dirPath, 'swingstore.sqlite');
167
+ filePath = dbFileInDirectory(dirPath);
378
168
  } else {
379
169
  filePath = ':memory:';
380
170
  }
@@ -413,9 +203,29 @@ function makeSwingStore(dirPath, forceReset, options = {}) {
413
203
  // mode that defers merge work for a later attempt rather than block any
414
204
  // potential readers or writers. See https://sqlite.org/wal.html for details.
415
205
 
206
+ // However we also allow opening the DB with journaling off, which is unsafe
207
+ // and doesn't support rollback, but avoids any overhead for large
208
+ // transactions like for during an import.
209
+
210
+ function setUnsafeFastMode(enabled) {
211
+ const journalMode = enabled ? 'off' : 'wal';
212
+ const synchronousMode = enabled ? 'normal' : 'full';
213
+ !db.inTransaction || Fail`must not be in a transaction`;
214
+
215
+ db.unsafeMode(!!enabled);
216
+ // The WAL mode is persistent so it's not possible to switch to a different
217
+ // mode for an existing DB.
218
+ const actualMode = db.pragma(`journal_mode=${journalMode}`, {
219
+ simple: true,
220
+ });
221
+ actualMode === journalMode ||
222
+ filePath === ':memory:' ||
223
+ Fail`Couldn't set swing-store DB to ${journalMode} mode (is ${actualMode})`;
224
+ db.pragma(`synchronous=${synchronousMode}`);
225
+ }
226
+
416
227
  // PRAGMAs have to happen outside a transaction
417
- db.exec(`PRAGMA journal_mode=WAL`);
418
- db.exec(`PRAGMA synchronous=FULL`);
228
+ setUnsafeFastMode(options.unsafeFastMode);
419
229
 
420
230
  // We use IMMEDIATE because the kernel is supposed to be the sole writer of
421
231
  // the DB, and if some other process is holding a write lock, we want to find
@@ -449,13 +259,6 @@ function makeSwingStore(dirPath, forceReset, options = {}) {
449
259
 
450
260
  // Perform all database initialization in a single transaction
451
261
  sqlBeginTransaction.run();
452
- db.exec(`
453
- CREATE TABLE IF NOT EXISTS kvStore (
454
- key TEXT,
455
- value TEXT,
456
- PRIMARY KEY (key)
457
- )
458
- `);
459
262
 
460
263
  db.exec(`
461
264
  CREATE TABLE IF NOT EXISTS pendingExports (
@@ -465,10 +268,32 @@ function makeSwingStore(dirPath, forceReset, options = {}) {
465
268
  )
466
269
  `);
467
270
 
271
+ let exportCallback;
272
+ function setExportCallback(cb) {
273
+ typeof cb === 'function' || Fail`callback must be a function`;
274
+ exportCallback = cb;
275
+ }
276
+ if (options.exportCallback) {
277
+ setExportCallback(options.exportCallback);
278
+ }
279
+
280
+ const sqlAddPendingExport = db.prepare(`
281
+ INSERT INTO pendingExports (key, value)
282
+ VALUES (?, ?)
283
+ ON CONFLICT DO UPDATE SET value = excluded.value
284
+ `);
285
+
286
+ function noteExport(key, value) {
287
+ if (exportCallback) {
288
+ sqlAddPendingExport.run(key, value);
289
+ }
290
+ }
291
+
292
+ const kvStore = makeKVStore(db, ensureTxn, trace);
293
+
468
294
  const { dumpTranscripts, ...transcriptStore } = makeTranscriptStore(
469
295
  db,
470
296
  ensureTxn,
471
- // eslint-disable-next-line no-use-before-define
472
297
  noteExport,
473
298
  {
474
299
  keepTranscripts,
@@ -478,7 +303,6 @@ function makeSwingStore(dirPath, forceReset, options = {}) {
478
303
  db,
479
304
  ensureTxn,
480
305
  makeSnapStoreIO(),
481
- // eslint-disable-next-line no-use-before-define
482
306
  noteExport,
483
307
  {
484
308
  keepSnapshots,
@@ -487,7 +311,6 @@ function makeSwingStore(dirPath, forceReset, options = {}) {
487
311
  const { dumpBundles, ...bundleStore } = makeBundleStore(
488
312
  db,
489
313
  ensureTxn,
490
- // eslint-disable-next-line no-use-before-define
491
314
  noteExport,
492
315
  );
493
316
 
@@ -496,20 +319,11 @@ function makeSwingStore(dirPath, forceReset, options = {}) {
496
319
  // At this point, all database initialization should be complete, so commit now.
497
320
  sqlCommit.run();
498
321
 
499
- let exportCallback;
500
- function setExportCallback(cb) {
501
- typeof cb === 'function' || Fail`callback must be a function`;
502
- exportCallback = cb;
503
- }
504
- if (options.exportCallback) {
505
- setExportCallback(options.exportCallback);
506
- }
507
-
508
322
  let inCrank = false;
509
323
 
510
324
  function diskUsage() {
511
325
  if (dirPath) {
512
- const dataFilePath = `${dirPath}/swingstore.sqlite`;
326
+ const dataFilePath = dbFileInDirectory(dirPath);
513
327
  const stat = fs.statSync(dataFilePath);
514
328
  return stat.size;
515
329
  } else {
@@ -517,154 +331,13 @@ function makeSwingStore(dirPath, forceReset, options = {}) {
517
331
  }
518
332
  }
519
333
 
520
- const sqlKVGet = db.prepare(`
521
- SELECT value
522
- FROM kvStore
523
- WHERE key = ?
524
- `);
525
- sqlKVGet.pluck(true);
526
-
527
- /**
528
- * Obtain the value stored for a given key.
529
- *
530
- * @param {string} key The key whose value is sought.
531
- *
532
- * @returns {string | undefined} the (string) value for the given key, or
533
- * undefined if there is no such value.
534
- *
535
- * @throws if key is not a string.
536
- */
537
- function get(key) {
538
- typeof key === 'string' || Fail`key must be a string`;
539
- return sqlKVGet.get(key);
540
- }
541
-
542
- const sqlKVGetNextKey = db.prepare(`
543
- SELECT key
544
- FROM kvStore
545
- WHERE key > ?
546
- LIMIT 1
547
- `);
548
- sqlKVGetNextKey.pluck(true);
549
-
550
- /**
551
- * getNextKey enables callers to iterate over all keys within a
552
- * given range. To build an iterator of all keys from start
553
- * (inclusive) to end (exclusive), do:
554
- *
555
- * function* iterate(start, end) {
556
- * if (kvStore.has(start)) {
557
- * yield start;
558
- * }
559
- * let prev = start;
560
- * while (true) {
561
- * let next = kvStore.getNextKey(prev);
562
- * if (!next || next >= end) {
563
- * break;
564
- * }
565
- * yield next;
566
- * prev = next;
567
- * }
568
- * }
569
- *
570
- * @param {string} previousKey The key returned will always be later than this one.
571
- *
572
- * @returns {string | undefined} a key string, or undefined if we reach the end of the store
573
- *
574
- * @throws if previousKey is not a string
575
- */
576
-
577
- function getNextKey(previousKey) {
578
- typeof previousKey === 'string' || Fail`previousKey must be a string`;
579
- return sqlKVGetNextKey.get(previousKey);
580
- }
581
-
582
- /**
583
- * Test if the state contains a value for a given key.
584
- *
585
- * @param {string} key The key that is of interest.
586
- *
587
- * @returns {boolean} true if a value is stored for the key, false if not.
588
- *
589
- * @throws if key is not a string.
590
- */
591
- function has(key) {
592
- typeof key === 'string' || Fail`key must be a string`;
593
- return get(key) !== undefined;
594
- }
595
-
596
- const sqlKVSet = db.prepare(`
597
- INSERT INTO kvStore (key, value)
598
- VALUES (?, ?)
599
- ON CONFLICT DO UPDATE SET value = excluded.value
600
- `);
601
-
602
- /**
603
- * Store a value for a given key. The value will replace any prior value if
604
- * there was one.
605
- *
606
- * @param {string} key The key whose value is being set.
607
- * @param {string} value The value to set the key to.
608
- *
609
- * @throws if either parameter is not a string.
610
- */
611
- function set(key, value) {
612
- typeof key === 'string' || Fail`key must be a string`;
613
- typeof value === 'string' || Fail`value must be a string`;
614
- // synchronous read after write within a transaction is safe
615
- // The transaction's overall success will be awaited during commit
616
- ensureTxn();
617
- sqlKVSet.run(key, value);
618
- trace('set', key, value);
619
- }
620
-
621
- const sqlKVDel = db.prepare(`
622
- DELETE FROM kvStore
623
- WHERE key = ?
624
- `);
625
-
626
- /**
627
- * Remove any stored value for a given key. It is permissible for there to
628
- * be no existing stored value for the key.
629
- *
630
- * @param {string} key The key whose value is to be deleted
631
- *
632
- * @throws if key is not a string.
633
- */
634
- function del(key) {
635
- typeof key === 'string' || Fail`key must be a string`;
636
- ensureTxn();
637
- sqlKVDel.run(key);
638
- trace('del', key);
639
- }
640
-
641
- const kvStore = {
642
- has,
643
- get,
644
- getNextKey,
645
- set,
646
- delete: del,
647
- };
648
-
649
- const sqlAddPendingExport = db.prepare(`
650
- INSERT INTO pendingExports (key, value)
651
- VALUES (?, ?)
652
- ON CONFLICT DO UPDATE SET value = excluded.value
653
- `);
654
-
655
- function noteExport(key, value) {
656
- if (exportCallback) {
657
- sqlAddPendingExport.run(key, value);
658
- }
659
- }
660
-
661
334
  const kernelKVStore = {
662
335
  ...kvStore,
663
336
  set(key, value) {
664
337
  typeof key === 'string' || Fail`key must be a string`;
665
338
  const keyType = getKeyType(key);
666
339
  keyType !== 'host' || Fail`kernelKVStore refuses host keys`;
667
- set(key, value);
340
+ kvStore.set(key, value);
668
341
  if (keyType === 'consensus') {
669
342
  noteExport(`kv.${key}`, value);
670
343
  crankhasher.add('add');
@@ -679,7 +352,7 @@ function makeSwingStore(dirPath, forceReset, options = {}) {
679
352
  typeof key === 'string' || Fail`key must be a string`;
680
353
  const keyType = getKeyType(key);
681
354
  keyType !== 'host' || Fail`kernelKVStore refuses host keys`;
682
- del(key);
355
+ kvStore.delete(key);
683
356
  if (keyType === 'consensus') {
684
357
  noteExport(`kv.${key}`, undefined);
685
358
  crankhasher.add('delete');
@@ -695,12 +368,12 @@ function makeSwingStore(dirPath, forceReset, options = {}) {
695
368
  set(key, value) {
696
369
  const keyType = getKeyType(key);
697
370
  keyType === 'host' || Fail`hostKVStore requires host keys`;
698
- set(key, value);
371
+ kvStore.set(key, value);
699
372
  },
700
373
  delete(key) {
701
374
  const keyType = getKeyType(key);
702
375
  keyType === 'host' || Fail`hostKVStore requires host keys`;
703
- del(key);
376
+ kvStore.delete(key);
704
377
  },
705
378
  };
706
379
 
@@ -717,6 +390,10 @@ function makeSwingStore(dirPath, forceReset, options = {}) {
717
390
  inCrank || Fail`establishCrankSavepoint outside of crank`;
718
391
  const savepointOrdinal = savepoints.length;
719
392
  savepoints.push(savepoint);
393
+ // We must be in a transaction when creating the savepoint or releasing it
394
+ // later will cause an autocommit.
395
+ // See https://github.com/Agoric/agoric-sdk/issues/8423
396
+ ensureTxn();
720
397
  const sql = db.prepare(`SAVEPOINT t${savepointOrdinal}`);
721
398
  sql.run();
722
399
  }
@@ -740,7 +417,7 @@ function makeSwingStore(dirPath, forceReset, options = {}) {
740
417
  resetCrankhash();
741
418
 
742
419
  // Get the old activityhash
743
- let oldActivityhash = get('activityhash');
420
+ let oldActivityhash = kvStore.get('activityhash');
744
421
  if (oldActivityhash === undefined) {
745
422
  oldActivityhash = '';
746
423
  }
@@ -756,7 +433,7 @@ function makeSwingStore(dirPath, forceReset, options = {}) {
756
433
 
757
434
  // Store the new activityhash
758
435
  const activityhash = hasher.finish();
759
- set('activityhash', activityhash);
436
+ kvStore.set('activityhash', activityhash);
760
437
  // Need to explicitly call noteExport here because activityhash is written
761
438
  // directly to the low-level store to avoid recursive hashing, which
762
439
  // bypasses the normal notification mechanism
@@ -766,7 +443,7 @@ function makeSwingStore(dirPath, forceReset, options = {}) {
766
443
  }
767
444
 
768
445
  function getActivityhash() {
769
- return get('activityhash') || '';
446
+ return kvStore.get('activityhash') || '';
770
447
  }
771
448
 
772
449
  const sqlExportsGet = db.prepare(`
@@ -823,6 +500,17 @@ function makeSwingStore(dirPath, forceReset, options = {}) {
823
500
  stopTrace();
824
501
  }
825
502
 
503
+ /** @type {import('./internal.js').SwingStoreInternal} */
504
+ const internal = harden({
505
+ snapStore,
506
+ transcriptStore,
507
+ bundleStore,
508
+ });
509
+
510
+ async function repairMetadata(exporter) {
511
+ return doRepairMetadata(internal, exporter);
512
+ }
513
+
826
514
  /**
827
515
  * Return a Buffer with the entire DB state, useful for cloning a
828
516
  * small swingstore in unit tests.
@@ -855,6 +543,10 @@ function makeSwingStore(dirPath, forceReset, options = {}) {
855
543
  });
856
544
  }
857
545
 
546
+ function getDatabase() {
547
+ return db;
548
+ }
549
+
858
550
  const transcriptStorePublic = {
859
551
  initTranscript: transcriptStore.initTranscript,
860
552
  rolloverSpan: transcriptStore.rolloverSpan,
@@ -894,6 +586,7 @@ function makeSwingStore(dirPath, forceReset, options = {}) {
894
586
  getActivityhash,
895
587
  };
896
588
  const hostStorage = {
589
+ repairMetadata,
897
590
  kvStore: hostKVStore,
898
591
  commit,
899
592
  close,
@@ -903,11 +596,7 @@ function makeSwingStore(dirPath, forceReset, options = {}) {
903
596
  const debug = {
904
597
  serialize,
905
598
  dump,
906
- };
907
- const internal = {
908
- snapStore,
909
- transcriptStore,
910
- bundleStore,
599
+ getDatabase,
911
600
  };
912
601
 
913
602
  return harden({
@@ -941,198 +630,6 @@ export function initSwingStore(dirPath = null, options = {}) {
941
630
  return makeSwingStore(dirPath, true, options);
942
631
  }
943
632
 
944
- function parseVatArtifactExportKey(key) {
945
- const parts = key.split('.');
946
- const [_type, vatID, rawPos] = parts;
947
- // prettier-ignore
948
- parts.length === 3 ||
949
- Fail`expected artifact name of the form '{type}.{vatID}.{pos}', saw ${q(key)}`;
950
- const isCurrent = rawPos === 'current';
951
- let pos;
952
- if (isCurrent) {
953
- pos = -1;
954
- } else {
955
- pos = Number(rawPos);
956
- }
957
-
958
- return { vatID, isCurrent, pos };
959
- }
960
-
961
- function artifactKey(type, vatID, pos) {
962
- return `${type}.${vatID}.${pos}`;
963
- }
964
-
965
- /**
966
- * @param {SwingStoreExporter} exporter
967
- * @param {string | null} [dirPath]
968
- * @param {object} options
969
- * @returns {Promise<SwingStore>}
970
- */
971
- export async function importSwingStore(exporter, dirPath = null, options = {}) {
972
- if (dirPath) {
973
- typeof dirPath === 'string' || Fail`dirPath must be a string`;
974
- }
975
- const { includeHistorical = false } = options;
976
- const store = makeSwingStore(dirPath, true, options);
977
- const { kernelStorage, internal } = store;
978
-
979
- // Artifact metadata, keyed as `${type}.${vatID}.${pos}`
980
- //
981
- // Note that this key is almost but not quite the artifact name, since the
982
- // names of transcript span artifacts also include the endPos, but the endPos
983
- // value is in flux until the span is complete.
984
- const artifactMetadata = new Map();
985
-
986
- // Each vat requires a transcript span and (usually) a snapshot. This table
987
- // tracks which of these we've seen, keyed by vatID.
988
- // vatID -> { snapshotKey: metadataKey, transcriptKey: metatdataKey }
989
- const vatArtifacts = new Map();
990
- const bundleArtifacts = new Map();
991
-
992
- for await (const [key, value] of exporter.getExportData()) {
993
- const [tag] = key.split('.', 1);
994
- const subKey = key.substring(tag.length + 1);
995
- if (tag === 'kv') {
996
- // 'kv' keys contain individual kvStore entries
997
- if (value == null) {
998
- // Note '==' rather than '===': any nullish value implies deletion
999
- kernelStorage.kvStore.delete(subKey);
1000
- } else {
1001
- kernelStorage.kvStore.set(subKey, value);
1002
- }
1003
- } else if (tag === 'bundle') {
1004
- // 'bundle' keys contain bundle IDs
1005
- if (value == null) {
1006
- bundleArtifacts.delete(key);
1007
- } else {
1008
- bundleArtifacts.set(key, value);
1009
- }
1010
- } else if (tag === 'transcript' || tag === 'snapshot') {
1011
- // 'transcript' and 'snapshot' keys contain artifact description info.
1012
- assert(value); // make TypeScript shut up
1013
- const { vatID, isCurrent, pos } = parseVatArtifactExportKey(key);
1014
- if (isCurrent) {
1015
- const vatInfo = vatArtifacts.get(vatID) || {};
1016
- if (tag === 'snapshot') {
1017
- // `export.snapshot.{vatID}.current` directly identifies the current snapshot artifact
1018
- vatInfo.snapshotKey = value;
1019
- } else if (tag === 'transcript') {
1020
- // `export.transcript.${vatID}.current` contains a metadata record for the current
1021
- // state of the current transcript span as of the time of export
1022
- const metadata = JSON.parse(value);
1023
- vatInfo.transcriptKey = artifactKey(tag, vatID, metadata.startPos);
1024
- artifactMetadata.set(vatInfo.transcriptKey, metadata);
1025
- }
1026
- vatArtifacts.set(vatID, vatInfo);
1027
- } else {
1028
- artifactMetadata.set(artifactKey(tag, vatID, pos), JSON.parse(value));
1029
- }
1030
- } else {
1031
- Fail`unknown artifact type tag ${q(tag)} on import`;
1032
- }
1033
- }
1034
-
1035
- // At this point we should have acquired the entire KV store state, plus
1036
- // sufficient metadata to identify the complete set of artifacts we'll need to
1037
- // fetch along with the information required to validate each of them after
1038
- // fetching.
1039
- //
1040
- // Depending on how the export was parameterized, the metadata may also include
1041
- // information about historical artifacts that we might or might not actually
1042
- // fetch depending on how this import was parameterized
1043
-
1044
- // Fetch the set of current artifacts.
1045
-
1046
- // Keep track of fetched artifacts in this set so we don't fetch them a second
1047
- // time if we are trying for historical artifacts also.
1048
- const fetchedArtifacts = new Set();
1049
-
1050
- for await (const [vatID, vatInfo] of vatArtifacts.entries()) {
1051
- // For each vat, we *must* have a transcript span. If this is not the very
1052
- // first transcript span in the history of that vat, then we also must have
1053
- // a snapshot for the state of the vat immediately prior to when the
1054
- // transcript span begins.
1055
- vatInfo.transcriptKey ||
1056
- Fail`missing current transcript key for vat ${q(vatID)}`;
1057
- const transcriptInfo = artifactMetadata.get(vatInfo.transcriptKey);
1058
- transcriptInfo || Fail`missing transcript metadata for vat ${q(vatID)}`;
1059
- let snapshotInfo;
1060
- if (vatInfo.snapshotKey) {
1061
- snapshotInfo = artifactMetadata.get(vatInfo.snapshotKey);
1062
- snapshotInfo || Fail`missing snapshot metadata for vat ${q(vatID)}`;
1063
- }
1064
- if (!snapshotInfo) {
1065
- transcriptInfo.startPos === 0 ||
1066
- Fail`missing current snapshot for vat ${q(vatID)}`;
1067
- } else {
1068
- snapshotInfo.snapPos + 1 === transcriptInfo.startPos ||
1069
- Fail`current transcript for vat ${q(vatID)} doesn't go with snapshot`;
1070
- fetchedArtifacts.add(vatInfo.snapshotKey);
1071
- }
1072
- await (!snapshotInfo ||
1073
- internal.snapStore.importSnapshot(
1074
- vatInfo.snapshotKey,
1075
- exporter,
1076
- snapshotInfo,
1077
- ));
1078
-
1079
- const transcriptArtifactName = `${vatInfo.transcriptKey}.${transcriptInfo.endPos}`;
1080
- await internal.transcriptStore.importSpan(
1081
- transcriptArtifactName,
1082
- exporter,
1083
- transcriptInfo,
1084
- );
1085
- fetchedArtifacts.add(transcriptArtifactName);
1086
- }
1087
- const bundleArtifactNames = Array.from(bundleArtifacts.keys()).sort();
1088
- for await (const bundleArtifactName of bundleArtifactNames) {
1089
- await internal.bundleStore.importBundle(
1090
- bundleArtifactName,
1091
- exporter,
1092
- bundleArtifacts.get(bundleArtifactName),
1093
- );
1094
- }
1095
-
1096
- if (!includeHistorical) {
1097
- // eslint-disable-next-line @jessie.js/no-nested-await
1098
- await exporter.close();
1099
- return store;
1100
- }
1101
-
1102
- // If we're also importing historical artifacts, have the exporter enumerate
1103
- // the complete set of artifacts it has and fetch all of them except for the
1104
- // ones we've already fetched.
1105
- for await (const artifactName of exporter.getArtifactNames()) {
1106
- if (fetchedArtifacts.has(artifactName)) {
1107
- continue;
1108
- }
1109
- let fetchedP;
1110
- if (artifactName.startsWith('snapshot.')) {
1111
- fetchedP = internal.snapStore.importSnapshot(
1112
- artifactName,
1113
- exporter,
1114
- artifactMetadata.get(artifactName),
1115
- );
1116
- } else if (artifactName.startsWith('transcript.')) {
1117
- // strip endPos off artifact name
1118
- const metadataKey = artifactName.split('.').slice(0, 3).join('.');
1119
- fetchedP = internal.transcriptStore.importSpan(
1120
- artifactName,
1121
- exporter,
1122
- artifactMetadata.get(metadataKey),
1123
- );
1124
- } else if (artifactName.startsWith('bundle.')) {
1125
- // already taken care of
1126
- continue;
1127
- } else {
1128
- Fail`unknown artifact type: ${artifactName}`;
1129
- }
1130
- await fetchedP;
1131
- }
1132
- await exporter.close();
1133
- return store;
1134
- }
1135
-
1136
633
  /**
1137
634
  * Open a persistent swingset store. If there is no existing store at the given
1138
635
  * `dirPath`, a new, empty store will be created.
@@ -1163,7 +660,7 @@ export function openSwingStore(dirPath, options = {}) {
1163
660
  export function isSwingStore(dirPath) {
1164
661
  typeof dirPath === 'string' || Fail`dirPath must be a string`;
1165
662
  if (fs.existsSync(dirPath)) {
1166
- const storeFile = path.resolve(dirPath, 'swingstore.sqlite');
663
+ const storeFile = dbFileInDirectory(dirPath);
1167
664
  if (fs.existsSync(storeFile)) {
1168
665
  return true;
1169
666
  }