@agoric/swing-store 0.9.2-dev-7bfadd3.0 → 0.9.2-dev-f2445bc.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/docs/data-export.md +20 -11
- package/package.json +4 -4
- package/src/assertComplete.js +5 -6
- package/src/bundleStore.js +4 -3
- package/src/exporter.js +18 -12
- package/src/importer.js +17 -13
- package/src/internal.js +10 -2
- package/src/repairMetadata.js +3 -1
- package/src/snapStore.js +11 -10
- package/src/transcriptStore.js +122 -57
- package/src/types.d.ts +10 -9
- package/test/test-export.js +78 -43
- package/test/test-exportImport.js +57 -16
- package/test/test-import.js +76 -47
- package/test/test-repair-metadata.js +10 -10
package/src/transcriptStore.js
CHANGED
|
@@ -11,6 +11,8 @@ import { createSHA256 } from './hasher.js';
|
|
|
11
11
|
*/
|
|
12
12
|
|
|
13
13
|
/**
|
|
14
|
+
* @typedef { import('./internal.js').ArtifactMode } ArtifactMode
|
|
15
|
+
*
|
|
14
16
|
* @typedef {{
|
|
15
17
|
* initTranscript: (vatID: string) => void,
|
|
16
18
|
* rolloverSpan: (vatID: string) => number,
|
|
@@ -24,10 +26,10 @@ import { createSHA256 } from './hasher.js';
|
|
|
24
26
|
* @typedef {{
|
|
25
27
|
* exportSpan: (name: string) => AsyncIterableIterator<Uint8Array>
|
|
26
28
|
* getExportRecords: (includeHistorical: boolean) => IterableIterator<readonly [key: string, value: string]>,
|
|
27
|
-
* getArtifactNames: (
|
|
29
|
+
* getArtifactNames: (artifactMode: ArtifactMode) => AsyncIterableIterator<string>,
|
|
28
30
|
* importTranscriptSpanRecord: (key: string, value: string) => void,
|
|
29
|
-
* populateTranscriptSpan: (name: string, makeChunkIterator: () => AnyIterableIterator<Uint8Array>, options: {
|
|
30
|
-
* assertComplete: (
|
|
31
|
+
* populateTranscriptSpan: (name: string, makeChunkIterator: () => AnyIterableIterator<Uint8Array>, options: { artifactMode: ArtifactMode }) => Promise<void>,
|
|
32
|
+
* assertComplete: (checkMode: Omit<ArtifactMode, 'debug'>) => void,
|
|
31
33
|
* repairTranscriptSpanRecord: (key: string, value: string) => void,
|
|
32
34
|
* readFullVatTranscript: (vatID: string) => Iterable<{position: number, item: string}>
|
|
33
35
|
* }} TranscriptStoreInternal
|
|
@@ -331,6 +333,13 @@ export function makeTranscriptStore(
|
|
|
331
333
|
ORDER BY vatID, startPos
|
|
332
334
|
`);
|
|
333
335
|
|
|
336
|
+
const sqlGetIncarnationSpanMetadata = db.prepare(`
|
|
337
|
+
SELECT vatID, startPos, endPos, hash, isCurrent, incarnation
|
|
338
|
+
FROM transcriptSpans
|
|
339
|
+
WHERE vatID=? AND incarnation=?
|
|
340
|
+
ORDER BY vatID, startPos
|
|
341
|
+
`);
|
|
342
|
+
|
|
334
343
|
const sqlGetCurrentSpanMetadata = db.prepare(`
|
|
335
344
|
SELECT vatID, startPos, endPos, hash, isCurrent, incarnation
|
|
336
345
|
FROM transcriptSpans
|
|
@@ -338,6 +347,11 @@ export function makeTranscriptStore(
|
|
|
338
347
|
ORDER BY vatID, startPos
|
|
339
348
|
`);
|
|
340
349
|
|
|
350
|
+
function dbRecToExportRec(dbRec) {
|
|
351
|
+
const { vatID, startPos, endPos, hash, isCurrent, incarnation } = dbRec;
|
|
352
|
+
return spanRec(vatID, startPos, endPos, hash, isCurrent, incarnation);
|
|
353
|
+
}
|
|
354
|
+
|
|
341
355
|
/**
|
|
342
356
|
* Obtain artifact metadata records for spans contained in this store.
|
|
343
357
|
*
|
|
@@ -360,63 +374,77 @@ export function makeTranscriptStore(
|
|
|
360
374
|
* replay will never be required or because such replay would be prohibitively
|
|
361
375
|
* expensive regardless of need and therefor other repair strategies employed.
|
|
362
376
|
*
|
|
377
|
+
* The only code path which could use 'false' would be `swingstore.dump()`,
|
|
378
|
+
* which takes the same flag.
|
|
379
|
+
*
|
|
363
380
|
* @yields {readonly [key: string, value: string]}
|
|
364
381
|
* @returns {IterableIterator<readonly [key: string, value: string]>}
|
|
365
382
|
* An iterator over pairs of [spanMetadataKey, rec], where `rec` is a
|
|
366
383
|
* JSON-encoded metadata record for the span named by `spanMetadataKey`.
|
|
367
384
|
*/
|
|
368
385
|
function* getExportRecords(includeHistorical = true) {
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
const
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
endPos,
|
|
378
|
-
hash,
|
|
379
|
-
isCurrent,
|
|
380
|
-
incarnation,
|
|
381
|
-
);
|
|
382
|
-
yield [spanMetadataKey(rec), JSON.stringify(exportRec)];
|
|
386
|
+
if (includeHistorical) {
|
|
387
|
+
for (const rec of sqlGetAllSpanMetadata.iterate()) {
|
|
388
|
+
yield [spanMetadataKey(rec), JSON.stringify(dbRecToExportRec(rec))];
|
|
389
|
+
}
|
|
390
|
+
} else {
|
|
391
|
+
for (const rec of sqlGetCurrentSpanMetadata.iterate()) {
|
|
392
|
+
yield [spanMetadataKey(rec), JSON.stringify(dbRecToExportRec(rec))];
|
|
393
|
+
}
|
|
383
394
|
}
|
|
384
395
|
}
|
|
385
396
|
|
|
386
|
-
|
|
387
|
-
// speaking this query doesn't need to filter on 'incarnation = ?',
|
|
388
|
-
// but this will catch problems like items with incorrect or missing
|
|
389
|
-
// incarnation values
|
|
390
|
-
|
|
391
|
-
const sqlCountPopulatedSpanItems = db.prepare(`
|
|
397
|
+
const sqlCountSpanItems = db.prepare(`
|
|
392
398
|
SELECT COUNT(*) FROM transcriptItems
|
|
393
|
-
WHERE vatID = ? AND
|
|
399
|
+
WHERE vatID = ? AND position >= ? AND position < ?
|
|
394
400
|
`);
|
|
395
|
-
|
|
401
|
+
sqlCountSpanItems.pluck();
|
|
396
402
|
|
|
397
403
|
/**
|
|
398
404
|
* Obtain artifact names for spans contained in this store.
|
|
399
405
|
*
|
|
400
|
-
* @param {
|
|
401
|
-
*
|
|
402
|
-
* the current
|
|
403
|
-
*
|
|
406
|
+
* @param {ArtifactMode} artifactMode Control which artifacts should be exported.
|
|
407
|
+
* At 'operational', only include current spans. At 'replay',
|
|
408
|
+
* include all spans of the current incarnation for each vat. At
|
|
409
|
+
* 'archival' and 'debug', include all spans.
|
|
404
410
|
* @yields {string}
|
|
405
411
|
* @returns {AsyncIterableIterator<string>} An iterator over the names of all the artifacts requested
|
|
406
412
|
*/
|
|
407
|
-
async function* getArtifactNames(
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
const
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
413
|
+
async function* getArtifactNames(artifactMode) {
|
|
414
|
+
// for all non-'debug' modes, the exporter asserts that all
|
|
415
|
+
// requested items are present (i.e. the artifacts will be
|
|
416
|
+
// complete), so we don't need to check that ourselves
|
|
417
|
+
if (artifactMode === 'operational') {
|
|
418
|
+
for (const rec of sqlGetCurrentSpanMetadata.iterate()) {
|
|
419
|
+
yield spanArtifactName(rec);
|
|
420
|
+
}
|
|
421
|
+
} else if (artifactMode === 'replay') {
|
|
422
|
+
for (const curRec of sqlGetCurrentSpanMetadata.iterate()) {
|
|
423
|
+
const { vatID, incarnation } = curRec;
|
|
424
|
+
for (const rec of sqlGetIncarnationSpanMetadata.iterate(
|
|
425
|
+
vatID,
|
|
426
|
+
incarnation,
|
|
427
|
+
)) {
|
|
428
|
+
yield spanArtifactName(rec);
|
|
429
|
+
}
|
|
430
|
+
}
|
|
431
|
+
} else if (artifactMode === 'archival') {
|
|
432
|
+
// everything
|
|
433
|
+
for (const rec of sqlGetAllSpanMetadata.iterate()) {
|
|
434
|
+
yield spanArtifactName(rec);
|
|
435
|
+
}
|
|
436
|
+
} else if (artifactMode === 'debug') {
|
|
437
|
+
// everything that is a complete span
|
|
438
|
+
for (const rec of sqlGetAllSpanMetadata.iterate()) {
|
|
439
|
+
const { vatID, startPos, endPos } = rec;
|
|
440
|
+
const count = sqlCountSpanItems.get(vatID, startPos, endPos);
|
|
441
|
+
if (count !== endPos - startPos) {
|
|
442
|
+
// skip incomplete spans, because the exporter did not
|
|
443
|
+
// already do a completeness check in 'debug' mode
|
|
444
|
+
continue;
|
|
445
|
+
}
|
|
446
|
+
yield spanArtifactName(rec);
|
|
417
447
|
}
|
|
418
|
-
|
|
419
|
-
yield spanArtifactName(rec);
|
|
420
448
|
}
|
|
421
449
|
}
|
|
422
450
|
|
|
@@ -590,19 +618,27 @@ export function makeTranscriptStore(
|
|
|
590
618
|
WHERE vatID = ? AND startPos = ?
|
|
591
619
|
`);
|
|
592
620
|
|
|
621
|
+
const sqlGetStartOfIncarnation = db.prepare(`
|
|
622
|
+
SELECT startPos
|
|
623
|
+
FROM transcriptSpans
|
|
624
|
+
WHERE vatID=? AND incarnation=?
|
|
625
|
+
ORDER BY startPos ASC LIMIT 1
|
|
626
|
+
`);
|
|
627
|
+
sqlGetStartOfIncarnation.pluck();
|
|
628
|
+
|
|
593
629
|
/**
|
|
594
630
|
* Import a transcript span from another store.
|
|
595
631
|
*
|
|
596
632
|
* @param {string} name Artifact Name of the transcript span
|
|
597
633
|
* @param {() => AnyIterableIterator<Uint8Array>} makeChunkIterator get an iterator of transcript byte chunks
|
|
598
634
|
* @param {object} options
|
|
599
|
-
* @param {
|
|
635
|
+
* @param {ArtifactMode} options.artifactMode
|
|
600
636
|
*
|
|
601
637
|
* @returns {Promise<void>}
|
|
602
638
|
*/
|
|
603
639
|
async function populateTranscriptSpan(name, makeChunkIterator, options) {
|
|
604
640
|
ensureTxn();
|
|
605
|
-
const {
|
|
641
|
+
const { artifactMode } = options;
|
|
606
642
|
const parts = name.split('.');
|
|
607
643
|
const [type, vatID, rawStartPos, rawEndPos] = parts;
|
|
608
644
|
// prettier-ignore
|
|
@@ -614,11 +650,23 @@ export function makeTranscriptStore(
|
|
|
614
650
|
const metadata =
|
|
615
651
|
sqlGetSpanMetadataFor.get(vatID, startPos) ||
|
|
616
652
|
Fail`no metadata for transcript span ${name}`;
|
|
617
|
-
if (!metadata.isCurrent && !includeHistorical) {
|
|
618
|
-
return; // ignore old spans
|
|
619
|
-
}
|
|
620
653
|
assert.equal(metadata.endPos, endPos);
|
|
621
654
|
|
|
655
|
+
if (artifactMode === 'operational') {
|
|
656
|
+
if (!metadata.isCurrent) {
|
|
657
|
+
return; // ignore old spans
|
|
658
|
+
}
|
|
659
|
+
}
|
|
660
|
+
if (artifactMode === 'replay') {
|
|
661
|
+
// ignore spans that aren't for the current incarnation
|
|
662
|
+
const { incarnation } = sqlGetCurrentSpanBounds.get(vatID);
|
|
663
|
+
const incStart = sqlGetStartOfIncarnation.get(vatID, incarnation);
|
|
664
|
+
if (startPos < incStart) {
|
|
665
|
+
return;
|
|
666
|
+
}
|
|
667
|
+
}
|
|
668
|
+
// 'archival' and 'debug' modes accept all spans
|
|
669
|
+
|
|
622
670
|
const artifactChunks = await makeChunkIterator();
|
|
623
671
|
const inStream = Readable.from(artifactChunks);
|
|
624
672
|
const lineTransform = new BufferLineTransform();
|
|
@@ -679,19 +727,36 @@ export function makeTranscriptStore(
|
|
|
679
727
|
}
|
|
680
728
|
}
|
|
681
729
|
|
|
682
|
-
function assertComplete(
|
|
683
|
-
assert
|
|
684
|
-
// every 'isCurrent' transcript span must have all items
|
|
730
|
+
function assertComplete(checkMode) {
|
|
731
|
+
assert(checkMode !== 'debug', checkMode);
|
|
685
732
|
for (const rec of sqlGetCurrentSpanMetadata.iterate()) {
|
|
686
733
|
const { vatID, startPos, endPos, incarnation } = rec;
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
endPos
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
734
|
+
|
|
735
|
+
if (checkMode === 'operational') {
|
|
736
|
+
// at 'operational', every 'isCurrent' transcript span must
|
|
737
|
+
// have all items
|
|
738
|
+
const count = sqlCountSpanItems.get(vatID, startPos, endPos);
|
|
739
|
+
if (count !== endPos - startPos) {
|
|
740
|
+
throw Fail`incomplete current transcript span: ${count} items, ${rec}`;
|
|
741
|
+
}
|
|
742
|
+
} else if (checkMode === 'replay') {
|
|
743
|
+
// at 'replay', every vat's current incarnation must be fully
|
|
744
|
+
// populated (which implies 'operational')
|
|
745
|
+
const incStart = sqlGetStartOfIncarnation.get(vatID, incarnation);
|
|
746
|
+
const incCount = sqlCountSpanItems.get(vatID, incStart, endPos);
|
|
747
|
+
if (incCount !== endPos - incStart) {
|
|
748
|
+
throw Fail`incomplete current incarnation transcript: ${incCount} items`;
|
|
749
|
+
}
|
|
750
|
+
} else if (checkMode === 'archival') {
|
|
751
|
+
// at 'archival', every incarnation must be fully populated,
|
|
752
|
+
// which means position=0 up through endPos-1 (which implies
|
|
753
|
+
// 'replay')
|
|
754
|
+
const arcCount = sqlCountSpanItems.get(vatID, 0, endPos);
|
|
755
|
+
if (arcCount !== endPos) {
|
|
756
|
+
throw Fail`incomplete archival transcript: ${arcCount} vs ${endPos}`;
|
|
757
|
+
}
|
|
758
|
+
} else {
|
|
759
|
+
throw Fail`unknown checkMode ${checkMode}`;
|
|
695
760
|
}
|
|
696
761
|
}
|
|
697
762
|
}
|
package/src/types.d.ts
CHANGED
|
@@ -2,13 +2,14 @@ export type {
|
|
|
2
2
|
SwingStore,
|
|
3
3
|
SwingStoreKernelStorage,
|
|
4
4
|
SwingStoreHostStorage,
|
|
5
|
-
} from './
|
|
6
|
-
export type { KVStore } from './
|
|
7
|
-
export type { BundleStore } from './
|
|
5
|
+
} from './swingStore.js';
|
|
6
|
+
export type { KVStore } from './kvStore.js';
|
|
7
|
+
export type { BundleStore } from './bundleStore.js';
|
|
8
|
+
export type { SnapStore, SnapshotResult, SnapshotInfo } from './snapStore.js';
|
|
9
|
+
export type { TranscriptStore } from './transcriptStore.js';
|
|
10
|
+
export type { ArtifactMode } from './internal.js';
|
|
11
|
+
export type { ImportSwingStoreOptions } from './importer.js';
|
|
8
12
|
export type {
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
} from './src/snapStore.js';
|
|
13
|
-
export type { TranscriptStore } from './src/transcriptStore.js';
|
|
14
|
-
export type { SwingStoreExporter, ExportMode } from './src/exporter.js';
|
|
13
|
+
SwingStoreExporter,
|
|
14
|
+
ExportSwingStoreOptions,
|
|
15
|
+
} from './exporter.js';
|
package/test/test-export.js
CHANGED
|
@@ -7,6 +7,14 @@ import { initSwingStore, makeSwingStoreExporter } from '../src/index.js';
|
|
|
7
7
|
|
|
8
8
|
import { tmpDir, getSnapshotStream, makeB0ID } from './util.js';
|
|
9
9
|
|
|
10
|
+
const rank = {
|
|
11
|
+
operational: 1,
|
|
12
|
+
replay: 2,
|
|
13
|
+
archival: 3,
|
|
14
|
+
debug: 4,
|
|
15
|
+
'debug-on-pruned': 4,
|
|
16
|
+
};
|
|
17
|
+
|
|
10
18
|
const snapshotData = 'snapshot data';
|
|
11
19
|
// this snapHash was computed manually
|
|
12
20
|
const snapHash =
|
|
@@ -28,48 +36,57 @@ const exportTest = test.macro(async (t, mode) => {
|
|
|
28
36
|
const ss1 = initSwingStore(dbDir, options);
|
|
29
37
|
const ks = ss1.kernelStorage;
|
|
30
38
|
|
|
31
|
-
// build a DB with
|
|
32
|
-
//
|
|
39
|
+
// build a DB with four spans (one in an old incarnation, two
|
|
40
|
+
// historical but current incarnation, only one inUse) and two
|
|
41
|
+
// snapshots (only one inUSe)
|
|
33
42
|
|
|
34
43
|
ks.kvStore.set('key1', 'value1');
|
|
35
44
|
ks.bundleStore.addBundle(bundle0ID, bundle0);
|
|
36
45
|
ks.transcriptStore.initTranscript('v1');
|
|
37
46
|
|
|
47
|
+
// incarnation 0
|
|
38
48
|
ks.transcriptStore.addItem('v1', 'start-worker'); // 0
|
|
39
|
-
ks.transcriptStore.addItem('v1', '
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
49
|
+
ks.transcriptStore.addItem('v1', 'shutdown-worker'); // 1
|
|
50
|
+
ks.transcriptStore.rolloverIncarnation('v1');
|
|
51
|
+
const spanHash0 =
|
|
52
|
+
'5bee0f44eca02f23eab03703e84ed2647d5d117fed99e1c30a3b424b7f082ab9';
|
|
53
|
+
|
|
54
|
+
// incarnation 1
|
|
55
|
+
ks.transcriptStore.addItem('v1', 'start-worker'); // 2
|
|
56
|
+
ks.transcriptStore.addItem('v1', 'delivery1'); // 3
|
|
57
|
+
await ks.snapStore.saveSnapshot('v1', 4, getSnapshotStream(snapshotData));
|
|
58
|
+
ks.transcriptStore.addItem('v1', 'save-snapshot'); // 4
|
|
59
|
+
ks.transcriptStore.rolloverSpan('v1'); // range= 2..5
|
|
43
60
|
const spanHash1 =
|
|
44
61
|
'57152efdd7fdf75c03371d2b4f1088d5bf3eae7fe643babce527ff81df38998c';
|
|
45
62
|
|
|
46
|
-
ks.transcriptStore.addItem('v1', 'load-snapshot'); //
|
|
47
|
-
ks.transcriptStore.addItem('v1', 'delivery2'); //
|
|
48
|
-
await ks.snapStore.saveSnapshot('v1',
|
|
49
|
-
ks.transcriptStore.addItem('v1', 'save-snapshot'); //
|
|
50
|
-
ks.transcriptStore.rolloverSpan('v1'); // range=
|
|
63
|
+
ks.transcriptStore.addItem('v1', 'load-snapshot'); // 5
|
|
64
|
+
ks.transcriptStore.addItem('v1', 'delivery2'); // 6
|
|
65
|
+
await ks.snapStore.saveSnapshot('v1', 7, getSnapshotStream(snapshotData));
|
|
66
|
+
ks.transcriptStore.addItem('v1', 'save-snapshot'); // 7
|
|
67
|
+
ks.transcriptStore.rolloverSpan('v1'); // range= 5..8
|
|
51
68
|
const spanHash2 =
|
|
52
69
|
'1947001e78e01bd1e773feb22b4ffc530447373b9de9274d5d5fbda3f23dbf2b';
|
|
53
70
|
|
|
54
|
-
ks.transcriptStore.addItem('v1', 'load-snapshot'); //
|
|
55
|
-
ks.transcriptStore.addItem('v1', 'delivery3'); //
|
|
71
|
+
ks.transcriptStore.addItem('v1', 'load-snapshot'); // 8
|
|
72
|
+
ks.transcriptStore.addItem('v1', 'delivery3'); // 9
|
|
56
73
|
const spanHash3 =
|
|
57
74
|
'e6b42c6a3fb94285a93162f25a9fc0145fd4c5bb144917dc572c50ae2d02ee69';
|
|
58
|
-
// current range=
|
|
75
|
+
// current range= 8..10
|
|
59
76
|
|
|
60
|
-
ss1.hostStorage.commit();
|
|
77
|
+
await ss1.hostStorage.commit();
|
|
61
78
|
|
|
62
79
|
// create an export, and assert that the pieces match what we
|
|
63
|
-
// expect.
|
|
80
|
+
// expect. artifactMode='operational' means we get all metadata, no
|
|
64
81
|
// historical transcript spans, and no historical snapshots
|
|
65
82
|
|
|
66
83
|
assert.typeof(mode, 'string');
|
|
67
|
-
/** @typedef {import('../src/
|
|
68
|
-
let
|
|
84
|
+
/** @typedef {import('../src/internal.js').ArtifactMode} ArtifactMode */
|
|
85
|
+
let artifactMode = /** @type {ArtifactMode} */ (mode);
|
|
69
86
|
if (mode === 'debug-on-pruned') {
|
|
70
|
-
|
|
87
|
+
artifactMode = 'debug';
|
|
71
88
|
}
|
|
72
|
-
const exporter = makeSwingStoreExporter(dbDir,
|
|
89
|
+
const exporter = makeSwingStoreExporter(dbDir, { artifactMode });
|
|
73
90
|
|
|
74
91
|
// exportData
|
|
75
92
|
{
|
|
@@ -90,36 +107,46 @@ const exportTest = test.macro(async (t, mode) => {
|
|
|
90
107
|
};
|
|
91
108
|
|
|
92
109
|
check('kv.key1', 'value1');
|
|
93
|
-
check('snapshot.v1.
|
|
110
|
+
check('snapshot.v1.4', {
|
|
94
111
|
vatID: 'v1',
|
|
95
|
-
snapPos:
|
|
112
|
+
snapPos: 4,
|
|
96
113
|
inUse: 0,
|
|
97
114
|
hash: snapHash,
|
|
98
115
|
});
|
|
99
|
-
check('snapshot.v1.
|
|
116
|
+
check('snapshot.v1.7', {
|
|
100
117
|
vatID: 'v1',
|
|
101
|
-
snapPos:
|
|
118
|
+
snapPos: 7,
|
|
102
119
|
inUse: 1,
|
|
103
120
|
hash: snapHash,
|
|
104
121
|
});
|
|
105
|
-
check('snapshot.v1.current', 'snapshot.v1.
|
|
106
|
-
const base = { vatID: 'v1',
|
|
122
|
+
check('snapshot.v1.current', 'snapshot.v1.7');
|
|
123
|
+
const base = { vatID: 'v1', isCurrent: 0 };
|
|
107
124
|
check('transcript.v1.0', {
|
|
108
125
|
...base,
|
|
126
|
+
incarnation: 0,
|
|
109
127
|
startPos: 0,
|
|
110
|
-
endPos:
|
|
128
|
+
endPos: 2,
|
|
129
|
+
hash: spanHash0,
|
|
130
|
+
});
|
|
131
|
+
check('transcript.v1.2', {
|
|
132
|
+
...base,
|
|
133
|
+
incarnation: 1,
|
|
134
|
+
startPos: 2,
|
|
135
|
+
endPos: 5,
|
|
111
136
|
hash: spanHash1,
|
|
112
137
|
});
|
|
113
|
-
check('transcript.v1.
|
|
138
|
+
check('transcript.v1.5', {
|
|
114
139
|
...base,
|
|
115
|
-
|
|
116
|
-
|
|
140
|
+
incarnation: 1,
|
|
141
|
+
startPos: 5,
|
|
142
|
+
endPos: 8,
|
|
117
143
|
hash: spanHash2,
|
|
118
144
|
});
|
|
119
145
|
check('transcript.v1.current', {
|
|
120
146
|
...base,
|
|
121
|
-
|
|
122
|
-
|
|
147
|
+
incarnation: 1,
|
|
148
|
+
startPos: 8,
|
|
149
|
+
endPos: 10,
|
|
123
150
|
isCurrent: 1,
|
|
124
151
|
hash: spanHash3,
|
|
125
152
|
});
|
|
@@ -153,29 +180,35 @@ const exportTest = test.macro(async (t, mode) => {
|
|
|
153
180
|
t.deepEqual(data, expected);
|
|
154
181
|
};
|
|
155
182
|
|
|
156
|
-
// export mode '
|
|
183
|
+
// export mode 'operational' means we omit historical snapshots and
|
|
157
184
|
// transcript spans
|
|
158
185
|
|
|
159
|
-
await check('snapshot.v1.
|
|
160
|
-
await check('transcript.v1.
|
|
186
|
+
await check('snapshot.v1.7', 'snapshot data');
|
|
187
|
+
await check('transcript.v1.8.10', 'load-snapshot\ndelivery3\n');
|
|
161
188
|
await check(`bundle.${bundle0ID}`, bundle0);
|
|
162
189
|
|
|
163
|
-
|
|
164
|
-
|
|
190
|
+
t.true(rank[mode] > 0);
|
|
191
|
+
if (rank[mode] >= rank.replay) {
|
|
192
|
+
// add the old transcript spans of the current incarnation
|
|
165
193
|
await check(
|
|
166
|
-
'transcript.v1.
|
|
194
|
+
'transcript.v1.2.5',
|
|
167
195
|
'start-worker\ndelivery1\nsave-snapshot\n',
|
|
168
196
|
);
|
|
169
197
|
await check(
|
|
170
|
-
'transcript.v1.
|
|
198
|
+
'transcript.v1.5.8',
|
|
171
199
|
'load-snapshot\ndelivery2\nsave-snapshot\n',
|
|
172
200
|
);
|
|
173
201
|
}
|
|
174
202
|
|
|
203
|
+
if (rank[mode] >= rank.archival) {
|
|
204
|
+
// add the spans of the old incarnation
|
|
205
|
+
await check('transcript.v1.0.2', 'start-worker\nshutdown-worker\n');
|
|
206
|
+
}
|
|
207
|
+
|
|
175
208
|
if (mode === 'debug') {
|
|
176
209
|
// adds the old snapshots, which are only present if
|
|
177
210
|
// initSwingStore() was given {keepSnapshots: true}
|
|
178
|
-
await check('snapshot.v1.
|
|
211
|
+
await check('snapshot.v1.4', 'snapshot data');
|
|
179
212
|
// mode='debug-on-pruned' exercises the keepSnapshots:false case
|
|
180
213
|
}
|
|
181
214
|
|
|
@@ -187,7 +220,8 @@ const exportTest = test.macro(async (t, mode) => {
|
|
|
187
220
|
}
|
|
188
221
|
});
|
|
189
222
|
|
|
190
|
-
test('export
|
|
223
|
+
test('export operational', exportTest, 'operational');
|
|
224
|
+
test('export replay', exportTest, 'replay');
|
|
191
225
|
test('export archival', exportTest, 'archival');
|
|
192
226
|
test('export debug', exportTest, 'debug');
|
|
193
227
|
test('export debug-on-pruned', exportTest, 'debug-on-pruned');
|
|
@@ -220,9 +254,10 @@ test('export omits pruned span artifacts', async t => {
|
|
|
220
254
|
const spanHash2 =
|
|
221
255
|
'b26c8faf425c3c2738e0c5a5e9a7cd71075c68f0c9f2d6cdfd83c68204801dbb';
|
|
222
256
|
|
|
223
|
-
ss1.hostStorage.commit();
|
|
257
|
+
await ss1.hostStorage.commit();
|
|
224
258
|
|
|
225
|
-
const
|
|
259
|
+
const artifactMode = 'debug';
|
|
260
|
+
const exporter = makeSwingStoreExporter(dbDir, { artifactMode });
|
|
226
261
|
|
|
227
262
|
// exportData
|
|
228
263
|
{
|
|
@@ -123,7 +123,8 @@ test('crank abort leaves no debris in export log', async t => {
|
|
|
123
123
|
await ssOut.hostStorage.commit();
|
|
124
124
|
}
|
|
125
125
|
|
|
126
|
-
const
|
|
126
|
+
const artifactMode = 'operational';
|
|
127
|
+
const exporter = makeSwingStoreExporter(dbDir, { artifactMode });
|
|
127
128
|
|
|
128
129
|
const exportData = [];
|
|
129
130
|
for await (const elem of exporter.getExportData()) {
|
|
@@ -172,12 +173,13 @@ async function testExportImport(
|
|
|
172
173
|
exportMode,
|
|
173
174
|
importMode,
|
|
174
175
|
expectedArtifactNames,
|
|
176
|
+
failureMode = 'none',
|
|
175
177
|
) {
|
|
176
178
|
const exportLog = makeExportLog();
|
|
177
179
|
const [dbDir, cleanup] = await tmpDir('testdb');
|
|
178
180
|
t.teardown(cleanup);
|
|
179
181
|
|
|
180
|
-
const keepTranscripts = runMode !== '
|
|
182
|
+
const keepTranscripts = runMode !== 'operational';
|
|
181
183
|
const keepSnapshots = runMode === 'debug';
|
|
182
184
|
const ssOut = initSwingStore(dbDir, {
|
|
183
185
|
exportCallback: exportLog.callback,
|
|
@@ -218,7 +220,15 @@ async function testExportImport(
|
|
|
218
220
|
await ssOut.hostStorage.commit();
|
|
219
221
|
}
|
|
220
222
|
|
|
221
|
-
const
|
|
223
|
+
const incomplete = 'incomplete archival transcript: 3 vs 12';
|
|
224
|
+
function doExport() {
|
|
225
|
+
return makeSwingStoreExporter(dbDir, { artifactMode: exportMode });
|
|
226
|
+
}
|
|
227
|
+
if (failureMode === 'export') {
|
|
228
|
+
await t.throws(doExport, { message: incomplete });
|
|
229
|
+
return;
|
|
230
|
+
}
|
|
231
|
+
const exporter = doExport();
|
|
222
232
|
|
|
223
233
|
const exportData = [];
|
|
224
234
|
for await (const elem of exporter.getExportData()) {
|
|
@@ -302,13 +312,36 @@ async function testExportImport(
|
|
|
302
312
|
}
|
|
303
313
|
t.deepEqual(artifactNames, expectedArtifactNames);
|
|
304
314
|
|
|
305
|
-
const includeHistorical = importMode !== 'current';
|
|
306
|
-
|
|
307
315
|
const beforeDump = debug.dump(keepSnapshots);
|
|
308
|
-
|
|
316
|
+
function doImport() {
|
|
317
|
+
return importSwingStore(exporter, null, { artifactMode: importMode });
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
if (failureMode === 'import') {
|
|
321
|
+
await t.throwsAsync(doImport, { message: incomplete });
|
|
322
|
+
return;
|
|
323
|
+
}
|
|
324
|
+
t.is(failureMode, 'none');
|
|
325
|
+
const ssIn = await doImport();
|
|
309
326
|
await ssIn.hostStorage.commit();
|
|
310
|
-
|
|
311
|
-
|
|
327
|
+
let dumpsShouldMatch = true;
|
|
328
|
+
if (runMode === 'operational') {
|
|
329
|
+
dumpsShouldMatch = true; // there's no data to lose
|
|
330
|
+
} else if (runMode === 'archival') {
|
|
331
|
+
if (exportMode === 'current') {
|
|
332
|
+
dumpsShouldMatch = false; // export omits some data
|
|
333
|
+
}
|
|
334
|
+
if (importMode === 'current') {
|
|
335
|
+
dumpsShouldMatch = false; // import ignores some data
|
|
336
|
+
}
|
|
337
|
+
} else if (runMode === 'debug') {
|
|
338
|
+
if (exportMode !== 'debug') {
|
|
339
|
+
dumpsShouldMatch = false; // export omits some data
|
|
340
|
+
}
|
|
341
|
+
if (importMode !== 'debug') {
|
|
342
|
+
dumpsShouldMatch = false; // import ignores some data
|
|
343
|
+
}
|
|
344
|
+
}
|
|
312
345
|
if (dumpsShouldMatch) {
|
|
313
346
|
const afterDump = ssIn.debug.dump(keepSnapshots);
|
|
314
347
|
t.deepEqual(beforeDump, afterDump);
|
|
@@ -343,35 +376,43 @@ const expectedDebugArtifacts = [
|
|
|
343
376
|
'transcript.vatB.5.10',
|
|
344
377
|
];
|
|
345
378
|
|
|
346
|
-
const C = 'current'
|
|
379
|
+
const C = 'operational'; // nee 'current'
|
|
380
|
+
// we don't try to test 'replay' here: see test-import.js and test-export.js
|
|
347
381
|
const A = 'archival';
|
|
348
382
|
const D = 'debug';
|
|
349
383
|
|
|
384
|
+
// importMode='archival' requires a non-pruned DB
|
|
385
|
+
// (runMode!=='current'), with exportMode as 'archival' or 'debug'
|
|
386
|
+
|
|
387
|
+
// the expected artifacts are a function of the runMode and exportMode, not importMode
|
|
388
|
+
|
|
350
389
|
test('export and import data for state sync - current->current->current', async t => {
|
|
351
390
|
await testExportImport(t, C, C, C, expectedCurrentArtifacts);
|
|
352
391
|
});
|
|
392
|
+
// so this one fails during import
|
|
353
393
|
test('export and import data for state sync - current->current->archival', async t => {
|
|
354
|
-
await testExportImport(t, C, C, A, expectedCurrentArtifacts);
|
|
394
|
+
await testExportImport(t, C, C, A, expectedCurrentArtifacts, 'import');
|
|
355
395
|
});
|
|
356
396
|
test('export and import data for state sync - current->current->debug', async t => {
|
|
357
397
|
await testExportImport(t, C, C, D, expectedCurrentArtifacts);
|
|
358
398
|
});
|
|
359
399
|
|
|
400
|
+
// these all throw an error during export, because 'archival' requires a non-pruned DB
|
|
360
401
|
test('export and import data for state sync - current->archival->current', async t => {
|
|
361
|
-
await testExportImport(t, C, A, C,
|
|
402
|
+
await testExportImport(t, C, A, C, [], 'export');
|
|
362
403
|
});
|
|
363
404
|
test('export and import data for state sync - current->archival->archival', async t => {
|
|
364
|
-
await testExportImport(t, C, A, A,
|
|
405
|
+
await testExportImport(t, C, A, A, [], 'export');
|
|
365
406
|
});
|
|
366
407
|
test('export and import data for state sync - current->archival->debug', async t => {
|
|
367
|
-
await testExportImport(t, C, A, D,
|
|
408
|
+
await testExportImport(t, C, A, D, [], 'export');
|
|
368
409
|
});
|
|
369
410
|
|
|
370
411
|
test('export and import data for state sync - current->debug->current', async t => {
|
|
371
412
|
await testExportImport(t, C, D, C, expectedCurrentArtifacts);
|
|
372
413
|
});
|
|
373
414
|
test('export and import data for state sync - current->debug->archival', async t => {
|
|
374
|
-
await testExportImport(t, C, D, A, expectedCurrentArtifacts);
|
|
415
|
+
await testExportImport(t, C, D, A, expectedCurrentArtifacts, 'import');
|
|
375
416
|
});
|
|
376
417
|
test('export and import data for state sync - current->debug->debug', async t => {
|
|
377
418
|
await testExportImport(t, C, D, D, expectedCurrentArtifacts);
|
|
@@ -383,7 +424,7 @@ test('export and import data for state sync - archival->current->current', async
|
|
|
383
424
|
await testExportImport(t, A, C, C, expectedCurrentArtifacts);
|
|
384
425
|
});
|
|
385
426
|
test('export and import data for state sync - archival->current->archival', async t => {
|
|
386
|
-
await testExportImport(t, A, C, A, expectedCurrentArtifacts);
|
|
427
|
+
await testExportImport(t, A, C, A, expectedCurrentArtifacts, 'import');
|
|
387
428
|
});
|
|
388
429
|
test('export and import data for state sync - archival->current->debug', async t => {
|
|
389
430
|
await testExportImport(t, A, C, D, expectedCurrentArtifacts);
|
|
@@ -415,7 +456,7 @@ test('export and import data for state sync - debug->current->current', async t
|
|
|
415
456
|
await testExportImport(t, D, C, C, expectedCurrentArtifacts);
|
|
416
457
|
});
|
|
417
458
|
test('export and import data for state sync - debug->current->archival', async t => {
|
|
418
|
-
await testExportImport(t, D, C, A, expectedCurrentArtifacts);
|
|
459
|
+
await testExportImport(t, D, C, A, expectedCurrentArtifacts, 'import');
|
|
419
460
|
});
|
|
420
461
|
test('export and import data for state sync - debug->current->debug', async t => {
|
|
421
462
|
await testExportImport(t, D, C, D, expectedCurrentArtifacts);
|