@agoric/swing-store 0.10.0 → 0.10.1-upgrade-23-dev-bd79330.0.bd79330
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +6 -6
- package/src/archiver.js +1 -1
- package/src/assertComplete.js +7 -2
- package/src/bundleStore.js +5 -5
- package/src/exporter.js +8 -5
- package/src/hasher.js +1 -1
- package/src/importer.js +9 -3
- package/src/index.js +2 -1
- package/src/internal.js +10 -6
- package/src/repairMetadata.js +9 -3
- package/src/snapStore.js +12 -7
- package/src/snapStoreIO.js +1 -1
- package/src/swingStore.js +32 -25
- package/src/transcriptStore.js +3 -3
- package/src/types-index.d.ts +1 -1
- package/src/util.js +21 -3
- package/test/bundles.test.js +8 -4
- package/test/deletion.test.js +5 -1
- package/test/export.test.js +6 -2
- package/test/exports.js +8 -3
- package/test/import.test.js +4 -4
- package/test/repair-metadata.test.js +1 -1
- package/tsconfig.json +2 -1
- package/src/kvStore.js +0 -173
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@agoric/swing-store",
|
|
3
|
-
"version": "0.10.0",
|
|
3
|
+
"version": "0.10.1-upgrade-23-dev-bd79330.0.bd79330",
|
|
4
4
|
"description": "Persistent storage for SwingSet",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./src/index.js",
|
|
@@ -18,10 +18,10 @@
|
|
|
18
18
|
"lint-fix": "yarn lint:eslint --fix",
|
|
19
19
|
"lint": "yarn run -T run-s --continue-on-error 'lint:*'",
|
|
20
20
|
"lint:types": "yarn run -T tsc",
|
|
21
|
-
"lint:eslint": "
|
|
21
|
+
"lint:eslint": "node ../../scripts/eslint-repo.mjs ."
|
|
22
22
|
},
|
|
23
23
|
"dependencies": {
|
|
24
|
-
"@agoric/internal": "0.4.0",
|
|
24
|
+
"@agoric/internal": "0.4.1-upgrade-23-dev-bd79330.0.bd79330",
|
|
25
25
|
"@endo/base64": "^1.0.12",
|
|
26
26
|
"@endo/bundle-source": "^4.1.2",
|
|
27
27
|
"@endo/check-bundle": "^1.0.17",
|
|
@@ -34,7 +34,7 @@
|
|
|
34
34
|
"@endo/promise-kit": "^1.1.13",
|
|
35
35
|
"@endo/stream": "^1.2.13",
|
|
36
36
|
"@types/better-sqlite3": "^7.6.13",
|
|
37
|
-
"ava": "^
|
|
37
|
+
"ava": "^6.4.1",
|
|
38
38
|
"c8": "^10.1.3",
|
|
39
39
|
"tmp": "^0.2.1"
|
|
40
40
|
},
|
|
@@ -51,10 +51,10 @@
|
|
|
51
51
|
"timeout": "2m"
|
|
52
52
|
},
|
|
53
53
|
"typeCoverage": {
|
|
54
|
-
"atLeast": 80.
|
|
54
|
+
"atLeast": 80.67
|
|
55
55
|
},
|
|
56
56
|
"engines": {
|
|
57
57
|
"node": "^20.9 || ^22.11"
|
|
58
58
|
},
|
|
59
|
-
"gitHead": "
|
|
59
|
+
"gitHead": "bd79330f78dae2faf9cc3d8b10063567700da07b"
|
|
60
60
|
}
|
package/src/archiver.js
CHANGED
|
@@ -42,7 +42,7 @@ export const makeArchiveSnapshot = (dirPath, powers) => {
|
|
|
42
42
|
const writer = fs.createWriteStream('', { fd, flush: true });
|
|
43
43
|
const reader = Readable.from(gzData);
|
|
44
44
|
const destroyReader = promisify(reader.destroy.bind(reader));
|
|
45
|
-
addCleanup(() => destroyReader(
|
|
45
|
+
addCleanup(() => destroyReader(undefined));
|
|
46
46
|
reader.pipe(writer);
|
|
47
47
|
await streamFinished(writer);
|
|
48
48
|
fs.renameSync(tmpName, destPath);
|
package/src/assertComplete.js
CHANGED
|
@@ -1,12 +1,17 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* @param {Pick<
|
|
3
|
-
* @param {Exclude<
|
|
2
|
+
* @param {Pick<SwingStoreInternal, 'bundleStore' | 'transcriptStore' | 'snapStore'>} internal
|
|
3
|
+
* @param {Exclude<ArtifactMode, 'debug'>} checkMode
|
|
4
4
|
* @returns {void}
|
|
5
5
|
*/
|
|
6
6
|
export function assertComplete(internal, checkMode) {
|
|
7
7
|
// every bundle must be populated
|
|
8
8
|
internal.bundleStore.assertComplete(checkMode);
|
|
9
9
|
|
|
10
|
+
/**
|
|
11
|
+
* @import {SwingStoreInternal} from './internal.js';
|
|
12
|
+
* @import {ArtifactMode} from './internal.js';
|
|
13
|
+
*/
|
|
14
|
+
|
|
10
15
|
// every 'isCurrent' transcript span must have all items
|
|
11
16
|
// TODO: every vat with any data must have a isCurrent transcript
|
|
12
17
|
// span
|
package/src/bundleStore.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// @ts-check
|
|
2
|
-
import { createHash } from 'crypto';
|
|
3
|
-
import { Readable } from 'stream';
|
|
4
|
-
import { Buffer } from 'buffer';
|
|
2
|
+
import { createHash } from 'node:crypto';
|
|
3
|
+
import { Readable } from 'node:stream';
|
|
4
|
+
import { Buffer } from 'node:buffer';
|
|
5
5
|
import { Fail, q } from '@endo/errors';
|
|
6
6
|
import { encodeBase64, decodeBase64 } from '@endo/base64';
|
|
7
7
|
import { checkBundle } from '@endo/check-bundle/lite.js';
|
|
@@ -15,8 +15,8 @@ import { createSHA256 } from './hasher.js';
|
|
|
15
15
|
* @typedef { EndoZipBase64Bundle | GetExportBundle | NestedEvaluateBundle } Bundle
|
|
16
16
|
*/
|
|
17
17
|
/**
|
|
18
|
-
* @
|
|
19
|
-
* @
|
|
18
|
+
* @import {SwingStoreExporter} from './exporter.js';
|
|
19
|
+
* @import {ArtifactMode} from './internal.js';
|
|
20
20
|
*
|
|
21
21
|
* @typedef {{
|
|
22
22
|
* addBundle: (bundleID: string, bundle: Bundle) => void;
|
package/src/exporter.js
CHANGED
|
@@ -2,8 +2,7 @@ import sqlite3 from 'better-sqlite3';
|
|
|
2
2
|
|
|
3
3
|
import { Fail, q } from '@endo/errors';
|
|
4
4
|
|
|
5
|
-
import { dbFileInDirectory } from './util.js';
|
|
6
|
-
import { getKeyType } from './kvStore.js';
|
|
5
|
+
import { dbFileInDirectory, getKVStoreKeyType } from './util.js';
|
|
7
6
|
import { makeBundleStore } from './bundleStore.js';
|
|
8
7
|
import { makeSnapStore } from './snapStore.js';
|
|
9
8
|
import { makeSnapStoreIO } from './snapStoreIO.js';
|
|
@@ -11,6 +10,10 @@ import { makeTranscriptStore } from './transcriptStore.js';
|
|
|
11
10
|
import { assertComplete } from './assertComplete.js';
|
|
12
11
|
import { validateArtifactMode } from './internal.js';
|
|
13
12
|
|
|
13
|
+
/**
|
|
14
|
+
* @import {ArtifactMode} from './internal.js';
|
|
15
|
+
*/
|
|
16
|
+
|
|
14
17
|
/**
|
|
15
18
|
* @template T
|
|
16
19
|
* @typedef { Iterable<T> | AsyncIterable<T> } AnyIterable
|
|
@@ -79,7 +82,7 @@ import { validateArtifactMode } from './internal.js';
|
|
|
79
82
|
|
|
80
83
|
/**
|
|
81
84
|
* @typedef { object } ExportSwingStoreOptions
|
|
82
|
-
* @property {
|
|
85
|
+
* @property { ArtifactMode } [artifactMode] What artifacts should/must the exporter provide?
|
|
83
86
|
*/
|
|
84
87
|
|
|
85
88
|
/**
|
|
@@ -129,7 +132,7 @@ export function makeSwingStoreExporter(dirPath, options = {}) {
|
|
|
129
132
|
* section
|
|
130
133
|
*/
|
|
131
134
|
function getHostKV(key) {
|
|
132
|
-
|
|
135
|
+
getKVStoreKeyType(key) === 'host' || Fail`getHostKV requires host keys`;
|
|
133
136
|
// @ts-expect-error unknown
|
|
134
137
|
return sqlKVGet.get(key);
|
|
135
138
|
}
|
|
@@ -147,7 +150,7 @@ export function makeSwingStoreExporter(dirPath, options = {}) {
|
|
|
147
150
|
*/
|
|
148
151
|
async function* getExportData() {
|
|
149
152
|
for (const { key, value } of sqlGetAllKVData.iterate()) {
|
|
150
|
-
if (
|
|
153
|
+
if (getKVStoreKeyType(key) === 'consensus') {
|
|
151
154
|
yield [`kv.${key}`, value];
|
|
152
155
|
}
|
|
153
156
|
}
|
package/src/hasher.js
CHANGED
package/src/importer.js
CHANGED
|
@@ -5,9 +5,15 @@ import { buffer } from './util.js';
|
|
|
5
5
|
import { validateArtifactMode } from './internal.js';
|
|
6
6
|
import { assertComplete } from './assertComplete.js';
|
|
7
7
|
|
|
8
|
+
/**
|
|
9
|
+
* @import {ArtifactMode} from './internal.js';
|
|
10
|
+
* @import {SwingStoreExporter} from './exporter.js';
|
|
11
|
+
* @import {SwingStore} from './swingStore.js';
|
|
12
|
+
*/
|
|
13
|
+
|
|
8
14
|
/**
|
|
9
15
|
* @typedef { object } ImportSwingStoreOptions
|
|
10
|
-
* @property {
|
|
16
|
+
* @property { ArtifactMode } [artifactMode] What artifacts should the importer use and require?
|
|
11
17
|
*/
|
|
12
18
|
|
|
13
19
|
/**
|
|
@@ -17,10 +23,10 @@ import { assertComplete } from './assertComplete.js';
|
|
|
17
23
|
* returned swingStore is not suitable for execution, and thus only contains
|
|
18
24
|
* the host facet for committing the populated swingStore.
|
|
19
25
|
*
|
|
20
|
-
* @param {
|
|
26
|
+
* @param {SwingStoreExporter} exporter
|
|
21
27
|
* @param {string | null} [dirPath]
|
|
22
28
|
* @param {ImportSwingStoreOptions} [options]
|
|
23
|
-
* @returns {Promise<Pick<
|
|
29
|
+
* @returns {Promise<Pick<SwingStore, 'hostStorage' | 'debug'>>}
|
|
24
30
|
*/
|
|
25
31
|
export async function importSwingStore(exporter, dirPath = null, options = {}) {
|
|
26
32
|
if (dirPath && typeof dirPath !== 'string') {
|
package/src/index.js
CHANGED
|
@@ -5,7 +5,8 @@ export { importSwingStore } from './importer.js';
|
|
|
5
5
|
export { makeArchiveSnapshot, makeArchiveTranscript } from './archiver.js';
|
|
6
6
|
|
|
7
7
|
// for the benefit of tools like SwingSet/misc-tools/replay-transcript.js
|
|
8
|
-
export { makeKVStore
|
|
8
|
+
export { makeKVStore } from '@agoric/internal/src/kv-store.js';
|
|
9
|
+
export { getKVStoreKeyType, getKVStoreKeyType as getKeyType } from './util.js';
|
|
9
10
|
export { makeTranscriptStore } from './transcriptStore.js';
|
|
10
11
|
export { makeSnapStore } from './snapStore.js';
|
|
11
12
|
export { makeSnapStoreIO } from './snapStoreIO.js';
|
package/src/internal.js
CHANGED
|
@@ -1,14 +1,18 @@
|
|
|
1
1
|
import { Fail, q } from '@endo/errors';
|
|
2
2
|
|
|
3
3
|
/**
|
|
4
|
-
* @
|
|
5
|
-
* @
|
|
6
|
-
* @
|
|
7
|
-
*
|
|
4
|
+
* @import {SnapStoreInternal} from './snapStore.js';
|
|
5
|
+
* @import {TranscriptStoreInternal} from './transcriptStore.js';
|
|
6
|
+
* @import {BundleStoreInternal} from './bundleStore.js';
|
|
7
|
+
* @import {KVStore} from '@agoric/internal/src/kv-store.js';
|
|
8
|
+
* @import {Database} from 'better-sqlite3';
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
/**
|
|
8
12
|
* @typedef {{
|
|
9
13
|
* dirPath: string | null,
|
|
10
|
-
* db:
|
|
11
|
-
* kvStore:
|
|
14
|
+
* db: Database,
|
|
15
|
+
* kvStore: KVStore,
|
|
12
16
|
* transcriptStore: TranscriptStoreInternal,
|
|
13
17
|
* snapStore: SnapStoreInternal,
|
|
14
18
|
* bundleStore: BundleStoreInternal,
|
package/src/repairMetadata.js
CHANGED
|
@@ -1,6 +1,12 @@
|
|
|
1
1
|
import { Fail, q } from '@endo/errors';
|
|
2
2
|
import { assertComplete } from './assertComplete.js';
|
|
3
3
|
|
|
4
|
+
/**
|
|
5
|
+
* @import {SwingStoreInternal} from './internal.js';
|
|
6
|
+
* @import {SwingStoreExporter} from './exporter.js';
|
|
7
|
+
* @import {ArtifactMode} from './internal.js';
|
|
8
|
+
*/
|
|
9
|
+
|
|
4
10
|
/**
|
|
5
11
|
* Given a pre-existing swingstore and a SwingStoreExporter, read in
|
|
6
12
|
* all the metadata from the exporter and use it to regenerate any
|
|
@@ -24,8 +30,8 @@ import { assertComplete } from './assertComplete.js';
|
|
|
24
30
|
* an open transaction. The caller is responsible for calling
|
|
25
31
|
* `hostStorage.commit()` when they are ready.
|
|
26
32
|
*
|
|
27
|
-
* @param {
|
|
28
|
-
* @param {
|
|
33
|
+
* @param {SwingStoreInternal} internal
|
|
34
|
+
* @param {SwingStoreExporter} exporter
|
|
29
35
|
* @returns {Promise<void>}
|
|
30
36
|
*/
|
|
31
37
|
export async function doRepairMetadata(internal, exporter) {
|
|
@@ -60,7 +66,7 @@ export async function doRepairMetadata(internal, exporter) {
|
|
|
60
66
|
}
|
|
61
67
|
|
|
62
68
|
// and do a completeness check
|
|
63
|
-
/** @type {
|
|
69
|
+
/** @type { ArtifactMode } */
|
|
64
70
|
const artifactMode = 'operational';
|
|
65
71
|
assertComplete(internal, artifactMode);
|
|
66
72
|
await exporter.close();
|
package/src/snapStore.js
CHANGED
|
@@ -1,8 +1,12 @@
|
|
|
1
1
|
// @ts-check
|
|
2
|
-
import { createHash } from 'crypto';
|
|
3
|
-
import {
|
|
4
|
-
|
|
5
|
-
|
|
2
|
+
import { createHash } from 'node:crypto';
|
|
3
|
+
import {
|
|
4
|
+
finished as finishedCallback,
|
|
5
|
+
PassThrough,
|
|
6
|
+
Readable,
|
|
7
|
+
} from 'node:stream';
|
|
8
|
+
import { promisify } from 'node:util';
|
|
9
|
+
import { createGzip, createGunzip } from 'node:zlib';
|
|
6
10
|
import { Fail, q } from '@endo/errors';
|
|
7
11
|
import { withDeferredCleanup } from '@agoric/internal';
|
|
8
12
|
import { buffer } from './util.js';
|
|
@@ -10,6 +14,7 @@ import { buffer } from './util.js';
|
|
|
10
14
|
/**
|
|
11
15
|
* @import { AnyIterable, SwingStoreExporter } from './exporter.js';
|
|
12
16
|
* @import { ArtifactMode } from './internal.js';
|
|
17
|
+
* @import {makeMeasureSeconds} from '@agoric/internal';
|
|
13
18
|
*/
|
|
14
19
|
|
|
15
20
|
/**
|
|
@@ -68,7 +73,7 @@ import { buffer } from './util.js';
|
|
|
68
73
|
* @callback SnapshotCallback
|
|
69
74
|
* Called with the gzipped contents of a new heap snapshot.
|
|
70
75
|
* @param {string} name an export key, e.g. `snapshot.${vatID}.${deliveryCount}`
|
|
71
|
-
* @param {Parameters<
|
|
76
|
+
* @param {Parameters<typeof Readable.from>[0]} compressedData
|
|
72
77
|
* @returns {Promise<void>}
|
|
73
78
|
*/
|
|
74
79
|
|
|
@@ -78,7 +83,7 @@ const finished = promisify(finishedCallback);
|
|
|
78
83
|
* @param {*} db
|
|
79
84
|
* @param {() => void} ensureTxn
|
|
80
85
|
* @param {{
|
|
81
|
-
* measureSeconds: ReturnType<typeof
|
|
86
|
+
* measureSeconds: ReturnType<typeof makeMeasureSeconds>,
|
|
82
87
|
* }} io
|
|
83
88
|
* @param {(key: string, value: string | undefined) => void} noteExport
|
|
84
89
|
* @param {object} [options]
|
|
@@ -227,7 +232,7 @@ export function makeSnapStore(
|
|
|
227
232
|
await measureSeconds(async () => {
|
|
228
233
|
const snapReader = Readable.from(snapshotStream);
|
|
229
234
|
const destroyReader = promisify(snapReader.destroy.bind(snapReader));
|
|
230
|
-
addCleanup(() => destroyReader(
|
|
235
|
+
addCleanup(() => destroyReader(undefined));
|
|
231
236
|
snapReader.on('data', chunk => {
|
|
232
237
|
uncompressedSize += chunk.length;
|
|
233
238
|
});
|
package/src/snapStoreIO.js
CHANGED
package/src/swingStore.js
CHANGED
|
@@ -1,17 +1,17 @@
|
|
|
1
1
|
// @ts-check
|
|
2
2
|
/* eslint-env node */
|
|
3
|
-
import * as fs from 'fs';
|
|
4
|
-
import * as pathlib from 'path';
|
|
3
|
+
import * as fs from 'node:fs';
|
|
4
|
+
import * as pathlib from 'node:path';
|
|
5
5
|
|
|
6
6
|
import sqlite3 from 'better-sqlite3';
|
|
7
7
|
|
|
8
8
|
import { Fail, q } from '@endo/errors';
|
|
9
9
|
|
|
10
10
|
import { attenuate } from '@agoric/internal';
|
|
11
|
+
import { makeKVStore } from '@agoric/internal/src/kv-store.js';
|
|
11
12
|
import { TRUE } from '@agoric/internal/src/js-utils.js';
|
|
12
13
|
|
|
13
|
-
import { dbFileInDirectory } from './util.js';
|
|
14
|
-
import { makeKVStore, getKeyType } from './kvStore.js';
|
|
14
|
+
import { dbFileInDirectory, getKVStoreKeyType } from './util.js';
|
|
15
15
|
import { makeTranscriptStore } from './transcriptStore.js';
|
|
16
16
|
import { makeSnapStore } from './snapStore.js';
|
|
17
17
|
import { makeBundleStore } from './bundleStore.js';
|
|
@@ -28,18 +28,25 @@ const IN_MEMORY = ':memory:';
|
|
|
28
28
|
*/
|
|
29
29
|
|
|
30
30
|
/**
|
|
31
|
-
* @
|
|
31
|
+
* @import {SwingStoreExporter} from './exporter.js';
|
|
32
|
+
* @import {SwingStoreInternal} from './internal.js';
|
|
33
|
+
* @import {SnapshotCallback} from './snapStore.js';
|
|
34
|
+
* @import {TranscriptCallback} from './transcriptStore.js';
|
|
35
|
+
*/
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* @import {KVStore} from '@agoric/internal/src/kv-store.js';
|
|
32
39
|
*
|
|
33
|
-
* @
|
|
34
|
-
* @
|
|
40
|
+
* @import {SnapStore} from './snapStore.js';
|
|
41
|
+
* @import {SnapshotResult} from './snapStore.js';
|
|
35
42
|
*
|
|
36
|
-
* @
|
|
37
|
-
* @
|
|
43
|
+
* @import {TranscriptStore} from './transcriptStore.js';
|
|
44
|
+
* @import {TranscriptStoreDebug} from './transcriptStore.js';
|
|
38
45
|
*
|
|
39
|
-
* @
|
|
40
|
-
* @
|
|
46
|
+
* @import {BundleStore} from './bundleStore.js';
|
|
47
|
+
* @import {BundleStoreDebug} from './bundleStore.js';
|
|
41
48
|
*
|
|
42
|
-
* @
|
|
49
|
+
* @import {KVPair} from './exporter.js';
|
|
43
50
|
*
|
|
44
51
|
* @typedef {{
|
|
45
52
|
* kvStore: KVStore, // a key-value API object to load and store data on behalf of the kernel
|
|
@@ -59,7 +66,7 @@ const IN_MEMORY = ':memory:';
|
|
|
59
66
|
* commit: () => Promise<void>, // commit changes made since the last commit
|
|
60
67
|
* close: () => Promise<void>, // shutdown the store, abandoning any uncommitted changes
|
|
61
68
|
* diskUsage?: () => number, // optional stats method
|
|
62
|
-
* repairMetadata: (exporter:
|
|
69
|
+
* repairMetadata: (exporter: SwingStoreExporter) => Promise<void>,
|
|
63
70
|
* }} SwingStoreHostStorage
|
|
64
71
|
*/
|
|
65
72
|
|
|
@@ -79,7 +86,7 @@ const IN_MEMORY = ':memory:';
|
|
|
79
86
|
* kernelStorage: SwingStoreKernelStorage,
|
|
80
87
|
* hostStorage: SwingStoreHostStorage,
|
|
81
88
|
* debug: SwingStoreDebugTools,
|
|
82
|
-
* internal:
|
|
89
|
+
* internal: SwingStoreInternal,
|
|
83
90
|
* }} SwingStore
|
|
84
91
|
*/
|
|
85
92
|
|
|
@@ -139,13 +146,13 @@ const IN_MEMORY = ':memory:';
|
|
|
139
146
|
* @property {string} [traceFile] Path at which to record KVStore set/delete activity
|
|
140
147
|
* @property {boolean} [keepSnapshots] Retain old heap snapshots
|
|
141
148
|
* @property {boolean} [keepTranscripts] Retain old transcript span items
|
|
142
|
-
* @property {
|
|
143
|
-
* @property {
|
|
149
|
+
* @property {SnapshotCallback} [archiveSnapshot] Called after creation of a new heap snapshot
|
|
150
|
+
* @property {TranscriptCallback} [archiveTranscript] Called after a formerly-current transcript span is finalized
|
|
144
151
|
* @property {(pendingExports: Iterable<[key: string, value: string | null]>) => void} [exportCallback]
|
|
145
|
-
* @property {Replacer<ReturnType<makeKVStore>>} [wrapKvStore]
|
|
146
|
-
* @property {Replacer<ReturnType<makeTranscriptStore>>} [wrapTranscriptStore]
|
|
147
|
-
* @property {Replacer<ReturnType<makeSnapStore>>} [wrapSnapStore]
|
|
148
|
-
* @property {Replacer<ReturnType<makeBundleStore>>} [wrapBundleStore]
|
|
152
|
+
* @property {Replacer<ReturnType<typeof makeKVStore>>} [wrapKvStore]
|
|
153
|
+
* @property {Replacer<ReturnType<typeof makeTranscriptStore>>} [wrapTranscriptStore]
|
|
154
|
+
* @property {Replacer<ReturnType<typeof makeSnapStore>>} [wrapSnapStore]
|
|
155
|
+
* @property {Replacer<ReturnType<typeof makeBundleStore>>} [wrapBundleStore]
|
|
149
156
|
*/
|
|
150
157
|
|
|
151
158
|
/**
|
|
@@ -351,7 +358,7 @@ export function makeSwingStore(path, forceReset, options = {}) {
|
|
|
351
358
|
const kernelKVStore = {
|
|
352
359
|
...kvStore,
|
|
353
360
|
set(key, value) {
|
|
354
|
-
const keyType =
|
|
361
|
+
const keyType = getKVStoreKeyType(key);
|
|
355
362
|
keyType !== 'host' || Fail`kernelKVStore refuses host keys`;
|
|
356
363
|
kvStore.set(key, value);
|
|
357
364
|
if (keyType === 'consensus') {
|
|
@@ -365,7 +372,7 @@ export function makeSwingStore(path, forceReset, options = {}) {
|
|
|
365
372
|
}
|
|
366
373
|
},
|
|
367
374
|
delete(key) {
|
|
368
|
-
const keyType =
|
|
375
|
+
const keyType = getKVStoreKeyType(key);
|
|
369
376
|
keyType !== 'host' || Fail`kernelKVStore refuses host keys`;
|
|
370
377
|
kvStore.delete(key);
|
|
371
378
|
if (keyType === 'consensus') {
|
|
@@ -381,12 +388,12 @@ export function makeSwingStore(path, forceReset, options = {}) {
|
|
|
381
388
|
const hostKVStore = {
|
|
382
389
|
...kvStore,
|
|
383
390
|
set(key, value) {
|
|
384
|
-
const keyType =
|
|
391
|
+
const keyType = getKVStoreKeyType(key);
|
|
385
392
|
keyType === 'host' || Fail`hostKVStore requires host keys`;
|
|
386
393
|
kvStore.set(key, value);
|
|
387
394
|
},
|
|
388
395
|
delete(key) {
|
|
389
|
-
const keyType =
|
|
396
|
+
const keyType = getKVStoreKeyType(key);
|
|
390
397
|
keyType === 'host' || Fail`hostKVStore requires host keys`;
|
|
391
398
|
kvStore.delete(key);
|
|
392
399
|
},
|
|
@@ -515,7 +522,7 @@ export function makeSwingStore(path, forceReset, options = {}) {
|
|
|
515
522
|
stopTrace();
|
|
516
523
|
}
|
|
517
524
|
|
|
518
|
-
/** @type {
|
|
525
|
+
/** @type {SwingStoreInternal} */
|
|
519
526
|
const internal = harden({
|
|
520
527
|
dirPath: asFile ? null : path,
|
|
521
528
|
asFile,
|
package/src/transcriptStore.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
// @ts-check
|
|
2
|
-
import { Readable } from 'stream';
|
|
3
|
-
import { Buffer } from 'buffer';
|
|
2
|
+
import { Readable } from 'node:stream';
|
|
3
|
+
import { Buffer } from 'node:buffer';
|
|
4
4
|
import { Fail, q } from '@endo/errors';
|
|
5
5
|
import BufferLineTransform from '@agoric/internal/src/node/buffer-line-transform.js';
|
|
6
6
|
import { createSHA256 } from './hasher.js';
|
|
@@ -374,7 +374,7 @@ export function makeTranscriptStore(
|
|
|
374
374
|
* transcript.${vatID}.current export record are responsibility of the caller.
|
|
375
375
|
*
|
|
376
376
|
* @param {string} vatID
|
|
377
|
-
* @param {ReturnType<getCurrentSpanBounds>} bounds
|
|
377
|
+
* @param {ReturnType<typeof getCurrentSpanBounds>} bounds
|
|
378
378
|
* @returns {{deferredTask: Promise<void>}}
|
|
379
379
|
*/
|
|
380
380
|
function closeSpan(vatID, bounds) {
|
package/src/types-index.d.ts
CHANGED
|
@@ -3,7 +3,7 @@ export type {
|
|
|
3
3
|
SwingStoreKernelStorage,
|
|
4
4
|
SwingStoreHostStorage,
|
|
5
5
|
} from './swingStore.js';
|
|
6
|
-
export type { KVStore } from '
|
|
6
|
+
export type { KVStore } from '@agoric/internal/src/kv-store.js';
|
|
7
7
|
export type { BundleStore } from './bundleStore.js';
|
|
8
8
|
export type {
|
|
9
9
|
SnapStore,
|
package/src/util.js
CHANGED
|
@@ -1,12 +1,17 @@
|
|
|
1
|
-
import
|
|
2
|
-
import
|
|
1
|
+
import { Buffer } from 'node:buffer';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
import { Fail } from '@endo/errors';
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* @import {AnyIterable} from './exporter.js';
|
|
7
|
+
*/
|
|
3
8
|
|
|
4
9
|
/**
|
|
5
10
|
* This is a polyfill for the `buffer` function from Node's
|
|
6
11
|
* 'stream/consumers' package, which unfortunately only exists in newer versions
|
|
7
12
|
* of Node.
|
|
8
13
|
*
|
|
9
|
-
* @param {
|
|
14
|
+
* @param {AnyIterable<Uint8Array>} inStream
|
|
10
15
|
*/
|
|
11
16
|
export const buffer = async inStream => {
|
|
12
17
|
const chunks = [];
|
|
@@ -20,3 +25,16 @@ export function dbFileInDirectory(dirPath) {
|
|
|
20
25
|
const filePath = path.resolve(dirPath, 'swingstore.sqlite');
|
|
21
26
|
return filePath;
|
|
22
27
|
}
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* @param {string} key
|
|
31
|
+
*/
|
|
32
|
+
export function getKVStoreKeyType(key) {
|
|
33
|
+
typeof key === 'string' || Fail`key must be a string`;
|
|
34
|
+
if (key.startsWith('local.')) {
|
|
35
|
+
return 'local';
|
|
36
|
+
} else if (key.startsWith('host.')) {
|
|
37
|
+
return 'host';
|
|
38
|
+
}
|
|
39
|
+
return 'consensus';
|
|
40
|
+
}
|
package/test/bundles.test.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
// @ts-check
|
|
2
2
|
import test from 'ava';
|
|
3
|
-
import { Buffer } from 'buffer';
|
|
3
|
+
import { Buffer } from 'node:buffer';
|
|
4
4
|
import tmp from 'tmp';
|
|
5
5
|
import { makeTempDirFactory } from '@agoric/internal/src/tmpDir.js';
|
|
6
6
|
import { createSHA256 } from '../src/hasher.js';
|
|
@@ -9,6 +9,10 @@ import { makeSwingStoreExporter } from '../src/exporter.js';
|
|
|
9
9
|
import { importSwingStore } from '../src/importer.js';
|
|
10
10
|
import { buffer } from '../src/util.js';
|
|
11
11
|
|
|
12
|
+
/**
|
|
13
|
+
* @import {Bundle} from '../src/bundleStore.js';
|
|
14
|
+
*/
|
|
15
|
+
|
|
12
16
|
const tmpDir = makeTempDirFactory(tmp);
|
|
13
17
|
|
|
14
18
|
function makeB0ID(bundle) {
|
|
@@ -19,7 +23,7 @@ test('b0 format', t => {
|
|
|
19
23
|
const { kernelStorage } = initSwingStore();
|
|
20
24
|
const { bundleStore } = kernelStorage;
|
|
21
25
|
|
|
22
|
-
/** @type {
|
|
26
|
+
/** @type {Bundle} */
|
|
23
27
|
const b0A = { moduleFormat: 'nestedEvaluate', source: '1+1' };
|
|
24
28
|
const idA = makeB0ID(b0A);
|
|
25
29
|
bundleStore.addBundle(idA, b0A);
|
|
@@ -32,7 +36,7 @@ test('b0 format', t => {
|
|
|
32
36
|
});
|
|
33
37
|
t.falsy(bundleStore.hasBundle(idBogus));
|
|
34
38
|
|
|
35
|
-
/** @type {
|
|
39
|
+
/** @type {Bundle} */
|
|
36
40
|
const b0B = { moduleFormat: 'getExport', source: '1+1' };
|
|
37
41
|
const idB = makeB0ID(b0B);
|
|
38
42
|
t.throws(() => bundleStore.addBundle(idB, b0B), {
|
|
@@ -73,7 +77,7 @@ test('b0 export', async t => {
|
|
|
73
77
|
});
|
|
74
78
|
const { bundleStore } = kernelStorage;
|
|
75
79
|
|
|
76
|
-
/** @type {
|
|
80
|
+
/** @type {Bundle} */
|
|
77
81
|
const b0A = { moduleFormat: 'nestedEvaluate', source: '1+1' };
|
|
78
82
|
const idA = makeB0ID(b0A);
|
|
79
83
|
bundleStore.addBundle(idA, b0A);
|
package/test/deletion.test.js
CHANGED
|
@@ -14,6 +14,10 @@ import { makeArchiveSnapshot, makeArchiveTranscript } from '../src/archiver.js';
|
|
|
14
14
|
import { makeSwingStoreExporter } from '../src/exporter.js';
|
|
15
15
|
import { importSwingStore } from '../src/importer.js';
|
|
16
16
|
|
|
17
|
+
/**
|
|
18
|
+
* @import {ExecutionContext} from 'ava';
|
|
19
|
+
*/
|
|
20
|
+
|
|
17
21
|
const tmpDir = makeTempDirFactory(tmp);
|
|
18
22
|
|
|
19
23
|
async function* getSnapshotStream() {
|
|
@@ -219,7 +223,7 @@ const setupTranscript = async (t, keepTranscripts) => {
|
|
|
219
223
|
};
|
|
220
224
|
|
|
221
225
|
/**
|
|
222
|
-
* @param {
|
|
226
|
+
* @param {ExecutionContext} t
|
|
223
227
|
* @param {{ keepTranscripts: boolean }} config
|
|
224
228
|
*/
|
|
225
229
|
const execSlowTranscriptDeletion = async (t, { keepTranscripts }) => {
|
package/test/export.test.js
CHANGED
|
@@ -8,6 +8,11 @@ import { initSwingStore, makeSwingStoreExporter } from '../src/index.js';
|
|
|
8
8
|
|
|
9
9
|
import { getSnapshotStream, makeB0ID } from './util.js';
|
|
10
10
|
|
|
11
|
+
/**
|
|
12
|
+
* @import {ArtifactMode} from '../src/internal.js'
|
|
13
|
+
* @import {Bundle} from '../src/bundleStore.js';
|
|
14
|
+
*/
|
|
15
|
+
|
|
11
16
|
const tmpDir = makeTempDirFactory(tmp);
|
|
12
17
|
|
|
13
18
|
const rank = {
|
|
@@ -23,7 +28,7 @@ const snapshotData = 'snapshot data';
|
|
|
23
28
|
const snapHash =
|
|
24
29
|
'e7dee7266896538616b630a5da40a90e007726a383e005a9c9c5dd0c2daf9329';
|
|
25
30
|
|
|
26
|
-
/** @type {
|
|
31
|
+
/** @type {Bundle} */
|
|
27
32
|
const bundle0 = { moduleFormat: 'nestedEvaluate', source: '1+1' };
|
|
28
33
|
const bundle0ID = makeB0ID(bundle0);
|
|
29
34
|
|
|
@@ -86,7 +91,6 @@ const exportTest = test.macro(async (t, mode) => {
|
|
|
86
91
|
// historical transcript spans, and no historical snapshots
|
|
87
92
|
|
|
88
93
|
assert.typeof(mode, 'string');
|
|
89
|
-
/** @import {ArtifactMode} from '../src/internal.js' */
|
|
90
94
|
let artifactMode = /** @type {ArtifactMode} */ (mode);
|
|
91
95
|
if (mode === 'debug-on-pruned') {
|
|
92
96
|
artifactMode = 'debug';
|
package/test/exports.js
CHANGED
|
@@ -1,12 +1,17 @@
|
|
|
1
|
-
import { Buffer } from 'buffer';
|
|
1
|
+
import { Buffer } from 'node:buffer';
|
|
2
2
|
import { makeB0ID } from './util.js';
|
|
3
3
|
|
|
4
|
+
/**
|
|
5
|
+
* @import {Bundle} from '../src/bundleStore.js';
|
|
6
|
+
* @import {KVPair} from '../src/exporter.js';
|
|
7
|
+
*/
|
|
8
|
+
|
|
4
9
|
export const snapshotData = 'snapshot data';
|
|
5
10
|
// this snapHash was computed manually
|
|
6
11
|
export const snapHash =
|
|
7
12
|
'e7dee7266896538616b630a5da40a90e007726a383e005a9c9c5dd0c2daf9329';
|
|
8
13
|
|
|
9
|
-
/** @type {
|
|
14
|
+
/** @type {Bundle} */
|
|
10
15
|
export const bundle0 = { moduleFormat: 'nestedEvaluate', source: '1+1' };
|
|
11
16
|
export const bundle0ID = makeB0ID(bundle0);
|
|
12
17
|
|
|
@@ -81,7 +86,7 @@ export function makeExporter(exportData, artifacts) {
|
|
|
81
86
|
},
|
|
82
87
|
async *getExportData() {
|
|
83
88
|
for (const [key, value] of exportData.entries()) {
|
|
84
|
-
/** @type {
|
|
89
|
+
/** @type { KVPair } */
|
|
85
90
|
const pair = [key, value];
|
|
86
91
|
yield pair;
|
|
87
92
|
}
|
package/test/import.test.js
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
// @ts-check
|
|
2
2
|
|
|
3
|
-
import path from 'path';
|
|
4
|
-
import { createGunzip } from 'zlib';
|
|
5
|
-
import { Readable } from 'stream';
|
|
6
|
-
import { Buffer } from 'buffer';
|
|
3
|
+
import path from 'node:path';
|
|
4
|
+
import { createGunzip } from 'node:zlib';
|
|
5
|
+
import { Readable } from 'node:stream';
|
|
6
|
+
import { Buffer } from 'node:buffer';
|
|
7
7
|
|
|
8
8
|
import sqlite3 from 'better-sqlite3';
|
|
9
9
|
import test from 'ava';
|
package/tsconfig.json
CHANGED
package/src/kvStore.js
DELETED
|
@@ -1,173 +0,0 @@
|
|
|
1
|
-
// @ts-check
|
|
2
|
-
import { Fail } from '@endo/errors';
|
|
3
|
-
|
|
4
|
-
/**
|
|
5
|
-
* @typedef {{
|
|
6
|
-
* has: (key: string) => boolean,
|
|
7
|
-
* get: (key: string) => string | undefined,
|
|
8
|
-
* getNextKey: (previousKey: string) => string | undefined,
|
|
9
|
-
* set: (key: string, value: string, bypassHash?: boolean ) => void,
|
|
10
|
-
* delete: (key: string) => void,
|
|
11
|
-
* }} KVStore
|
|
12
|
-
*/
|
|
13
|
-
|
|
14
|
-
/**
|
|
15
|
-
* @param {string} key
|
|
16
|
-
*/
|
|
17
|
-
export function getKeyType(key) {
|
|
18
|
-
typeof key === 'string' || Fail`key must be a string`;
|
|
19
|
-
if (key.startsWith('local.')) {
|
|
20
|
-
return 'local';
|
|
21
|
-
} else if (key.startsWith('host.')) {
|
|
22
|
-
return 'host';
|
|
23
|
-
}
|
|
24
|
-
return 'consensus';
|
|
25
|
-
}
|
|
26
|
-
|
|
27
|
-
/**
|
|
28
|
-
* @param {object} db The SQLite database connection.
|
|
29
|
-
* @param {() => void} ensureTxn Called before mutating methods to establish a DB transaction
|
|
30
|
-
* @param {(...args: string[]) => void} trace Called after set/delete to record a debug log
|
|
31
|
-
* @returns { KVStore }
|
|
32
|
-
*/
|
|
33
|
-
|
|
34
|
-
export function makeKVStore(db, ensureTxn, trace) {
|
|
35
|
-
db.exec(`
|
|
36
|
-
CREATE TABLE IF NOT EXISTS kvStore (
|
|
37
|
-
key TEXT,
|
|
38
|
-
value TEXT,
|
|
39
|
-
PRIMARY KEY (key)
|
|
40
|
-
)
|
|
41
|
-
`);
|
|
42
|
-
|
|
43
|
-
const sqlKVGet = db.prepare(`
|
|
44
|
-
SELECT value
|
|
45
|
-
FROM kvStore
|
|
46
|
-
WHERE key = ?
|
|
47
|
-
`);
|
|
48
|
-
sqlKVGet.pluck(true);
|
|
49
|
-
|
|
50
|
-
/**
|
|
51
|
-
* Obtain the value stored for a given key.
|
|
52
|
-
*
|
|
53
|
-
* @param {string} key The key whose value is sought.
|
|
54
|
-
*
|
|
55
|
-
* @returns {string | undefined} the (string) value for the given key, or
|
|
56
|
-
* undefined if there is no such value.
|
|
57
|
-
*
|
|
58
|
-
* @throws if key is not a string.
|
|
59
|
-
*/
|
|
60
|
-
function get(key) {
|
|
61
|
-
typeof key === 'string' || Fail`key must be a string`;
|
|
62
|
-
return sqlKVGet.get(key);
|
|
63
|
-
}
|
|
64
|
-
|
|
65
|
-
const sqlKVGetNextKey = db.prepare(`
|
|
66
|
-
SELECT key
|
|
67
|
-
FROM kvStore
|
|
68
|
-
WHERE key > ?
|
|
69
|
-
LIMIT 1
|
|
70
|
-
`);
|
|
71
|
-
sqlKVGetNextKey.pluck(true);
|
|
72
|
-
|
|
73
|
-
/**
|
|
74
|
-
* getNextKey enables callers to iterate over all keys within a
|
|
75
|
-
* given range. To build an iterator of all keys from start
|
|
76
|
-
* (inclusive) to end (exclusive), do:
|
|
77
|
-
*
|
|
78
|
-
* function* iterate(start, end) {
|
|
79
|
-
* if (kvStore.has(start)) {
|
|
80
|
-
* yield start;
|
|
81
|
-
* }
|
|
82
|
-
* let prev = start;
|
|
83
|
-
* while (true) {
|
|
84
|
-
* let next = kvStore.getNextKey(prev);
|
|
85
|
-
* if (!next || next >= end) {
|
|
86
|
-
* break;
|
|
87
|
-
* }
|
|
88
|
-
* yield next;
|
|
89
|
-
* prev = next;
|
|
90
|
-
* }
|
|
91
|
-
* }
|
|
92
|
-
*
|
|
93
|
-
* @param {string} previousKey The key returned will always be later than this one.
|
|
94
|
-
*
|
|
95
|
-
* @returns {string | undefined} a key string, or undefined if we reach the end of the store
|
|
96
|
-
*
|
|
97
|
-
* @throws if previousKey is not a string
|
|
98
|
-
*/
|
|
99
|
-
|
|
100
|
-
function getNextKey(previousKey) {
|
|
101
|
-
typeof previousKey === 'string' || Fail`previousKey must be a string`;
|
|
102
|
-
return sqlKVGetNextKey.get(previousKey);
|
|
103
|
-
}
|
|
104
|
-
|
|
105
|
-
/**
|
|
106
|
-
* Test if the state contains a value for a given key.
|
|
107
|
-
*
|
|
108
|
-
* @param {string} key The key that is of interest.
|
|
109
|
-
*
|
|
110
|
-
* @returns {boolean} true if a value is stored for the key, false if not.
|
|
111
|
-
*
|
|
112
|
-
* @throws if key is not a string.
|
|
113
|
-
*/
|
|
114
|
-
function has(key) {
|
|
115
|
-
typeof key === 'string' || Fail`key must be a string`;
|
|
116
|
-
return get(key) !== undefined;
|
|
117
|
-
}
|
|
118
|
-
|
|
119
|
-
const sqlKVSet = db.prepare(`
|
|
120
|
-
INSERT INTO kvStore (key, value)
|
|
121
|
-
VALUES (?, ?)
|
|
122
|
-
ON CONFLICT DO UPDATE SET value = excluded.value
|
|
123
|
-
`);
|
|
124
|
-
|
|
125
|
-
/**
|
|
126
|
-
* Store a value for a given key. The value will replace any prior value if
|
|
127
|
-
* there was one.
|
|
128
|
-
*
|
|
129
|
-
* @param {string} key The key whose value is being set.
|
|
130
|
-
* @param {string} value The value to set the key to.
|
|
131
|
-
*
|
|
132
|
-
* @throws if either parameter is not a string.
|
|
133
|
-
*/
|
|
134
|
-
function set(key, value) {
|
|
135
|
-
typeof key === 'string' || Fail`key must be a string`;
|
|
136
|
-
typeof value === 'string' || Fail`value must be a string`;
|
|
137
|
-
// synchronous read after write within a transaction is safe
|
|
138
|
-
// The transaction's overall success will be awaited during commit
|
|
139
|
-
ensureTxn();
|
|
140
|
-
sqlKVSet.run(key, value);
|
|
141
|
-
trace('set', key, value);
|
|
142
|
-
}
|
|
143
|
-
|
|
144
|
-
const sqlKVDel = db.prepare(`
|
|
145
|
-
DELETE FROM kvStore
|
|
146
|
-
WHERE key = ?
|
|
147
|
-
`);
|
|
148
|
-
|
|
149
|
-
/**
|
|
150
|
-
* Remove any stored value for a given key. It is permissible for there to
|
|
151
|
-
* be no existing stored value for the key.
|
|
152
|
-
*
|
|
153
|
-
* @param {string} key The key whose value is to be deleted
|
|
154
|
-
*
|
|
155
|
-
* @throws if key is not a string.
|
|
156
|
-
*/
|
|
157
|
-
function del(key) {
|
|
158
|
-
typeof key === 'string' || Fail`key must be a string`;
|
|
159
|
-
ensureTxn();
|
|
160
|
-
sqlKVDel.run(key);
|
|
161
|
-
trace('del', key);
|
|
162
|
-
}
|
|
163
|
-
|
|
164
|
-
const kvStore = {
|
|
165
|
-
has,
|
|
166
|
-
get,
|
|
167
|
-
getNextKey,
|
|
168
|
-
set,
|
|
169
|
-
delete: del,
|
|
170
|
-
};
|
|
171
|
-
|
|
172
|
-
return kvStore;
|
|
173
|
-
}
|