@agoric/swing-store 0.9.2-other-dev-3eb1a1d.0 → 0.9.2-other-dev-d15096d.0.d15096d

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@agoric/swing-store",
-  "version": "0.9.2-other-dev-3eb1a1d.0+3eb1a1d",
+  "version": "0.9.2-other-dev-d15096d.0.d15096d",
   "description": "Persistent storage for SwingSet",
   "type": "module",
   "main": "./src/index.js",
@@ -13,27 +13,29 @@
   "scripts": {
     "build": "exit 0",
     "test": "ava",
-    "test:c8": "c8 --all $C8_OPTIONS ava",
+    "test:c8": "c8 --all ${C8_OPTIONS:-} ava",
     "test:xs": "exit 0",
     "lint-fix": "yarn lint:eslint --fix",
-    "lint": "run-s --continue-on-error lint:*",
-    "lint:types": "tsc",
-    "lint:eslint": "eslint ."
+    "lint": "yarn run -T run-s --continue-on-error 'lint:*'",
+    "lint:types": "yarn run -T tsc",
+    "lint:eslint": "yarn run -T eslint ."
   },
   "dependencies": {
-    "@agoric/internal": "0.3.3-other-dev-3eb1a1d.0+3eb1a1d",
-    "@endo/base64": "^1.0.9",
-    "@endo/bundle-source": "^3.5.0",
-    "@endo/check-bundle": "^1.0.12",
-    "@endo/errors": "^1.2.8",
-    "@endo/nat": "^5.0.13",
-    "better-sqlite3": "^9.1.1"
+    "@agoric/internal": "0.3.3-other-dev-d15096d.0.d15096d",
+    "@endo/base64": "^1.0.12",
+    "@endo/bundle-source": "^4.1.2",
+    "@endo/check-bundle": "^1.0.17",
+    "@endo/errors": "^1.2.13",
+    "@endo/nat": "^5.1.3",
+    "better-sqlite3": "^10.1.0"
   },
   "devDependencies": {
-    "@endo/init": "^1.1.7",
-    "@types/better-sqlite3": "^7.6.9",
+    "@endo/init": "^1.1.12",
+    "@endo/promise-kit": "^1.1.13",
+    "@endo/stream": "^1.2.13",
+    "@types/better-sqlite3": "^7.6.13",
     "ava": "^5.3.0",
-    "c8": "^10.1.2",
+    "c8": "^10.1.3",
     "tmp": "^0.2.1"
   },
   "publishConfig": {
@@ -49,7 +51,10 @@
     "timeout": "2m"
   },
   "typeCoverage": {
-    "atLeast": 79.04
+    "atLeast": 80.49
   },
-  "gitHead": "3eb1a1d2d75b2b4a94807cd3bf759bc9fc531f05"
+  "engines": {
+    "node": "^20.9 || ^22.11"
+  },
+  "gitHead": "d15096dc4ff8b96e9b6cd11954c20d3a9efbb393"
 }

package/src/archiver.js

@@ -1,11 +1,16 @@
 import { finished as streamFinishedCallback, Readable } from 'node:stream';
+import { pipeline } from 'node:stream/promises';
 import { promisify } from 'node:util';
 import { createGzip } from 'node:zlib';
 import { withDeferredCleanup } from '@agoric/internal';
 
+/**
+ * @import {AnyIterable} from './exporter.js';
+ */
+
 const streamFinished = promisify(streamFinishedCallback);
 
-/*
+/**
  * @param {string} dirPath
  * @param {object} powers
  * @param {Pick<import('fs'), 'createWriteStream' | 'mkdirSync' | 'renameSync'>} powers.fs
@@ -15,6 +20,11 @@ const streamFinished = promisify(streamFinishedCallback);
 export const makeArchiveSnapshot = (dirPath, powers) => {
   const { fs, path, tmp } = powers;
   fs.mkdirSync(dirPath, { recursive: true });
+  /**
+   * @param {string} name
+   * @param {AnyIterable<Uint8Array>} gzData
+   * @returns {Promise<void>}
+   */
   const archiveSnapshot = (name, gzData) => {
     const destPath = path.join(dirPath, `${name}.gz`);
     return withDeferredCleanup(async addCleanup => {
@@ -28,7 +38,7 @@ export const makeArchiveSnapshot = (dirPath, powers) => {
         postfix: '.gz.tmp',
         detachDescriptor: true,
       });
-      addCleanup(() => removeCallback());
+      addCleanup(async () => removeCallback());
       const writer = fs.createWriteStream('', { fd, flush: true });
       const reader = Readable.from(gzData);
       const destroyReader = promisify(reader.destroy.bind(reader));
@@ -42,7 +52,7 @@ export const makeArchiveSnapshot = (dirPath, powers) => {
 };
 harden(makeArchiveSnapshot);
 
-/*
+/**
  * @param {string} dirPath
  * @param {object} powers
  * @param {Pick<import('fs'), 'createWriteStream' | 'mkdirSync' | 'renameSync'>} powers.fs
@@ -52,6 +62,11 @@ harden(makeArchiveSnapshot);
 export const makeArchiveTranscript = (dirPath, powers) => {
   const { fs, path, tmp } = powers;
   fs.mkdirSync(dirPath, { recursive: true });
+  /**
+   * @param {string} spanName
+   * @param {AnyIterable<Uint8Array>} entries
+   * @returns {Promise<void>}
+   */
   const archiveTranscript = (spanName, entries) => {
     const destPath = path.join(dirPath, `${spanName}.gz`);
     return withDeferredCleanup(async addCleanup => {
@@ -65,15 +80,12 @@ export const makeArchiveTranscript = (dirPath, powers) => {
         postfix: '.gz.tmp',
         detachDescriptor: true,
       });
-      addCleanup(() => removeCallback());
-      const writer = fs.createWriteStream('', { fd, flush: true });
-      const gzip = createGzip();
-      gzip.pipe(writer);
-      const reader = Readable.from(entries);
-      const destroyReader = promisify(reader.destroy.bind(reader));
-      addCleanup(() => destroyReader(null));
-      reader.pipe(gzip);
-      await streamFinished(gzip);
+      addCleanup(async () => removeCallback());
+      await pipeline(
+        entries,
+        createGzip(),
+        fs.createWriteStream('', { fd, flush: true }),
+      );
       fs.renameSync(tmpName, destPath);
     });
   };

package/src/assertComplete.js

@@ -1,12 +1,17 @@
 /**
- * @param {import('./internal.js').SwingStoreInternal} internal
- * @param {Omit<import('./internal.js').ArtifactMode, 'debug'>} checkMode
+ * @param {Pick<SwingStoreInternal, 'bundleStore' | 'transcriptStore' | 'snapStore'>} internal
+ * @param {Exclude<ArtifactMode, 'debug'>} checkMode
  * @returns {void}
  */
 export function assertComplete(internal, checkMode) {
   // every bundle must be populated
   internal.bundleStore.assertComplete(checkMode);
 
+  /**
+   * @import {SwingStoreInternal} from './internal.js';
+   * @import {ArtifactMode} from './internal.js';
+   */
+
   // every 'isCurrent' transcript span must have all items
   // TODO: every vat with any data must have a isCurrent transcript
   // span

package/src/bundleStore.js

@@ -15,8 +15,8 @@ import { createSHA256 } from './hasher.js';
  * @typedef { EndoZipBase64Bundle | GetExportBundle | NestedEvaluateBundle } Bundle
  */
 /**
- * @typedef { import('./exporter.js').SwingStoreExporter } SwingStoreExporter
- * @typedef { import('./internal.js').ArtifactMode } ArtifactMode
+ * @import {SwingStoreExporter} from './exporter.js';
+ * @import {ArtifactMode} from './internal.js';
  *
  * @typedef {{
  *   addBundle: (bundleID: string, bundle: Bundle) => void;
@@ -42,7 +42,7 @@ import { createSHA256 } from './hasher.js';
  *
  */
 
-function bundleIDFromName(name) {
+export const bundleIDFromName = name => {
   typeof name === 'string' || Fail`artifact name must be a string`;
   const [tag, ...pieces] = name.split('.');
   if (tag !== 'bundle' || pieces.length !== 1) {
@@ -52,7 +52,8 @@ function bundleIDFromName(name) {
   }
   const bundleID = pieces[0];
   return bundleID;
-}
+};
+harden(bundleIDFromName);
 
 /**
  * @param {*} db

package/src/exporter.js

@@ -12,12 +12,12 @@ import { assertComplete } from './assertComplete.js';
 import { validateArtifactMode } from './internal.js';
 
 /**
- * @template T
- * @typedef  { Iterable<T> | AsyncIterable<T> } AnyIterable
+ * @import {ArtifactMode} from './internal.js';
  */
+
 /**
  * @template T
- * @typedef  { IterableIterator<T> | AsyncIterableIterator<T> } AnyIterableIterator
+ * @typedef  { Iterable<T> | AsyncIterable<T> } AnyIterable
  */
 
 /**
@@ -41,7 +41,7 @@ import { validateArtifactMode } from './internal.js';
  * Retrieve a value from the "host" portion of the kvStore, just like
  * hostStorage.hostKVStore.get() would do.
  *
- * @property {() => AnyIterableIterator<KVPair>} getExportData
+ * @property {() => AnyIterable<KVPair>} getExportData
  *
  * Get a full copy of the first-stage export data (key-value pairs) from the
  * swingStore. This represents both the contents of the KVStore (excluding host
@@ -56,7 +56,7 @@ import { validateArtifactMode } from './internal.js';
  * - transcript.${vatID}.${startPos} = ${{ vatID, startPos, endPos, hash }}
  * - transcript.${vatID}.current = ${{ vatID, startPos, endPos, hash }}
  *
- * @property {() => AnyIterableIterator<string>} getArtifactNames
+ * @property {() => AnyIterable<string>} getArtifactNames
  *
  * Get a list of name of artifacts available from the swingStore.  A name
  * returned by this method guarantees that a call to `getArtifact` on the same
@@ -69,7 +69,7 @@ import { validateArtifactMode } from './internal.js';
  * - snapshot.${vatID}.${snapPos}
  * - bundle.${bundleID}
  *
- * @property {(name: string) => AnyIterableIterator<Uint8Array>} getArtifact
+ * @property {(name: string) => AnyIterable<Uint8Array>} getArtifact
  *
  * Retrieve an artifact by name as a sequence of binary chunks.  May throw if
  * the artifact is not available, which can occur if the artifact is historical
@@ -83,7 +83,7 @@ import { validateArtifactMode } from './internal.js';
 
 /**
  * @typedef { object } ExportSwingStoreOptions
- * @property { import('./internal.js').ArtifactMode } [artifactMode]  What artifacts should/must the exporter provide?
+ * @property { ArtifactMode } [artifactMode]  What artifacts should/must the exporter provide?
  */
 
 /**

package/src/importer.js

@@ -5,9 +5,15 @@ import { buffer } from './util.js';
 import { validateArtifactMode } from './internal.js';
 import { assertComplete } from './assertComplete.js';
 
+/**
+ * @import {ArtifactMode} from './internal.js';
+ * @import {SwingStoreExporter} from './exporter.js';
+ * @import {SwingStore} from './swingStore.js';
+ */
+
 /**
  * @typedef { object } ImportSwingStoreOptions
- * @property { import('./internal.js').ArtifactMode } [artifactMode]  What artifacts should the importer use and require?
+ * @property { ArtifactMode } [artifactMode]  What artifacts should the importer use and require?
  */
 
 /**
@@ -17,10 +23,10 @@ import { assertComplete } from './assertComplete.js';
  * returned swingStore is not suitable for execution, and thus only contains
  * the host facet for committing the populated swingStore.
  *
- * @param {import('./exporter.js').SwingStoreExporter} exporter
+ * @param {SwingStoreExporter} exporter
  * @param {string | null} [dirPath]
  * @param {ImportSwingStoreOptions} [options]
- * @returns {Promise<Pick<import('./swingStore.js').SwingStore, 'hostStorage' | 'debug'>>}
+ * @returns {Promise<Pick<SwingStore, 'hostStorage' | 'debug'>>}
  */
 export async function importSwingStore(exporter, dirPath = null, options = {}) {
   if (dirPath && typeof dirPath !== 'string') {

package/src/index.js

@@ -4,10 +4,12 @@ export { importSwingStore } from './importer.js';
 
 export { makeArchiveSnapshot, makeArchiveTranscript } from './archiver.js';
 
-// temporary, for the benefit of SwingSet/misc-tools/replay-transcript.js
+// for the benefit of tools like SwingSet/misc-tools/replay-transcript.js
+export { makeKVStore, getKeyType } from './kvStore.js';
+export { makeTranscriptStore } from './transcriptStore.js';
 export { makeSnapStore } from './snapStore.js';
-// and less temporary, for SwingSet/test/vat-warehouse/test-reload-snapshot.js
 export { makeSnapStoreIO } from './snapStoreIO.js';
+export { makeBundleStore, bundleIDFromName } from './bundleStore.js';
 
 // eslint-disable-next-line import/export
 export * from './types-index.js';

package/src/internal.js

@@ -1,12 +1,18 @@
 import { Fail, q } from '@endo/errors';
 
 /**
- * @typedef { import('./snapStore.js').SnapStoreInternal } SnapStoreInternal
- * @typedef { import('./transcriptStore.js').TranscriptStoreInternal } TranscriptStoreInternal
- * @typedef { import('./bundleStore.js').BundleStoreInternal } BundleStoreInternal
+ * @import {KVStore} from './kvStore.js';
+ */
+
+/**
+ * @import {SnapStoreInternal} from './snapStore.js';
+ * @import {TranscriptStoreInternal} from './transcriptStore.js';
+ * @import {BundleStoreInternal} from './bundleStore.js';
  *
  * @typedef {{
  *    dirPath: string | null,
+ *    db: ReturnType<import('better-sqlite3')>,
+ *    kvStore: KVStore,
  *    transcriptStore: TranscriptStoreInternal,
  *    snapStore: SnapStoreInternal,
  *    bundleStore: BundleStoreInternal,

package/src/kvStore.js

@@ -27,7 +27,7 @@ export function getKeyType(key) {
 /**
  * @param {object} db  The SQLite database connection.
  * @param {() => void} ensureTxn  Called before mutating methods to establish a DB transaction
- * @param {(...args: string[]) => void} trace  Called after sets/gets to record a debug log
+ * @param {(...args: string[]) => void} trace  Called after set/delete to record a debug log
  * @returns { KVStore }
  */
 

package/src/repairMetadata.js

@@ -1,6 +1,12 @@
 import { Fail, q } from '@endo/errors';
 import { assertComplete } from './assertComplete.js';
 
+/**
+ * @import {SwingStoreInternal} from './internal.js';
+ * @import {SwingStoreExporter} from './exporter.js';
+ * @import {ArtifactMode} from './internal.js';
+ */
+
 /**
  * Given a pre-existing swingstore and a SwingStoreExporter, read in
  * all the metadata from the exporter and use it to regenerate any
@@ -24,8 +30,8 @@ import { assertComplete } from './assertComplete.js';
  * an open transaction. The caller is responsible for calling
  * `hostStorage.commit()` when they are ready.
  *
- * @param {import('./internal.js').SwingStoreInternal} internal
- * @param {import('./exporter.js').SwingStoreExporter} exporter
+ * @param {SwingStoreInternal} internal
+ * @param {SwingStoreExporter} exporter
  * @returns {Promise<void>}
  */
 export async function doRepairMetadata(internal, exporter) {
@@ -60,7 +66,7 @@ export async function doRepairMetadata(internal, exporter) {
   }
 
   // and do a completeness check
-  /** @type { import('./internal.js').ArtifactMode } */
+  /** @type { ArtifactMode } */
   const artifactMode = 'operational';
   assertComplete(internal, artifactMode);
   await exporter.close();

package/src/snapStore.js

@@ -7,6 +7,12 @@ import { Fail, q } from '@endo/errors';
 import { withDeferredCleanup } from '@agoric/internal';
 import { buffer } from './util.js';
 
+/**
+ * @import { AnyIterable, SwingStoreExporter } from './exporter.js';
+ * @import { ArtifactMode } from './internal.js';
+ * @import {makeMeasureSeconds} from '@agoric/internal';
+ */
+
 /**
  * @typedef {object} SnapshotResult
  * @property {string} hash sha256 hash of (uncompressed) snapshot
@@ -26,13 +32,6 @@ import { buffer } from './util.js';
  */
 
 /**
- * @import {AnyIterableIterator} from './exporter.js'
- */
-
-/**
- * @typedef { import('./exporter.js').SwingStoreExporter } SwingStoreExporter
- * @typedef { import('./internal.js').ArtifactMode } ArtifactMode
- *
  * @typedef {{
  *   loadSnapshot: (vatID: string) => AsyncIterableIterator<Uint8Array>,
  *   saveSnapshot: (vatID: string, snapPos: number, snapshotStream: AsyncIterable<Uint8Array>) => Promise<SnapshotResult>,
@@ -47,17 +46,31 @@ import { buffer } from './util.js';
  *   getExportRecords: (includeHistorical: boolean) => IterableIterator<readonly [key: string, value: string]>,
  *   getArtifactNames: (artifactMode: ArtifactMode) => AsyncIterableIterator<string>,
  *   importSnapshotRecord: (key: string, value: string) => void,
- *   populateSnapshot: (name: string, makeChunkIterator: () => AnyIterableIterator<Uint8Array>, options: { artifactMode: ArtifactMode }) => Promise<void>,
+ *   populateSnapshot: (name: string, makeChunkIterator: () => AnyIterable<Uint8Array>, options: { artifactMode: ArtifactMode }) => Promise<void>,
  *   assertComplete: (checkMode: Omit<ArtifactMode, 'debug'>) => void,
  *   repairSnapshotRecord: (key: string, value: string) => void,
  * }} SnapStoreInternal
  *
+ * @typedef {object} SnapshotRow
+ * @property {string} vatID
+ * @property {number} snapPos
+ * @property {string} [hash]
+ * @property {number | null} [inUse]
+ * @property {number} uncompressedSize
+ * @property {number} compressedSize
+ *
  * @typedef {{
  *   hasHash: (vatID: string, hash: string) => boolean,
- *   dumpSnapshots: (includeHistorical?: boolean) => {},
+ *   listAllSnapshots: () => Iterable<SnapshotRow>,
+ *   dumpSnapshots: (includeHistorical?: boolean) => Record<string, Array<{snapPos: number, hash: string, compressedSnapshot: Buffer, inUse: (null | 0 | 1)}>>,
  *   deleteSnapshotByHash: (vatID: string, hash: string) => void,
  * }} SnapStoreDebug
  *
+ * @callback SnapshotCallback
+ * Called with the gzipped contents of a new heap snapshot.
+ * @param {string} name  an export key, e.g. `snapshot.${vatID}.${deliveryCount}`
+ * @param {Parameters<typeof Readable.from>[0]} compressedData
+ * @returns {Promise<void>}
  */
 
 const finished = promisify(finishedCallback);
@@ -66,12 +79,12 @@ const finished = promisify(finishedCallback);
  * @param {*} db
  * @param {() => void} ensureTxn
  * @param {{
- *   measureSeconds: ReturnType<typeof import('@agoric/internal').makeMeasureSeconds>,
+ *   measureSeconds: ReturnType<typeof makeMeasureSeconds>,
  * }} io
  * @param {(key: string, value: string | undefined) => void} noteExport
  * @param {object} [options]
  * @param {boolean | undefined} [options.keepSnapshots]
- * @param {(name: string, compressedData: Parameters<import('stream').Readable.from>[0]) => Promise<void>} [options.archiveSnapshot]
+ * @param {SnapshotCallback} [options.archiveSnapshot]
  * @returns {SnapStore & SnapStoreInternal & SnapStoreDebug}
  */
 export function makeSnapStore(
@@ -605,7 +618,7 @@ export function makeSnapStore(
 
   /**
    * @param {string} name  Artifact name of the snapshot
-   * @param {() => AnyIterableIterator<Uint8Array>} makeChunkIterator  get an iterator of snapshot byte chunks
+   * @param {() => AnyIterable<Uint8Array>} makeChunkIterator  get an iterator of snapshot byte chunks
    * @param {object} options
    * @param {ArtifactMode} options.artifactMode
    * @returns {Promise<void>}
@@ -677,7 +690,7 @@ export function makeSnapStore(
 
   /**
    * debug function to list all snapshots
-   *
+   * @returns {Iterable<SnapshotRow>}
    */
   function* listAllSnapshots() {
     yield* sqlListAllSnapshots.iterate();
@@ -706,6 +719,7 @@ export function makeSnapStore(
     const sql = includeHistorical
       ? sqlDumpAllSnapshots
       : sqlDumpCurrentSnapshots;
+    /** @type {Record<string, Array<{snapPos: number, hash: string, compressedSnapshot: Buffer, inUse: (null | 0 | 1)}>>} */
     const dump = {};
     for (const row of sql.iterate()) {
       const { vatID, snapPos, hash, compressedSnapshot, inUse } = row;