@agoric/swing-store 0.9.2-dev-bf1e645.0 → 0.9.2-dev-07f6885.0

package/docs/transcriptstore.md

@@ -32,14 +32,14 @@ Pruned spans are not available for export artifacts, of course, because the data
 
 Every transcript span, both current and historic, gets an export-data record. The record name is different for the two types of spans.
 
-Historical spans, which are "closed" and no longer growing, use a record name of
-`transcript.${vatID}.${startPos}`, where `startPos` is the delivery number of the first delivery included in the span. The value is a JSON-serialized record of `{ vatID, startPos, endPos, hash, isCurrent, incarnation }` (where `isCurrent = 0`).
+Historical spans, which are "closed" and no longer growing, use a record name of `transcript.${vatID}.${startPos}.${endPos}`, where `startPos` is the delivery number of the first delivery included in the span and `endPos` is the the delivery number of the first delivery included in the **next** span (i.e., the former is an inclusive lower bound and the latter is an exclusive upper bound).
+The value is a JSON-serialized record of `{ vatID, startPos, endPos, hash, isCurrent, incarnation }` (where `isCurrent = 0`).
 
 The current span, if any, uses a record name of `transcript.${vatID}.current`, and has the same value as historical spans (except `isCurrent = 1`). Current spans are growing: new transcript items are added as more deliveries are made, until the span is closed off (becomes historical) and replaced with a new current span. There is at most one current span per vatID.
 
 The available export *artifacts* will depend upon the export mode, and upon the swingstore's `keepTranscripts` setting. Each export artifact corresponds to a single span, and the artifact names are always `transcript.${vatID}.${startPos}.${endPos}` (for both historical and current spans).
 
-In the most-minimal `operational` mode, the export includes one artifact for each active (non-terminated) vat: just the current span. If `keepTranscripts` is true, these will be the only available artifacts anyways.
+In the most-minimal `operational` mode, the export includes one artifact for each active (non-terminated) vat: just the current span. If `keepTranscripts` is not true, these will be the only available artifacts anyways.
 
 The `replay` mode includes all spans for each vat's current incarnation, but omits spans from earlier incarnations. The `archival` mode includes all spans from all incarnations.
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@agoric/swing-store",
-  "version": "0.9.2-dev-bf1e645.0+bf1e645",
+  "version": "0.9.2-dev-07f6885.0+07f6885",
   "description": "Persistent storage for SwingSet",
   "type": "module",
   "main": "./src/index.js",
@@ -21,7 +21,7 @@
     "lint:eslint": "eslint ."
   },
   "dependencies": {
-    "@agoric/internal": "0.3.3-dev-bf1e645.0+bf1e645",
+    "@agoric/internal": "0.3.3-dev-07f6885.0+07f6885",
     "@endo/base64": "^1.0.7",
     "@endo/bundle-source": "^3.4.0",
     "@endo/check-bundle": "^1.0.9",
@@ -51,5 +51,5 @@
   "typeCoverage": {
     "atLeast": 78.77
   },
-  "gitHead": "bf1e6453eee701f3e465120a03877cf44d458cc6"
+  "gitHead": "07f6885c560333981d6564581b2a00735a6c428c"
 }

package/src/archiver.js

@@ -0,0 +1,80 @@
+import { finished as streamFinishedCallback, Readable } from 'node:stream';
+import { promisify } from 'node:util';
+import { createGzip } from 'node:zlib';
+import { withDeferredCleanup } from '@agoric/internal';
+
+const streamFinished = promisify(streamFinishedCallback);
+
+/*
+ * @param {string} dirPath
+ * @param {object} powers
+ * @param {Pick<import('fs'), 'createWriteStream' | 'mkdirSync' | 'renameSync'>} powers.fs
+ * @param {Pick<import('path'), 'join'>} powers.path
+ * @param {Pick<import('tmp'), 'fileSync'>} powers.tmp
+ */
+export const makeArchiveSnapshot = (dirPath, powers) => {
+  const { fs, path, tmp } = powers;
+  fs.mkdirSync(dirPath, { recursive: true });
+  const archiveSnapshot = (name, gzData) => {
+    const destPath = path.join(dirPath, `${name}.gz`);
+    return withDeferredCleanup(async addCleanup => {
+      const {
+        name: tmpName,
+        fd,
+        removeCallback,
+      } = tmp.fileSync({
+        prefix: name,
+        postfix: '.gz',
+        detachDescriptor: true,
+      });
+      addCleanup(() => removeCallback());
+      const writer = fs.createWriteStream('', { fd, flush: true });
+      const reader = Readable.from(gzData);
+      const destroyReader = promisify(reader.destroy.bind(reader));
+      addCleanup(() => destroyReader(null));
+      reader.pipe(writer);
+      await streamFinished(writer);
+      fs.renameSync(tmpName, destPath);
+    });
+  };
+  return archiveSnapshot;
+};
+harden(makeArchiveSnapshot);
+
+/*
+ * @param {string} dirPath
+ * @param {object} powers
+ * @param {Pick<import('fs'), 'createWriteStream' | 'mkdirSync' | 'renameSync'>} powers.fs
+ * @param {Pick<import('path'), 'join'>} powers.path
+ * @param {Pick<import('tmp'), 'fileSync'>} powers.tmp
+ */
+export const makeArchiveTranscript = (dirPath, powers) => {
+  const { fs, path, tmp } = powers;
+  fs.mkdirSync(dirPath, { recursive: true });
+  const archiveTranscript = (spanName, entries) => {
+    const destPath = path.join(dirPath, `${spanName}.gz`);
+    return withDeferredCleanup(async addCleanup => {
+      const {
+        name: tmpName,
+        fd,
+        removeCallback,
+      } = tmp.fileSync({
+        prefix: spanName,
+        postfix: '.gz',
+        detachDescriptor: true,
+      });
+      addCleanup(() => removeCallback());
+      const writer = fs.createWriteStream('', { fd, flush: true });
+      const gzip = createGzip();
+      gzip.pipe(writer);
+      const reader = Readable.from(entries);
+      const destroyReader = promisify(reader.destroy.bind(reader));
+      addCleanup(() => destroyReader(null));
+      reader.pipe(gzip);
+      await streamFinished(gzip);
+      fs.renameSync(tmpName, destPath);
+    });
+  };
+  return archiveTranscript;
+};
+harden(makeArchiveTranscript);

package/src/index.js

@@ -2,6 +2,8 @@ export { initSwingStore, openSwingStore, isSwingStore } from './swingStore.js';
 export { makeSwingStoreExporter } from './exporter.js';
 export { importSwingStore } from './importer.js';
 
+export { makeArchiveSnapshot, makeArchiveTranscript } from './archiver.js';
+
 // temporary, for the benefit of SwingSet/misc-tools/replay-transcript.js
 export { makeSnapStore } from './snapStore.js';
 // and less temporary, for SwingSet/test/vat-warehouse/test-reload-snapshot.js

package/src/snapStore.js

@@ -4,10 +4,7 @@ import { finished as finishedCallback, PassThrough, Readable } from 'stream';
 import { promisify } from 'util';
 import { createGzip, createGunzip } from 'zlib';
 import { Fail, q } from '@endo/errors';
-import {
-  aggregateTryFinally,
-  PromiseAllOrErrors,
-} from '@agoric/internal/src/node/utils.js';
+import { withDeferredCleanup } from '@agoric/internal';
 import { buffer } from './util.js';
 
 /**
@@ -17,6 +14,7 @@ import { buffer } from './util.js';
  * @property {number} dbSaveSeconds time to write snapshot in DB
  * @property {number} compressedSize size of (compressed) snapshot
  * @property {number} compressSeconds time to generate and compress the snapshot
+ * @property {number} [archiveWriteSeconds] time to write an archive to disk (if applicable)
  */
 
 /**
@@ -73,6 +71,7 @@ const finished = promisify(finishedCallback);
  * @param {(key: string, value: string | undefined) => void} noteExport
  * @param {object} [options]
  * @param {boolean | undefined} [options.keepSnapshots]
+ * @param {(name: string, compressedData: Parameters<import('stream').Readable.from>[0]) => Promise<void>} [options.archiveSnapshot]
  * @returns {SnapStore & SnapStoreInternal & SnapStoreDebug}
  */
 export function makeSnapStore(
@@ -80,7 +79,7 @@ export function makeSnapStore(
   ensureTxn,
   { measureSeconds },
   noteExport = () => {},
-  { keepSnapshots = false } = {},
+  { keepSnapshots = false, archiveSnapshot } = {},
 ) {
   db.exec(`
     CREATE TABLE IF NOT EXISTS snapshots (
@@ -196,7 +195,8 @@ export function makeSnapStore(
 
   /**
    * Generates a new XS heap snapshot, stores a gzipped copy of it into the
-   * snapshots table, and reports information about the process, including
+   * snapshots table (and also to an archiveSnapshot callback if provided for
+   * e.g. disk archival), and reports information about the process, including
    * snapshot size and timing metrics.
    *
    * @param {string} vatID
@@ -205,74 +205,62 @@ export function makeSnapStore(
    * @returns {Promise<SnapshotResult>}
    */
   async function saveSnapshot(vatID, snapPos, snapshotStream) {
-    const cleanup = [];
-    return aggregateTryFinally(
-      async () => {
-        const hashStream = createHash('sha256');
-        const gzip = createGzip();
-        let compressedSize = 0;
-        let uncompressedSize = 0;
-
-        const { duration: compressSeconds, result: compressedSnapshot } =
-          await measureSeconds(async () => {
-            const snapReader = Readable.from(snapshotStream);
-            cleanup.push(
-              () =>
-                new Promise((resolve, reject) =>
-                  snapReader.destroy(
-                    null,
-                    // @ts-expect-error incorrect types
-                    err => (err ? reject(err) : resolve()),
-                  ),
-                ),
-            );
-
-            snapReader.on('data', chunk => {
-              uncompressedSize += chunk.length;
-            });
-            snapReader.pipe(hashStream);
-            const compressedSnapshotData = await buffer(snapReader.pipe(gzip));
-            await finished(snapReader);
-            return compressedSnapshotData;
+    return withDeferredCleanup(async addCleanup => {
+      const hashStream = createHash('sha256');
+      const gzip = createGzip();
+      let compressedSize = 0;
+      let uncompressedSize = 0;
+
+      const { duration: compressSeconds, result: compressedSnapshot } =
+        await measureSeconds(async () => {
+          const snapReader = Readable.from(snapshotStream);
+          const destroyReader = promisify(snapReader.destroy.bind(snapReader));
+          addCleanup(() => destroyReader(null));
+          snapReader.on('data', chunk => {
+            uncompressedSize += chunk.length;
           });
-        const hash = hashStream.digest('hex');
-
-        const { duration: dbSaveSeconds } = await measureSeconds(async () => {
-          ensureTxn();
-          stopUsingLastSnapshot(vatID);
-          compressedSize = compressedSnapshot.length;
-          sqlSaveSnapshot.run(
-            vatID,
-            snapPos,
-            1,
-            hash,
-            uncompressedSize,
-            compressedSize,
-            compressedSnapshot,
-          );
-          const rec = snapshotRec(vatID, snapPos, hash, 1);
-          const exportKey = snapshotMetadataKey(rec);
-          noteExport(exportKey, JSON.stringify(rec));
-          noteExport(
-            currentSnapshotMetadataKey(rec),
-            snapshotArtifactName(rec),
-          );
+          snapReader.pipe(hashStream);
+          const compressedSnapshotData = await buffer(snapReader.pipe(gzip));
+          await finished(snapReader);
+          return compressedSnapshotData;
         });
+      const hash = hashStream.digest('hex');
+      const rec = snapshotRec(vatID, snapPos, hash, 1);
+      const exportKey = snapshotMetadataKey(rec);
 
-        return harden({
+      const { duration: dbSaveSeconds } = await measureSeconds(async () => {
+        ensureTxn();
+        stopUsingLastSnapshot(vatID);
+        compressedSize = compressedSnapshot.length;
+        sqlSaveSnapshot.run(
+          vatID,
+          snapPos,
+          1,
           hash,
           uncompressedSize,
-          compressSeconds,
-          dbSaveSeconds,
           compressedSize,
-        });
-      },
-      async () => {
-        await PromiseAllOrErrors(
-          cleanup.reverse().map(fn => Promise.resolve().then(() => fn())),
+          compressedSnapshot,
         );
-      },
-    );
+        noteExport(exportKey, JSON.stringify(rec));
+        noteExport(currentSnapshotMetadataKey(rec), snapshotArtifactName(rec));
+      });
+
+      let archiveWriteSeconds;
+      if (archiveSnapshot) {
+        ({ duration: archiveWriteSeconds } = await measureSeconds(async () => {
+          await archiveSnapshot(exportKey, compressedSnapshot);
+        }));
+      }
+
+      return harden({
+        hash,
+        uncompressedSize,
+        compressSeconds,
+        dbSaveSeconds,
+        archiveWriteSeconds,
+        compressedSize,
+      });
+    });
   }
 
   const sqlGetSnapshot = db.prepare(`

package/src/swingStore.js

@@ -169,7 +169,13 @@ export function makeSwingStore(dirPath, forceReset, options = {}) {
     filePath = ':memory:';
   }
 
-  const { traceFile, keepSnapshots, keepTranscripts } = options;
+  const {
+    traceFile,
+    keepSnapshots,
+    keepTranscripts,
+    archiveSnapshot,
+    archiveTranscript,
+  } = options;
 
   let traceOutput = traceFile
     ? fs.createWriteStream(path.resolve(traceFile), {
@@ -297,6 +303,7 @@ export function makeSwingStore(dirPath, forceReset, options = {}) {
     noteExport,
     {
       keepTranscripts,
+      archiveTranscript,
     },
   );
   const { dumpSnapshots, ...snapStore } = makeSnapStore(
@@ -306,6 +313,7 @@ export function makeSwingStore(dirPath, forceReset, options = {}) {
     noteExport,
     {
       keepSnapshots,
+      archiveSnapshot,
     },
   );
   const { dumpBundles, ...bundleStore } = makeBundleStore(