@agoric/swing-store 0.9.2-dev-914bee2.0 → 0.9.2-dev-07f6885.0

package/docs/transcriptstore.md

@@ -32,14 +32,14 @@ Pruned spans are not available for export artifacts, of course, because the data
 
 Every transcript span, both current and historic, gets an export-data record. The record name is different for the two types of spans.
 
-Historical spans, which are "closed" and no longer growing, use a record name of
-`transcript.${vatID}.${startPos}`, where `startPos` is the delivery number of the first delivery included in the span. The value is a JSON-serialized record of `{ vatID, startPos, endPos, hash, isCurrent, incarnation }` (where `isCurrent = 0`).
+Historical spans, which are "closed" and no longer growing, use a record name of `transcript.${vatID}.${startPos}.${endPos}`, where `startPos` is the delivery number of the first delivery included in the span and `endPos` is the the delivery number of the first delivery included in the **next** span (i.e., the former is an inclusive lower bound and the latter is an exclusive upper bound).
+The value is a JSON-serialized record of `{ vatID, startPos, endPos, hash, isCurrent, incarnation }` (where `isCurrent = 0`).
 
 The current span, if any, uses a record name of `transcript.${vatID}.current`, and has the same value as historical spans (except `isCurrent = 1`). Current spans are growing: new transcript items are added as more deliveries are made, until the span is closed off (becomes historical) and replaced with a new current span. There is at most one current span per vatID.
 
 The available export *artifacts* will depend upon the export mode, and upon the swingstore's `keepTranscripts` setting. Each export artifact corresponds to a single span, and the artifact names are always `transcript.${vatID}.${startPos}.${endPos}` (for both historical and current spans).
 
-In the most-minimal `operational` mode, the export includes one artifact for each active (non-terminated) vat: just the current span. If `keepTranscripts` is true, these will be the only available artifacts anyways.
+In the most-minimal `operational` mode, the export includes one artifact for each active (non-terminated) vat: just the current span. If `keepTranscripts` is not true, these will be the only available artifacts anyways.
 
 The `replay` mode includes all spans for each vat's current incarnation, but omits spans from earlier incarnations. The `archival` mode includes all spans from all incarnations.
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@agoric/swing-store",
-  "version": "0.9.2-dev-914bee2.0+914bee2",
+  "version": "0.9.2-dev-07f6885.0+07f6885",
   "description": "Persistent storage for SwingSet",
   "type": "module",
   "main": "./src/index.js",
@@ -21,7 +21,7 @@
     "lint:eslint": "eslint ."
   },
   "dependencies": {
-    "@agoric/internal": "0.3.3-dev-914bee2.0+914bee2",
+    "@agoric/internal": "0.3.3-dev-07f6885.0+07f6885",
     "@endo/base64": "^1.0.7",
     "@endo/bundle-source": "^3.4.0",
     "@endo/check-bundle": "^1.0.9",
@@ -51,5 +51,5 @@
   "typeCoverage": {
     "atLeast": 78.77
   },
-  "gitHead": "914bee2aa7472039ba230267d72027b940bf8a1c"
+  "gitHead": "07f6885c560333981d6564581b2a00735a6c428c"
 }

package/src/archiver.js

@@ -0,0 +1,80 @@
+import { finished as streamFinishedCallback, Readable } from 'node:stream';
+import { promisify } from 'node:util';
+import { createGzip } from 'node:zlib';
+import { withDeferredCleanup } from '@agoric/internal';
+
+const streamFinished = promisify(streamFinishedCallback);
+
+/*
+ * @param {string} dirPath
+ * @param {object} powers
+ * @param {Pick<import('fs'), 'createWriteStream' | 'mkdirSync' | 'renameSync'>} powers.fs
+ * @param {Pick<import('path'), 'join'>} powers.path
+ * @param {Pick<import('tmp'), 'fileSync'>} powers.tmp
+ */
+export const makeArchiveSnapshot = (dirPath, powers) => {
+  const { fs, path, tmp } = powers;
+  fs.mkdirSync(dirPath, { recursive: true });
+  const archiveSnapshot = (name, gzData) => {
+    const destPath = path.join(dirPath, `${name}.gz`);
+    return withDeferredCleanup(async addCleanup => {
+      const {
+        name: tmpName,
+        fd,
+        removeCallback,
+      } = tmp.fileSync({
+        prefix: name,
+        postfix: '.gz',
+        detachDescriptor: true,
+      });
+      addCleanup(() => removeCallback());
+      const writer = fs.createWriteStream('', { fd, flush: true });
+      const reader = Readable.from(gzData);
+      const destroyReader = promisify(reader.destroy.bind(reader));
+      addCleanup(() => destroyReader(null));
+      reader.pipe(writer);
+      await streamFinished(writer);
+      fs.renameSync(tmpName, destPath);
+    });
+  };
+  return archiveSnapshot;
+};
+harden(makeArchiveSnapshot);
+
+/*
+ * @param {string} dirPath
+ * @param {object} powers
+ * @param {Pick<import('fs'), 'createWriteStream' | 'mkdirSync' | 'renameSync'>} powers.fs
+ * @param {Pick<import('path'), 'join'>} powers.path
+ * @param {Pick<import('tmp'), 'fileSync'>} powers.tmp
+ */
+export const makeArchiveTranscript = (dirPath, powers) => {
+  const { fs, path, tmp } = powers;
+  fs.mkdirSync(dirPath, { recursive: true });
+  const archiveTranscript = (spanName, entries) => {
+    const destPath = path.join(dirPath, `${spanName}.gz`);
+    return withDeferredCleanup(async addCleanup => {
+      const {
+        name: tmpName,
+        fd,
+        removeCallback,
+      } = tmp.fileSync({
+        prefix: spanName,
+        postfix: '.gz',
+        detachDescriptor: true,
+      });
+      addCleanup(() => removeCallback());
+      const writer = fs.createWriteStream('', { fd, flush: true });
+      const gzip = createGzip();
+      gzip.pipe(writer);
+      const reader = Readable.from(entries);
+      const destroyReader = promisify(reader.destroy.bind(reader));
+      addCleanup(() => destroyReader(null));
+      reader.pipe(gzip);
+      await streamFinished(gzip);
+      fs.renameSync(tmpName, destPath);
+    });
+  };
+  return archiveTranscript;
+};
+harden(makeArchiveTranscript);

package/src/index.js

@@ -2,6 +2,8 @@ export { initSwingStore, openSwingStore, isSwingStore } from './swingStore.js';
 export { makeSwingStoreExporter } from './exporter.js';
 export { importSwingStore } from './importer.js';
 
+export { makeArchiveSnapshot, makeArchiveTranscript } from './archiver.js';
+
 // temporary, for the benefit of SwingSet/misc-tools/replay-transcript.js
 export { makeSnapStore } from './snapStore.js';
 // and less temporary, for SwingSet/test/vat-warehouse/test-reload-snapshot.js

package/src/snapStore.js

@@ -4,10 +4,7 @@ import { finished as finishedCallback, PassThrough, Readable } from 'stream';
 import { promisify } from 'util';
 import { createGzip, createGunzip } from 'zlib';
 import { Fail, q } from '@endo/errors';
-import {
-  aggregateTryFinally,
-  PromiseAllOrErrors,
-} from '@agoric/internal/src/node/utils.js';
+import { withDeferredCleanup } from '@agoric/internal';
 import { buffer } from './util.js';
 
 /**
@@ -17,6 +14,7 @@ import { buffer } from './util.js';
  * @property {number} dbSaveSeconds time to write snapshot in DB
  * @property {number} compressedSize size of (compressed) snapshot
  * @property {number} compressSeconds time to generate and compress the snapshot
+ * @property {number} [archiveWriteSeconds] time to write an archive to disk (if applicable)
  */
 
 /**
@@ -73,6 +71,7 @@ const finished = promisify(finishedCallback);
  * @param {(key: string, value: string | undefined) => void} noteExport
  * @param {object} [options]
  * @param {boolean | undefined} [options.keepSnapshots]
+ * @param {(name: string, compressedData: Parameters<import('stream').Readable.from>[0]) => Promise<void>} [options.archiveSnapshot]
  * @returns {SnapStore & SnapStoreInternal & SnapStoreDebug}
  */
 export function makeSnapStore(
@@ -80,7 +79,7 @@ export function makeSnapStore(
   ensureTxn,
   { measureSeconds },
   noteExport = () => {},
-  { keepSnapshots = false } = {},
+  { keepSnapshots = false, archiveSnapshot } = {},
 ) {
   db.exec(`
     CREATE TABLE IF NOT EXISTS snapshots (
@@ -196,7 +195,8 @@ export function makeSnapStore(
 
   /**
    * Generates a new XS heap snapshot, stores a gzipped copy of it into the
-   * snapshots table, and reports information about the process, including
+   * snapshots table (and also to an archiveSnapshot callback if provided for
+   * e.g. disk archival), and reports information about the process, including
    * snapshot size and timing metrics.
    *
    * @param {string} vatID
@@ -205,74 +205,62 @@ export function makeSnapStore(
    * @returns {Promise<SnapshotResult>}
    */
   async function saveSnapshot(vatID, snapPos, snapshotStream) {
-    const cleanup = [];
-    return aggregateTryFinally(
-      async () => {
-        const hashStream = createHash('sha256');
-        const gzip = createGzip();
-        let compressedSize = 0;
-        let uncompressedSize = 0;
-
-        const { duration: compressSeconds, result: compressedSnapshot } =
-          await measureSeconds(async () => {
-            const snapReader = Readable.from(snapshotStream);
-            cleanup.push(
-              () =>
-                new Promise((resolve, reject) =>
-                  snapReader.destroy(
-                    null,
-                    // @ts-expect-error incorrect types
-                    err => (err ? reject(err) : resolve()),
-                  ),
-                ),
-            );
-
-            snapReader.on('data', chunk => {
-              uncompressedSize += chunk.length;
-            });
-            snapReader.pipe(hashStream);
-            const compressedSnapshotData = await buffer(snapReader.pipe(gzip));
-            await finished(snapReader);
-            return compressedSnapshotData;
+    return withDeferredCleanup(async addCleanup => {
+      const hashStream = createHash('sha256');
+      const gzip = createGzip();
+      let compressedSize = 0;
+      let uncompressedSize = 0;
+
+      const { duration: compressSeconds, result: compressedSnapshot } =
+        await measureSeconds(async () => {
+          const snapReader = Readable.from(snapshotStream);
+          const destroyReader = promisify(snapReader.destroy.bind(snapReader));
+          addCleanup(() => destroyReader(null));
+          snapReader.on('data', chunk => {
+            uncompressedSize += chunk.length;
           });
-        const hash = hashStream.digest('hex');
-
-        const { duration: dbSaveSeconds } = await measureSeconds(async () => {
-          ensureTxn();
-          stopUsingLastSnapshot(vatID);
-          compressedSize = compressedSnapshot.length;
-          sqlSaveSnapshot.run(
-            vatID,
-            snapPos,
-            1,
-            hash,
-            uncompressedSize,
-            compressedSize,
-            compressedSnapshot,
-          );
-          const rec = snapshotRec(vatID, snapPos, hash, 1);
-          const exportKey = snapshotMetadataKey(rec);
-          noteExport(exportKey, JSON.stringify(rec));
-          noteExport(
-            currentSnapshotMetadataKey(rec),
-            snapshotArtifactName(rec),
-          );
+          snapReader.pipe(hashStream);
+          const compressedSnapshotData = await buffer(snapReader.pipe(gzip));
+          await finished(snapReader);
+          return compressedSnapshotData;
         });
+      const hash = hashStream.digest('hex');
+      const rec = snapshotRec(vatID, snapPos, hash, 1);
+      const exportKey = snapshotMetadataKey(rec);
 
-        return harden({
+      const { duration: dbSaveSeconds } = await measureSeconds(async () => {
+        ensureTxn();
+        stopUsingLastSnapshot(vatID);
+        compressedSize = compressedSnapshot.length;
+        sqlSaveSnapshot.run(
+          vatID,
+          snapPos,
+          1,
           hash,
           uncompressedSize,
-          compressSeconds,
-          dbSaveSeconds,
           compressedSize,
-        });
-      },
-      async () => {
-        await PromiseAllOrErrors(
-          cleanup.reverse().map(fn => Promise.resolve().then(() => fn())),
+          compressedSnapshot,
         );
-      },
-    );
+        noteExport(exportKey, JSON.stringify(rec));
+        noteExport(currentSnapshotMetadataKey(rec), snapshotArtifactName(rec));
+      });
+
+      let archiveWriteSeconds;
+      if (archiveSnapshot) {
+        ({ duration: archiveWriteSeconds } = await measureSeconds(async () => {
+          await archiveSnapshot(exportKey, compressedSnapshot);
+        }));
+      }
+
+      return harden({
+        hash,
+        uncompressedSize,
+        compressSeconds,
+        dbSaveSeconds,
+        archiveWriteSeconds,
+        compressedSize,
+      });
+    });
   }
 
   const sqlGetSnapshot = db.prepare(`

package/src/swingStore.js

@@ -169,7 +169,13 @@ export function makeSwingStore(dirPath, forceReset, options = {}) {
     filePath = ':memory:';
   }
 
-  const { traceFile, keepSnapshots, keepTranscripts } = options;
+  const {
+    traceFile,
+    keepSnapshots,
+    keepTranscripts,
+    archiveSnapshot,
+    archiveTranscript,
+  } = options;
 
   let traceOutput = traceFile
     ? fs.createWriteStream(path.resolve(traceFile), {
@@ -297,6 +303,7 @@ export function makeSwingStore(dirPath, forceReset, options = {}) {
     noteExport,
     {
       keepTranscripts,
+      archiveTranscript,
     },
   );
   const { dumpSnapshots, ...snapStore } = makeSnapStore(
@@ -306,6 +313,7 @@ export function makeSwingStore(dirPath, forceReset, options = {}) {
     noteExport,
     {
       keepSnapshots,
+      archiveSnapshot,
     },
   );
   const { dumpBundles, ...bundleStore } = makeBundleStore(

package/src/transcriptStore.js

@@ -15,10 +15,10 @@ import { createSHA256 } from './hasher.js';
  *
  * @typedef {{
  *   initTranscript: (vatID: string) => void,
- *   rolloverSpan: (vatID: string) => number,
- *   rolloverIncarnation: (vatID: string) => number,
+ *   rolloverSpan: (vatID: string) => Promise<number>,
+ *   rolloverIncarnation: (vatID: string) => Promise<number>,
  *   getCurrentSpanBounds: (vatID: string) => { startPos: number, endPos: number, hash: string, incarnation: number },
- *   stopUsingTranscript: (vatID: string) => void,
+ *   stopUsingTranscript: (vatID: string) => Promise<void>,
  *   deleteVatTranscripts: (vatID: string, budget?: number) => { done: boolean, cleanups: number },
  *   addItem: (vatID: string, item: string) => void,
  *   readSpan: (vatID: string, startPos?: number) => IterableIterator<string>,
@@ -61,14 +61,15 @@ function insistTranscriptPosition(position) {
  * @param {() => void} ensureTxn
  * @param {(key: string, value: string | undefined ) => void} noteExport
  * @param {object} [options]
- * @param {boolean | undefined} [options.keepTranscripts]
+ * @param {boolean} [options.keepTranscripts]
+ * @param {(spanName: string, entries: ReturnType<exportSpan>) => Promise<void>} [options.archiveTranscript]
  * @returns { TranscriptStore & TranscriptStoreInternal & TranscriptStoreDebug }
  */
 export function makeTranscriptStore(
   db,
   ensureTxn,
   noteExport = () => {},
-  { keepTranscripts = true } = {},
+  { keepTranscripts = true, archiveTranscript } = {},
 ) {
   db.exec(`
     CREATE TABLE IF NOT EXISTS transcriptItems (
@@ -246,6 +247,102 @@ export function makeTranscriptStore(
     return bounds;
   }
 
+  const sqlGetSpanEndPos = db.prepare(`
+    SELECT endPos
+    FROM transcriptSpans
+    WHERE vatID = ? AND startPos = ?
+  `);
+  sqlGetSpanEndPos.pluck(true);
+
+  const sqlReadSpanItems = db.prepare(`
+    SELECT item
+    FROM transcriptItems
+    WHERE vatID = ? AND ? <= position AND position < ?
+    ORDER BY position
+  `);
+
+  /**
+   * Read the items in a transcript span
+   *
+   * @param {string} vatID  The vat whose transcript is being read
+   * @param {number} [startPos] A start position identifying the span to be
+   *    read; defaults to the current span, whatever it is
+   *
+   * @returns {IterableIterator<string>}  An iterator over the items in the indicated span
+   */
+  function readSpan(vatID, startPos) {
+    /** @type {number | undefined} */
+    let endPos;
+    if (startPos === undefined) {
+      ({ startPos, endPos } = getCurrentSpanBounds(vatID));
+    } else {
+      insistTranscriptPosition(startPos);
+      endPos = sqlGetSpanEndPos.get(vatID, startPos);
+      if (typeof endPos !== 'number') {
+        throw Fail`no transcript span for ${q(vatID)} at ${q(startPos)}`;
+      }
+    }
+    startPos <= endPos || Fail`${q(startPos)} <= ${q(endPos)}}`;
+    const expectedCount = endPos - startPos;
+
+    function* reader() {
+      let count = 0;
+      for (const { item } of sqlReadSpanItems.iterate(
+        vatID,
+        startPos,
+        endPos,
+      )) {
+        yield item;
+        count += 1;
+      }
+      count === expectedCount ||
+        Fail`read ${q(count)} transcript entries (expected ${q(
+          expectedCount,
+        )})`;
+    }
+    harden(reader);
+
+    if (startPos === endPos) {
+      return empty();
+    }
+
+    return reader();
+  }
+
+  const sqlGetSpanIsCurrent = db.prepare(`
+    SELECT isCurrent
+    FROM transcriptSpans
+    WHERE vatID = ? AND startPos = ?
+  `);
+  sqlGetSpanIsCurrent.pluck(true);
+
+  /**
+   * Read a transcript span and return it as a stream of data suitable for
+   * export to another store.  Transcript items are terminated by newlines.
+   *
+   * Transcript span artifact names should be strings of the form:
+   *   `transcript.${vatID}.${startPos}.${endPos}`
+   *
+   * @param {string} name  The name of the transcript artifact to be read
+   * @returns {AsyncIterableIterator<Uint8Array>}
+   * @yields {Uint8Array}
+   */
+  async function* exportSpan(name) {
+    typeof name === 'string' || Fail`artifact name must be a string`;
+    const parts = name.split('.');
+    const [type, vatID, pos] = parts;
+    // prettier-ignore
+    (parts.length === 4 && type === 'transcript') ||
+      Fail`expected artifact name of the form 'transcript.{vatID}.{startPos}.{endPos}', saw ${q(name)}`;
+    const isCurrent = sqlGetSpanIsCurrent.get(vatID, pos);
+    isCurrent !== undefined || Fail`transcript span ${q(name)} not available`;
+    const startPos = Number(pos);
+    for (const entry of readSpan(vatID, startPos)) {
+      yield Buffer.from(`${entry}\n`);
+    }
+  }
+  harden(exportSpan);
+
   const sqlEndCurrentSpan = db.prepare(`
     UPDATE transcriptSpans
     SET isCurrent = null
@@ -267,7 +364,7 @@ export function makeTranscriptStore(
    * @param {string} vatID
    * @param {ReturnType<getCurrentSpanBounds>} bounds
    */
-  function closeSpan(vatID, bounds) {
+  async function closeSpan(vatID, bounds) {
     ensureTxn();
     const { startPos, endPos, hash, incarnation } = bounds;
     const rec = spanRec(vatID, startPos, endPos, hash, false, incarnation);
@@ -278,6 +375,12 @@ export function makeTranscriptStore(
     // and change its DB row to isCurrent=null
     sqlEndCurrentSpan.run(vatID);
 
+    await null;
+    if (archiveTranscript) {
+      const spanName = spanArtifactName(rec);
+      const entries = exportSpan(spanName);
+      await archiveTranscript(spanName, entries);
+    }
     if (!keepTranscripts) {
       // Delete items of the previously-current span.
       // There may still be items associated with even older spans, but we leave
@@ -289,13 +392,13 @@ export function makeTranscriptStore(
     }
   }
 
-  function doSpanRollover(vatID, isNewIncarnation) {
+  async function doSpanRollover(vatID, isNewIncarnation) {
     ensureTxn();
     const bounds = getCurrentSpanBounds(vatID);
     const { endPos, incarnation } = bounds;
 
     // deal with the now-old span
-    closeSpan(vatID, bounds);
+    await closeSpan(vatID, bounds);
 
     // create a new (empty) DB row, with isCurrent=1
     const newSpanIncarnation = isNewIncarnation ? incarnation + 1 : incarnation;
@@ -322,9 +425,9 @@ export function makeTranscriptStore(
    * @param {string} vatID  The vat whose transcript is to rollover to a new
    *    span.
    *
-   * @returns {number} the new incarnation number
+   * @returns {Promise<number>} the new incarnation number
    */
-  function rolloverIncarnation(vatID) {
+  async function rolloverIncarnation(vatID) {
     return doSpanRollover(vatID, true);
   }
 
@@ -335,9 +438,9 @@ export function makeTranscriptStore(
    * @param {string} vatID  The vat whose transcript is to rollover to a new
    *    span.
    *
-   * @returns {number} the incarnation number
+   * @returns {Promise<number>} the incarnation number
    */
-  function rolloverSpan(vatID) {
+  async function rolloverSpan(vatID) {
     return doSpanRollover(vatID, false);
   }
 
@@ -387,13 +490,13 @@ export function makeTranscriptStore(
    *
    * @param {string} vatID  The vat being terminated/deleted.
    */
-  function stopUsingTranscript(vatID) {
+  async function stopUsingTranscript(vatID) {
     ensureTxn();
     const bounds = sqlGetCurrentSpanBounds.get(vatID);
     if (!bounds) return;
 
     // deal with the now-old span
-    closeSpan(vatID, bounds);
+    await closeSpan(vatID, bounds);
 
     // remove the transcript.${vatID}.current record
     noteExport(spanMetadataKey({ vatID, isCurrent: true }), undefined);
@@ -584,102 +687,6 @@ export function makeTranscriptStore(
   }
   harden(getArtifactNames);
 
-  const sqlGetSpanEndPos = db.prepare(`
-    SELECT endPos
-    FROM transcriptSpans
-    WHERE vatID = ? AND startPos = ?
-  `);
-  sqlGetSpanEndPos.pluck(true);
-
-  const sqlReadSpanItems = db.prepare(`
-    SELECT item
-    FROM transcriptItems
-    WHERE vatID = ? AND ? <= position AND position < ?
-    ORDER BY position
-  `);
-
-  /**
-   * Read the items in a transcript span
-   *
-   * @param {string} vatID  The vat whose transcript is being read
-   * @param {number} [startPos] A start position identifying the span to be
-   *    read; defaults to the current span, whatever it is
-   *
-   * @returns {IterableIterator<string>}  An iterator over the items in the indicated span
-   */
-  function readSpan(vatID, startPos) {
-    /** @type {number | undefined} */
-    let endPos;
-    if (startPos === undefined) {
-      ({ startPos, endPos } = getCurrentSpanBounds(vatID));
-    } else {
-      insistTranscriptPosition(startPos);
-      endPos = sqlGetSpanEndPos.get(vatID, startPos);
-      if (typeof endPos !== 'number') {
-        throw Fail`no transcript span for ${q(vatID)} at ${q(startPos)}`;
-      }
-    }
-    startPos <= endPos || Fail`${q(startPos)} <= ${q(endPos)}}`;
-    const expectedCount = endPos - startPos;
-
-    function* reader() {
-      let count = 0;
-      for (const { item } of sqlReadSpanItems.iterate(
-        vatID,
-        startPos,
-        endPos,
-      )) {
-        yield item;
-        count += 1;
-      }
-      count === expectedCount ||
-        Fail`read ${q(count)} transcript entries (expected ${q(
-          expectedCount,
-        )})`;
-    }
-    harden(reader);
-
-    if (startPos === endPos) {
-      return empty();
-    }
-
-    return reader();
-  }
-
-  const sqlGetSpanIsCurrent = db.prepare(`
-    SELECT isCurrent
-    FROM transcriptSpans
-    WHERE vatID = ? AND startPos = ?
-  `);
-  sqlGetSpanIsCurrent.pluck(true);
-
-  /**
-   * Read a transcript span and return it as a stream of data suitable for
-   * export to another store.  Transcript items are terminated by newlines.
-   *
-   * Transcript span artifact names should be strings of the form:
-   *   `transcript.${vatID}.${startPos}.${endPos}`
-   *
-   * @param {string} name  The name of the transcript artifact to be read
-   * @returns {AsyncIterableIterator<Uint8Array>}
-   * @yields {Uint8Array}
-   */
-  async function* exportSpan(name) {
-    typeof name === 'string' || Fail`artifact name must be a string`;
-    const parts = name.split('.');
-    const [type, vatID, pos] = parts;
-    // prettier-ignore
-    (parts.length === 4 && type === 'transcript') ||
-      Fail`expected artifact name of the form 'transcript.{vatID}.{startPos}.{endPos}', saw ${q(name)}`;
-    const isCurrent = sqlGetSpanIsCurrent.get(vatID, pos);
-    isCurrent !== undefined || Fail`transcript span ${q(name)} not available`;
-    const startPos = Number(pos);
-    for (const entry of readSpan(vatID, startPos)) {
-      yield Buffer.from(`${entry}\n`);
-    }
-  }
-  harden(exportSpan);
-
   const sqlAddItem = db.prepare(`
     INSERT INTO transcriptItems (vatID, item, position, incarnation)
     VALUES (?, ?, ?, ?)

package/test/deletion.test.js

@@ -1,12 +1,16 @@
 // @ts-check
 import test from 'ava';
-import path from 'path';
 
 import { Buffer } from 'node:buffer';
+import path from 'node:path';
+import fs from 'node:fs';
+import zlib from 'node:zlib';
 import sqlite3 from 'better-sqlite3';
+import tmp from 'tmp';
 import { arrayIsLike } from '@agoric/internal/tools/ava-assertions.js';
 import { tmpDir } from './util.js';
 import { initSwingStore } from '../src/swingStore.js';
+import { makeArchiveSnapshot, makeArchiveTranscript } from '../src/archiver.js';
 import { makeSwingStoreExporter } from '../src/exporter.js';
 import { importSwingStore } from '../src/importer.js';
 
@@ -95,7 +99,7 @@ test('delete transcripts with export callback', async t => {
   transcriptStore.addItem('v1', 'aaa');
   transcriptStore.addItem('v1', 'bbb');
   transcriptStore.addItem('v1', 'ccc');
-  transcriptStore.rolloverSpan('v1');
+  await transcriptStore.rolloverSpan('v1');
   transcriptStore.addItem('v1', 'ddd');
   transcriptStore.addItem('v1', 'eee');
   transcriptStore.addItem('v1', 'fff');
@@ -168,7 +172,17 @@ const setupTranscript = async (t, keepTranscripts) => {
   };
   const [dbDir, cleanup] = await tmpDir('testdb');
   t.teardown(cleanup);
-  const store = initSwingStore(dbDir, { exportCallback, keepTranscripts });
+  const [archiveDir, cleanupArchives] = await tmpDir('archives');
+  t.teardown(cleanupArchives);
+  const fsPowers = { fs, path, tmp };
+  const archiveSnapshot = makeArchiveSnapshot(archiveDir, fsPowers);
+  const archiveTranscript = makeArchiveTranscript(archiveDir, fsPowers);
+  const store = initSwingStore(dbDir, {
+    exportCallback,
+    keepTranscripts,
+    archiveSnapshot,
+    archiveTranscript,
+  });
   const { kernelStorage, hostStorage } = store;
   const { transcriptStore } = kernelStorage;
   const { commit } = hostStorage;
@@ -179,13 +193,13 @@ const setupTranscript = async (t, keepTranscripts) => {
   transcriptStore.initTranscript(vatID);
   transcriptStore.addItem(vatID, 'aaa');
   transcriptStore.addItem(vatID, 'bbb');
-  transcriptStore.rolloverSpan(vatID);
+  await transcriptStore.rolloverSpan(vatID);
   transcriptStore.addItem(vatID, 'ccc');
   transcriptStore.addItem(vatID, 'ddd');
-  transcriptStore.rolloverIncarnation(vatID);
+  await transcriptStore.rolloverIncarnation(vatID);
   transcriptStore.addItem(vatID, 'eee');
   transcriptStore.addItem(vatID, 'fff');
-  transcriptStore.rolloverSpan(vatID);
+  await transcriptStore.rolloverSpan(vatID);
   transcriptStore.addItem(vatID, 'ggg');
   transcriptStore.addItem(vatID, 'hhh');
   await commit();
@@ -193,6 +207,7 @@ const setupTranscript = async (t, keepTranscripts) => {
   return {
     db,
     dbDir,
+    archiveDir,
     commit,
     transcriptStore,
     exportLog,
@@ -212,6 +227,7 @@ const execSlowTranscriptDeletion = async (t, { keepTranscripts }) => {
   const {
     db,
     dbDir,
+    archiveDir,
     commit,
     transcriptStore,
     exportLog,
@@ -241,12 +257,13 @@ const execSlowTranscriptDeletion = async (t, { keepTranscripts }) => {
     'transcript.v1.4': t4,
     'transcript.v1.6': t6,
   };
-  const expectedArtifactNames = [
-    'transcript.v1.0.2',
-    'transcript.v1.2.4',
-    'transcript.v1.4.6',
-    'transcript.v1.6.8',
-  ];
+  const expectedArtifactContents = {
+    'transcript.v1.0.2': 'aaa\nbbb\n',
+    'transcript.v1.2.4': 'ccc\nddd\n',
+    'transcript.v1.4.6': 'eee\nfff\n',
+    'transcript.v1.6.8': 'ggg\nhhh\n',
+  };
+  const expectedArtifactNames = Object.keys(expectedArtifactContents);
 
   t.deepEqual(stripHashes(currentExportData), expectedLiveExportData);
   t.is(
@@ -255,6 +272,17 @@ const execSlowTranscriptDeletion = async (t, { keepTranscripts }) => {
   );
   t.is(db.prepare('SELECT COUNT(*) FROM transcriptSpans').pluck().get(), 4);
 
+  // verify archived transcripts
+  t.deepEqual(
+    fs.readdirSync(archiveDir),
+    expectedArtifactNames.slice(0, -1).map(name => `${name}.gz`),
+  );
+  for (const name of expectedArtifactNames.slice(0, -1)) {
+    const filePath = path.join(archiveDir, `${name}.gz`);
+    const contents = zlib.gunzipSync(fs.readFileSync(filePath)).toString();
+    t.is(contents, expectedArtifactContents[name], `${filePath} contents`);
+  }
+
   // an "operational"-mode export should list all spans, but only have
   // artifacts for the current one
   {
@@ -284,15 +312,24 @@ const execSlowTranscriptDeletion = async (t, { keepTranscripts }) => {
   // deletes the .current record (i.e. it transforms .current into a
   // closed record)
   {
-    transcriptStore.stopUsingTranscript(vatID);
+    await transcriptStore.stopUsingTranscript(vatID);
     await commit();
     t.deepEqual(stripHashes(currentExportData), expectedStoppedExportData);
     exportLog.length = 0;
     // stopUsingTranscript is idempotent
-    transcriptStore.stopUsingTranscript(vatID);
+    await transcriptStore.stopUsingTranscript(vatID);
     await commit();
     t.deepEqual(exportLog, []);
   }
+  t.deepEqual(
+    fs.readdirSync(archiveDir),
+    expectedArtifactNames.map(name => `${name}.gz`),
+  );
+  for (const name of expectedArtifactNames) {
+    const filePath = path.join(archiveDir, `${name}.gz`);
+    const contents = zlib.gunzipSync(fs.readFileSync(filePath)).toString();
+    t.is(contents, expectedArtifactContents[name], `${filePath} contents`);
+  }
 
   // All exports (debug and non-debug) in this "terminated but not
   // deleted" state will still have the export-data keys. Only

package/test/export.test.js

@@ -47,7 +47,7 @@ const exportTest = test.macro(async (t, mode) => {
   // incarnation 0
   ks.transcriptStore.addItem('v1', 'start-worker'); // 0
   ks.transcriptStore.addItem('v1', 'shutdown-worker'); // 1
-  ks.transcriptStore.rolloverIncarnation('v1');
+  await ks.transcriptStore.rolloverIncarnation('v1');
   const spanHash0 =
     '5bee0f44eca02f23eab03703e84ed2647d5d117fed99e1c30a3b424b7f082ab9';
 
@@ -56,7 +56,7 @@ const exportTest = test.macro(async (t, mode) => {
   ks.transcriptStore.addItem('v1', 'delivery1'); // 3
   await ks.snapStore.saveSnapshot('v1', 4, getSnapshotStream(snapshotData));
   ks.transcriptStore.addItem('v1', 'save-snapshot'); // 4
-  ks.transcriptStore.rolloverSpan('v1'); // range= 2..5
+  await ks.transcriptStore.rolloverSpan('v1'); // range= 2..5
   const spanHash1 =
     '57152efdd7fdf75c03371d2b4f1088d5bf3eae7fe643babce527ff81df38998c';
 
@@ -64,7 +64,7 @@ const exportTest = test.macro(async (t, mode) => {
   ks.transcriptStore.addItem('v1', 'delivery2'); // 6
   await ks.snapStore.saveSnapshot('v1', 7, getSnapshotStream(snapshotData));
   ks.transcriptStore.addItem('v1', 'save-snapshot'); // 7
-  ks.transcriptStore.rolloverSpan('v1'); // range= 5..8
+  await ks.transcriptStore.rolloverSpan('v1'); // range= 5..8
   const spanHash2 =
     '1947001e78e01bd1e773feb22b4ffc530447373b9de9274d5d5fbda3f23dbf2b';
 
@@ -251,7 +251,7 @@ test('export omits pruned span artifacts', async t => {
   ks.transcriptStore.addItem('v1', 'delivery1'); // 1
   await ks.snapStore.saveSnapshot('v1', 2, getSnapshotStream(snapshotData));
   ks.transcriptStore.addItem('v1', 'save-snapshot'); // 2
-  ks.transcriptStore.rolloverSpan('v1'); // range= 0..3
+  await ks.transcriptStore.rolloverSpan('v1'); // range= 0..3
   const spanHash1 =
     '57152efdd7fdf75c03371d2b4f1088d5bf3eae7fe643babce527ff81df38998c';
   // rolloverSpan prunes the contents of the old span

package/test/exportImport.test.js

@@ -87,7 +87,7 @@ async function fakeAVatSnapshot(vat, ks) {
   await ks.snapStore.saveSnapshot(vat.vatID, vat.endPos, getSnapshotStream());
   ks.transcriptStore.addItem(vat.vatID, 'save-snapshot');
   vat.endPos += 1;
-  ks.transcriptStore.rolloverSpan(vat.vatID);
+  await ks.transcriptStore.rolloverSpan(vat.vatID);
   ks.transcriptStore.addItem(vat.vatID, 'load-snapshot');
   vat.endPos += 1;
 }

package/test/snapstore.test.js

@@ -1,12 +1,17 @@
 // @ts-check
+import test from 'ava';
 
 import { Buffer } from 'node:buffer';
-import zlib from 'zlib';
+import fs from 'node:fs';
+import path from 'node:path';
+import zlib from 'node:zlib';
 import sqlite3 from 'better-sqlite3';
+import tmp from 'tmp';
 
-import test from 'ava';
 import { makeMeasureSeconds } from '@agoric/internal';
 import { makeSnapStore } from '../src/snapStore.js';
+import { makeArchiveSnapshot } from '../src/archiver.js';
+import { tmpDir } from './util.js';
 
 function makeExportLog() {
   const exportLog = [];
@@ -31,6 +36,10 @@ harden(getSnapshotStream);
 test('compress to cache file; closes snapshot stream', async t => {
   const db = sqlite3(':memory:');
   const exportLog = makeExportLog();
+  const [archiveDir, cleanupArchives] = await tmpDir('archives');
+  t.teardown(cleanupArchives);
+  const fsPowers = { fs, path, tmp };
+  const archiveSnapshot = makeArchiveSnapshot(archiveDir, fsPowers);
   const store = makeSnapStore(
     db,
     ensureTxn,
@@ -38,6 +47,7 @@ test('compress to cache file; closes snapshot stream', async t => {
       measureSeconds: makeMeasureSeconds(() => 0),
     },
     exportLog.noteExport,
+    { archiveSnapshot },
   );
 
   const snapshotStream = getSnapshotStream('abc');
@@ -50,6 +60,7 @@ test('compress to cache file; closes snapshot stream', async t => {
     uncompressedSize: 3,
     compressedSize: 23,
     dbSaveSeconds: 0,
+    archiveWriteSeconds: 0,
     compressSeconds: 0,
   });
   const snapshotInfo = store.getSnapshotInfo('fakeVatID');
@@ -89,6 +100,26 @@ test('compress to cache file; closes snapshot stream', async t => {
     ['snapshot.fakeVatID.47', JSON.stringify(logInfo)],
     ['snapshot.fakeVatID.current', `snapshot.fakeVatID.47`],
   ]);
+
+  // verify disk archive
+  t.deepEqual(
+    fs.readdirSync(archiveDir),
+    ['snapshot.fakeVatID.47.gz'],
+    'archive must be written to disk',
+  );
+  const fileContents = fs.readFileSync(
+    path.join(archiveDir, 'snapshot.fakeVatID.47.gz'),
+  );
+  t.is(
+    fileContents.length,
+    dbInfo.compressedSize,
+    'file size must match database compressedSize',
+  );
+  t.is(
+    zlib.gunzipSync(fileContents).toString(),
+    'abc',
+    'gunzip(fileContents) must match input data',
+  );
 });
 
 test('snapStore prepare / commit delete is robust', async t => {
@@ -96,11 +127,17 @@ test('snapStore prepare / commit delete is robust', async t => {
     measureSeconds: makeMeasureSeconds(() => 0),
   };
   const db = sqlite3(':memory:');
+  const [archiveDir, cleanupArchives] = await tmpDir('archives');
+  t.teardown(cleanupArchives);
+  const fsPowers = { fs, path, tmp };
+  const archiveSnapshot = makeArchiveSnapshot(archiveDir, fsPowers);
   const store = makeSnapStore(db, ensureTxn, io, () => {}, {
     keepSnapshots: true,
+    archiveSnapshot,
   });
 
   const hashes = [];
+  const expectedFiles = [];
   for (let i = 0; i < 5; i += 1) {
     const { hash } = await store.saveSnapshot(
       'fakeVatID2',
@@ -108,6 +145,7 @@ test('snapStore prepare / commit delete is robust', async t => {
       getSnapshotStream(`file ${i}`),
     );
     hashes.push(hash);
+    expectedFiles.push(`snapshot.fakeVatID2.${i}.gz`);
   }
   const sqlCountSnapshots = db.prepare(`
     SELECT COUNT(*)
@@ -116,19 +154,24 @@ test('snapStore prepare / commit delete is robust', async t => {
   sqlCountSnapshots.pluck(true);
 
   t.is(sqlCountSnapshots.get(), 5);
+  t.deepEqual(fs.readdirSync(archiveDir), expectedFiles);
 
   store.deleteSnapshotByHash('fakeVatID2', hashes[2]);
   t.is(sqlCountSnapshots.get(), 4);
+  t.deepEqual(fs.readdirSync(archiveDir), expectedFiles);
 
   // Restore (re-save) between prepare and commit.
   store.deleteSnapshotByHash('fakeVatID2', hashes[3]);
   await store.saveSnapshot('fakeVatID3', 29, getSnapshotStream(`file 3`));
+  expectedFiles.push(`snapshot.fakeVatID3.29.gz`);
   t.true(store.hasHash('fakeVatID3', hashes[3]));
+  t.deepEqual(fs.readdirSync(archiveDir), expectedFiles);
 
   store.deleteVatSnapshots('fakeVatID2');
   t.is(sqlCountSnapshots.get(), 1);
   store.deleteVatSnapshots('fakeVatID3');
   t.is(sqlCountSnapshots.get(), 0);
+  t.deepEqual(fs.readdirSync(archiveDir), expectedFiles);
 
   for (let i = 0; i < 5; i += 1) {
     const { hash } = await store.saveSnapshot(
@@ -137,8 +180,10 @@ test('snapStore prepare / commit delete is robust', async t => {
       getSnapshotStream(`file ${i}`),
     );
     hashes.push(hash);
+    expectedFiles.push(`snapshot.fakeVatID4.${i}.gz`);
   }
   t.is(sqlCountSnapshots.get(), 5);
   store.deleteAllUnusedSnapshots();
   t.is(sqlCountSnapshots.get(), 1);
+  t.deepEqual(fs.readdirSync(archiveDir), expectedFiles);
 });

package/test/state.test.js

@@ -181,10 +181,10 @@ const testTranscriptStore = test.macro({
     transcriptStore.initTranscript('st1');
     transcriptStore.initTranscript('st2');
     transcriptStore.addItem('st1', 'zeroth');
-    transcriptStore.rolloverSpan('st1');
+    await transcriptStore.rolloverSpan('st1');
     transcriptStore.addItem('st1', 'first');
     transcriptStore.addItem('st1', 'second');
-    transcriptStore.rolloverSpan('st1');
+    await transcriptStore.rolloverSpan('st1');
     transcriptStore.addItem('st1', 'third');
     transcriptStore.addItem('st2', 'oneth');
     transcriptStore.addItem('st1', 'fourth');