@agoric/swing-store 0.9.2-dev-dde639f.0.dde639f → 0.9.2-dev-63f4816.0.63f4816

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@agoric/swing-store",
-  "version": "0.9.2-dev-dde639f.0.dde639f",
+  "version": "0.9.2-dev-63f4816.0.63f4816",
   "description": "Persistent storage for SwingSet",
   "type": "module",
   "main": "./src/index.js",
@@ -21,7 +21,7 @@
     "lint:eslint": "yarn run -T eslint ."
   },
   "dependencies": {
-    "@agoric/internal": "0.3.3-dev-dde639f.0.dde639f",
+    "@agoric/internal": "0.3.3-dev-63f4816.0.63f4816",
     "@endo/base64": "^1.0.12",
     "@endo/bundle-source": "^4.1.2",
     "@endo/check-bundle": "^1.0.17",
@@ -31,6 +31,8 @@
   },
   "devDependencies": {
     "@endo/init": "^1.1.12",
+    "@endo/promise-kit": "^1.1.13",
+    "@endo/stream": "^1.2.13",
     "@types/better-sqlite3": "^7.6.13",
     "ava": "^5.3.0",
     "c8": "^10.1.3",
@@ -51,5 +53,5 @@
   "typeCoverage": {
     "atLeast": 79.33
   },
-  "gitHead": "dde639f7be5981f6dba721b9d4ea86fe27ba3c1a"
+  "gitHead": "63f481671ced3c1174fa533194d28f64b0e36c50"
 }

package/src/archiver.js

@@ -4,9 +4,13 @@ import { promisify } from 'node:util';
 import { createGzip } from 'node:zlib';
 import { withDeferredCleanup } from '@agoric/internal';
 
+/**
+ * @import {AnyIterable} from './exporter.js';
+ */
+
 const streamFinished = promisify(streamFinishedCallback);
 
-/*
+/**
  * @param {string} dirPath
  * @param {object} powers
  * @param {Pick<import('fs'), 'createWriteStream' | 'mkdirSync' | 'renameSync'>} powers.fs
@@ -16,6 +20,11 @@ const streamFinished = promisify(streamFinishedCallback);
 export const makeArchiveSnapshot = (dirPath, powers) => {
   const { fs, path, tmp } = powers;
   fs.mkdirSync(dirPath, { recursive: true });
+  /**
+   * @param {string} name
+   * @param {AnyIterable<Uint8Array>} gzData
+   * @returns {Promise<void>}
+   */
   const archiveSnapshot = (name, gzData) => {
     const destPath = path.join(dirPath, `${name}.gz`);
     return withDeferredCleanup(async addCleanup => {
@@ -29,7 +38,7 @@ export const makeArchiveSnapshot = (dirPath, powers) => {
         postfix: '.gz.tmp',
         detachDescriptor: true,
       });
-      addCleanup(() => removeCallback());
+      addCleanup(async () => removeCallback());
       const writer = fs.createWriteStream('', { fd, flush: true });
       const reader = Readable.from(gzData);
       const destroyReader = promisify(reader.destroy.bind(reader));
@@ -43,7 +52,7 @@ export const makeArchiveSnapshot = (dirPath, powers) => {
 };
 harden(makeArchiveSnapshot);
 
-/*
+/**
  * @param {string} dirPath
  * @param {object} powers
  * @param {Pick<import('fs'), 'createWriteStream' | 'mkdirSync' | 'renameSync'>} powers.fs
@@ -53,6 +62,11 @@ harden(makeArchiveSnapshot);
 export const makeArchiveTranscript = (dirPath, powers) => {
   const { fs, path, tmp } = powers;
   fs.mkdirSync(dirPath, { recursive: true });
+  /**
+   * @param {string} spanName
+   * @param {AnyIterable<Uint8Array>} entries
+   * @returns {Promise<void>}
+   */
   const archiveTranscript = (spanName, entries) => {
     const destPath = path.join(dirPath, `${spanName}.gz`);
     return withDeferredCleanup(async addCleanup => {
@@ -66,7 +80,7 @@ export const makeArchiveTranscript = (dirPath, powers) => {
         postfix: '.gz.tmp',
         detachDescriptor: true,
       });
-      addCleanup(() => removeCallback());
+      addCleanup(async () => removeCallback());
       await pipeline(
         entries,
         createGzip(),

package/src/transcriptStore.js

@@ -331,6 +331,16 @@ export function makeTranscriptStore(
    * @yields {Uint8Array}
    */
   async function* exportSpan(name) {
+    yield* syncExportSpan(name);
+  }
+  harden(exportSpan);
+
+  /**
+   * @param {string} name  The name of the transcript artifact to be read
+   * @returns {IterableIterator<Uint8Array>}
+   * @yields {Uint8Array}
+   */
+  function* syncExportSpan(name) {
     typeof name === 'string' || Fail`artifact name must be a string`;
     const parts = name.split('.');
     const [type, vatID, pos] = parts;
@@ -344,7 +354,6 @@ export function makeTranscriptStore(
       yield Buffer.from(`${entry}\n`);
     }
   }
-  harden(exportSpan);
 
   const sqlEndCurrentSpan = db.prepare(`
     UPDATE transcriptSpans
@@ -366,8 +375,9 @@ export function makeTranscriptStore(
    *
    * @param {string} vatID
    * @param {ReturnType<getCurrentSpanBounds>} bounds
+   * @returns {{deferredTask: Promise<void>}}
    */
-  async function closeSpan(vatID, bounds) {
+  function closeSpan(vatID, bounds) {
     ensureTxn();
     const { startPos, endPos, hash, incarnation } = bounds;
     const rec = spanRec(vatID, startPos, endPos, hash, false, incarnation);
@@ -378,11 +388,11 @@ export function makeTranscriptStore(
     // and change its DB row to isCurrent=null
     sqlEndCurrentSpan.run(vatID);
 
-    await null;
+    let archiveResult = Promise.resolve();
     if (archiveTranscript) {
       const spanName = spanArtifactName(rec);
-      const entries = exportSpan(spanName);
-      await archiveTranscript(spanName, entries);
+      const entries = [...syncExportSpan(spanName)];
+      archiveResult = archiveTranscript(spanName, entries);
     }
     if (!keepTranscripts) {
       // Delete items of the previously-current span.
@@ -393,15 +403,23 @@ export function makeTranscriptStore(
       // that doesn't include them.
       sqlDeleteOldItems.run(vatID, startPos, endPos);
     }
+
+    return harden({ deferredTask: archiveResult });
   }
 
+  /**
+   * This function guarantees that all store operations are done synchronously.
+   *
+   * @param {string} vatID
+   * @param {boolean} isNewIncarnation
+   */
   async function doSpanRollover(vatID, isNewIncarnation) {
     ensureTxn();
     const bounds = getCurrentSpanBounds(vatID);
     const { endPos, incarnation } = bounds;
 
     // deal with the now-old span
-    await closeSpan(vatID, bounds);
+    const { deferredTask } = closeSpan(vatID, bounds);
 
     // create a new (empty) DB row, with isCurrent=1
     const newSpanIncarnation = isNewIncarnation ? incarnation + 1 : incarnation;
@@ -418,6 +436,8 @@ export function makeTranscriptStore(
     );
     noteExport(spanMetadataKey(rec), JSON.stringify(rec));
 
+    await deferredTask;
+
     return newSpanIncarnation;
   }
 
@@ -425,6 +445,8 @@ export function makeTranscriptStore(
    * End the current transcript span for a vat and start a new span in a new
    * incarnation (e.g., after a vat upgrade).
    *
+   * This function guarantees that all store operations are done synchronously.
+   *
    * @param {string} vatID  The vat whose transcript is to rollover to a new
    *    span.
    *
@@ -438,6 +460,8 @@ export function makeTranscriptStore(
    * End the current transcript span for a vat and start a new span in the
    * current incarnation (e.g., after a heap snapshot event).
    *
+   * This function guarantees that all store operations are done synchronously.
+   *
    * @param {string} vatID  The vat whose transcript is to rollover to a new
    *    span.
    *
@@ -491,6 +515,8 @@ export function makeTranscriptStore(
    * Prepare for vat deletion by marking the isCurrent=1 span as not current.
    * Idempotent.
    *
+   * This function guarantees that all store operations are done synchronously.
+   *
    * @param {string} vatID  The vat being terminated/deleted.
    */
   async function stopUsingTranscript(vatID) {
@@ -499,10 +525,12 @@ export function makeTranscriptStore(
     if (!bounds) return;
 
     // deal with the now-old span
-    await closeSpan(vatID, bounds);
+    const { deferredTask } = closeSpan(vatID, bounds);
 
     // remove the transcript.${vatID}.current record
     noteExport(spanMetadataKey({ vatID, isCurrent: true }), undefined);
+
+    await deferredTask;
   }
 
   /**

package/test/transcriptStore.test.js

@@ -0,0 +1,178 @@
+// @ts-check
+import test from 'ava';
+
+import sqlite3 from 'better-sqlite3';
+import { makePromiseKit } from '@endo/promise-kit';
+import { makeQueue } from '@endo/stream';
+
+import { makeTranscriptStore } from '../src/transcriptStore.js';
+
+/**
+ * @import {AsyncQueue} from '@endo/stream';
+ * @import {PromiseKit} from '@endo/promise-kit';
+ */
+
+function makeExportLog() {
+  const exportLog = [];
+  return {
+    noteExport(key, value) {
+      exportLog.push([key, value]);
+    },
+    getLog() {
+      return exportLog.splice(0);
+    },
+  };
+}
+
+/**
+ * @typedef {{name: string; entries: Uint8Array[]}} TranscriptArtifact
+ */
+
+/**
+ * A test kit that consumes the transcript archive entries on demand
+ */
+function makeArchiveTranscript() {
+  /** @type {AsyncQueue<(transcriptArtifact: TranscriptArtifact) => void>} */
+  const queue = makeQueue();
+
+  return {
+    archiveTranscript: async (name, rawEntries) => {
+      const resolve = await queue.get();
+      // TODO: use Array.fromAsync once we're on Node 22
+      const entries = [];
+      for await (const entry of rawEntries) {
+        entries.push(entry);
+      }
+      resolve({ name, entries });
+    },
+    getTranscriptArtifact: () => {
+      /** @type {PromiseKit<TranscriptArtifact>} */
+      const { resolve, promise } = makePromiseKit();
+      queue.put(resolve);
+      return promise;
+    },
+  };
+}
+
+function ensureTxn() {}
+
+const closeSpanMacro = test.macro(async (t, useArchiver) => {
+  const db = sqlite3(':memory:');
+  const exportLog = makeExportLog();
+  const { archiveTranscript, getTranscriptArtifact } = useArchiver
+    ? makeArchiveTranscript()
+    : {};
+  const store = makeTranscriptStore(db, ensureTxn, exportLog.noteExport, {
+    keepTranscripts: false,
+    archiveTranscript,
+  });
+
+  const vatID = 'v1';
+
+  await null;
+
+  store.initTranscript(vatID);
+  store.addItem(vatID, 'foo');
+  store.addItem(vatID, 'bar');
+
+  t.deepEqual(exportLog.getLog(), [
+    [
+      'transcript.v1.current',
+      '{"vatID":"v1","startPos":0,"endPos":0,"hash":"43e6be43a3a34d60c0ebeb8498b5849b094fc20fc68483a7aeb3624fa10f79f6","isCurrent":1,"incarnation":0}',
+    ],
+    [
+      'transcript.v1.current',
+      '{"vatID":"v1","startPos":0,"endPos":1,"hash":"3a676f1da91fe6da8268853f21e64e3366da45811b830f9d92f39d4c01d69291","isCurrent":1,"incarnation":0}',
+    ],
+    [
+      'transcript.v1.current',
+      '{"vatID":"v1","startPos":0,"endPos":2,"hash":"92bf47a2498333897b80e299e6248c3a9f180a68d9f8ced4d1408956afc48e55","isCurrent":1,"incarnation":0}',
+    ],
+  ]);
+
+  const spanRolloverResult = store.rolloverSpan(vatID);
+
+  t.deepEqual(exportLog.getLog(), [
+    [
+      'transcript.v1.0',
+      '{"vatID":"v1","startPos":0,"endPos":2,"hash":"92bf47a2498333897b80e299e6248c3a9f180a68d9f8ced4d1408956afc48e55","isCurrent":0,"incarnation":0}',
+    ],
+    [
+      'transcript.v1.current',
+      '{"vatID":"v1","startPos":2,"endPos":2,"hash":"43e6be43a3a34d60c0ebeb8498b5849b094fc20fc68483a7aeb3624fa10f79f6","isCurrent":1,"incarnation":0}',
+    ],
+  ]);
+
+  if (useArchiver) {
+    t.deepEqual(await getTranscriptArtifact?.(), {
+      name: 'transcript.v1.0.2',
+      entries: [Buffer.from('foo\n'), Buffer.from('bar\n')],
+    });
+  }
+
+  await spanRolloverResult;
+
+  t.deepEqual(exportLog.getLog(), []);
+
+  store.addItem(vatID, 'foo');
+  const incarnationRolloverResult = store.rolloverIncarnation(vatID);
+
+  t.deepEqual(exportLog.getLog(), [
+    [
+      'transcript.v1.current',
+      '{"vatID":"v1","startPos":2,"endPos":3,"hash":"3a676f1da91fe6da8268853f21e64e3366da45811b830f9d92f39d4c01d69291","isCurrent":1,"incarnation":0}',
+    ],
+    [
+      'transcript.v1.2',
+      '{"vatID":"v1","startPos":2,"endPos":3,"hash":"3a676f1da91fe6da8268853f21e64e3366da45811b830f9d92f39d4c01d69291","isCurrent":0,"incarnation":0}',
+    ],
+    [
+      'transcript.v1.current',
+      '{"vatID":"v1","startPos":3,"endPos":3,"hash":"43e6be43a3a34d60c0ebeb8498b5849b094fc20fc68483a7aeb3624fa10f79f6","isCurrent":1,"incarnation":1}',
+    ],
+  ]);
+
+  if (useArchiver) {
+    t.deepEqual(await getTranscriptArtifact?.(), {
+      name: 'transcript.v1.2.3',
+      entries: [Buffer.from('foo\n')],
+    });
+  }
+
+  await incarnationRolloverResult;
+
+  t.deepEqual(exportLog.getLog(), []);
+
+  store.addItem(vatID, 'foo');
+  const stopResult = store.stopUsingTranscript(vatID);
+
+  t.deepEqual(exportLog.getLog(), [
+    [
+      'transcript.v1.current',
+      '{"vatID":"v1","startPos":3,"endPos":4,"hash":"3a676f1da91fe6da8268853f21e64e3366da45811b830f9d92f39d4c01d69291","isCurrent":1,"incarnation":1}',
+    ],
+    [
+      'transcript.v1.3',
+      '{"vatID":"v1","startPos":3,"endPos":4,"hash":"3a676f1da91fe6da8268853f21e64e3366da45811b830f9d92f39d4c01d69291","isCurrent":0,"incarnation":1}',
+    ],
+    ['transcript.v1.current', undefined],
+  ]);
+
+  if (useArchiver) {
+    t.deepEqual(await getTranscriptArtifact?.(), {
+      name: 'transcript.v1.3.4',
+      entries: [Buffer.from('foo\n')],
+    });
+  }
+
+  await stopResult;
+
+  t.deepEqual(exportLog.getLog(), []);
+});
+
+test('span close operations are sync (with archiver)', closeSpanMacro, true);
+test(
+  'span close operations are sync (without archiver)',
+  closeSpanMacro,
+  false,
+);