@agoric/swing-store 0.9.1 → 0.9.2-calypso-dev-84eb287.0

package/src/transcriptStore.js

@@ -6,7 +6,12 @@ import BufferLineTransform from '@agoric/internal/src/node/buffer-line-transform
 import { createSHA256 } from './hasher.js';
 
 /**
- * @typedef { import('./swingStore').SwingStoreExporter } SwingStoreExporter
+ * @template T
+ *  @typedef  { IterableIterator<T> | AsyncIterableIterator<T> } AnyIterableIterator<T>
+ */
+
+/**
+ * @typedef { import('./internal.js').ArtifactMode } ArtifactMode
  *
  * @typedef {{
  *   initTranscript: (vatID: string) => void,
@@ -19,10 +24,13 @@ import { createSHA256 } from './hasher.js';
  * }} TranscriptStore
  *
  * @typedef {{
- *   exportSpan: (name: string, includeHistorical: boolean) => AsyncIterableIterator<Uint8Array>
- *   importSpan: (artifactName: string, exporter: SwingStoreExporter, artifactMetadata: Map) => Promise<void>,
+ *   exportSpan: (name: string) => AsyncIterableIterator<Uint8Array>
  *   getExportRecords: (includeHistorical: boolean) => IterableIterator<readonly [key: string, value: string]>,
- *   getArtifactNames: (includeHistorical: boolean) => AsyncIterableIterator<string>,
+ *   getArtifactNames: (artifactMode: ArtifactMode) => AsyncIterableIterator<string>,
+ *   importTranscriptSpanRecord: (key: string, value: string) => void,
+ *   populateTranscriptSpan: (name: string, makeChunkIterator: () => AnyIterableIterator<Uint8Array>, options: { artifactMode: ArtifactMode }) => Promise<void>,
+ *   assertComplete: (checkMode: Omit<ArtifactMode, 'debug'>) => void,
+ *   repairTranscriptSpanRecord: (key: string, value: string) => void,
  *   readFullVatTranscript: (vatID: string) => Iterable<{position: number, item: string}>
  * }} TranscriptStoreInternal
  *
@@ -35,6 +43,7 @@ import { createSHA256 } from './hasher.js';
 function* empty() {
   // Yield nothing
 }
+harden(empty);
 
 /**
  * @param {number} position
@@ -82,7 +91,7 @@ export function makeTranscriptStore(
   //
   // The transcriptItems associated with historical spans may or may not exist,
   // depending on pruning.  However, the items associated with the current span
-  // must always be present
+  // must always be present.
 
   db.exec(`
     CREATE TABLE IF NOT EXISTS transcriptSpans (
@@ -150,6 +159,7 @@ export function makeTranscriptStore(
       }
     }
   }
+  harden(readFullVatTranscript);
 
   function spanArtifactName(rec) {
     return `transcript.${rec.vatID}.${rec.startPos}.${rec.endPos}`;
@@ -325,6 +335,13 @@ export function makeTranscriptStore(
     ORDER BY vatID, startPos
   `);
 
+  const sqlGetIncarnationSpanMetadata = db.prepare(`
+    SELECT vatID, startPos, endPos, hash, isCurrent, incarnation
+    FROM transcriptSpans
+    WHERE vatID=? AND incarnation=?
+    ORDER BY vatID, startPos
+  `);
+
   const sqlGetCurrentSpanMetadata = db.prepare(`
     SELECT vatID, startPos, endPos, hash, isCurrent, incarnation
     FROM transcriptSpans
@@ -332,6 +349,11 @@ export function makeTranscriptStore(
     ORDER BY vatID, startPos
   `);
 
+  function dbRecToExportRec(dbRec) {
+    const { vatID, startPos, endPos, hash, isCurrent, incarnation } = dbRec;
+    return spanRec(vatID, startPos, endPos, hash, isCurrent, incarnation);
+  }
+
   /**
    * Obtain artifact metadata records for spans contained in this store.
    *
@@ -354,47 +376,81 @@ export function makeTranscriptStore(
    * replay will never be required or because such replay would be prohibitively
    * expensive regardless of need and therefor other repair strategies employed.
    *
+   * The only code path which could use 'false' would be `swingstore.dump()`,
+   * which takes the same flag.
+   *
    * @yields {readonly [key: string, value: string]}
    * @returns {IterableIterator<readonly [key: string, value: string]>}
    *    An iterator over pairs of [spanMetadataKey, rec], where `rec` is a
    *    JSON-encoded metadata record for the span named by `spanMetadataKey`.
    */
   function* getExportRecords(includeHistorical = true) {
-    const sql = includeHistorical
-      ? sqlGetAllSpanMetadata
-      : sqlGetCurrentSpanMetadata;
-    for (const rec of sql.iterate()) {
-      const { vatID, startPos, endPos, hash, isCurrent, incarnation } = rec;
-      const exportRec = spanRec(
-        vatID,
-        startPos,
-        endPos,
-        hash,
-        isCurrent,
-        incarnation,
-      );
-      yield [spanMetadataKey(rec), JSON.stringify(exportRec)];
+    if (includeHistorical) {
+      for (const rec of sqlGetAllSpanMetadata.iterate()) {
+        yield [spanMetadataKey(rec), JSON.stringify(dbRecToExportRec(rec))];
+      }
+    } else {
+      for (const rec of sqlGetCurrentSpanMetadata.iterate()) {
+        yield [spanMetadataKey(rec), JSON.stringify(dbRecToExportRec(rec))];
+      }
     }
   }
+  harden(getExportRecords);
+
+  const sqlCountSpanItems = db.prepare(`
+    SELECT COUNT(*) FROM transcriptItems
+      WHERE vatID = ? AND position >= ? AND position < ?
+  `);
+  sqlCountSpanItems.pluck();
 
   /**
    * Obtain artifact names for spans contained in this store.
    *
-   * @param {boolean} includeHistorical If true, include all spans that are
-   * present in the store regardless of their currency; if false, only include
-   * the current span for each vat.
-   *
+   * @param {ArtifactMode} artifactMode Control which artifacts should be exported.
+   *   At 'operational', only include current spans. At 'replay',
+   *   include all spans of the current incarnation for each vat. At
+   *   'archival' and 'debug', include all spans.
    * @yields {string}
    * @returns {AsyncIterableIterator<string>}  An iterator over the names of all the artifacts requested
    */
-  async function* getArtifactNames(includeHistorical) {
-    const sql = includeHistorical
-      ? sqlGetAllSpanMetadata
-      : sqlGetCurrentSpanMetadata;
-    for (const rec of sql.iterate()) {
-      yield spanArtifactName(rec);
+  async function* getArtifactNames(artifactMode) {
+    // for all non-'debug' modes, the exporter asserts that all
+    // requested items are present (i.e. the artifacts will be
+    // complete), so we don't need to check that ourselves
+    if (artifactMode === 'operational') {
+      for (const rec of sqlGetCurrentSpanMetadata.iterate()) {
+        yield spanArtifactName(rec);
+      }
+    } else if (artifactMode === 'replay') {
+      for (const curRec of sqlGetCurrentSpanMetadata.iterate()) {
+        const { vatID, incarnation } = curRec;
+        for (const rec of sqlGetIncarnationSpanMetadata.iterate(
+          vatID,
+          incarnation,
+        )) {
+          yield spanArtifactName(rec);
+        }
+      }
+    } else if (artifactMode === 'archival') {
+      // everything
+      for (const rec of sqlGetAllSpanMetadata.iterate()) {
+        yield spanArtifactName(rec);
+      }
+    } else if (artifactMode === 'debug') {
+      // everything that is a complete span
+      for (const rec of sqlGetAllSpanMetadata.iterate()) {
+        const { vatID, startPos, endPos } = rec;
+        const count = sqlCountSpanItems.get(vatID, startPos, endPos);
+        if (count !== endPos - startPos) {
+          // skip incomplete spans, because the exporter did not
+          // already do a completeness check in 'debug' mode
+          continue;
+        }
+        yield spanArtifactName(rec);
+      }
     }
   }
+  harden(getArtifactNames);
 
   const sqlGetSpanEndPos = db.prepare(`
     SELECT endPos
@@ -432,16 +488,24 @@ export function makeTranscriptStore(
       }
     }
     startPos <= endPos || Fail`${q(startPos)} <= ${q(endPos)}}`;
+    const expectedCount = endPos - startPos;
 
     function* reader() {
+      let count = 0;
       for (const { item } of sqlReadSpanItems.iterate(
         vatID,
         startPos,
         endPos,
       )) {
         yield item;
+        count += 1;
       }
+      count === expectedCount ||
+        Fail`read ${q(count)} transcript entries (expected ${q(
+          expectedCount,
+        )})`;
     }
+    harden(reader);
 
     if (startPos === endPos) {
       return empty();
@@ -465,12 +529,10 @@ export function makeTranscriptStore(
    *   `transcript.${vatID}.${startPos}.${endPos}`
    *
    * @param {string} name  The name of the transcript artifact to be read
-   * @param {boolean} includeHistorical  If true, allow non-current spans to be fetched
-   *
    * @returns {AsyncIterableIterator<Uint8Array>}
    * @yields {Uint8Array}
    */
-  async function* exportSpan(name, includeHistorical) {
+  async function* exportSpan(name) {
     typeof name === 'string' || Fail`artifact name must be a string`;
     const parts = name.split('.');
     const [type, vatID, pos] = parts;
@@ -479,14 +541,12 @@ export function makeTranscriptStore(
       Fail`expected artifact name of the form 'transcript.{vatID}.{startPos}.{endPos}', saw ${q(name)}`;
     const isCurrent = sqlGetSpanIsCurrent.get(vatID, pos);
     isCurrent !== undefined || Fail`transcript span ${q(name)} not available`;
-    isCurrent ||
-      includeHistorical ||
-      Fail`transcript span ${q(name)} not available`;
     const startPos = Number(pos);
     for (const entry of readSpan(vatID, startPos)) {
       yield Buffer.from(`${entry}\n`);
     }
   }
+  harden(exportSpan);
 
   const sqlAddItem = db.prepare(`
     INSERT INTO transcriptItems (vatID, item, position, incarnation)
@@ -516,33 +576,104 @@ export function makeTranscriptStore(
     noteExport(spanMetadataKey(rec), JSON.stringify(rec));
   };
 
+  function importTranscriptSpanRecord(key, value) {
+    ensureTxn();
+    const [tag, keyVatID, keyStartPos] = key.split('.');
+    assert.equal(tag, 'transcript');
+    const metadata = JSON.parse(value);
+    if (key.endsWith('.current') !== Boolean(metadata.isCurrent)) {
+      throw Fail`transcript key ${key} mismatches metadata ${metadata}`;
+    }
+    const { vatID, startPos, endPos, hash, isCurrent, incarnation } = metadata;
+    vatID || Fail`transcript metadata missing vatID: ${metadata}`;
+    startPos !== undefined ||
+      Fail`transcript metadata missing startPos: ${metadata}`;
+    endPos !== undefined ||
+      Fail`transcript metadata missing endPos: ${metadata}`;
+    hash || Fail`transcript metadata missing hash: ${metadata}`;
+    isCurrent !== undefined ||
+      Fail`transcript metadata missing isCurrent: ${metadata}`;
+    incarnation !== undefined ||
+      Fail`transcript metadata missing incarnation: ${metadata}`;
+    if (keyStartPos !== 'current') {
+      if (Number(keyStartPos) !== startPos) {
+        Fail`transcript key ${key} mismatches metadata ${metadata}`;
+      }
+    }
+    keyVatID === vatID ||
+      Fail`transcript key ${key} mismatches metadata ${metadata}`;
+
+    // sqlWriteSpan is an INSERT, so the PRIMARY KEY (vatID, position)
+    // constraint will catch broken export-data errors like trying to
+    // add two different versions of the same span (e.g. one holding
+    // items 4..8, a second holding 4..9)
+
+    sqlWriteSpan.run(
+      vatID,
+      startPos,
+      endPos,
+      hash,
+      isCurrent ? 1 : null,
+      incarnation,
+    );
+  }
+
+  const sqlGetSpanMetadataFor = db.prepare(`
+    SELECT hash, isCurrent, incarnation, endPos
+      FROM transcriptSpans
+      WHERE vatID = ? AND startPos = ?
+  `);
+
+  const sqlGetStartOfIncarnation = db.prepare(`
+    SELECT startPos
+      FROM transcriptSpans
+      WHERE vatID=? AND incarnation=?
+      ORDER BY startPos ASC LIMIT 1
+  `);
+  sqlGetStartOfIncarnation.pluck();
+
   /**
    * Import a transcript span from another store.
    *
    * @param {string} name  Artifact Name of the transcript span
-   * @param {SwingStoreExporter} exporter  Exporter from which to get the span data
-   * @param {object} info  Metadata describing the span
+   * @param {() => AnyIterableIterator<Uint8Array>} makeChunkIterator  get an iterator of transcript byte chunks
+   * @param {object} options
+   * @param {ArtifactMode} options.artifactMode
    *
    * @returns {Promise<void>}
    */
-  async function importSpan(name, exporter, info) {
+  async function populateTranscriptSpan(name, makeChunkIterator, options) {
+    ensureTxn();
+    const { artifactMode } = options;
     const parts = name.split('.');
     const [type, vatID, rawStartPos, rawEndPos] = parts;
     // prettier-ignore
     parts.length === 4 && type === 'transcript' ||
       Fail`expected artifact name of the form 'transcript.{vatID}.{startPos}.{endPos}', saw '${q(name)}'`;
-    // prettier-ignore
-    info.vatID === vatID ||
-      Fail`artifact name says vatID ${q(vatID)}, metadata says ${q(info.vatID)}`;
     const startPos = Number(rawStartPos);
-    // prettier-ignore
-    info.startPos === startPos ||
-      Fail`artifact name says startPos ${q(startPos)}, metadata says ${q(info.startPos)}`;
     const endPos = Number(rawEndPos);
-    // prettier-ignore
-    info.endPos === endPos ||
-      Fail`artifact name says endPos ${q(endPos)}, metadata says ${q(info.endPos)}`;
-    const artifactChunks = exporter.getArtifact(name);
+
+    const metadata =
+      sqlGetSpanMetadataFor.get(vatID, startPos) ||
+      Fail`no metadata for transcript span ${name}`;
+    assert.equal(metadata.endPos, endPos);
+
+    if (artifactMode === 'operational') {
+      if (!metadata.isCurrent) {
+        return; // ignore old spans
+      }
+    }
+    if (artifactMode === 'replay') {
+      // ignore spans that aren't for the current incarnation
+      const { incarnation } = sqlGetCurrentSpanBounds.get(vatID);
+      const incStart = sqlGetStartOfIncarnation.get(vatID, incarnation);
+      if (startPos < incStart) {
+        return;
+      }
+    }
+    // 'archival' and 'debug' modes accept all spans
+
+    const artifactChunks = await makeChunkIterator();
     const inStream = Readable.from(artifactChunks);
     const lineTransform = new BufferLineTransform();
     const lineStream = inStream.pipe(lineTransform).setEncoding('utf8');
@@ -550,21 +681,90 @@ export function makeTranscriptStore(
     let pos = startPos;
     for await (const line of lineStream) {
       const item = line.trimEnd();
-      sqlAddItem.run(vatID, item, pos, info.incarnation);
+      sqlAddItem.run(vatID, item, pos, metadata.incarnation);
       hash = updateSpanHash(hash, item);
       pos += 1;
     }
-    pos === endPos || Fail`artifact ${name} is not available`;
-    info.hash === hash ||
-      Fail`artifact ${name} hash is ${q(hash)}, metadata says ${q(info.hash)}`;
-    sqlWriteSpan.run(
-      info.vatID,
-      info.startPos,
-      info.endPos,
-      info.hash,
-      info.isCurrent ? 1 : null,
-      info.incarnation,
-    );
+    pos === endPos || Fail`artifact ${name} is not complete`;
+
+    // validate against the previously-established metadata
+
+    // prettier-ignore
+    metadata.hash === hash ||
+      Fail`artifact ${name} hash is ${q(hash)}, metadata says ${q(metadata.hash)}`;
+
+    // If that passes, the not-yet-committed data is good. If it
+    // fails, the thrown error will flunk the import and inhibit a
+    // commit. So we're done.
+  }
+
+  function repairTranscriptSpanRecord(key, value) {
+    ensureTxn();
+    const [tag, keyVatID, keyStartPos] = key.split('.');
+    assert.equal(tag, 'transcript');
+    const metadata = JSON.parse(value);
+    const { vatID, startPos, endPos, hash, isCurrent, incarnation } = metadata;
+    assert.equal(keyVatID, vatID);
+    if (keyStartPos !== 'current') {
+      if (Number(keyStartPos) !== startPos) {
+        Fail`transcript key ${key} mismatches metadata ${metadata}`;
+      }
+    }
+
+    const existing = sqlGetSpanMetadataFor.get(vatID, startPos);
+    if (existing) {
+      if (
+        Boolean(existing.isCurrent) !== Boolean(isCurrent) ||
+        existing.hash !== hash ||
+        existing.incarnation !== incarnation ||
+        existing.endPos !== endPos
+      ) {
+        throw Fail`repairTranscriptSpanRecord metadata mismatch: ${existing} vs ${metadata}`;
+      }
+    } else {
+      sqlWriteSpan.run(
+        vatID,
+        startPos,
+        endPos,
+        hash,
+        isCurrent ? 1 : null,
+        incarnation,
+      );
+    }
+  }
+
+  function assertComplete(checkMode) {
+    assert(checkMode !== 'debug', checkMode);
+    for (const rec of sqlGetCurrentSpanMetadata.iterate()) {
+      const { vatID, startPos, endPos, incarnation } = rec;
+
+      if (checkMode === 'operational') {
+        // at 'operational', every 'isCurrent' transcript span must
+        // have all items
+        const count = sqlCountSpanItems.get(vatID, startPos, endPos);
+        if (count !== endPos - startPos) {
+          throw Fail`incomplete current transcript span: ${count} items, ${rec}`;
+        }
+      } else if (checkMode === 'replay') {
+        // at 'replay', every vat's current incarnation must be fully
+        // populated (which implies 'operational')
+        const incStart = sqlGetStartOfIncarnation.get(vatID, incarnation);
+        const incCount = sqlCountSpanItems.get(vatID, incStart, endPos);
+        if (incCount !== endPos - incStart) {
+          throw Fail`incomplete current incarnation transcript: ${incCount} items`;
+        }
+      } else if (checkMode === 'archival') {
+        // at 'archival', every incarnation must be fully populated,
+        // which means position=0 up through endPos-1 (which implies
+        // 'replay')
+        const arcCount = sqlCountSpanItems.get(vatID, 0, endPos);
+        if (arcCount !== endPos) {
+          throw Fail`incomplete archival transcript: ${arcCount} vs ${endPos}`;
+        }
+      } else {
+        throw Fail`unknown checkMode ${checkMode}`;
+      }
+    }
   }
 
   return harden({
@@ -577,10 +777,14 @@ export function makeTranscriptStore(
     deleteVatTranscripts,
 
     exportSpan,
-    importSpan,
     getExportRecords,
     getArtifactNames,
 
+    importTranscriptSpanRecord,
+    populateTranscriptSpan,
+    assertComplete,
+    repairTranscriptSpanRecord,
+
     dumpTranscripts,
     readFullVatTranscript,
   });

package/src/types.d.ts

@@ -0,0 +1,15 @@
+export type {
+  SwingStore,
+  SwingStoreKernelStorage,
+  SwingStoreHostStorage,
+} from './swingStore.js';
+export type { KVStore } from './kvStore.js';
+export type { BundleStore } from './bundleStore.js';
+export type { SnapStore, SnapshotResult, SnapshotInfo } from './snapStore.js';
+export type { TranscriptStore } from './transcriptStore.js';
+export type { ArtifactMode } from './internal.js';
+export type { ImportSwingStoreOptions } from './importer.js';
+export type {
+  SwingStoreExporter,
+  ExportSwingStoreOptions,
+} from './exporter.js';

package/src/types.js

@@ -0,0 +1,6 @@
+// Types for the public API
+
+// Everything this "exports" actually comes from the neighboring types.d.ts file
+
+// Ensure this is a module.
+export {};

package/src/util.js

@@ -1,3 +1,4 @@
+import path from 'path';
 import { Buffer } from 'buffer';
 
 /**
@@ -5,7 +6,7 @@ import { Buffer } from 'buffer';
  * 'stream/consumers' package, which unfortunately only exists in newer versions
  * of Node.
  *
- * @param {import('./swingStore').AnyIterable<Uint8Array>} inStream
+ * @param {import('./exporter.js').AnyIterable<Uint8Array>} inStream
  */
 export const buffer = async inStream => {
   const chunks = [];
@@ -14,3 +15,8 @@ export const buffer = async inStream => {
   }
   return Buffer.concat(chunks);
 };
+
+export function dbFileInDirectory(dirPath) {
+  const filePath = path.resolve(dirPath, 'swingstore.sqlite');
+  return filePath;
+}

package/test/{test-bundles.js → bundles.test.js}

@@ -1,14 +1,11 @@
 // @ts-check
-import '@endo/init/debug.js';
 import test from 'ava';
 import tmp from 'tmp';
 import { Buffer } from 'buffer';
 import { createSHA256 } from '../src/hasher.js';
-import {
-  importSwingStore,
-  initSwingStore,
-  makeSwingStoreExporter,
-} from '../src/swingStore.js';
+import { initSwingStore } from '../src/swingStore.js';
+import { makeSwingStoreExporter } from '../src/exporter.js';
+import { importSwingStore } from '../src/importer.js';
 import { buffer } from '../src/util.js';
 
 function makeB0ID(bundle) {
@@ -109,6 +106,9 @@ test('b0 import', async t => {
   const idA = makeB0ID(b0A);
   const nameA = `bundle.${idA}`;
   const exporter = {
+    getHostKV(_key) {
+      return undefined;
+    },
     async *getExportData() {
       yield /** @type {const} */ ([nameA, idA]);
     },
@@ -116,10 +116,16 @@ test('b0 import', async t => {
       t.is(name, nameA);
       yield Buffer.from(JSON.stringify(b0A));
     },
-    getArtifactNames: () => assert.fail('import should not query all names'),
+    async *getArtifactNames() {
+      yield* [nameA];
+    },
     close: async () => undefined,
   };
-  const { kernelStorage } = await importSwingStore(exporter);
+  const ss = await importSwingStore(exporter);
+  t.teardown(ss.hostStorage.close);
+  await ss.hostStorage.commit();
+  const serialized = ss.debug.serialize();
+  const { kernelStorage } = initSwingStore(null, { serialized });
   const { bundleStore } = kernelStorage;
   t.truthy(bundleStore.hasBundle(idA));
   t.deepEqual(bundleStore.getBundle(idA), b0A);
@@ -131,6 +137,9 @@ test('b0 bad import', async t => {
   const idA = makeB0ID(b0A);
   const nameA = `bundle.${idA}`;
   const exporter = {
+    getHostKV(_key) {
+      return undefined;
+    },
     async *getExportData() {
       yield /** @type {const} */ ([nameA, idA]);
     },
@@ -138,7 +147,9 @@ test('b0 bad import', async t => {
       t.is(name, nameA);
       yield Buffer.from(JSON.stringify(b0Abogus));
     },
-    getArtifactNames: () => assert.fail('import should not query all names'),
+    async *getArtifactNames() {
+      yield* [nameA];
+    },
     close: async () => undefined,
   };
   await t.throwsAsync(async () => importSwingStore(exporter), {
@@ -150,7 +161,7 @@ test('unknown format', t => {
   const { kernelStorage } = initSwingStore();
   const { bundleStore } = kernelStorage;
   const unknownID = 'b1999-whoa-futuristic';
-  /** @typedef {import('../src/bundleStore.js').Bundle} Bundle */
+  /** @import {Bundle} from '../src/bundleStore.js' */
   t.throws(() => bundleStore.addBundle(unknownID, /** @type {Bundle} */ ({})), {
     message: /unsupported BundleID/,
   });

package/test/{test-deletion.js → deletion.test.js}

@@ -1,12 +1,12 @@
 // @ts-check
 import test from 'ava';
-import '@endo/init/debug.js';
 import { Buffer } from 'node:buffer';
 import { initSwingStore } from '../src/swingStore.js';
 
 async function* getSnapshotStream() {
   yield Buffer.from('abc');
 }
+harden(getSnapshotStream);
 
 test('delete snapshots with export callback', async t => {
   const exportLog = [];