@agoric/swing-store 0.9.2-dev-2f092c3.0 → 0.9.2-dev-aa10ecd.0

package/src/transcriptStore.js

@@ -6,8 +6,11 @@ import BufferLineTransform from '@agoric/internal/src/node/buffer-line-transform
 import { createSHA256 } from './hasher.js';
 
 /**
- * @typedef { import('./swingStore').SwingStoreExporter } SwingStoreExporter
- *
+ * @template T
+ *  @typedef  { IterableIterator<T> | AsyncIterableIterator<T> } AnyIterableIterator<T>
+ */
+
+/**
  * @typedef {{
  *   initTranscript: (vatID: string) => void,
  *   rolloverSpan: (vatID: string) => number,
@@ -19,10 +22,13 @@ import { createSHA256 } from './hasher.js';
  * }} TranscriptStore
  *
  * @typedef {{
- *   exportSpan: (name: string, includeHistorical: boolean) => AsyncIterableIterator<Uint8Array>
- *   importSpan: (artifactName: string, exporter: SwingStoreExporter, artifactMetadata: Map) => Promise<void>,
+ *   exportSpan: (name: string) => AsyncIterableIterator<Uint8Array>
  *   getExportRecords: (includeHistorical: boolean) => IterableIterator<readonly [key: string, value: string]>,
  *   getArtifactNames: (includeHistorical: boolean) => AsyncIterableIterator<string>,
+ *   importTranscriptSpanRecord: (key: string, value: string) => void,
+ *   populateTranscriptSpan: (name: string, makeChunkIterator: () => AnyIterableIterator<Uint8Array>, options: { includeHistorical: boolean }) => Promise<void>,
+ *   assertComplete: (level: 'operational') => void,
+ *   repairTranscriptSpanRecord: (key: string, value: string) => void,
  *   readFullVatTranscript: (vatID: string) => Iterable<{position: number, item: string}>
  * }} TranscriptStoreInternal
  *
@@ -82,7 +88,7 @@ export function makeTranscriptStore(
   //
   // The transcriptItems associated with historical spans may or may not exist,
   // depending on pruning.  However, the items associated with the current span
-  // must always be present
+  // must always be present.
 
   db.exec(`
     CREATE TABLE IF NOT EXISTS transcriptSpans (
@@ -377,6 +383,17 @@ export function makeTranscriptStore(
     }
   }
 
+  // 'position' is not recycled across incarnations, so strictly
+  // speaking this query doesn't need to filter on 'incarnation = ?',
+  // but this will catch problems like items with incorrect or missing
+  // incarnation values
+
+  const sqlCountPopulatedSpanItems = db.prepare(`
+    SELECT COUNT(*) FROM transcriptItems
+      WHERE vatID = ? AND incarnation = ? AND position >= ? AND position < ?
+  `);
+  sqlCountPopulatedSpanItems.pluck();
+
   /**
    * Obtain artifact names for spans contained in this store.
    *
@@ -392,6 +409,13 @@ export function makeTranscriptStore(
       ? sqlGetAllSpanMetadata
       : sqlGetCurrentSpanMetadata;
     for (const rec of sql.iterate()) {
+      const { vatID, incarnation, startPos, endPos } = rec;
+      if (
+        !sqlCountPopulatedSpanItems.get(vatID, incarnation, startPos, endPos)
+      ) {
+        continue;
+      }
+
       yield spanArtifactName(rec);
     }
   }
@@ -432,15 +456,22 @@ export function makeTranscriptStore(
       }
     }
     startPos <= endPos || Fail`${q(startPos)} <= ${q(endPos)}}`;
+    const expectedCount = endPos - startPos;
 
     function* reader() {
+      let count = 0;
       for (const { item } of sqlReadSpanItems.iterate(
         vatID,
         startPos,
         endPos,
       )) {
         yield item;
+        count += 1;
       }
+      count === expectedCount ||
+        Fail`read ${q(count)} transcript entries (expected ${q(
+          expectedCount,
+        )})`;
     }
 
     if (startPos === endPos) {
@@ -465,12 +496,10 @@ export function makeTranscriptStore(
    *   `transcript.${vatID}.${startPos}.${endPos}`
    *
    * @param {string} name  The name of the transcript artifact to be read
-   * @param {boolean} includeHistorical  If true, allow non-current spans to be fetched
-   *
    * @returns {AsyncIterableIterator<Uint8Array>}
    * @yields {Uint8Array}
    */
-  async function* exportSpan(name, includeHistorical) {
+  async function* exportSpan(name) {
     typeof name === 'string' || Fail`artifact name must be a string`;
     const parts = name.split('.');
     const [type, vatID, pos] = parts;
@@ -479,9 +508,6 @@ export function makeTranscriptStore(
       Fail`expected artifact name of the form 'transcript.{vatID}.{startPos}.{endPos}', saw ${q(name)}`;
     const isCurrent = sqlGetSpanIsCurrent.get(vatID, pos);
     isCurrent !== undefined || Fail`transcript span ${q(name)} not available`;
-    isCurrent ||
-      includeHistorical ||
-      Fail`transcript span ${q(name)} not available`;
     const startPos = Number(pos);
     for (const entry of readSpan(vatID, startPos)) {
       yield Buffer.from(`${entry}\n`);
@@ -516,33 +542,84 @@ export function makeTranscriptStore(
     noteExport(spanMetadataKey(rec), JSON.stringify(rec));
   };
 
+  function importTranscriptSpanRecord(key, value) {
+    ensureTxn();
+    const [tag, keyVatID, keyStartPos] = key.split('.');
+    assert.equal(tag, 'transcript');
+    const metadata = JSON.parse(value);
+    if (key.endsWith('.current') !== Boolean(metadata.isCurrent)) {
+      throw Fail`transcript key ${key} mismatches metadata ${metadata}`;
+    }
+    const { vatID, startPos, endPos, hash, isCurrent, incarnation } = metadata;
+    vatID || Fail`transcript metadata missing vatID: ${metadata}`;
+    startPos !== undefined ||
+      Fail`transcript metadata missing startPos: ${metadata}`;
+    endPos !== undefined ||
+      Fail`transcript metadata missing endPos: ${metadata}`;
+    hash || Fail`transcript metadata missing hash: ${metadata}`;
+    isCurrent !== undefined ||
+      Fail`transcript metadata missing isCurrent: ${metadata}`;
+    incarnation !== undefined ||
+      Fail`transcript metadata missing incarnation: ${metadata}`;
+    if (keyStartPos !== 'current') {
+      if (Number(keyStartPos) !== startPos) {
+        Fail`transcript key ${key} mismatches metadata ${metadata}`;
+      }
+    }
+    keyVatID === vatID ||
+      Fail`transcript key ${key} mismatches metadata ${metadata}`;
+
+    // sqlWriteSpan is an INSERT, so the PRIMARY KEY (vatID, position)
+    // constraint will catch broken export-data errors like trying to
+    // add two different versions of the same span (e.g. one holding
+    // items 4..8, a second holding 4..9)
+
+    sqlWriteSpan.run(
+      vatID,
+      startPos,
+      endPos,
+      hash,
+      isCurrent ? 1 : null,
+      incarnation,
+    );
+  }
+
+  const sqlGetSpanMetadataFor = db.prepare(`
+    SELECT hash, isCurrent, incarnation, endPos
+      FROM transcriptSpans
+      WHERE vatID = ? AND startPos = ?
+  `);
+
   /**
    * Import a transcript span from another store.
    *
    * @param {string} name  Artifact Name of the transcript span
-   * @param {SwingStoreExporter} exporter  Exporter from which to get the span data
-   * @param {object} info  Metadata describing the span
+   * @param {() => AnyIterableIterator<Uint8Array>} makeChunkIterator  get an iterator of transcript byte chunks
+   * @param {object} options
+   * @param {boolean} options.includeHistorical
    *
    * @returns {Promise<void>}
    */
-  async function importSpan(name, exporter, info) {
+  async function populateTranscriptSpan(name, makeChunkIterator, options) {
+    ensureTxn();
+    const { includeHistorical } = options;
     const parts = name.split('.');
     const [type, vatID, rawStartPos, rawEndPos] = parts;
     // prettier-ignore
     parts.length === 4 && type === 'transcript' ||
       Fail`expected artifact name of the form 'transcript.{vatID}.{startPos}.{endPos}', saw '${q(name)}'`;
-    // prettier-ignore
-    info.vatID === vatID ||
-      Fail`artifact name says vatID ${q(vatID)}, metadata says ${q(info.vatID)}`;
     const startPos = Number(rawStartPos);
-    // prettier-ignore
-    info.startPos === startPos ||
-      Fail`artifact name says startPos ${q(startPos)}, metadata says ${q(info.startPos)}`;
     const endPos = Number(rawEndPos);
-    // prettier-ignore
-    info.endPos === endPos ||
-      Fail`artifact name says endPos ${q(endPos)}, metadata says ${q(info.endPos)}`;
-    const artifactChunks = exporter.getArtifact(name);
+
+    const metadata =
+      sqlGetSpanMetadataFor.get(vatID, startPos) ||
+      Fail`no metadata for transcript span ${name}`;
+    if (!metadata.isCurrent && !includeHistorical) {
+      return; // ignore old spans
+    }
+    assert.equal(metadata.endPos, endPos);
+
+    const artifactChunks = await makeChunkIterator();
     const inStream = Readable.from(artifactChunks);
     const lineTransform = new BufferLineTransform();
     const lineStream = inStream.pipe(lineTransform).setEncoding('utf8');
@@ -550,21 +627,73 @@ export function makeTranscriptStore(
     let pos = startPos;
     for await (const line of lineStream) {
       const item = line.trimEnd();
-      sqlAddItem.run(vatID, item, pos, info.incarnation);
+      sqlAddItem.run(vatID, item, pos, metadata.incarnation);
       hash = updateSpanHash(hash, item);
       pos += 1;
     }
-    pos === endPos || Fail`artifact ${name} is not available`;
-    info.hash === hash ||
-      Fail`artifact ${name} hash is ${q(hash)}, metadata says ${q(info.hash)}`;
-    sqlWriteSpan.run(
-      info.vatID,
-      info.startPos,
-      info.endPos,
-      info.hash,
-      info.isCurrent ? 1 : null,
-      info.incarnation,
-    );
+    pos === endPos || Fail`artifact ${name} is not complete`;
+
+    // validate against the previously-established metadata
+
+    // prettier-ignore
+    metadata.hash === hash ||
+      Fail`artifact ${name} hash is ${q(hash)}, metadata says ${q(metadata.hash)}`;
+
+    // If that passes, the not-yet-committed data is good. If it
+    // fails, the thrown error will flunk the import and inhibit a
+    // commit. So we're done.
+  }
+
+  function repairTranscriptSpanRecord(key, value) {
+    ensureTxn();
+    const [tag, keyVatID, keyStartPos] = key.split('.');
+    assert.equal(tag, 'transcript');
+    const metadata = JSON.parse(value);
+    const { vatID, startPos, endPos, hash, isCurrent, incarnation } = metadata;
+    assert.equal(keyVatID, vatID);
+    if (keyStartPos !== 'current') {
+      if (Number(keyStartPos) !== startPos) {
+        Fail`transcript key ${key} mismatches metadata ${metadata}`;
+      }
+    }
+
+    const existing = sqlGetSpanMetadataFor.get(vatID, startPos);
+    if (existing) {
+      if (
+        Boolean(existing.isCurrent) !== Boolean(isCurrent) ||
+        existing.hash !== hash ||
+        existing.incarnation !== incarnation ||
+        existing.endPos !== endPos
+      ) {
+        throw Fail`repairTranscriptSpanRecord metadata mismatch: ${existing} vs ${metadata}`;
+      }
+    } else {
+      sqlWriteSpan.run(
+        vatID,
+        startPos,
+        endPos,
+        hash,
+        isCurrent ? 1 : null,
+        incarnation,
+      );
+    }
+  }
+
+  function assertComplete(level) {
+    assert.equal(level, 'operational'); // for now
+    // every 'isCurrent' transcript span must have all items
+    for (const rec of sqlGetCurrentSpanMetadata.iterate()) {
+      const { vatID, startPos, endPos, incarnation } = rec;
+      const count = sqlCountPopulatedSpanItems.get(
+        vatID,
+        incarnation,
+        startPos,
+        endPos,
+      );
+      if (count !== endPos - startPos) {
+        throw Fail`incomplete current transcript span: ${count} items, ${rec}`;
+      }
+    }
   }
 
   return harden({
@@ -577,10 +706,14 @@ export function makeTranscriptStore(
     deleteVatTranscripts,
 
     exportSpan,
-    importSpan,
     getExportRecords,
     getArtifactNames,
 
+    importTranscriptSpanRecord,
+    populateTranscriptSpan,
+    assertComplete,
+    repairTranscriptSpanRecord,
+
     dumpTranscripts,
     readFullVatTranscript,
   });

package/src/types.d.ts

@@ -0,0 +1,14 @@
+export type {
+  SwingStore,
+  SwingStoreKernelStorage,
+  SwingStoreHostStorage,
+} from './src/swingStore.js';
+export type { KVStore } from './src/kvStore.js';
+export type { BundleStore } from './src/bundleStore.js';
+export type {
+  SnapStore,
+  SnapshotResult,
+  SnapshotInfo,
+} from './src/snapStore.js';
+export type { TranscriptStore } from './src/transcriptStore.js';
+export type { SwingStoreExporter, ExportMode } from './src/exporter.js';

package/src/types.js

@@ -0,0 +1,6 @@
+// Types for the public API
+
+// Everything this "exports" actually comes from the neighboring types.d.ts file
+
+// Ensure this is a module.
+export {};

package/src/util.js

@@ -1,3 +1,4 @@
+import path from 'path';
 import { Buffer } from 'buffer';
 
 /**
@@ -5,7 +6,7 @@ import { Buffer } from 'buffer';
  * 'stream/consumers' package, which unfortunately only exists in newer versions
  * of Node.
  *
- * @param {import('./swingStore').AnyIterable<Uint8Array>} inStream
+ * @param {import('./exporter').AnyIterable<Uint8Array>} inStream
  */
 export const buffer = async inStream => {
   const chunks = [];
@@ -14,3 +15,8 @@ export const buffer = async inStream => {
   }
   return Buffer.concat(chunks);
 };
+
+export function dbFileInDirectory(dirPath) {
+  const filePath = path.resolve(dirPath, 'swingstore.sqlite');
+  return filePath;
+}

package/test/test-bundles.js

@@ -4,11 +4,9 @@ import test from 'ava';
 import tmp from 'tmp';
 import { Buffer } from 'buffer';
 import { createSHA256 } from '../src/hasher.js';
-import {
-  importSwingStore,
-  initSwingStore,
-  makeSwingStoreExporter,
-} from '../src/swingStore.js';
+import { initSwingStore } from '../src/swingStore.js';
+import { makeSwingStoreExporter } from '../src/exporter.js';
+import { importSwingStore } from '../src/importer.js';
 import { buffer } from '../src/util.js';
 
 function makeB0ID(bundle) {
@@ -116,7 +114,9 @@ test('b0 import', async t => {
       t.is(name, nameA);
       yield Buffer.from(JSON.stringify(b0A));
     },
-    getArtifactNames: () => assert.fail('import should not query all names'),
+    async *getArtifactNames() {
+      yield* [nameA];
+    },
     close: async () => undefined,
   };
   const { kernelStorage } = await importSwingStore(exporter);
@@ -138,7 +138,9 @@ test('b0 bad import', async t => {
       t.is(name, nameA);
       yield Buffer.from(JSON.stringify(b0Abogus));
     },
-    getArtifactNames: () => assert.fail('import should not query all names'),
+    async *getArtifactNames() {
+      yield* [nameA];
+    },
     close: async () => undefined,
   };
   await t.throwsAsync(async () => importSwingStore(exporter), {