@agoric/swing-store 0.9.1 → 0.9.2-calypso-dev-84eb287.0

package/src/index.js

@@ -0,0 +1,11 @@
+export { initSwingStore, openSwingStore, isSwingStore } from './swingStore.js';
+export { makeSwingStoreExporter } from './exporter.js';
+export { importSwingStore } from './importer.js';
+
+// temporary, for the benefit of SwingSet/misc-tools/replay-transcript.js
+export { makeSnapStore } from './snapStore.js';
+// and less temporary, for SwingSet/test/vat-warehouse/test-reload-snapshot.js
+export { makeSnapStoreIO } from './snapStoreIO.js';
+
+// eslint-disable-next-line import/export
+export * from './types.js';

package/src/internal.js

@@ -0,0 +1,22 @@
+import { Fail, q } from '@agoric/assert';
+
+/**
+ * @typedef { import('./snapStore.js').SnapStoreInternal } SnapStoreInternal
+ * @typedef { import('./transcriptStore.js').TranscriptStoreInternal } TranscriptStoreInternal
+ * @typedef { import('./bundleStore.js').BundleStoreInternal } BundleStoreInternal
+ *
+ * @typedef {{
+ *    transcriptStore: TranscriptStoreInternal,
+ *    snapStore: SnapStoreInternal,
+ *    bundleStore: BundleStoreInternal,
+ * }} SwingStoreInternal
+ *
+ * @typedef {'operational' | 'replay' | 'archival' | 'debug'} ArtifactMode
+ */
+
+export const artifactModes = ['operational', 'replay', 'archival', 'debug'];
+export function validateArtifactMode(artifactMode) {
+  if (!artifactModes.includes(artifactMode)) {
+    Fail`invalid artifactMode ${q(artifactMode)}`;
+  }
+}

package/src/kvStore.js

@@ -0,0 +1,172 @@
+// @ts-check
+import { Fail } from '@agoric/assert';
+
+/**
+ * @typedef {{
+ *   has: (key: string) => boolean,
+ *   get: (key: string) => string | undefined,
+ *   getNextKey: (previousKey: string) => string | undefined,
+ *   set: (key: string, value: string, bypassHash?: boolean ) => void,
+ *   delete: (key: string) => void,
+ * }} KVStore
+ */
+
+/**
+ * @param {string} key
+ */
+export function getKeyType(key) {
+  if (key.startsWith('local.')) {
+    return 'local';
+  } else if (key.startsWith('host.')) {
+    return 'host';
+  }
+  return 'consensus';
+}
+
+/**
+ * @param {object} db  The SQLite database connection.
+ * @param {() => void} ensureTxn  Called before mutating methods to establish a DB transaction
+ * @param {(...args: string[]) => void} trace  Called after sets/gets to record a debug log
+ * @returns { KVStore }
+ */
+
+export function makeKVStore(db, ensureTxn, trace) {
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS kvStore (
+      key TEXT,
+      value TEXT,
+      PRIMARY KEY (key)
+    )
+  `);
+
+  const sqlKVGet = db.prepare(`
+    SELECT value
+    FROM kvStore
+    WHERE key = ?
+  `);
+  sqlKVGet.pluck(true);
+
+  /**
+   * Obtain the value stored for a given key.
+   *
+   * @param {string} key  The key whose value is sought.
+   *
+   * @returns {string | undefined} the (string) value for the given key, or
+   *    undefined if there is no such value.
+   *
+   * @throws if key is not a string.
+   */
+  function get(key) {
+    typeof key === 'string' || Fail`key must be a string`;
+    return sqlKVGet.get(key);
+  }
+
+  const sqlKVGetNextKey = db.prepare(`
+    SELECT key
+    FROM kvStore
+    WHERE key > ?
+    LIMIT 1
+  `);
+  sqlKVGetNextKey.pluck(true);
+
+  /**
+   * getNextKey enables callers to iterate over all keys within a
+   * given range. To build an iterator of all keys from start
+   * (inclusive) to end (exclusive), do:
+   *
+   * function* iterate(start, end) {
+   *   if (kvStore.has(start)) {
+   *     yield start;
+   *   }
+   *   let prev = start;
+   *   while (true) {
+   *     let next = kvStore.getNextKey(prev);
+   *     if (!next || next >= end) {
+   *       break;
+   *     }
+   *     yield next;
+   *     prev = next;
+   *   }
+   * }
+   *
+   * @param {string} previousKey  The key returned will always be later than this one.
+   *
+   * @returns {string | undefined} a key string, or undefined if we reach the end of the store
+   *
+   * @throws if previousKey is not a string
+   */
+
+  function getNextKey(previousKey) {
+    typeof previousKey === 'string' || Fail`previousKey must be a string`;
+    return sqlKVGetNextKey.get(previousKey);
+  }
+
+  /**
+   * Test if the state contains a value for a given key.
+   *
+   * @param {string} key  The key that is of interest.
+   *
+   * @returns {boolean} true if a value is stored for the key, false if not.
+   *
+   * @throws if key is not a string.
+   */
+  function has(key) {
+    typeof key === 'string' || Fail`key must be a string`;
+    return get(key) !== undefined;
+  }
+
+  const sqlKVSet = db.prepare(`
+    INSERT INTO kvStore (key, value)
+    VALUES (?, ?)
+    ON CONFLICT DO UPDATE SET value = excluded.value
+  `);
+
+  /**
+   * Store a value for a given key.  The value will replace any prior value if
+   * there was one.
+   *
+   * @param {string} key  The key whose value is being set.
+   * @param {string} value  The value to set the key to.
+   *
+   * @throws if either parameter is not a string.
+   */
+  function set(key, value) {
+    typeof key === 'string' || Fail`key must be a string`;
+    typeof value === 'string' || Fail`value must be a string`;
+    // synchronous read after write within a transaction is safe
+    // The transaction's overall success will be awaited during commit
+    ensureTxn();
+    sqlKVSet.run(key, value);
+    trace('set', key, value);
+  }
+
+  const sqlKVDel = db.prepare(`
+    DELETE FROM kvStore
+    WHERE key = ?
+  `);
+
+  /**
+   * Remove any stored value for a given key.  It is permissible for there to
+   * be no existing stored value for the key.
+   *
+   * @param {string} key  The key whose value is to be deleted
+   *
+   * @throws if key is not a string.
+   */
+  function del(key) {
+    typeof key === 'string' || Fail`key must be a string`;
+    ensureTxn();
+    sqlKVDel.run(key);
+    trace('del', key);
+  }
+
+  const kvStore = {
+    has,
+    get,
+    getNextKey,
+    set,
+    delete: del,
+  };
+
+  return kvStore;
+}

package/src/repairMetadata.js

@@ -0,0 +1,67 @@
+import { Fail, q } from '@agoric/assert';
+import { assertComplete } from './assertComplete.js';
+
+/**
+ * Given a pre-existing swingstore and a SwingStoreExporter, read in
+ * all the metadata from the exporter and use it to regenerate any
+ * missing metadata records. This can be used to fix the damage caused
+ * by #8025.
+ *
+ * The repair method will call `exporter.getExportData` and examine
+ * all entries to do one of three things:
+ *
+ * 1: kvStore records are ignored (they are not metadata)
+ * 2: bundle/snapshot/transcript records whose keys already exist will
+ *    be compared against the existing data, and an error thrown if
+ *    they do not match
+ * 3: new snapshot/transcript records will be silently added to
+ *    the swingstore (new bundle records are an error, since we do not
+ *    tolerate pruned bundles)
+ *
+ * It will not call `exporter.getArtifactNames` or `getArtifacts`.
+ *
+ * At the end of the process, the DB will contain pending changes in
+ * an open transaction. The caller is responsible for calling
+ * `hostStorage.commit()` when they are ready.
+ *
+ * @param {import('./internal.js').SwingStoreInternal} internal
+ * @param {import('./exporter.js').SwingStoreExporter} exporter
+ * @returns {Promise<void>}
+ */
+export async function doRepairMetadata(internal, exporter) {
+  // first we strip kvStore entries and deduplicate the rest
+
+  const allMetadata = new Map();
+
+  for await (const [key, value] of exporter.getExportData()) {
+    const [tag] = key.split('.', 1);
+    if (tag === 'kv') {
+      continue;
+    } else if (value == null) {
+      allMetadata.delete(key);
+    } else {
+      allMetadata.set(key, value);
+    }
+  }
+
+  // then process the metadata records
+
+  for (const [key, value] of allMetadata.entries()) {
+    const [tag] = key.split('.', 1);
+    if (tag === 'bundle') {
+      internal.bundleStore.repairBundleRecord(key, value);
+    } else if (tag === 'snapshot') {
+      internal.snapStore.repairSnapshotRecord(key, value);
+    } else if (tag === 'transcript') {
+      internal.transcriptStore.repairTranscriptSpanRecord(key, value);
+    } else {
+      Fail`unknown export-data type in key ${q(key)} on repairMetadata`;
+    }
+  }
+
+  // and do a completeness check
+  /** @type { import('./internal.js').ArtifactMode } */
+  const artifactMode = 'operational';
+  assertComplete(internal, artifactMode);
+  await exporter.close();
+}

package/src/snapStore.js

@@ -4,7 +4,10 @@ import { finished as finishedCallback, PassThrough, Readable } from 'stream';
 import { promisify } from 'util';
 import { createGzip, createGunzip } from 'zlib';
 import { Fail, q } from '@agoric/assert';
-import { aggregateTryFinally, PromiseAllOrErrors } from '@agoric/internal';
+import {
+  aggregateTryFinally,
+  PromiseAllOrErrors,
+} from '@agoric/internal/src/node/utils.js';
 import { buffer } from './util.js';
 
 /**
@@ -25,7 +28,12 @@ import { buffer } from './util.js';
  */
 
 /**
- * @typedef { import('./swingStore').SwingStoreExporter } SwingStoreExporter
+ * @import {AnyIterableIterator} from './exporter.js'
+ */
+
+/**
+ * @typedef { import('./exporter.js').SwingStoreExporter } SwingStoreExporter
+ * @typedef { import('./internal.js').ArtifactMode } ArtifactMode
  *
  * @typedef {{
  *   loadSnapshot: (vatID: string) => AsyncIterableIterator<Uint8Array>,
@@ -37,10 +45,13 @@ import { buffer } from './util.js';
  * }} SnapStore
  *
  * @typedef {{
- *   exportSnapshot: (name: string, includeHistorical: boolean) => AsyncIterableIterator<Uint8Array>,
- *   importSnapshot: (artifactName: string, exporter: SwingStoreExporter, artifactMetadata: Map) => void,
+ *   exportSnapshot: (name: string) => AsyncIterableIterator<Uint8Array>,
  *   getExportRecords: (includeHistorical: boolean) => IterableIterator<readonly [key: string, value: string]>,
- *   getArtifactNames: (includeHistorical: boolean) => AsyncIterableIterator<string>,
+ *   getArtifactNames: (artifactMode: ArtifactMode) => AsyncIterableIterator<string>,
+ *   importSnapshotRecord: (key: string, value: string) => void,
+ *   populateSnapshot: (name: string, makeChunkIterator: () => AnyIterableIterator<Uint8Array>, options: { artifactMode: ArtifactMode }) => Promise<void>,
+ *   assertComplete: (checkMode: Omit<ArtifactMode, 'debug'>) => void,
+ *   repairSnapshotRecord: (key: string, value: string) => void,
  * }} SnapStoreInternal
  *
  * @typedef {{
@@ -81,11 +92,26 @@ export function makeSnapStore(
       compressedSize INTEGER,
       compressedSnapshot BLOB,
       PRIMARY KEY (vatID, snapPos),
-      UNIQUE (vatID, inUse),
-      CHECK(compressedSnapshot is not null or inUse is null)
+      UNIQUE (vatID, inUse)
     )
   `);
 
+  // NOTE: there are two versions of this schema. The original, which
+  // we'll call "version 1A", has a:
+  //   CHECK(compressedSnapshot is not null or inUse is null)
+  // in the table. Version 1B is missing that constraint. Any DB
+  // created by the original code will use 1A. Any DB created by the
+  // new version will use 1B. The import process needs to temporarily
+  // violate that check, but any DB created by `importSwingStore` is
+  // (by definition) new, so it will use 1B, which doesn't enforce the
+  // check. We expect to implement schema migration
+  // (https://github.com/Agoric/agoric-sdk/issues/8089) soon, which
+  // will upgrade both 1A and 1B to "version 2", which will omit the
+  // check (in addition to any other changes we need at that point)
+
+  // pruned snapshots will have compressedSnapshot of NULL, and might
+  // also have NULL for uncompressedSize and compressedSize
+
   const sqlDeleteAllUnusedSnapshots = db.prepare(`
     DELETE FROM snapshots
     WHERE inUse is null
@@ -98,6 +124,12 @@ export function makeSnapStore(
   function deleteAllUnusedSnapshots() {
     ensureTxn();
     sqlDeleteAllUnusedSnapshots.run();
+
+    // NOTE: this is more than pruning the snapshot data, it deletes
+    // the metadata/hash as well, making it impossible to safely
+    // repopulate the snapshot data from an untrusted source. We need
+    // to replace this with a method that merely nulls out the
+    // 'compressedSnapshot' field.
   }
 
   function snapshotArtifactName(rec) {
@@ -255,10 +287,9 @@ export function makeSnapStore(
    *   `snapshot.${vatID}.${startPos}`
    *
    * @param {string} name
-   * @param {boolean} includeHistorical
    * @returns {AsyncIterableIterator<Uint8Array>}
    */
-  function exportSnapshot(name, includeHistorical) {
+  function exportSnapshot(name) {
     typeof name === 'string' || Fail`artifact name must be a string`;
     const parts = name.split('.');
     const [type, vatID, pos] = parts;
@@ -268,9 +299,8 @@ export function makeSnapStore(
     const snapPos = Number(pos);
     const snapshotInfo = sqlGetSnapshot.get(vatID, snapPos);
     snapshotInfo || Fail`snapshot ${q(name)} not available`;
-    const { inUse, compressedSnapshot } = snapshotInfo;
+    const { compressedSnapshot } = snapshotInfo;
     compressedSnapshot || Fail`artifact ${q(name)} is not available`;
-    inUse || includeHistorical || Fail`artifact ${q(name)} is not available`;
     // weird construct here is because we need to be able to throw before the generator starts
     async function* exporter() {
       const gzReader = Readable.from(compressedSnapshot);
@@ -278,6 +308,7 @@ export function makeSnapStore(
       const snapshotReader = gzReader.pipe(unzipper);
       yield* snapshotReader;
     }
+    harden(exporter);
     return exporter();
   }
 
@@ -305,17 +336,18 @@ export function makeSnapStore(
     snapReader.pipe(hashStream);
     snapReader.pipe(output);
 
+    await null;
     try {
       yield* output;
     } finally {
       gzReader.destroy();
-      // eslint-disable-next-line @jessie.js/no-nested-await
       await finished(gzReader);
       const hash = hashStream.digest('hex');
       hash === snapshotID ||
         Fail`actual hash ${q(hash)} !== expected ${q(snapshotID)}`;
     }
   }
+  harden(loadSnapshot);
 
   const sqlDeleteVatSnapshots = db.prepare(`
     DELETE FROM snapshots
@@ -412,6 +444,13 @@ export function makeSnapStore(
     ORDER BY vatID, snapPos
   `);
 
+  const sqlGetAvailableSnapshots = db.prepare(`
+    SELECT vatID, snapPos, hash, uncompressedSize, compressedSize, inUse
+    FROM snapshots
+    WHERE inUse IS ? AND compressedSnapshot is not NULL
+    ORDER BY vatID, snapPos
+  `);
+
   /**
    * Obtain artifact metadata records for spanshots contained in this store.
    *
@@ -446,42 +485,116 @@ export function makeSnapStore(
       }
     }
   }
+  harden(getExportRecords);
 
-  async function* getArtifactNames(includeHistorical) {
-    for (const rec of sqlGetSnapshotMetadata.iterate(1)) {
+  async function* getArtifactNames(artifactMode) {
+    for (const rec of sqlGetAvailableSnapshots.iterate(1)) {
       yield snapshotArtifactName(rec);
     }
-    if (includeHistorical) {
-      for (const rec of sqlGetSnapshotMetadata.iterate(null)) {
+    if (artifactMode === 'debug') {
+      for (const rec of sqlGetAvailableSnapshots.iterate(null)) {
         yield snapshotArtifactName(rec);
       }
     }
   }
+  harden(getArtifactNames);
+
+  const sqlAddSnapshotRecord = db.prepare(`
+    INSERT INTO snapshots (vatID, snapPos, hash, inUse)
+    VALUES (?, ?, ?, ?)
+  `);
+
+  function importSnapshotRecord(key, value) {
+    ensureTxn();
+    const [tag, ...pieces] = key.split('.');
+    assert.equal(tag, 'snapshot');
+    const [_vatID, endPos] = pieces;
+    if (endPos === 'current') {
+      // metadata['snapshot.v1.current'] = 'snapshot.v1.5' , i.e. it
+      // points to the name of the current artifact. We could
+      // conceivably remember this and compare it against the .inUse
+      // property of that record, but it's not worth the effort (we
+      // might encounter the records in either order).
+      return;
+    }
+    const metadata = JSON.parse(value);
+    const { vatID, snapPos, hash, inUse } = metadata;
+    vatID || Fail`snapshot metadata missing vatID: ${metadata}`;
+    snapPos !== undefined ||
+      Fail`snapshot metadata missing snapPos: ${metadata}`;
+    hash || Fail`snapshot metadata missing hash: ${metadata}`;
+    inUse !== undefined || Fail`snapshot metadata missing inUse: ${metadata}`;
+
+    sqlAddSnapshotRecord.run(vatID, snapPos, hash, inUse ? 1 : null);
+  }
+
+  const sqlGetSnapshotHashFor = db.prepare(`
+    SELECT hash, inUse
+    FROM snapshots
+    WHERE vatID = ? AND snapPos = ?
+  `);
+
+  function repairSnapshotRecord(key, value) {
+    ensureTxn();
+    const [tag, keyVatID, keySnapPos] = key.split('.');
+    assert.equal(tag, 'snapshot');
+    if (keySnapPos === 'current') {
+      // "snapshot.${vatID}.current" entries are meta-metadata: they
+      // point to the metadata key of the current snapshot, to avoid
+      // the need for an expensive search
+      return;
+    }
+    const metadata = JSON.parse(value);
+    const { vatID, snapPos, hash, inUse } = metadata;
+    assert.equal(keyVatID, vatID);
+    assert.equal(Number(keySnapPos), snapPos);
+    const existing = sqlGetSnapshotHashFor.get(vatID, snapPos);
+    if (existing) {
+      if (
+        Boolean(existing.inUse) !== Boolean(inUse) ||
+        existing.hash !== hash
+      ) {
+        throw Fail`repairSnapshotRecord metadata mismatch: ${existing} vs ${metadata}`;
+      }
+    } else {
+      sqlAddSnapshotRecord.run(vatID, snapPos, hash, inUse ? 1 : null);
+    }
+  }
+
+  const sqlPopulateSnapshot = db.prepare(`
+    UPDATE snapshots SET
+      uncompressedSize = ?, compressedSize = ?, compressedSnapshot = ?
+    WHERE vatID = ? AND snapPos = ?
+  `);
 
   /**
    * @param {string} name  Artifact name of the snapshot
-   * @param {SwingStoreExporter} exporter  Whence to get the bits
-   * @param {object} info  Metadata describing the artifact
+   * @param {() => AnyIterableIterator<Uint8Array>} makeChunkIterator  get an iterator of snapshot byte chunks
+   * @param {object} options
+   * @param {ArtifactMode} options.artifactMode
    * @returns {Promise<void>}
    */
-  async function importSnapshot(name, exporter, info) {
+  async function populateSnapshot(name, makeChunkIterator, options) {
+    ensureTxn();
+    const { artifactMode } = options;
     const parts = name.split('.');
     const [type, vatID, rawEndPos] = parts;
     // prettier-ignore
     parts.length === 3 && type === 'snapshot' ||
       Fail`expected snapshot name of the form 'snapshot.{vatID}.{snapPos}', saw '${q(name)}'`;
-    // prettier-ignore
-    info.vatID === vatID ||
-      Fail`snapshot name says vatID ${q(vatID)}, metadata says ${q(info.vatID)}`;
     const snapPos = Number(rawEndPos);
-    // prettier-ignore
-    info.snapPos === snapPos ||
-      Fail`snapshot name says snapPos ${q(snapPos)}, metadata says ${q(info.snapPos)}`;
+    const metadata =
+      sqlGetSnapshotHashFor.get(vatID, snapPos) ||
+      Fail`no metadata for snapshot ${name}`;
 
-    const artifactChunks = exporter.getArtifact(name);
+    if (!metadata.inUse && artifactMode !== 'debug') {
+      return; // ignore old snapshots
+    }
+
+    const artifactChunks = makeChunkIterator();
     const inStream = Readable.from(artifactChunks);
-    let size = 0;
-    inStream.on('data', chunk => (size += chunk.length));
+    let uncompressedSize = 0;
+    inStream.on('data', chunk => (uncompressedSize += chunk.length));
     const hashStream = createHash('sha256');
     const gzip = createGzip();
     inStream.pipe(hashStream);
@@ -489,21 +602,37 @@ export function makeSnapStore(
     const compressedArtifact = await buffer(gzip);
     await finished(inStream);
     const hash = hashStream.digest('hex');
+
+    // validate against the previously-established metadata
     // prettier-ignore
-    info.hash === hash ||
-      Fail`snapshot ${q(name)} hash is ${q(hash)}, metadata says ${q(info.hash)}`;
-    ensureTxn();
-    sqlSaveSnapshot.run(
-      vatID,
-      snapPos,
-      info.inUse ? 1 : null,
-      info.hash,
-      size,
+    metadata.hash === hash ||
+      Fail`snapshot ${q(name)} hash is ${q(hash)}, metadata says ${q(metadata.hash)}`;
+
+    sqlPopulateSnapshot.run(
+      uncompressedSize,
       compressedArtifact.length,
       compressedArtifact,
+      vatID,
+      snapPos,
     );
   }
 
+  const sqlListPrunedCurrentSnapshots = db.prepare(`
+    SELECT vatID FROM snapshots
+      WHERE inUse = 1 AND compressedSnapshot IS NULL
+      ORDER BY vatID
+  `);
+  sqlListPrunedCurrentSnapshots.pluck();
+
+  function assertComplete(checkMode) {
+    assert(checkMode !== 'debug', checkMode);
+    // every 'inUse' snapshot must be populated
+    const vatIDs = sqlListPrunedCurrentSnapshots.all();
+    if (vatIDs.length) {
+      throw Fail`current snapshots are pruned for vats ${vatIDs.join(',')}`;
+    }
+  }
+
   const sqlListAllSnapshots = db.prepare(`
     SELECT vatID, snapPos, inUse, hash, uncompressedSize, compressedSize
     FROM snapshots
@@ -517,6 +646,7 @@ export function makeSnapStore(
   function* listAllSnapshots() {
     yield* sqlListAllSnapshots.iterate();
   }
+  harden(listAllSnapshots);
 
   const sqlDumpCurrentSnapshots = db.prepare(`
     SELECT vatID, snapPos, hash, compressedSnapshot, inUse
@@ -563,10 +693,15 @@ export function makeSnapStore(
     deleteVatSnapshots,
     stopUsingLastSnapshot,
     getSnapshotInfo,
+
     getExportRecords,
     getArtifactNames,
     exportSnapshot,
-    importSnapshot,
+
+    importSnapshotRecord,
+    populateSnapshot,
+    assertComplete,
+    repairSnapshotRecord,
 
     hasHash,
     listAllSnapshots,

package/src/snapStoreIO.js

@@ -0,0 +1,8 @@
+import { performance } from 'perf_hooks';
+import { makeMeasureSeconds } from '@agoric/internal';
+
+export function makeSnapStoreIO() {
+  return {
+    measureSeconds: makeMeasureSeconds(performance.now.bind(performance)),
+  };
+}