@agoric/swing-store 0.9.1 → 0.9.2-calypso-dev-84eb287.0

package/test/exports.js

@@ -0,0 +1,102 @@
+import { Buffer } from 'buffer';
+import { makeB0ID } from './util.js';
+
+export const snapshotData = 'snapshot data';
+// this snapHash was computed manually
+export const snapHash =
+  'e7dee7266896538616b630a5da40a90e007726a383e005a9c9c5dd0c2daf9329';
+
+/** @type {import('../src/bundleStore.js').Bundle} */
+export const bundle0 = { moduleFormat: 'nestedEvaluate', source: '1+1' };
+export const bundle0ID = makeB0ID(bundle0);
+
+export function buildData() {
+  // build an export manually
+  const exportData = new Map();
+  const artifacts = new Map();
+
+  // shadow kvStore
+  exportData.set('kv.key1', 'value1');
+
+  // now add artifacts and metadata in pairs
+
+  artifacts.set(`bundle.${bundle0ID}`, JSON.stringify(bundle0));
+  exportData.set(`bundle.${bundle0ID}`, bundle0ID);
+
+  const sbase = { vatID: 'v1', hash: snapHash, inUse: 0 };
+  const tbase = { vatID: 'v1', startPos: 0, isCurrent: 0, incarnation: 1 };
+  const addTS = (key, obj) =>
+    exportData.set(key, JSON.stringify({ ...tbase, ...obj }));
+  const t0hash =
+    '5bee0f44eca02f23eab03703e84ed2647d5d117fed99e1c30a3b424b7f082ab9';
+  const t2hash =
+    '57152efdd7fdf75c03371d2b4f1088d5bf3eae7fe643babce527ff81df38998c';
+  const t5hash =
+    '1947001e78e01bd1e773feb22b4ffc530447373b9de9274d5d5fbda3f23dbf2b';
+  const t8hash =
+    'e6b42c6a3fb94285a93162f25a9fc0145fd4c5bb144917dc572c50ae2d02ee69';
+
+  addTS(`transcript.v1.0`, { incarnation: 0, endPos: 2, hash: t0hash });
+  artifacts.set(`transcript.v1.0.2`, 'start-worker\nshutdown-worker\n');
+
+  addTS(`transcript.v1.2`, { startPos: 2, endPos: 5, hash: t2hash });
+  artifacts.set(
+    `transcript.v1.2.5`,
+    'start-worker\ndelivery1\nsave-snapshot\n',
+  );
+  exportData.set(`snapshot.v1.4`, JSON.stringify({ ...sbase, snapPos: 4 }));
+  artifacts.set(`snapshot.v1.4`, snapshotData);
+
+  addTS(`transcript.v1.5`, { startPos: 5, endPos: 8, hash: t5hash });
+  artifacts.set(
+    'transcript.v1.5.8',
+    'load-snapshot\ndelivery2\nsave-snapshot\n',
+  );
+  exportData.set(
+    `snapshot.v1.7`,
+    JSON.stringify({ ...sbase, snapPos: 7, inUse: 1 }),
+  );
+  artifacts.set(`snapshot.v1.7`, snapshotData);
+
+  artifacts.set('transcript.v1.8.10', 'load-snapshot\ndelivery3\n');
+  exportData.set(`snapshot.v1.current`, 'snapshot.v1.7');
+  addTS(`transcript.v1.current`, {
+    startPos: 8,
+    endPos: 10,
+    isCurrent: 1,
+    hash: t8hash,
+  });
+
+  return { exportData, artifacts, t0hash, t2hash, t5hash, t8hash };
+}
+
+/**
+ * @param { Map<string, string | null> } exportData
+ * @param { Map<string, string> } artifacts
+ */
+export function makeExporter(exportData, artifacts) {
+  return {
+    getHostKV(_key) {
+      return undefined;
+    },
+    async *getExportData() {
+      for (const [key, value] of exportData.entries()) {
+        /** @type { import('../src/exporter.js').KVPair } */
+        const pair = [key, value];
+        yield pair;
+      }
+    },
+    async *getArtifactNames() {
+      for (const name of artifacts.keys()) {
+        yield name;
+      }
+    },
+    async *getArtifact(name) {
+      const data = artifacts.get(name);
+      assert(data, `missing artifact ${name}`);
+      yield Buffer.from(data);
+    },
+    // eslint-disable-next-line no-empty-function
+    async close() {},
+  };
+}

package/test/{test-hasher.js → hasher.test.js}

@@ -1,7 +1,5 @@
-import '@endo/init/debug.js';
 import test from 'ava';
 
-// eslint-disable-next-line import/order
 import { createSHA256 } from '../src/hasher.js';
 
 test('createSHA256', t => {

package/test/import.test.js

@@ -0,0 +1,413 @@
+// @ts-check
+
+import path from 'path';
+import { createGunzip } from 'zlib';
+import { Readable } from 'stream';
+import { Buffer } from 'buffer';
+
+import sqlite3 from 'better-sqlite3';
+import test from 'ava';
+import { decodeBase64 } from '@endo/base64';
+
+import { buffer } from '../src/util.js';
+import { importSwingStore, makeSwingStoreExporter } from '../src/index.js';
+
+import { tmpDir } from './util.js';
+import {
+  buildData,
+  bundle0,
+  bundle0ID,
+  makeExporter,
+  snapHash,
+  snapshotData,
+} from './exports.js';
+
+const rank = {
+  operational: 1,
+  replay: 2,
+  archival: 3,
+  debug: 4,
+};
+
+function convert(orig) {
+  const bundles = Object.fromEntries(
+    Object.entries(orig.bundles).map(([bundleID, encBundle]) => {
+      const s = new TextDecoder().decode(decodeBase64(encBundle));
+      assert(bundleID.startsWith('b0-'), bundleID);
+      const bundle = JSON.parse(s);
+      return [bundleID, bundle];
+    }),
+  );
+  return { ...orig, bundles };
+}
+
+test('import empty', async t => {
+  const [dbDir, cleanup] = await tmpDir('testdb');
+  t.teardown(cleanup);
+  const exporter = makeExporter(new Map(), new Map());
+  const ss = await importSwingStore(exporter, dbDir);
+  t.teardown(ss.hostStorage.close);
+  await ss.hostStorage.commit();
+  const data = convert(ss.debug.dump());
+  t.deepEqual(data, {
+    kvEntries: {},
+    transcripts: {},
+    snapshots: {},
+    bundles: {},
+  });
+});
+
+const importTest = test.macro(async (t, mode) => {
+  /** @import {ArtifactMode} from '../src/internal.js' */
+  const artifactMode = /** @type {ArtifactMode} */ (mode);
+
+  const [dbDir, cleanup] = await tmpDir('testdb');
+  t.teardown(cleanup);
+
+  const { exportData, artifacts, t0hash, t2hash, t5hash, t8hash } = buildData();
+
+  const exporter = makeExporter(exportData, artifacts);
+
+  // now import
+  const ss = await importSwingStore(exporter, dbDir, { artifactMode });
+  t.teardown(ss.hostStorage.close);
+  await ss.hostStorage.commit();
+  const data = convert(ss.debug.dump());
+
+  const convertTranscript = (items, startPos = 0) => {
+    const out = {};
+    let pos = startPos;
+    for (const item of items) {
+      out[pos] = item;
+      pos += 1;
+    }
+    return out;
+  };
+
+  const convertSnapshots = async allVatSnapshots => {
+    const out = {};
+    for await (const [vatID, snapshots] of Object.entries(allVatSnapshots)) {
+      const convertedSnapshots = [];
+      for await (const snapshot of snapshots) {
+        if (!snapshot.compressedSnapshot) {
+          continue;
+        }
+        const gzReader = Readable.from(snapshot.compressedSnapshot);
+        const unzipper = createGunzip();
+        const snapshotReader = gzReader.pipe(unzipper);
+        const uncompressedSnapshot = await buffer(snapshotReader);
+        const converted = { ...snapshot, uncompressedSnapshot };
+        delete converted.compressedSnapshot;
+        convertedSnapshots.push(converted);
+      }
+      out[vatID] = convertedSnapshots;
+    }
+    return out;
+  };
+
+  t.deepEqual(data.kvEntries, { key1: 'value1' });
+  let ts = [];
+  if (rank[artifactMode] >= rank.archival) {
+    // only 'archival' and 'debug' get the old incarnation's span
+    ts = ts.concat(['start-worker', 'shutdown-worker']); // 0,1
+  }
+  if (rank[artifactMode] >= rank.replay) {
+    // those, or 'replay', get the current incarnation's old spans
+    ts = ts.concat(['start-worker', 'delivery1', 'save-snapshot']); // 2,3,4
+    ts = ts.concat(['load-snapshot', 'delivery2', 'save-snapshot']); // 5,6,7
+  }
+  ts = ts.concat(['load-snapshot', 'delivery3']); // 8,9
+
+  let tsStart;
+  if (artifactMode === 'archival' || artifactMode === 'debug') {
+    tsStart = 0;
+  } else if (artifactMode === 'replay') {
+    tsStart = 2;
+  } else {
+    tsStart = 8;
+  }
+
+  const expectedTranscript = convertTranscript(ts, tsStart);
+  t.deepEqual(data.transcripts, { v1: expectedTranscript });
+  const uncompressedSnapshot = Buffer.from(snapshotData);
+  const expectedSnapshots = [];
+  if (artifactMode === 'debug') {
+    expectedSnapshots.push({
+      uncompressedSnapshot,
+      hash: snapHash,
+      inUse: 0,
+      snapPos: 4,
+    });
+  }
+  expectedSnapshots.push({
+    uncompressedSnapshot,
+    hash: snapHash,
+    inUse: 1,
+    snapPos: 7,
+  });
+  t.deepEqual(await convertSnapshots(data.snapshots), {
+    v1: expectedSnapshots,
+  });
+  t.deepEqual(data.bundles, { [bundle0ID]: bundle0 });
+
+  // look directly at the DB to confirm presence of metadata rows
+  const db = sqlite3(path.join(dbDir, 'swingstore.sqlite'));
+  const spanRows = [
+    ...db.prepare('SELECT * FROM transcriptSpans ORDER BY startPos').iterate(),
+  ];
+  t.deepEqual(
+    // @ts-expect-error unknown
+    spanRows.map(sr => sr.startPos),
+    [0, 2, 5, 8],
+  );
+
+  // and a new export should include all metadata, regardless of import mode
+
+  const reExporter = makeSwingStoreExporter(dbDir);
+  const reExportData = new Map();
+  for await (const [key, value] of reExporter.getExportData()) {
+    reExportData.set(key, value);
+  }
+  // console.log(reExportData);
+
+  const check = (key, expected) => {
+    t.true(reExportData.has(key), `missing exportData ${key}`);
+    let value = reExportData.get(key);
+    reExportData.delete(key);
+    if (typeof expected === 'object') {
+      value = JSON.parse(value);
+    }
+    t.deepEqual(value, expected);
+  };
+
+  check('kv.key1', 'value1');
+  check('snapshot.v1.4', { vatID: 'v1', snapPos: 4, inUse: 0, hash: snapHash });
+  check('snapshot.v1.7', { vatID: 'v1', snapPos: 7, inUse: 1, hash: snapHash });
+  check('snapshot.v1.current', 'snapshot.v1.7');
+  const base0 = { vatID: 'v1', incarnation: 0, isCurrent: 0 };
+  const base1 = { vatID: 'v1', incarnation: 1, isCurrent: 0 };
+  check('transcript.v1.0', { ...base0, startPos: 0, endPos: 2, hash: t0hash });
+  check('transcript.v1.2', { ...base1, startPos: 2, endPos: 5, hash: t2hash });
+  check('transcript.v1.5', { ...base1, startPos: 5, endPos: 8, hash: t5hash });
+  check('transcript.v1.current', {
+    ...base1,
+    startPos: 8,
+    endPos: 10,
+    isCurrent: 1,
+    hash: t8hash,
+  });
+  check(`bundle.${bundle0ID}`, bundle0ID);
+
+  // the above list is supposed to be exhaustive
+  if (reExportData.size) {
+    console.log(reExportData);
+    t.fail('unexpected exportData keys');
+  }
+});
+
+test('import operational', importTest, 'operational');
+test('import replay', importTest, 'replay');
+test('import archival', importTest, 'archival');
+test('import debug', importTest, 'debug');
+
+test('import is missing bundle', async t => {
+  const [dbDir, cleanup] = await tmpDir('testdb');
+  t.teardown(cleanup);
+
+  const exportData = new Map();
+  exportData.set(`bundle.${bundle0ID}`, bundle0ID);
+  // but there is no artifact to match
+  const exporter = makeExporter(exportData, new Map());
+  await t.throwsAsync(async () => importSwingStore(exporter, dbDir), {
+    message: /missing bundles for:/,
+  });
+});
+
+test('import is missing snapshot', async t => {
+  const [dbDir, cleanup] = await tmpDir('testdb');
+  t.teardown(cleanup);
+
+  const exportData = new Map();
+  exportData.set(
+    `snapshot.v1.2`,
+    JSON.stringify({ vatID: 'v1', hash: snapHash, inUse: 1, snapPos: 2 }),
+  );
+  // but there is no artifact to match
+  const exporter = makeExporter(exportData, new Map());
+  await t.throwsAsync(async () => importSwingStore(exporter, dbDir), {
+    message: /current snapshots are pruned for vats/,
+  });
+});
+
+test('import is missing transcript span', async t => {
+  const [dbDir, cleanup] = await tmpDir('testdb');
+  t.teardown(cleanup);
+
+  const exportData = new Map();
+  const t0hash =
+    '57152efdd7fdf75c03371d2b4f1088d5bf3eae7fe643babce527ff81df38998c';
+  exportData.set(
+    `transcript.v1.current`,
+    JSON.stringify({
+      vatID: 'v1',
+      startPos: 0,
+      endPos: 3,
+      hash: t0hash,
+      isCurrent: 1,
+      incarnation: 0,
+    }),
+  );
+  // but there is no artifact to match
+  const exporter = makeExporter(exportData, new Map());
+  await t.throwsAsync(async () => importSwingStore(exporter, dbDir), {
+    message: /incomplete current transcript/,
+  });
+});
+
+test('import has mismatched transcript span', async t => {
+  const [dbDir, cleanup] = await tmpDir('testdb');
+  t.teardown(cleanup);
+
+  const exportData = new Map();
+  const t0hash =
+    '57152efdd7fdf75c03371d2b4f1088d5bf3eae7fe643babce527ff81df38998c';
+  exportData.set(
+    `transcript.v1.current`,
+    JSON.stringify({
+      vatID: 'v1',
+      startPos: 0,
+      endPos: 3,
+      hash: t0hash,
+      isCurrent: 0, // mismatch
+      incarnation: 0,
+    }),
+  );
+  const exporter = makeExporter(exportData, new Map());
+  await t.throwsAsync(async () => importSwingStore(exporter, dbDir), {
+    message: /transcript key "transcript.v1.current" mismatches metadata/,
+  });
+});
+
+test('import has incomplete transcript span', async t => {
+  const [dbDir, cleanup] = await tmpDir('testdb');
+  t.teardown(cleanup);
+
+  const exportData = new Map();
+  const artifacts = new Map();
+  const t0hash =
+    '57152efdd7fdf75c03371d2b4f1088d5bf3eae7fe643babce527ff81df38998c';
+  exportData.set(
+    `transcript.v1.current`,
+    JSON.stringify({
+      vatID: 'v1',
+      startPos: 0,
+      endPos: 4, // expect 4 items
+      hash: t0hash,
+      isCurrent: 1,
+      incarnation: 0,
+    }),
+  );
+  // but artifact only contains 3
+  artifacts.set(
+    `transcript.v1.0.4`,
+    'start-worker\ndelivery1\nsave-snapshot\n',
+  );
+
+  const exporter = makeExporter(exportData, artifacts);
+  await t.throwsAsync(async () => importSwingStore(exporter, dbDir), {
+    message: /artifact "transcript.v1.0.4" is not complete/,
+  });
+});
+
+test('import has corrupt transcript span', async t => {
+  const [dbDir, cleanup] = await tmpDir('testdb');
+  t.teardown(cleanup);
+
+  const exportData = new Map();
+  const artifacts = new Map();
+  const t0hash =
+    '57152efdd7fdf75c03371d2b4f1088d5bf3eae7fe643babce527ff81df38998c';
+  exportData.set(
+    `transcript.v1.current`,
+    JSON.stringify({
+      vatID: 'v1',
+      startPos: 0,
+      endPos: 3,
+      hash: t0hash,
+      isCurrent: 1,
+      incarnation: 0,
+    }),
+  );
+  artifacts.set(
+    `transcript.v1.0.3`,
+    'start-worker\nBAD-DELIVERY1\nsave-snapshot\n',
+  );
+
+  const exporter = makeExporter(exportData, artifacts);
+  await t.throwsAsync(async () => importSwingStore(exporter, dbDir), {
+    message: /artifact "transcript.v1.0.3" hash is.*metadata says/,
+  });
+});
+
+test('import has corrupt snapshot', async t => {
+  const [dbDir, cleanup] = await tmpDir('testdb');
+  t.teardown(cleanup);
+
+  const exportData = new Map();
+  const artifacts = new Map();
+  exportData.set(
+    `snapshot.v1.2`,
+    JSON.stringify({
+      vatID: 'v1',
+      snapPos: 2,
+      hash: snapHash,
+      inUse: 1,
+    }),
+  );
+  artifacts.set('snapshot.v1.2', `${snapshotData}WRONG`);
+
+  const exporter = makeExporter(exportData, artifacts);
+  await t.throwsAsync(async () => importSwingStore(exporter, dbDir), {
+    message: /snapshot "snapshot.v1.2" hash is.*metadata says/,
+  });
+});
+
+test('import has corrupt bundle', async t => {
+  const [dbDir, cleanup] = await tmpDir('testdb');
+  t.teardown(cleanup);
+
+  const exportData = new Map();
+  const artifacts = new Map();
+  exportData.set(`bundle.${bundle0ID}`, bundle0ID);
+  const badBundle = { ...bundle0, source: 'WRONG' };
+  artifacts.set(`bundle.${bundle0ID}`, JSON.stringify(badBundle));
+
+  const exporter = makeExporter(exportData, artifacts);
+  await t.throwsAsync(async () => importSwingStore(exporter, dbDir), {
+    message: /bundleID ".*" does not match bundle artifact/,
+  });
+});
+
+test('import has unknown metadata tag', async t => {
+  const [dbDir, cleanup] = await tmpDir('testdb');
+  t.teardown(cleanup);
+
+  const exportData = new Map();
+  exportData.set(`unknown.v1.current`, 'value');
+  const exporter = makeExporter(exportData, new Map());
+  await t.throwsAsync(async () => importSwingStore(exporter, dbDir), {
+    message: /unknown export-data type "unknown" on import/,
+  });
+});
+
+test('import has unknown artifact tag', async t => {
+  const [dbDir, cleanup] = await tmpDir('testdb');
+  t.teardown(cleanup);
+
+  const artifacts = new Map();
+  artifacts.set('unknown.v1.current', 'value');
+  const exporter = makeExporter(new Map(), artifacts);
+  await t.throwsAsync(async () => importSwingStore(exporter, dbDir), {
+    message: /unknown artifact type "unknown" on import/,
+  });
+});

package/test/repair-metadata.test.js

@@ -0,0 +1,143 @@
+// @ts-check
+
+import path from 'path';
+import test from 'ava';
+import sqlite3 from 'better-sqlite3';
+
+import { importSwingStore, openSwingStore } from '../src/index.js';
+
+import { makeExporter, buildData } from './exports.js';
+import { tmpDir } from './util.js';
+
+test('repair metadata', async t => {
+  const [dbDir, cleanup] = await tmpDir('testdb');
+  t.teardown(cleanup);
+
+  const { exportData, artifacts } = buildData();
+
+  // simulate a swingstore broken by #8025 by importing everything,
+  // then manually deleting the historical metadata entries from the
+  // DB
+  const exporter = makeExporter(exportData, artifacts);
+  const ssi = await importSwingStore(exporter, dbDir);
+  await ssi.hostStorage.commit();
+  await ssi.hostStorage.close();
+
+  const filePath = path.join(dbDir, 'swingstore.sqlite');
+  const db = sqlite3(filePath);
+
+  const getTS = db.prepare(
+    'SELECT startPos FROM transcriptSpans WHERE vatID = ? ORDER BY startPos',
+  );
+  getTS.pluck();
+  const getSS = db.prepare(
+    'SELECT snapPos FROM snapshots WHERE vatID = ? ORDER BY snapPos',
+  );
+  getSS.pluck();
+
+  // assert that all the metadata is there at first
+  const ts1 = getTS.all('v1');
+  t.deepEqual(ts1, [0, 2, 5, 8]); // four spans
+  const ss1 = getSS.all('v1');
+  t.deepEqual(ss1, [4, 7]); // two snapshots
+
+  // now clobber them to simulate #8025 (note: these auto-commit)
+  db.prepare('DELETE FROM transcriptSpans WHERE isCurrent IS NULL').run();
+  db.prepare('DELETE FROM snapshots WHERE inUSE IS NULL').run();
+
+  // confirm that we clobbered them
+  const ts2 = getTS.all('v1');
+  t.deepEqual(ts2, [8]); // only the latest
+  const ss2 = getSS.all('v1');
+  t.deepEqual(ss2, [7]);
+
+  // now fix it
+  const ss = openSwingStore(dbDir);
+  t.teardown(ss.hostStorage.close);
+  await ss.hostStorage.repairMetadata(exporter);
+  await ss.hostStorage.commit();
+
+  // and check that the metadata is back
+  const ts3 = getTS.all('v1');
+  t.deepEqual(ts3, [0, 2, 5, 8]); // all four again
+  const ss3 = getSS.all('v1');
+  t.deepEqual(ss3, [4, 7]);
+
+  // repair should be idempotent
+  await ss.hostStorage.repairMetadata(exporter);
+  await ss.hostStorage.commit();
+
+  const ts4 = getTS.all('v1');
+  t.deepEqual(ts4, [0, 2, 5, 8]); // still there
+  const ss4 = getSS.all('v1');
+  t.deepEqual(ss4, [4, 7]);
+});
+
+test('repair metadata ignores kvStore entries', async t => {
+  const [dbDir, cleanup] = await tmpDir('testdb');
+  t.teardown(cleanup);
+
+  const { exportData, artifacts } = buildData();
+
+  const exporter = makeExporter(exportData, artifacts);
+  const ssi = await importSwingStore(exporter, dbDir);
+  await ssi.hostStorage.commit();
+  await ssi.hostStorage.close();
+
+  // perform the repair with spurious kv entries
+  exportData.set('kv.key2', 'value2');
+
+  const ss = openSwingStore(dbDir);
+  t.teardown(ss.hostStorage.close);
+  await ss.hostStorage.repairMetadata(exporter);
+  await ss.hostStorage.commit();
+
+  // the spurious kv entry should be ignored
+  t.deepEqual(ss.debug.dump().kvEntries, { key1: 'value1' });
+});
+
+test('repair metadata rejects mismatched snapshot entries', async t => {
+  const [dbDir, cleanup] = await tmpDir('testdb');
+  t.teardown(cleanup);
+
+  const { exportData, artifacts } = buildData();
+
+  const exporter = makeExporter(exportData, artifacts);
+  const ssi = await importSwingStore(exporter, dbDir);
+  await ssi.hostStorage.commit();
+  await ssi.hostStorage.close();
+
+  // perform the repair with mismatched snapshot entry
+  const old = JSON.parse(exportData.get('snapshot.v1.4'));
+  const wrong = { ...old, hash: 'wrong' };
+  exportData.set('snapshot.v1.4', JSON.stringify(wrong));
+
+  const ss = openSwingStore(dbDir);
+  t.teardown(ss.hostStorage.close);
+  await t.throwsAsync(async () => ss.hostStorage.repairMetadata(exporter), {
+    message: /repairSnapshotRecord metadata mismatch/,
+  });
+});
+
+test('repair metadata rejects mismatched transcript span', async t => {
+  const [dbDir, cleanup] = await tmpDir('testdb');
+  t.teardown(cleanup);
+
+  const { exportData, artifacts } = buildData();
+
+  const exporter = makeExporter(exportData, artifacts);
+  const ssi = await importSwingStore(exporter, dbDir);
+  await ssi.hostStorage.commit();
+  await ssi.hostStorage.close();
+
+  // perform the repair with mismatched transcript span entry
+  const old = JSON.parse(exportData.get('transcript.v1.0'));
+  const wrong = { ...old, hash: 'wrong' };
+  exportData.set('transcript.v1.0', JSON.stringify(wrong));
+
+  const ss = openSwingStore(dbDir);
+  t.teardown(ss.hostStorage.close);
+  await t.throwsAsync(async () => ss.hostStorage.repairMetadata(exporter), {
+    message: /repairTranscriptSpanRecord metadata mismatch/,
+  });
+});