@agoric/cosmic-swingset 0.41.4-dev-c6d8a27.0 → 0.41.4-dev-f2445bc.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@agoric/cosmic-swingset",
-  "version": "0.41.4-dev-c6d8a27.0+c6d8a27",
+  "version": "0.41.4-dev-f2445bc.0+f2445bc",
   "description": "Agoric's Cosmos blockchain integration",
   "type": "module",
   "bin": {
@@ -22,15 +22,15 @@
   "author": "Agoric",
   "license": "Apache-2.0",
   "dependencies": {
-    "@agoric/assert": "0.6.1-dev-c6d8a27.0+c6d8a27",
-    "@agoric/cosmos": "0.34.2-dev-c6d8a27.0+c6d8a27",
-    "@agoric/deploy-script-support": "0.10.4-dev-c6d8a27.0+c6d8a27",
-    "@agoric/internal": "0.3.3-dev-c6d8a27.0+c6d8a27",
-    "@agoric/store": "0.9.3-dev-c6d8a27.0+c6d8a27",
-    "@agoric/swing-store": "0.9.2-dev-c6d8a27.0+c6d8a27",
-    "@agoric/swingset-vat": "0.32.3-dev-c6d8a27.0+c6d8a27",
-    "@agoric/telemetry": "0.6.3-dev-c6d8a27.0+c6d8a27",
-    "@agoric/vm-config": "0.1.1-dev-c6d8a27.0+c6d8a27",
+    "@agoric/assert": "0.6.1-dev-f2445bc.0+f2445bc",
+    "@agoric/cosmos": "0.34.2-dev-f2445bc.0+f2445bc",
+    "@agoric/deploy-script-support": "0.10.4-dev-f2445bc.0+f2445bc",
+    "@agoric/internal": "0.3.3-dev-f2445bc.0+f2445bc",
+    "@agoric/store": "0.9.3-dev-f2445bc.0+f2445bc",
+    "@agoric/swing-store": "0.9.2-dev-f2445bc.0+f2445bc",
+    "@agoric/swingset-vat": "0.32.3-dev-f2445bc.0+f2445bc",
+    "@agoric/telemetry": "0.6.3-dev-f2445bc.0+f2445bc",
+    "@agoric/vm-config": "0.1.1-dev-f2445bc.0+f2445bc",
     "@endo/bundle-source": "^2.5.2",
     "@endo/far": "^0.2.19",
     "@endo/import-bundle": "^0.3.5",
@@ -65,5 +65,5 @@
     ],
     "timeout": "20m"
   },
-  "gitHead": "c6d8a278ad7d7c0ad758f57c7e4db858d6a67e53"
+  "gitHead": "f2445bcb471c26169ea8052d04e96d51ef9f1487"
 }

package/src/export-kernel-db.js

@@ -23,30 +23,64 @@ import { makeProcessValue } from './helpers/process-value.js';
 // with the golang SwingStoreExportsHandler in golang/cosmos/x/swingset/keeper/swing_store_exports_handler.go
 export const ExportManifestFileName = 'export-manifest.json';
 
-/** @typedef {'current' | 'archival' | 'debug'} SwingStoreExportMode */
+/**
+ * @typedef {'none'  // No artifacts included
+ *  | import("@agoric/swing-store").ArtifactMode
+ * } SwingStoreArtifactMode
+ */
 
 /**
- * @param {SwingStoreExportMode | undefined} exportMode
+ * @typedef {'skip'      // Do not include any "export data" (artifacts only)
+ *   | 'repair-metadata' // Add missing artifact metadata (import only)
+ *   | 'all'             // Include all export data, create new swing-store on import
+ * } SwingStoreExportDataMode
+ */
+
+/**
+ * @param {SwingStoreArtifactMode | undefined} artifactMode
  * @returns {import("@agoric/swing-store").ArtifactMode}
  */
-const getArtifactModeFromExportMode = exportMode => {
-  switch (exportMode) {
-    case 'current':
-    case undefined:
+export const getEffectiveArtifactMode = artifactMode => {
+  switch (artifactMode) {
+    case 'none':
+    case 'operational':
       return 'operational';
+    case undefined:
+    case 'replay':
+      return 'replay';
     case 'archival':
-      return 'archival';
     case 'debug':
-      return 'debug';
+      return artifactMode;
     default:
-      throw Fail`Invalid value ${q(exportMode)} for "export-mode"`;
+      throw Fail`Invalid value ${q(artifactMode)} for "artifact-mode"`;
   }
 };
 
+/** @type {(artifactMode: string | undefined) => asserts artifactMode is SwingStoreArtifactMode | undefined} */
+export const checkArtifactMode = getEffectiveArtifactMode;
+
 /**
- * @type {(exportMode: string | undefined) => asserts exportMode is SwingStoreExportMode}
+ * @param {string | undefined} mode
+ * @param {boolean} [isImport]
+ * @returns {asserts mode is SwingStoreExportDataMode | undefined}
  */
-const checkExportMode = getArtifactModeFromExportMode;
+export const checkExportDataMode = (mode, isImport = false) => {
+  switch (mode) {
+    case 'skip':
+    case undefined:
+      break;
+    case 'all':
+      break;
+    case 'repair-metadata': {
+      if (isImport) {
+        break;
+      }
+      // Fall through
+    }
+    default:
+      throw Fail`Invalid value ${q(mode)} for "export-data-mode"`;
+  }
+};
 
 /**
  * A state-sync manifest is a representation of the information contained in a
@@ -60,7 +94,7 @@ const checkExportMode = getArtifactModeFromExportMode;
  *
  * @typedef {object} StateSyncManifest
  * @property {number} blockHeight the block height corresponding to this export
- * @property {SwingStoreExportMode} [mode]
+ * @property {SwingStoreArtifactMode} [artifactMode]
  * @property {string} [data] file name containing the swingStore "export data"
  * @property {Array<[artifactName: string, fileName: string]>} artifacts
  *   List of swingStore export artifacts which can be validated by the export data
@@ -79,8 +113,8 @@ const checkExportMode = getArtifactModeFromExportMode;
  * @property {string} stateDir the directory containing the SwingStore to export
  * @property {string} exportDir the directory in which to place the exported artifacts and manifest
  * @property {number} [blockHeight] block height to check for
- * @property {SwingStoreExportMode} [exportMode] whether to include historical or debug artifacts in the export
- * @property {boolean} [includeExportData] whether to include an artifact for the export data in the export
+ * @property {SwingStoreArtifactMode} [artifactMode] the level of artifacts to include in the export
+ * @property {SwingStoreExportDataMode} [exportDataMode] include a synthetic artifact for the export data in the export
  */
 
 /**
@@ -96,10 +130,12 @@ export const validateExporterOptions = options => {
   options.blockHeight == null ||
     typeof options.blockHeight === 'number' ||
     Fail`optional blockHeight option not a number`;
-  checkExportMode(options.exportMode);
-  options.includeExportData == null ||
-    typeof options.includeExportData === 'boolean' ||
-    Fail`optional includeExportData option not a boolean`;
+  checkArtifactMode(options.artifactMode);
+  checkExportDataMode(options.exportDataMode);
+
+  options.includeExportData === undefined ||
+    Fail`deprecated includeExportData option found`;
+  options.exportMode === undefined || Fail`deprecated exportMode option found`;
 };
 
 /**
@@ -113,7 +149,7 @@ export const validateExporterOptions = options => {
  * @returns {StateSyncExporter}
  */
 export const initiateSwingStoreExport = (
-  { stateDir, exportDir, blockHeight, exportMode, includeExportData },
+  { stateDir, exportDir, blockHeight, artifactMode, exportDataMode },
   {
     fs: { open, writeFile },
     pathResolve,
@@ -122,8 +158,7 @@ export const initiateSwingStoreExport = (
     log = console.log,
   },
 ) => {
-  const artifactMode = getArtifactModeFromExportMode(exportMode);
-
+  const effectiveArtifactMode = getEffectiveArtifactMode(artifactMode);
   /** @type {number | undefined} */
   let savedBlockHeight;
 
@@ -143,7 +178,9 @@ export const initiateSwingStoreExport = (
     const manifestFile = await open(manifestPath, 'wx');
     cleanup.push(async () => manifestFile.close());
 
-    const swingStoreExporter = makeExporter(stateDir, { artifactMode });
+    const swingStoreExporter = makeExporter(stateDir, {
+      artifactMode: effectiveArtifactMode,
+    });
     cleanup.push(async () => swingStoreExporter.close());
 
     const { hostStorage } = openDB(stateDir);
@@ -153,7 +190,9 @@ export const initiateSwingStoreExport = (
 
     if (blockHeight) {
       blockHeight === savedBlockHeight ||
-        Fail`DB at unexpected block height ${savedBlockHeight} (expected ${blockHeight})`;
+        Fail`DB at unexpected block height ${q(savedBlockHeight)} (expected ${q(
+          blockHeight,
+        )})`;
     }
 
     abortIfStopped();
@@ -163,11 +202,11 @@ export const initiateSwingStoreExport = (
     /** @type {StateSyncManifest} */
     const manifest = {
       blockHeight: savedBlockHeight,
-      mode: exportMode,
+      artifactMode: artifactMode || effectiveArtifactMode,
       artifacts: [],
     };
 
-    if (includeExportData) {
+    if (exportDataMode === 'all') {
       log?.(`Writing Export Data`);
       const fileName = `export-data.jsonl`;
       const exportDataFile = await open(pathResolve(exportDir, fileName), 'wx');
@@ -181,14 +220,16 @@ export const initiateSwingStoreExport = (
     }
     abortIfStopped();
 
-    for await (const artifactName of swingStoreExporter.getArtifactNames()) {
-      abortIfStopped();
-      log?.(`Writing artifact: ${artifactName}`);
-      const artifactData = swingStoreExporter.getArtifact(artifactName);
-      // Use artifactName as the file name as we trust swingStore to generate
-      // artifact names that are valid file names.
-      await writeFile(pathResolve(exportDir, artifactName), artifactData);
-      manifest.artifacts.push([artifactName, artifactName]);
+    if (artifactMode !== 'none') {
+      for await (const artifactName of swingStoreExporter.getArtifactNames()) {
+        abortIfStopped();
+        log?.(`Writing artifact: ${artifactName}`);
+        const artifactData = swingStoreExporter.getArtifact(artifactName);
+        // Use artifactName as the file name as we trust swingStore to generate
+        // artifact names that are valid file names.
+        await writeFile(pathResolve(exportDir, artifactName), artifactData);
+        manifest.artifacts.push([artifactName, artifactName]);
+      }
     }
 
     await manifestFile.write(JSON.stringify(manifest, null, 2));
@@ -272,11 +313,22 @@ export const main = async (
     /** @type {string} */ (processValue.getFlag('export-dir', '.')),
   );
 
-  const includeExportData = processValue.getBoolean({
-    flagName: 'include-export-data',
-  });
-  const exportMode = processValue.getFlag('export-mode');
-  checkExportMode(exportMode);
+  const artifactMode = /** @type {SwingStoreArtifactMode | undefined} */ (
+    processValue.getFlag('artifact-mode')
+  );
+  checkArtifactMode(artifactMode);
+
+  const exportDataMode = processValue.getFlag('export-data-mode');
+  checkExportDataMode(exportDataMode);
+
+  if (
+    processValue.getBoolean({ flagName: 'include-export-data' }) !== undefined
+  ) {
+    throw Fail`deprecated "include-export-data" options, use "export-data-mode" instead`;
+  }
+  if (processValue.getFlag('export-mode') !== undefined) {
+    throw Fail`deprecated "export-mode" options, use "artifact-mode" instead`;
+  }
 
   const checkBlockHeight = processValue.getInteger({
     flagName: 'check-block-height',
@@ -293,8 +345,8 @@ export const main = async (
       stateDir,
       exportDir,
       blockHeight: checkBlockHeight,
-      exportMode,
-      includeExportData,
+      artifactMode,
+      exportDataMode,
     },
     {
       fs,
@@ -335,7 +387,7 @@ export const main = async (
  * @returns {StateSyncExporter}
  */
 export const spawnSwingStoreExport = (
-  { stateDir, exportDir, blockHeight, exportMode, includeExportData },
+  { stateDir, exportDir, blockHeight, artifactMode, exportDataMode },
   { fork, verbose },
 ) => {
   const args = ['--state-dir', stateDir, '--export-dir', exportDir];
@@ -344,12 +396,12 @@ export const spawnSwingStoreExport = (
     args.push('--check-block-height', String(blockHeight));
   }
 
-  if (exportMode) {
-    args.push('--export-mode', exportMode);
+  if (artifactMode) {
+    args.push('--artifact-mode', artifactMode);
   }
 
-  if (includeExportData) {
-    args.push('--include-export-data');
+  if (exportDataMode) {
+    args.push('--export-data-mode', exportDataMode);
   }
 
   if (verbose) {
@@ -401,7 +453,7 @@ export const spawnSwingStoreExport = (
       }
       default: {
         // @ts-expect-error exhaustive check
-        Fail`Unexpected ${msg.type} message`;
+        Fail`Unexpected ${q(msg.type)} message`;
       }
     }
   };

package/src/import-kernel-db.js

@@ -12,19 +12,24 @@ import fsPromisesPower from 'fs/promises';
 import pathPower from 'path';
 
 import BufferLineTransform from '@agoric/internal/src/node/buffer-line-transform.js';
-import { Fail } from '@agoric/assert';
-import { importSwingStore } from '@agoric/swing-store';
+import { Fail, q } from '@agoric/assert';
+import { importSwingStore, openSwingStore } from '@agoric/swing-store';
 
 import { isEntrypoint } from './helpers/is-entrypoint.js';
 import { makeProcessValue } from './helpers/process-value.js';
-import { ExportManifestFileName } from './export-kernel-db.js';
+import {
+  ExportManifestFileName,
+  checkExportDataMode,
+  checkArtifactMode,
+} from './export-kernel-db.js';
 
 /**
  * @typedef {object} StateSyncImporterOptions
  * @property {string} stateDir the directory containing the SwingStore to export
  * @property {string} exportDir the directory where to place the exported artifacts and manifest
  * @property {number} [blockHeight] block height to check for
- * @property {boolean} [includeHistorical] whether to include historical artifacts in the export
+ * @property {import('./export-kernel-db.js').SwingStoreExportDataMode} [exportDataMode] how to handle export data
+ * @property {import('./export-kernel-db.js').SwingStoreArtifactMode} [artifactMode] the level of historical artifacts to import
  */
 
 /**
@@ -40,9 +45,57 @@ export const validateImporterOptions = options => {
   options.blockHeight == null ||
     typeof options.blockHeight === 'number' ||
     Fail`optional blockHeight option not a number`;
-  options.includeHistorical == null ||
-    typeof options.includeHistorical === 'boolean' ||
-    Fail`optional includeHistorical option not a boolean`;
+  checkExportDataMode(options.exportDataMode, true);
+  checkArtifactMode(options.artifactMode);
+  options.includeHistorical === undefined ||
+    Fail`deprecated includeHistorical option found`;
+};
+
+/**
+ * @param {Pick<StateSyncImporterOptions, 'artifactMode' | 'exportDataMode' >} options
+ * @param {Readonly<import('./export-kernel-db.js').StateSyncManifest>} manifest
+ * @returns {import('@agoric/swing-store').ImportSwingStoreOptions}
+ */
+const checkAndGetImportSwingStoreOptions = (options, manifest) => {
+  typeof manifest.blockHeight === 'number' ||
+    Fail`Cannot restore snapshot without block height`;
+
+  manifest.data || Fail`State-sync manifest missing export data`;
+
+  const { artifactMode = manifest.artifactMode || 'replay' } = options;
+
+  if (artifactMode === 'none') {
+    throw Fail`Cannot import "export data" without at least "operational" artifacts`;
+  }
+
+  manifest.artifacts?.length ||
+    Fail`State-sync manifest missing required artifacts`;
+
+  switch (artifactMode) {
+    case 'debug':
+    // eslint-disable-next-line no-fallthrough
+    case 'operational':
+      if (manifest.artifactMode === 'operational') break;
+    // eslint-disable-next-line no-fallthrough
+    case 'replay':
+      if (manifest.artifactMode === 'replay') break;
+    // eslint-disable-next-line no-fallthrough
+    case 'archival':
+      if (manifest.artifactMode === 'archival') break;
+      if (
+        manifest.artifactMode === undefined ||
+        manifest.artifactMode === 'debug'
+      ) {
+        // assume the export has sufficient data
+        break;
+      }
+      throw Fail`State-sync manifest has insufficient artifacts: requested import artifact mode: ${q(
+        artifactMode,
+      )}, manifest has ${q(manifest.artifactMode)} artifacts`;
+    default:
+      throw Fail`Unexpected artifactMode ${q(artifactMode)}`;
+  }
+  return { artifactMode };
 };
 
 /**
@@ -51,15 +104,17 @@ export const validateImporterOptions = options => {
  * @param {Pick<import('fs/promises'), 'readFile'> & Pick<import('fs'), 'createReadStream'>} powers.fs
  * @param {import('path')['resolve']} powers.pathResolve
  * @param {typeof import('@agoric/swing-store')['importSwingStore']} [powers.importSwingStore]
+ * @param {typeof import('@agoric/swing-store')['openSwingStore']} [powers.openSwingStore]
  * @param {null | ((...args: any[]) => void)} [powers.log]
  * @returns {Promise<void>}
  */
 export const performStateSyncImport = async (
-  { stateDir, exportDir, blockHeight, includeHistorical },
+  { stateDir, exportDir, blockHeight, exportDataMode = 'all', artifactMode },
   {
     fs: { createReadStream, readFile },
     pathResolve,
     importSwingStore: importDB = importSwingStore,
+    openSwingStore: openDB = openSwingStore,
     log = console.log,
   },
 ) => {
@@ -67,7 +122,9 @@ export const performStateSyncImport = async (
   const safeExportFileResolve = allegedRelativeFilename => {
     const resolvedPath = pathResolve(exportDir, allegedRelativeFilename);
     resolvedPath.startsWith(exportDir) ||
-      Fail`Exported file ${allegedRelativeFilename} must be in export dir ${exportDir}`;
+      Fail`Exported file ${q(
+        allegedRelativeFilename,
+      )} must be in export dir ${q(exportDir)}`;
     return resolvedPath;
   };
 
@@ -78,26 +135,12 @@ export const performStateSyncImport = async (
   );
 
   if (blockHeight !== undefined && manifest.blockHeight !== blockHeight) {
-    Fail`State-sync manifest for unexpected block height ${manifest.blockHeight} (expected ${blockHeight})`;
-  }
-
-  if (!manifest.data) {
-    throw Fail`State-sync manifest missing export data`;
-  }
-
-  if (!manifest.artifacts) {
-    throw Fail`State-sync manifest missing required artifacts`;
+    Fail`State-sync manifest for unexpected block height ${q(
+      manifest.blockHeight,
+    )} (expected ${q(blockHeight)})`;
   }
 
-  const artifacts = harden(Object.fromEntries(manifest.artifacts));
-
-  if (
-    includeHistorical &&
-    manifest.mode !== 'archival' &&
-    manifest.mode !== 'debug'
-  ) {
-    throw Fail`State-sync manifest missing historical artifacts`;
-  }
+  const artifacts = harden(Object.fromEntries(manifest.artifacts || []));
 
   // Represent the data in `exportDir` as a SwingSetExporter object.
   /** @type {import('@agoric/swing-store').SwingStoreExporter} */
@@ -128,7 +171,7 @@ export const performStateSyncImport = async (
       log?.(`importing artifact ${name}`);
       const fileName = artifacts[name];
       if (!fileName) {
-        Fail`invalid artifact ${name}`;
+        Fail`invalid artifact ${q(name)}`;
       }
       const stream = createReadStream(safeExportFileResolve(fileName));
       yield* stream;
@@ -141,17 +184,47 @@ export const performStateSyncImport = async (
     },
   });
 
-  const artifactMode = includeHistorical
-    ? 'debug' // for now don't enforce completeness but allow importing all provided artifacts
-    : 'operational';
+  if (exportDataMode === 'all') {
+    const importOptions = checkAndGetImportSwingStoreOptions(
+      { artifactMode, exportDataMode },
+      manifest,
+    );
+
+    const swingstore = await importDB(exporter, stateDir, importOptions);
+
+    const { hostStorage } = swingstore;
+
+    hostStorage.kvStore.set('host.height', String(manifest.blockHeight));
+    await hostStorage.commit();
+    await hostStorage.close();
+  } else if (exportDataMode === 'repair-metadata') {
+    blockHeight !== 0 || Fail`repair metadata requires a block height`;
+
+    manifest.data || Fail`State-sync manifest missing export data`;
+
+    artifactMode === 'none' ||
+      Fail`Cannot restore artifacts while repairing metadata`;
+
+    const { hostStorage } = openDB(stateDir);
 
-  const swingstore = await importDB(exporter, stateDir, { artifactMode });
+    const savedBlockHeight =
+      Number(hostStorage.kvStore.get('host.height')) || 0;
 
-  const { hostStorage } = swingstore;
+    if (blockHeight !== savedBlockHeight) {
+      throw Fail`block height doesn't match. requested=${q(
+        blockHeight,
+      )}, current=${q(savedBlockHeight)}`;
+    }
 
-  hostStorage.kvStore.set('host.height', String(manifest.blockHeight));
-  await hostStorage.commit();
-  await hostStorage.close();
+    await hostStorage.repairMetadata(exporter);
+
+    await hostStorage.commit();
+    await hostStorage.close();
+  } else if (exportDataMode === 'skip') {
+    throw Fail`Repopulation of artifacts not yet supported`;
+  } else {
+    throw Fail`Unknown export-data-mode ${exportDataMode}`;
+  }
 };
 
 /**
@@ -186,9 +259,20 @@ export const main = async (
     /** @type {string} */ (processValue.getFlag('export-dir', '.')),
   );
 
-  const includeHistorical = processValue.getBoolean({
-    flagName: 'include-historical',
-  });
+  const artifactMode =
+    /** @type {import('./export-kernel-db.js').SwingStoreArtifactMode | undefined} */ (
+      processValue.getFlag('artifact-mode')
+    );
+  checkArtifactMode(artifactMode);
+
+  const exportDataMode = processValue.getFlag('export-data-mode');
+  checkExportDataMode(exportDataMode, true);
+
+  if (
+    processValue.getBoolean({ flagName: 'include-historical' }) !== undefined
+  ) {
+    throw Fail`deprecated "include-historical" options, use "artifact-mode" instead`;
+  }
 
   const checkBlockHeight = processValue.getInteger({
     flagName: 'check-block-height',
@@ -203,7 +287,8 @@ export const main = async (
       stateDir,
       exportDir,
       blockHeight: checkBlockHeight,
-      includeHistorical,
+      artifactMode,
+      exportDataMode,
     },
     {
       fs,