@agoric/cosmic-swingset 0.40.0 → 0.40.1-dev-800c2bb.0

package/src/helpers/bufferedStorage.js

@@ -0,0 +1,194 @@
+// @ts-check
+
+import { assert, Fail } from '@agoric/assert';
+
+// XXX Do these "StorageAPI" functions belong in their own package?
+
+/**
+ * @template {unknown} [T=unknown]
+ * @typedef {{
+ *   has: (key: string) => boolean,
+ *   get: (key: string) => T | undefined,
+ *   getNextKey: (previousKey: string) => string | undefined,
+ *   set: (key: string, value: T ) => void,
+ *   delete: (key: string) => void,
+ * }} KVStore
+ */
+
+/**
+ * Assert function to ensure that an object implements the StorageAPI
+ * interface: methods { has, getNextKey, get, set, delete }
+ * (cf. packages/SwingSet/docs/state.md#transactions).
+ *
+ * @param {*} kvStore  The object to be tested
+ *
+ * @throws {Error} if, upon inspection, the parameter does not satisfy the above
+ *   criteria.
+ *
+ * @returns {void}
+ */
+export function insistStorageAPI(kvStore) {
+  for (const n of ['has', 'getNextKey', 'get', 'set', 'delete']) {
+    n in kvStore || Fail`kvStore.${n} is missing, cannot use`;
+  }
+}
+
+/**
+ * Create a StorageAPI object that buffers writes to a wrapped StorageAPI object
+ * until told to commit (or abort) them.
+ *
+ * @template {unknown} [T=unknown]
+ * @param {KVStore<T>} kvStore  The StorageAPI object to wrap
+ * @param {{
+ *   onGet?: (key: string, value: T) => void, // a callback invoked after getting a value from kvStore
+ *   onPendingSet?: (key: string, value: T) => void, // a callback invoked after a new uncommitted value is set
+ *   onPendingDelete?: (key: string) => void, // a callback invoked after a new uncommitted delete
+ *   onCommit?: () => void, // a callback invoked after pending operations have been committed
+ *   onAbort?: () => void, // a callback invoked after pending operations have been aborted
+ * }} listeners  Optional callbacks to be invoked when respective events occur
+ *
+ * @returns {{kvStore: KVStore<T>, commit: () => void, abort: () => void}}
+ */
+export function makeBufferedStorage(kvStore, listeners = {}) {
+  insistStorageAPI(kvStore);
+
+  const { onGet, onPendingSet, onPendingDelete, onCommit, onAbort } = listeners;
+
+  // To avoid confusion, additions and deletions are prevented from sharing
+  // the same key at any given time.
+  const additions = new Map();
+  const deletions = new Set();
+
+  const buffered = {
+    has(key) {
+      assert.typeof(key, 'string');
+      if (additions.has(key)) return true;
+      if (deletions.has(key)) return false;
+      return kvStore.has(key);
+    },
+    get(key) {
+      assert.typeof(key, 'string');
+      if (additions.has(key)) return additions.get(key);
+      if (deletions.has(key)) return undefined;
+      const value = kvStore.get(key);
+      // @ts-expect-error value may be undefined
+      if (onGet !== undefined) onGet(key, value);
+      return value;
+    },
+    set(key, value) {
+      assert.typeof(key, 'string');
+      additions.set(key, value);
+      deletions.delete(key);
+      if (onPendingSet !== undefined) onPendingSet(key, value);
+    },
+    delete(key) {
+      assert.typeof(key, 'string');
+      additions.delete(key);
+      deletions.add(key);
+      if (onPendingDelete !== undefined) onPendingDelete(key);
+    },
+
+    /**
+     * @param {string} previousKey
+     */
+    getNextKey(previousKey) {
+      assert.typeof(previousKey, 'string');
+      throw Error('not implemented');
+    },
+  };
+  function commit() {
+    for (const [key, value] of additions) {
+      kvStore.set(key, value);
+    }
+    for (const key of deletions) {
+      kvStore.delete(key);
+    }
+    additions.clear();
+    deletions.clear();
+    if (onCommit !== undefined) onCommit();
+  }
+  function abort() {
+    additions.clear();
+    deletions.clear();
+    if (onAbort !== undefined) onAbort();
+  }
+  return { kvStore: buffered, commit, abort };
+}
+
+/**
+ * @template {unknown} [T=unknown]
+ * @param {KVStore<T>} kvStore
+ */
+export const makeReadCachingStorage = kvStore => {
+  // In addition to the wrapping write buffer, keep a simple cache of
+  // read values for has and get.
+  let cache;
+  function resetCache() {
+    cache = new Map();
+  }
+  resetCache();
+
+  const deleted = Symbol('deleted');
+  const undef = Symbol('undefined');
+
+  /** @type {KVStore<T>} */
+  const storage = harden({
+    has(key) {
+      const value = cache.get(key);
+      if (value !== undefined) {
+        return value !== deleted;
+      } else {
+        const ret = kvStore.has(key);
+        if (!ret) {
+          cache.set(key, deleted);
+        }
+        return ret;
+      }
+    },
+    get(key) {
+      let value = cache.get(key);
+      if (value !== undefined) {
+        return value === deleted || value === undef ? undefined : value;
+      }
+
+      // Fetch the value and cache it until the next commit or abort.
+      value = kvStore.get(key);
+      cache.set(key, value === undefined ? undef : value);
+      return value;
+    },
+    set(key, value) {
+      // Set the value and cache it until the next commit or abort (which is
+      // expected immediately, since the buffered wrapper only calls set
+      // *during* a commit).
+      // `undefined` is a valid value technically different than deletion,
+      // depending on how the underlying store does its serialization
+      cache.set(key, value === undefined ? undef : value);
+      kvStore.set(key, value);
+    },
+    delete(key) {
+      // Delete the value and cache the deletion until next commit or abort.
+      // Deletion results in `undefined` on `get`, but `false` on `has`
+      cache.set(key, deleted);
+      kvStore.delete(key);
+    },
+    getNextKey(_previousKey) {
+      throw Error('not implemented');
+    },
+  });
+  const {
+    kvStore: buffered,
+    commit,
+    abort,
+  } = makeBufferedStorage(storage, {
+    // Enqueue a write of any retrieved value, to handle callers like mailbox.js
+    // that expect local mutations to be automatically written back.
+    onGet(key, value) {
+      buffered.set(key, value);
+    },
+
+    // Reset the read cache upon commit or abort.
+    onCommit: resetCache,
+    onAbort: resetCache,
+  });
+  return harden({ ...buffered, commit, abort });
+};

package/src/{is-entrypoint.js → helpers/is-entrypoint.js}

@@ -1,3 +1,5 @@
+// @jessie-check
+
 // Detect if this is run as a script.
 import url from 'url';
 import process from 'process';

package/src/{json-stable-stringify.js → helpers/json-stable-stringify.js}

@@ -74,7 +74,7 @@ export default function stableStringify(obj, opts) {
     }
     if (seen.indexOf(node) !== -1) {
       if (cycles) return JSON.stringify('__cycle__');
-      throw new TypeError('Converting circular structure to JSON');
+      throw TypeError('Converting circular structure to JSON');
     } else seen.push(node);
 
     const keys = objectKeys(node).sort(cmp && cmp(node));

package/src/helpers/make-queue.js

@@ -0,0 +1,134 @@
+// @ts-check
+
+import { Fail } from '@agoric/assert';
+
+/**
+ * @typedef {object} QueueStorage
+ * @property {() => void} commit
+ * @property {() => void} abort
+ * @property {(key: string) => string | undefined} get
+ * @property {(key: string, value: string) => void} set
+ * @property {(key: string) => void} delete
+ */
+
+/**
+ * @typedef {{[idx: number]: string | undefined, head?: string, tail?: string}} QueueStorageDump
+ * @param {QueueStorageDump} [init]
+ * @returns {{storage: QueueStorage; dump: () => QueueStorageDump}}
+ */
+export const makeQueueStorageMock = init => {
+  const storage = new Map(init && Object.entries(init));
+  return harden({
+    storage: {
+      get(key) {
+        return storage.get(key);
+      },
+      set(key, value) {
+        typeof value === 'string' || Fail`invalid value type ${value}`;
+        storage.set(key, value);
+      },
+      delete(key) {
+        storage.delete(key);
+      },
+      commit() {},
+      abort() {},
+    },
+    dump: () => harden(Object.fromEntries(storage.entries())),
+  });
+};
+
+/**
+ * Create a queue backed by some sort of scoped storage.
+ *
+ * The queue writes the following bare keys, and expect any prefixing/scoping
+ * to be handled by the storage:
+ * - `head`: the index of the first entry of the queue.
+ * - `tail`: the index *past* the last entry in the queue.
+ * - `<index>`: the contents of the queue at the given index.
+ *
+ * For the cosmos inbound queues (`actionQueue` or `highPriorityQueue`), the
+ * golang side will push into the queue, updating the index stored at key
+ * `${queuePath}.tail` and setting data for key `${queuePath}.${index}`.
+ * The JS side will shift the queue, updating the index at key
+ * `${queuePath}.head` and reading and deleting `${queuePath}.${index}`.
+ *
+ * Parallel access is not supported, only a single outstanding operation at a
+ * time.
+ *
+ * @template {unknown} [T=unknown]
+ * @param {QueueStorage} storage a scoped queue storage
+ */
+export const makeQueue = storage => {
+  const getHead = () => BigInt(storage.get('head') || 0);
+  const getTail = () => BigInt(storage.get('tail') || 0);
+
+  /** @type {IterableIterator<T> | null} */
+  let currentIterator = null;
+  const queue = {
+    size: () => {
+      return Number(getTail() - getHead());
+    },
+    /** @param {T} obj */
+    push: obj => {
+      const tail = getTail();
+      storage.set('tail', String(tail + 1n));
+      storage.set(`${tail}`, JSON.stringify(obj));
+      storage.commit();
+      currentIterator = null;
+    },
+    /** @returns {IterableIterator<T>} */
+    consumeAll: () => {
+      let head = getHead();
+      const tail = getTail();
+      let done = !(head < tail);
+      const iterator = {
+        [Symbol.iterator]: () => iterator,
+        next: () => {
+          currentIterator === iterator || Fail`invalid iterator`;
+          if (!done) {
+            if (head < tail) {
+              // Still within the queue.
+              const headKey = `${head}`;
+              const value = JSON.parse(
+                /** @type {string} */ (storage.get(headKey)),
+              );
+              storage.delete(headKey);
+              head += 1n;
+              return { value, done };
+            }
+            // Reached the end, so clean up our indices.
+            storage.delete('head');
+            storage.delete('tail');
+            storage.commit();
+            done = true;
+          }
+          return { value: undefined, done };
+        },
+        return: () => {
+          currentIterator === iterator || Fail`invalid iterator`;
+          if (!done) {
+            // We're done consuming, so save our state.
+            storage.set('head', String(head));
+            storage.commit();
+            done = true;
+          }
+          return { value: undefined, done };
+        },
+        throw: err => {
+          currentIterator === iterator || Fail`invalid iterator`;
+          if (!done) {
+            // Don't change our state.
+            storage.abort();
+            done = true;
+            throw err;
+          }
+          return { value: undefined, done };
+        },
+      };
+      currentIterator = iterator;
+      return iterator;
+    },
+  };
+  return queue;
+};
+harden(makeQueue);

package/src/helpers/process-value.js

@@ -0,0 +1,133 @@
+// @ts-check
+
+import { resolve as pathResolve } from 'path';
+
+export const makeProcessValue = ({ env, args }) => {
+  /**
+   * @param {string} flagName
+   * @param {string | undefined} [deflt]
+   * @returns {string | undefined}
+   */
+  const getFlag = (flagName, deflt) => {
+    let flagValue = deflt;
+    const envValue =
+      env[`AG_CHAIN_COSMOS_${flagName.toUpperCase().replace(/-/g, '_')}`];
+    if (envValue !== undefined) {
+      flagValue = envValue;
+    }
+    const flag = `--${flagName}`;
+    const flagEquals = `--${flagName}=`;
+    for (let i = 0; i < args.length; i += 1) {
+      const arg = args[i];
+      if (arg === flag) {
+        if (args.length > i + 1 && !args[i + 1].startsWith('--')) {
+          i += 1;
+          flagValue = args[i];
+        } else {
+          flagValue = '';
+        }
+      } else if (arg.startsWith(flagEquals)) {
+        flagValue = arg.substr(flagEquals.length);
+      }
+    }
+    return flagValue;
+  };
+
+  /**
+   * @param {object} options
+   * @param {string} [options.envName]
+   * @param {string} [options.flagName]
+   * @param {boolean} [options.emptyFlagIsTrue]
+   * @returns {boolean | undefined}
+   */
+  const getBoolean = ({ envName, flagName, emptyFlagIsTrue = true }) => {
+    let option;
+    if (flagName) {
+      option = getFlag(flagName);
+    } else if (envName) {
+      option = env[envName];
+    } else {
+      return undefined;
+    }
+
+    switch (option) {
+      case '0':
+      case 'false':
+      case false:
+        return false;
+      case '1':
+      case 'true':
+      case true:
+        return true;
+      default:
+        if (option === '' && flagName && emptyFlagIsTrue) {
+          return true;
+        }
+        if (option === undefined && envName && flagName) {
+          return getBoolean({ envName, emptyFlagIsTrue });
+        }
+        return undefined;
+    }
+  };
+
+  /**
+   * @param {object} options
+   * @param {string} [options.envName]
+   * @param {string} [options.flagName]
+   * @returns {number | undefined}
+   */
+  const getInteger = ({ envName, flagName }) => {
+    let option;
+    if (flagName) {
+      option = getFlag(flagName);
+    } else if (envName) {
+      option = env[envName];
+    } else {
+      return undefined;
+    }
+
+    if (option) {
+      const value = Number.parseInt(option, 10);
+      return String(value) === option ? value : undefined;
+    } else if (option === undefined && flagName && envName) {
+      return getInteger({ envName });
+    }
+
+    return undefined;
+  };
+
+  /**
+   * @param {object} options
+   * @param {string} [options.envName]
+   * @param {string} [options.flagName]
+   * @param {string} [options.trueValue]
+   * @returns {string | undefined}
+   */
+  const getPath = ({ envName, flagName, trueValue }) => {
+    if (trueValue !== undefined) {
+      const boolValue = getBoolean({ envName, flagName });
+      if (boolValue !== undefined) {
+        return boolValue ? trueValue : undefined;
+      }
+    }
+
+    let option;
+    if (flagName) {
+      option = getFlag(flagName);
+    } else if (envName) {
+      option = env[envName];
+    } else {
+      return undefined;
+    }
+
+    if (option) {
+      return pathResolve(option);
+    } else if (trueValue && envName && flagName) {
+      return getPath({ envName, trueValue });
+    } else {
+      return undefined;
+    }
+  };
+
+  return harden({ getFlag, getBoolean, getInteger, getPath });
+};

package/src/import-kernel-db.js

@@ -0,0 +1,207 @@
+#! /usr/bin/env node
+// @ts-check
+// @jessie-check
+
+import '@endo/init/unsafe-fast.js';
+
+import os from 'os';
+import process from 'process';
+import { Transform } from 'stream';
+import fsPower from 'fs';
+import fsPromisesPower from 'fs/promises';
+import pathPower from 'path';
+
+import BufferLineTransform from '@agoric/internal/src/node/buffer-line-transform.js';
+import { Fail } from '@agoric/assert';
+import { importSwingStore } from '@agoric/swing-store';
+
+import { isEntrypoint } from './helpers/is-entrypoint.js';
+import { makeProcessValue } from './helpers/process-value.js';
+
+/**
+ * @typedef {object} StateSyncImporterOptions
+ * @property {string} stateDir the directory containing the SwingStore to export
+ * @property {string} exportDir the directory where to place the exported artifacts and manifest
+ * @property {number} [blockHeight] block height to check for
+ * @property {boolean} [includeHistorical] whether to include historical artifacts in the export
+ */
+
+/**
+ * @param {StateSyncImporterOptions} options
+ * @param {object} powers
+ * @param {Pick<import('fs/promises'), 'readFile'> & Pick<import('fs'), 'createReadStream'>} powers.fs
+ * @param {import('path')['resolve']} powers.pathResolve
+ * @param {typeof import('@agoric/swing-store')['importSwingStore']} [powers.importSwingStore]
+ * @param {null | ((...args: any[]) => void)} [powers.log]
+ * @returns {Promise<void>}
+ */
+export const performStateSyncImport = async (
+  { stateDir, exportDir, blockHeight, includeHistorical },
+  {
+    fs: { createReadStream, readFile },
+    pathResolve,
+    importSwingStore: importDB = importSwingStore,
+    log = console.log,
+  },
+) => {
+  /** @param {string} allegedRelativeFilename */
+  const safeExportFileResolve = allegedRelativeFilename => {
+    const resolvedPath = pathResolve(exportDir, allegedRelativeFilename);
+    resolvedPath.startsWith(exportDir) ||
+      Fail`Exported file ${allegedRelativeFilename} must be in export dir ${exportDir}`;
+    return resolvedPath;
+  };
+
+  const manifestPath = safeExportFileResolve('export-manifest.json');
+  /** @type {Readonly<import('./export-kernel-db.js').StateSyncManifest>} */
+  const manifest = await readFile(manifestPath, { encoding: 'utf-8' }).then(
+    data => JSON.parse(data),
+  );
+
+  if (blockHeight !== undefined && manifest.blockHeight !== blockHeight) {
+    Fail`State-sync manifest for unexpected block height ${manifest.blockHeight} (expected ${blockHeight})`;
+  }
+
+  if (!manifest.data) {
+    throw Fail`State-sync manifest missing export data`;
+  }
+
+  if (!manifest.artifacts) {
+    throw Fail`State-sync manifest missing required artifacts`;
+  }
+
+  const artifacts = harden(Object.fromEntries(manifest.artifacts));
+
+  if (
+    includeHistorical &&
+    manifest.mode !== 'archival' &&
+    manifest.mode !== 'debug'
+  ) {
+    throw Fail`State-sync manifest missing historical artifacts`;
+  }
+
+  // Represent the data in `exportDir` as a SwingSetExporter object.
+  /** @type {import('@agoric/swing-store').SwingStoreExporter} */
+  const exporter = harden({
+    async *getExportData() {
+      log?.('importing export data');
+      const exportData = createReadStream(
+        safeExportFileResolve(/** @type {string} */ (manifest.data)),
+      );
+      const exportDataEntries = exportData
+        .pipe(new BufferLineTransform())
+        .setEncoding('utf8')
+        .pipe(
+          new Transform({
+            objectMode: true,
+            transform(data, encoding, callback) {
+              try {
+                callback(null, JSON.parse(data));
+              } catch (error) {
+                callback(error);
+              }
+            },
+          }),
+        );
+      yield* exportDataEntries;
+    },
+    async *getArtifact(name) {
+      log?.(`importing artifact ${name}`);
+      const fileName = artifacts[name];
+      if (!fileName) {
+        Fail`invalid artifact ${name}`;
+      }
+      const stream = createReadStream(safeExportFileResolve(fileName));
+      yield* stream;
+    },
+    async *getArtifactNames() {
+      yield* Object.keys(artifacts);
+    },
+    async close() {
+      // TODO
+    },
+  });
+
+  const swingstore = await importDB(exporter, stateDir, { includeHistorical });
+
+  const { hostStorage } = swingstore;
+
+  hostStorage.kvStore.set('host.height', String(manifest.blockHeight));
+  await hostStorage.commit();
+  await hostStorage.close();
+};
+
+/**
+ * @param {string[]} args
+ * @param {object} powers
+ * @param {Partial<Record<string, string>>} powers.env
+ * @param {string} powers.homedir
+ * @param {Console} powers.console
+ * @param {Pick<import('fs/promises'), 'readFile' | 'stat'> & Pick<import('fs'), 'createReadStream'>} powers.fs
+ * @param {import('path')['resolve']} powers.pathResolve
+ */
+export const main = async (
+  args,
+  { env, homedir, console, fs, pathResolve },
+) => {
+  const processValue = makeProcessValue({ env, args });
+
+  const stateDir =
+    processValue.getFlag('state-dir') ||
+    // We try to find the actual cosmos state directory (default=~/.ag-chain-cosmos)
+    `${processValue.getFlag(
+      'home',
+      `${homedir}/.ag-chain-cosmos`,
+    )}/data/agoric`;
+
+  const stateDirStat = await fs.stat(stateDir);
+  if (!stateDirStat.isDirectory()) {
+    throw Error('state-dir must be an exiting directory');
+  }
+
+  const exportDir = pathResolve(
+    /** @type {string} */ (processValue.getFlag('export-dir', '.')),
+  );
+
+  const includeHistorical = processValue.getBoolean({
+    flagName: 'include-historical',
+  });
+
+  const checkBlockHeight = processValue.getInteger({
+    flagName: 'check-block-height',
+  });
+
+  const verbose = processValue.getBoolean({
+    flagName: 'verbose',
+  });
+
+  await performStateSyncImport(
+    {
+      stateDir,
+      exportDir,
+      blockHeight: checkBlockHeight,
+      includeHistorical,
+    },
+    {
+      fs,
+      pathResolve,
+      log: verbose ? console.log : undefined,
+    },
+  );
+};
+
+if (isEntrypoint(import.meta.url)) {
+  main(process.argv.splice(2), {
+    homedir: os.homedir(),
+    env: process.env,
+    console,
+    fs: { ...fsPower, ...fsPromisesPower },
+    pathResolve: pathPower.resolve,
+  }).then(
+    _res => 0,
+    rej => {
+      console.error(`error running export-kernel-db:`, rej);
+      process.exit(process.exitCode || rej.exitCode || 1);
+    },
+  );
+}