@agoric/xsnap 0.14.3-u14.0 → 0.14.3-u16.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@agoric/xsnap",
-  "version": "0.14.3-u14.0",
+  "version": "0.14.3-u16.0",
   "description": "Snapshotting VM worker based on Moddable's XS Javascript engine",
   "author": "Agoric",
   "license": "Apache-2.0",
@@ -13,53 +13,70 @@
   "scripts": {
     "repl": "node src/xsrepl.js",
     "build:bin": "if test -d ./test; then node src/build.js; else yarn build:from-env; fi",
-    "build:env": "test -d ./test && node src/build.js --show-env > build.env",
+    "build:env": "node src/build.js --show-env > build.env",
     "build:from-env": "{ cat build.env; echo node src/build.js; } | xargs env",
     "build": "yarn build:bin && yarn build:env",
-    "postinstall": "yarn build:from-env",
+    "check-version": "xsnap_version=$(./scripts/get_xsnap_version.sh); if test \"${npm_package_version}\" != \"${xsnap_version}\"; then echo \"xsnap version mismatch; expected '${npm_package_version}', got '${xsnap_version}'\"; exit 1; fi",
+    "postinstall": "npm run build:from-env",
     "clean": "rm -rf xsnap-native/xsnap/build",
     "lint": "run-s --continue-on-error lint:*",
     "lint:js": "eslint 'src/**/*.js' 'test/**/*.js' api.js",
-    "lint:types": "tsc -p jsconfig.json",
+    "lint:types": "tsc",
     "lint-fix": "eslint --fix 'src/**/*.js' 'test/**/*.js' api.js",
     "test": "ava",
     "test:c8": "c8 $C8_OPTIONS ava --config=ava-nesm.config.js",
     "test:xs": "exit 0"
   },
   "dependencies": {
-    "@agoric/assert": "^0.6.1-u11wf.0",
-    "@agoric/internal": "^0.4.0-u14.0",
-    "@agoric/xsnap-lockdown": "^0.14.1-u13.0",
-    "@endo/bundle-source": "2.5.2-upstream-rollup",
-    "@endo/eventual-send": "0.17.2",
-    "@endo/init": "0.5.56",
-    "@endo/netstring": "0.3.26",
-    "@endo/promise-kit": "0.2.56",
-    "@endo/stream": "0.3.25",
-    "@endo/stream-node": "0.2.26",
+    "@agoric/assert": "^0.6.1-u16.0",
+    "@agoric/internal": "^0.4.0-u16.0",
+    "@agoric/xsnap-lockdown": "^0.14.1-u16.0",
+    "@endo/bundle-source": "^3.2.3",
+    "@endo/eventual-send": "^1.2.2",
+    "@endo/init": "^1.1.2",
+    "@endo/netstring": "^1.0.7",
+    "@endo/promise-kit": "^1.1.2",
+    "@endo/stream": "^1.2.2",
+    "@endo/stream-node": "^1.1.2",
     "glob": "^7.1.6",
     "tmp": "^0.2.1"
   },
   "devDependencies": {
-    "@endo/base64": "0.2.31",
-    "ava": "^5.2.0",
-    "c8": "^7.13.0"
+    "@endo/base64": "^1.0.5",
+    "@endo/nat": "^5.0.7",
+    "@types/glob": "^8.1.0",
+    "ava": "^5.3.0",
+    "c8": "^9.1.0"
   },
   "files": [
     "LICENSE*",
     "api.js",
     "build.env",
-    "src"
+    "moddable/modules/data",
+    "moddable/xs/includes",
+    "moddable/xs/makefiles",
+    "moddable/xs/platforms/*.h",
+    "moddable/xs/sources",
+    "scripts",
+    "src",
+    "xsnap-native/xsnap/makefiles",
+    "xsnap-native/xsnap/sources"
   ],
   "publishConfig": {
     "access": "public"
   },
   "ava": {
     "files": [
-      "test/**/test-*.js"
+      "test/**/*.test.*"
+    ],
+    "require": [
+      "@endo/init/debug.js"
     ],
     "timeout": "2m",
     "workerThreads": false
   },
-  "gitHead": "b3a6f3374cb3bddab39fc6d6f426429cae6c29c6"
+  "typeCoverage": {
+    "atLeast": 93.95
+  },
+  "gitHead": "bbdf652c3f413381cb352a8a360db1063974fafd"
 }

package/scripts/get_xsnap_version.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+set -ueo pipefail
+
+# the xsnap binary lives in a platform-specific directory
+unameOut="$(uname -s)"
+case "${unameOut}" in
+  Linux*) platform=lin ;;
+  Darwin*) platform=mac ;;
+  *) platform=win ;;
+esac
+
+# extract the xsnap package version from the long version printed by xsnap-worker
+"./xsnap-native/xsnap/build/bin/${platform}/release/xsnap-worker" -v | sed -e 's/^xsnap \([^ ]*\) (XS [^)]*)$/\1/g'

package/scripts/test-package.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+# Verifies that files in package.json covers everything xsnap needs to compile
+# from sources out of an npm package.
+set -xueo pipefail
+
+TEMP=$(mktemp -d)
+# function cleanup() {
+#   rm -rf "$TEMP"
+# }
+# trap cleanup EXIT
+
+yarn pack -f "$TEMP/package.tar"
+(
+  cd "$TEMP"
+  tar xvf package.tar
+  cd package
+  time yarn
+  time yarn
+  time yarn
+  time yarn
+)

package/src/avaAssertXS.js

@@ -1,5 +1,4 @@
 /* global globalThis */
-/* eslint-disable no-await-in-loop, @jessie.js/no-nested-await -- test code */
 /** global print */
 
 const { assign, freeze, keys } = Object;
@@ -67,7 +66,7 @@ function deepDifference(x, y) {
  * @param {(msg: TapMessage) => void} send
  *
  * @typedef { ReturnType<typeof tapFormat> } TapFormat
- * @typedef {import('./avaXS').TapMessage} TapMessage
+ * @import {TapMessage} from './avaXS.js'
  */
 function tapFormat(send) {
   return freeze({
@@ -229,6 +228,8 @@ function makeTester(htest, out) {
     fail(message) {
       assert(false, message);
     },
+    // Not the SES or @endo/errors `assert`
+    // See https://github.com/Agoric/agoric-sdk/issues/9515
     assert,
     truthy,
     /**
@@ -300,6 +301,7 @@ function makeTester(htest, out) {
       expectation,
       message = `should reject like ${expectation}`,
     ) {
+      await null;
       try {
         await (typeof thrower === 'function' ? thrower() : thrower);
         assert(false, message);
@@ -310,6 +312,7 @@ function makeTester(htest, out) {
     },
     /** @type {(thrower: () => Promise<unknown>, message?: string) => Promise<void> } */
     async notThrowsAsync(nonThrower, message) {
+      await null;
       try {
         await (typeof nonThrower === 'function' ? nonThrower() : nonThrower);
       } catch (ex) {
@@ -335,6 +338,7 @@ const test = (label, run, htestOpt) => {
   htest.queue(label, async () => {
     const out = tapFormat(htest.send);
     const t = makeTester(htest, out);
+    await null;
     try {
       // out.diagnostic('start', label);
       await run(t);

package/src/avaHandler.cjs

@@ -12,7 +12,6 @@ HandledPromise is defined by eventual send shim.
 /// <reference types="ses" />
 /// <reference types="@endo/eventual-send" />
 
-// @ts-expect-error cannot redeclare encoder
 const encoder = new TextEncoder();
 const decoder = new TextDecoder();
 
@@ -20,7 +19,7 @@ const decoder = new TextDecoder();
  * @param {{ testNames: string[]} |
  *          { bundleSource: [string, ...unknown[]] } |
  *          TapMessage | Summary } item
- * @typedef {import('./avaXS').Summary} Summary
+ * @import {Summary} from './avaXS'
  */
 function send(item) {
   const msg = encoder.encode(JSON.stringify(item)).buffer;
@@ -33,7 +32,6 @@ function send(item) {
  */
 const bundleSource = async (startFilename, ...args) => {
   const msg = await send({ bundleSource: [startFilename, ...args] });
-  // @ts-expect-error send() returns void
   return JSON.parse(decoder.decode(msg));
 };
 
@@ -79,7 +77,7 @@ function handler(rawMessage) {
       const { source } = msg;
       const virtualObjectGlobals =
         // @ts-expect-error
-        // eslint-disable-next-line no-undef
+
         typeof VatData !== 'undefined' ? { VatData } : {};
       const c = new Compartment({
         require: testRequire,
@@ -87,7 +85,6 @@ function handler(rawMessage) {
         __filename,
         console,
         assert,
-        // @ts-expect-error
         HandledPromise,
         URL: class URLStub {
           constructor(url, base) {

package/src/avaXS.js

@@ -2,11 +2,10 @@
 
 Usage:
 
-  node avaXS.js [--debug] test-*.js
+  node avaXS.js [--debug] *.test.js
 
 */
 
-/* eslint-disable no-await-in-loop, @jessie.js/no-nested-await -- test code */
 import '@endo/init';
 
 import fs from 'fs';
@@ -69,8 +68,6 @@ function isMatch(specimen, pattern) {
  *
  * It also calls back if a test calls `bundleSource`.
  *
- * @typedef {{ moduleFormat: string, source: string }} Bundle
- *
  * And finally it reports back a summary of assertion results.
  *
  * @typedef {{
@@ -84,7 +81,7 @@ function isMatch(specimen, pattern) {
  * @param {{ verbose?: boolean, titleMatch?: string }} options
  * @param {{
  *   spawnXSnap: (opts: object) => XSnap,
- *   bundleSource: (...args: [string, ...unknown[]]) => Promise<Bundle>,
+ *   bundleSource: import('@endo/bundle-source').BundleSource,
  *   resolve: ResolveFn,
  *   dirname: typeof import('path').dirname,
  *   basename: typeof import('path').basename,
@@ -93,7 +90,7 @@ function isMatch(specimen, pattern) {
  *
  * @typedef {{ total: number, pass: number, fail: { filename: string, name: string }[] }} TestResults
  * @typedef { 'ok' | 'not ok' | 'SKIP' } Status
- * @typedef {ReturnType<typeof import('./xsnap').xsnap>} XSnap
+ * @typedef {ReturnType<typeof import('./xsnap.js').xsnap>} XSnap
  */
 async function runTestScript(
   filename,
@@ -120,7 +117,7 @@ async function runTestScript(
     /**
      * See also send() in avaHandler.cjs
      *
-     * @type { TapMessage | { testNames: string[] } | { bundleSource: [string, ...unknown[]] } | Summary }
+     * @type { TapMessage | { testNames: string[] } | { bundleSource: Parameters<import('@endo/bundle-source').BundleSource> } | Summary }
      */
     const msg = JSON.parse(decoder.decode(message));
     // console.log(input, msg, qty, byStatus);
@@ -129,6 +126,7 @@ async function runTestScript(
       testNames = msg.testNames;
     }
 
+    await null;
     if ('bundleSource' in msg) {
       const [startFilename, ...rest] = msg.bundleSource;
       // see also makeBundleResolve() below
@@ -244,6 +242,7 @@ async function avaConfig(args, options, { glob, readFile }) {
   let debug = false;
   let verbose = false;
   let titleMatch;
+  await null;
   while (args.length > 0) {
     const arg = args.shift();
     assert.typeof(arg, 'string');
@@ -301,7 +300,7 @@ async function avaConfig(args, options, { glob, readFile }) {
 /**
  * @param {string[]} args - CLI args (excluding node interpreter, script name)
  * @param {{
- *   bundleSource: typeof import('@endo/bundle-source').default,
+ *   bundleSource: import('@endo/bundle-source').BundleSource,
  *   spawn: typeof import('child_process')['spawn'],
  *   osType: typeof import('os')['type'],
  *   readFile: typeof import('fs')['promises']['readFile'],

package/src/build.js

@@ -1,6 +1,5 @@
 #!/usr/bin/env node
 /* global process */
-/* eslint-disable @jessie.js/no-nested-await -- test/build code */
 import * as childProcessTop from 'child_process';
 import fsTop from 'fs';
 import osTop from 'os';
@@ -140,7 +139,7 @@ const makeSubmodule = (path, repoUrl, { git }) => {
 };
 
 /**
- * @param {string[]} args
+ * @param {boolean} showEnv
  * @param {{
  *   env: Record<string, string | undefined>,
  *   stdout: typeof process.stdout,
@@ -150,12 +149,9 @@ const makeSubmodule = (path, repoUrl, { git }) => {
  *     rmdirSync: typeof import('fs').rmdirSync,
  *     readFile: typeof import('fs').promises.readFile,
  *   },
- *   os: {
- *     type: typeof import('os').type,
- *   }
  * }} io
  */
-async function main(args, { env, stdout, spawn, fs, os }) {
+const updateSubmodules = async (showEnv, { env, stdout, spawn, fs }) => {
   const git = makeCLI('git', { spawn });
 
   // When changing/adding entries here, make sure to search the whole project
@@ -176,13 +172,13 @@ async function main(args, { env, stdout, spawn, fs, os }) {
     },
   ];
 
-  if (args.includes('--show-env')) {
+  await null;
+  if (showEnv) {
     for (const submodule of submodules) {
       const { path, envPrefix, commitHash } = submodule;
       if (!commitHash) {
         // We need to glean the commitHash and url from Git.
         const sm = makeSubmodule(path, '?', { git });
-        // eslint-disable-next-line no-await-in-loop
         const [[{ hash }], url] = await Promise.all([
           sm.status(),
           sm.config('url'),
@@ -208,20 +204,50 @@ async function main(args, { env, stdout, spawn, fs, os }) {
         // ignore
       }
       if (!fs.existsSync(submodule.path)) {
-        // eslint-disable-next-line no-await-in-loop
         await submodule.clone();
       }
-      // eslint-disable-next-line no-await-in-loop
       await submodule.checkout(commitHash);
     } else {
-      // eslint-disable-next-line no-await-in-loop
       await submodule.init();
     }
   }
+};
 
+/**
+ * @param {{
+ *   spawn: typeof import('child_process').spawn,
+ *   fs: {
+ *     existsSync: typeof import('fs').existsSync,
+ *     rmdirSync: typeof import('fs').rmdirSync,
+ *     readFile: typeof import('fs').promises.readFile,
+ *     writeFile: typeof import('fs').promises.writeFile,
+ *   },
+ *   os: {
+ *     type: typeof import('os').type,
+ *   }
+ * }} io
+ * @param {object} [options]
+ * @param {boolean} [options.forceBuild]
+ */
+const makeXsnap = async ({ spawn, fs, os }, { forceBuild = false } = {}) => {
   const pjson = await fs.readFile(asset('../package.json'), 'utf-8');
   const pkg = JSON.parse(pjson);
 
+  const configEnvs = [
+    `XSNAP_VERSION=${pkg.version}`,
+    `CC=cc "-D__has_builtin(x)=1"`,
+  ];
+
+  const configEnvFile = asset('../build.config.env');
+  const existingConfigEnvs = fs.existsSync(configEnvFile)
+    ? await fs.readFile(configEnvFile, 'utf-8')
+    : '';
+
+  const expectedConfigEnvs = configEnvs.concat('').join('\n');
+  if (forceBuild || existingConfigEnvs.trim() !== expectedConfigEnvs.trim()) {
+    await fs.writeFile(configEnvFile, expectedConfigEnvs);
+  }
+
   const platform = ModdableSDK.platforms[os.type()];
   if (!platform) {
     throw Error(`Unsupported OS found: ${os.type()}`);
@@ -229,13 +255,14 @@ async function main(args, { env, stdout, spawn, fs, os }) {
 
   const make = makeCLI(platform.make || 'make', { spawn });
   for (const goal of ModdableSDK.buildGoals) {
-    // eslint-disable-next-line no-await-in-loop
     await make.run(
       [
         `MODDABLE=${ModdableSDK.MODDABLE}`,
         `GOAL=${goal}`,
-        `XSNAP_VERSION=${pkg.version}`,
-        `CC=cc "-D__has_builtin(x)=1"`,
+        // Any other configuration variables that affect the build output
+        // should be placed in `configEnvs` to force a rebuild if they change
+        ...configEnvs,
+        `EXTRA_DEPS=${configEnvFile}`,
         '-f',
         'xsnap-worker.mk',
       ],
@@ -244,21 +271,127 @@ async function main(args, { env, stdout, spawn, fs, os }) {
       },
     );
   }
+};
+
+/**
+ * @param {string[]} args
+ * @param {{
+ *   env: Record<string, string | undefined>,
+ *   stdout: typeof process.stdout,
+ *   spawn: typeof import('child_process').spawn,
+ *   fs: {
+ *     existsSync: typeof import('fs').existsSync,
+ *     rmdirSync: typeof import('fs').rmdirSync,
+ *     readFile: typeof import('fs').promises.readFile,
+ *     writeFile: typeof import('fs').promises.writeFile,
+ *   },
+ *   os: {
+ *     type: typeof import('os').type,
+ *   }
+ * }} io
+ */
+async function main(args, { env, stdout, spawn, fs, os }) {
+  // I solemnly swear I will do no synchronous work followed by a variable
+  // number turns of the event loop.
+  await null;
+
+  const osType = os.type();
+  const platform = {
+    Linux: 'lin',
+    Darwin: 'mac',
+    // Windows_NT: 'win', // One can dream.
+  }[osType];
+  if (platform === undefined) {
+    throw Error(`xsnap does not support platform ${osType}`);
+  }
+
+  // If this is a working copy of xsnap in a checkout of agoric-sdk, we need to
+  // either clone or update submodules.
+  // Otherwise, we are running from an extracted npm tarball and we should not
+  // attempt to update Git submodules and should make the binary from the
+  // published source.
+  //
+  // These steps will avoid rebuilding native xsnap in the common case for end
+  // users.
+  //
+  //                    ||      | X    || git
+  //                    || X    | X    || make
+  //                    || ---- | ---- || ----
+  // | bin | src | .git || pack | work ||
+  // | --- | --- | ---- || ---- | ---- ||
+  // |     |     |      ||      | X    ||
+  // |     |     | X    ||      |      ||
+  // |     | X   |      || X    |      ||
+  // |     | X   | X    ||      | X    ||
+  // | X   |     |      ||      |      ||
+  // | X   |     | X    ||      |      ||
+  // | X   | X   |      || X    |      ||
+  // | X   | X   | X    ||      | X    ||
+  //
+  // We build both release and debug, so checking for one should suffice.
+  // XXX This will need to account for the .exe extension if we recover support
+  // for Windows.
+  const hasBin = fs.existsSync(
+    asset(`../xsnap-native/xsnap/build/bin/${platform}/release/xsnap-worker`),
+  );
+  let hasSource = fs.existsSync(asset('../moddable/xs/includes/xs.h'));
+  const hasGit = fs.existsSync(asset('../moddable/.git'));
+  const isWorkingCopy = hasGit || (!hasSource && !hasBin);
+  const showEnv = args.includes('--show-env');
+
+  if (isWorkingCopy || showEnv) {
+    if (showEnv && !isWorkingCopy) {
+      throw new Error('XSnap requires a working copy and git to --show-env');
+    }
+    await updateSubmodules(showEnv, { env, stdout, spawn, fs });
+    hasSource = true;
+  }
+
+  if (!showEnv) {
+    if (hasSource) {
+      // Force a rebuild if for some reason the binary is out of date
+      // Since the make checks may not always detect that situation
+      let forceBuild = !hasBin;
+      if (hasBin) {
+        const npm = makeCLI('npm', { spawn });
+        await npm
+          .run(['run', '-s', 'check-version'], { cwd: asset('..') })
+          .catch(() => {
+            forceBuild = true;
+          });
+      }
+      await makeXsnap({ spawn, fs, os }, { forceBuild });
+    } else if (!hasBin) {
+      throw new Error(
+        'XSnap has neither sources nor a pre-built binary. Docker? .dockerignore? npm files?',
+      );
+    }
+  }
 }
 
-main(process.argv.slice(2), {
-  env: { ...process.env },
-  stdout: process.stdout,
-  spawn: childProcessTop.spawn,
-  fs: {
-    readFile: fsTop.promises.readFile,
-    existsSync: fsTop.existsSync,
-    rmdirSync: fsTop.rmdirSync,
+const run = () =>
+  main(process.argv.slice(2), {
+    env: { ...process.env },
+    stdout: process.stdout,
+    spawn: childProcessTop.spawn,
+    fs: {
+      readFile: fsTop.promises.readFile,
+      writeFile: fsTop.promises.writeFile,
+      existsSync: fsTop.existsSync,
+      rmdirSync: fsTop.rmdirSync,
+    },
+    os: {
+      type: osTop.type,
+    },
+  });
+
+process.exitCode = 1;
+run().then(
+  () => {
+    process.exitCode = 0;
   },
-  os: {
-    type: osTop.type,
+  err => {
+    console.error('Failed with', err);
+    process.exit(process.exitCode || 1);
   },
-}).catch(e => {
-  console.error(e);
-  process.exit(1);
-});
+);

package/src/replay.js

@@ -227,14 +227,12 @@ export async function replayXSnap(
       const seq = parseInt(digits, 10);
       console.log(folder, seq, kind);
       if (running && !['command', 'reply'].includes(kind)) {
-        // eslint-disable-next-line @jessie.js/no-nested-await
         await running;
         running = undefined;
       }
       const file = rd.file(step);
       switch (kind) {
         case 'isReady':
-          // eslint-disable-next-line @jessie.js/no-nested-await
           await it.isReady();
           break;
         case 'evaluate':
@@ -254,7 +252,6 @@ export async function replayXSnap(
             console.log(folder, step, 'ignoring remaining steps from', folder);
             return;
           } else {
-            // eslint-disable-next-line @jessie.js/no-nested-await
             await (async () => {
               const snapshotPath = file.getText();
               const snapFile = await opts.fs.open(snapshotPath, 'w');