@agoric/xsnap 0.14.3-other-dev-8f8782b.0 → 0.14.3-other-dev-3eb1a1d.0

package/src/xsnap.js

@@ -1,27 +1,23 @@
-/* global process */
+/* eslint-env node */
 /* eslint no-await-in-loop: ["off"] */
 
-/**
- * @typedef {typeof import('child_process').spawn} Spawn
- * @typedef {import('stream').Writable} Writable
- */
-
-/**
- * @template T
- * @typedef {import('./defer').Deferred<T>} Deferred
- */
-
 import { finished } from 'stream/promises';
 import { PassThrough, Readable } from 'stream';
 import { promisify } from 'util';
+import { fileURLToPath } from 'url';
+import { Fail, q } from '@endo/errors';
 import { makeNetstringReader, makeNetstringWriter } from '@endo/netstring';
 import { makeNodeReader, makeNodeWriter } from '@endo/stream-node';
 import { makePromiseKit, racePromises } from '@endo/promise-kit';
 import { forever } from '@agoric/internal';
 import { ErrorCode, ErrorSignal, ErrorMessage, METER_TYPE } from '../api.js';
-import { defer } from './defer.js';
 
-const { Fail, quote: q } = assert;
+/** @import {PromiseKit} from '@endo/promise-kit' */
+
+/**
+ * @typedef {typeof import('child_process').spawn} Spawn
+ * @import {Writable} from 'stream'
+ */
 
 // This will need adjustment, but seems to be fine for a start.
 export const DEFAULT_CRANK_METERING_LIMIT = 1e8;
@@ -55,6 +51,81 @@ function echoCommand(arg) {
 const safeHintFromDescription = description =>
   description.replaceAll(/[^a-zA-Z0-9_.-]/g, '-');
 
+/**
+ * @typedef {object} SnapshotLoader
+ * @property {string} snapPath
+ *   where XS can load the snapshot from, either a filesystem path or a string
+ *   like `@${fileDescriptorNumber}:${readableDescription}`
+ * @property {(destStream?: Writable) => Promise<void>} afterSpawn
+ *   callback for providing a destination stream to which the data should be
+ *   piped (only relevant for a stream-based loader)
+ * @property {() => Promise<void>} cleanup
+ *   callback to free resources when the loader is no longer needed
+ */
+
+/**
+ * @callback MakeSnapshotLoader
+ * @param {AsyncIterable<Uint8Array>} sourceBytes
+ * @param {string} description
+ * @param {{fs: Pick<typeof import('fs/promises'), 'open' | 'unlink'>, ptmpName: (opts: import('tmp').TmpNameOptions) => Promise<string>}} ioPowers
+ * @returns {Promise<SnapshotLoader>}
+ */
+
+/** @type {MakeSnapshotLoader} */
+const makeSnapshotLoaderWithFS = async (
+  sourceBytes,
+  description,
+  { fs, ptmpName },
+) => {
+  const snapPath = await ptmpName({
+    template: `load-snapshot-${safeHintFromDescription(description)}-XXXXXX.xss`,
+  });
+
+  const afterSpawn = async () => {};
+  const cleanup = async () => fs.unlink(snapPath);
+
+  try {
+    const tmpSnap = await fs.open(snapPath, 'w');
+    // @ts-expect-error incorrect typings; writeFile does support AsyncIterable
+    await tmpSnap.writeFile(sourceBytes);
+    await tmpSnap.close();
+  } catch (e) {
+    await cleanup();
+    throw e;
+  }
+
+  return harden({
+    snapPath,
+    afterSpawn,
+    cleanup,
+  });
+};
+
+/** @type {MakeSnapshotLoader} */
+const makeSnapshotLoaderWithPipe = async (
+  sourceBytes,
+  description,
+  _ioPowers,
+) => {
+  let done = Promise.resolve();
+
+  const cleanup = async () => done;
+
+  const afterSpawn = async destStream => {
+    const sourceStream = Readable.from(sourceBytes);
+    sourceStream.pipe(destStream, { end: false });
+
+    done = finished(sourceStream);
+    void done.catch(noop).then(() => sourceStream.unpipe(destStream));
+  };
+
+  return harden({
+    snapPath: `@${SNAPSHOT_LOAD_FD}:${safeHintFromDescription(description)}`,
+    afterSpawn,
+    cleanup,
+  });
+};
+
 /**
  * @param {XSnapOptions} options
  *
@@ -70,8 +141,8 @@ const safeHintFromDescription = description =>
  * @property {AsyncIterable<Uint8Array>} [snapshotStream]
  * @property {string} [snapshotDescription]
  * @property {boolean} [snapshotUseFs]
- * @property {'ignore' | 'inherit'} [stdout]
- * @property {'ignore' | 'inherit'} [stderr]
+ * @property {'ignore' | 'inherit' | 'pipe'} [stdout]
+ * @property {'ignore' | 'inherit' | 'pipe'} [stderr]
  * @property {number} [meteringLimit]
  * @property {Record<string, string>} [env]
  */
@@ -86,7 +157,7 @@ export async function xsnap(options) {
     netstringMaxChunkSize = undefined,
     parserBufferSize = undefined,
     snapshotStream,
-    snapshotDescription = snapshotStream && 'unknown',
+    snapshotDescription = 'unknown',
     snapshotUseFs = false,
     stdout = 'ignore',
     stderr = 'ignore',
@@ -97,97 +168,39 @@ export async function xsnap(options) {
   const platform = {
     Linux: 'lin',
     Darwin: 'mac',
-    Windows_NT: 'win',
+    // Windows_NT: 'win', // One can dream.
   }[os];
 
   if (platform === undefined) {
     throw Error(`xsnap does not support platform ${os}`);
   }
 
-  /** @type {(opts: import('tmp').TmpNameOptions) => Promise<string>} */
-  const ptmpName = fs.tmpName && promisify(fs.tmpName);
-
-  const makeLoadSnapshotHandlerWithFS = async () => {
-    assert(snapshotStream);
-    const snapPath = await ptmpName({
-      template: `load-snapshot-${safeHintFromDescription(
-        snapshotDescription,
-      )}-XXXXXX.xss`,
-    });
-
-    const afterSpawn = async () => {};
-    const cleanup = async () => fs.unlink(snapPath);
-
-    try {
-      // eslint-disable-next-line @jessie.js/no-nested-await
-      const tmpSnap = await fs.open(snapPath, 'w');
-      // eslint-disable-next-line @jessie.js/no-nested-await
-      await tmpSnap.writeFile(
-        // @ts-expect-error incorrect typings, does support AsyncIterable
-        snapshotStream,
-      );
-      // eslint-disable-next-line @jessie.js/no-nested-await
-      await tmpSnap.close();
-    } catch (e) {
-      // eslint-disable-next-line @jessie.js/no-nested-await
-      await cleanup();
-      throw e;
-    }
-
-    return harden({
-      snapPath,
-      afterSpawn,
-      cleanup,
-    });
-  };
-
-  const makeLoadSnapshotHandlerWithPipe = async () => {
-    let done = Promise.resolve();
-
-    const cleanup = async () => done;
-
-    /** @param {Writable} loadSnapshotsStream */
-    const afterSpawn = async loadSnapshotsStream => {
-      assert(snapshotStream);
-      const destStream = loadSnapshotsStream;
-
-      const sourceStream = Readable.from(snapshotStream);
-      sourceStream.pipe(destStream, { end: false });
-
-      done = finished(sourceStream);
-      done.catch(noop).then(() => sourceStream.unpipe(destStream));
-    };
-
-    return harden({
-      snapPath: `@${SNAPSHOT_LOAD_FD}:${safeHintFromDescription(
-        snapshotDescription,
-      )}`,
-      afterSpawn,
-      cleanup,
-    });
-  };
-
-  let bin = new URL(
-    `../xsnap-native/xsnap/build/bin/${platform}/${
-      debug ? 'debug' : 'release'
-    }/xsnap-worker`,
-    import.meta.url,
-  ).pathname;
+  let bin = fileURLToPath(
+    new URL(
+      `../xsnap-native/xsnap/build/bin/${platform}/${
+        debug ? 'debug' : 'release'
+      }/xsnap-worker`,
+      import.meta.url,
+    ),
+  );
 
-  /** @type {Deferred<void>} */
-  const vatExit = defer();
+  /** @type {PromiseKit<void>} */
+  const vatExit = makePromiseKit();
 
   assert(!/^-/.test(name), `name '${name}' cannot start with hyphen`);
 
-  let loadSnapshotHandler = await (snapshotStream &&
-    (snapshotUseFs
-      ? makeLoadSnapshotHandlerWithFS
-      : makeLoadSnapshotHandlerWithPipe)());
+  /** @type {(opts: import('tmp').TmpNameOptions) => Promise<string>} */
+  const ptmpName = fs.tmpName && promisify(fs.tmpName);
+  const makeSnapshotLoader = snapshotUseFs
+    ? makeSnapshotLoaderWithFS
+    : makeSnapshotLoaderWithPipe;
+  let snapshotLoader = await (snapshotStream &&
+    makeSnapshotLoader(snapshotStream, snapshotDescription, { fs, ptmpName }));
 
   let args = [name];
 
-  if (loadSnapshotHandler) {
-    args.push('-r', loadSnapshotHandler.snapPath);
+  if (snapshotLoader) {
+    args.push('-r', snapshotLoader.snapPath);
   }
 
   if (meteringLimit) {
@@ -258,13 +271,13 @@ export async function xsnap(options) {
   const snapshotSaveStream = xsnapProcessStdio[SNAPSHOT_SAVE_FD];
   const snapshotLoadStream = xsnapProcessStdio[SNAPSHOT_LOAD_FD];
 
-  await loadSnapshotHandler?.afterSpawn(snapshotLoadStream);
+  await snapshotLoader?.afterSpawn(snapshotLoadStream);
 
-  if (loadSnapshotHandler) {
-    vatExit.promise.catch(noop).then(() => {
-      if (loadSnapshotHandler) {
-        const { cleanup } = loadSnapshotHandler;
-        loadSnapshotHandler = undefined;
+  if (snapshotLoader) {
+    void vatExit.promise.catch(noop).then(() => {
+      if (snapshotLoader) {
+        const { cleanup } = snapshotLoader;
+        snapshotLoader = undefined;
         return cleanup();
       }
     });
@@ -277,7 +290,7 @@ export async function xsnap(options) {
    * @template T
    * @typedef {object} RunResult
    * @property {T} reply
-   * @property {{ meterType: string, allocate: number|null, compute: number|null, timestamps: number[]|null }} meterUsage
+   * @property {{ meterType: string, allocate: number|null, compute: number|null, currentHeapCount: number|null, timestamps: number[]|null }} meterUsage
    */
 
   /**
@@ -286,10 +299,9 @@ export async function xsnap(options) {
   async function runToIdle() {
     for await (const _ of forever) {
       const iteration = await messagesFromXsnap.next(undefined);
-      if (loadSnapshotHandler) {
-        const { cleanup } = loadSnapshotHandler;
-        loadSnapshotHandler = undefined;
-        // eslint-disable-next-line @jessie.js/no-nested-await
+      if (snapshotLoader) {
+        const { cleanup } = snapshotLoader;
+        snapshotLoader = undefined;
         await cleanup();
       }
       if (iteration.done) {
@@ -303,7 +315,12 @@ export async function xsnap(options) {
         xsnapProcess.kill();
         throw Error('xsnap protocol error: received empty message');
       } else if (message[0] === OK) {
-        let meterInfo = { compute: null, allocate: null, timestamps: [] };
+        let meterInfo = {
+          compute: null,
+          allocate: null,
+          currentHeapCount: null,
+          timestamps: [],
+        };
         const meterSeparator = message.indexOf(OK_SEPARATOR, 1);
         if (meterSeparator >= 0) {
           // The message is `.meterdata\1reply`.
@@ -328,9 +345,7 @@ export async function xsnap(options) {
           )}`,
         );
       } else if (message[0] === QUERY) {
-        // eslint-disable-next-line @jessie.js/no-nested-await
         const commandResult = await handleCommand(message.subarray(1));
-        // eslint-disable-next-line @jessie.js/no-nested-await
         await messagesToXsnap.next([QUERY_RESPONSE_BUF, commandResult]);
       } else {
         // unrecognized responses also kill the process
@@ -429,6 +444,7 @@ export async function xsnap(options) {
     let snapshotReadSize = 0;
     /** @type {number | undefined} */
     let snapshotSize;
+    await null;
     try {
       /** @type {string} */
       let snapPath;
@@ -444,7 +460,6 @@ export async function xsnap(options) {
 
       if (snapshotUseFs) {
         // TODO: Refactor to use tmpFile rather than tmpName.
-        // eslint-disable-next-line @jessie.js/no-nested-await
         snapPath = await ptmpName({
           template: `make-snapshot-${safeHintFromDescription(
             description,
@@ -461,14 +476,16 @@ export async function xsnap(options) {
         // then wait for the command response to pipe the file stream into the
         // output, causing the file read to begin.
 
-        // eslint-disable-next-line @jessie.js/no-nested-await
         const handle = await fs.open(snapPath, 'w+');
         // @ts-expect-error 'close' event added in Node 15.4
         handle.on('close', () => {
-          fs.unlink(snapPath);
+          // Safe to ignore the result because we are skipping to to clean up the temp directory.
+          void fs.unlink(snapPath);
         });
         sourceStream = handle.createReadStream();
-        finished(output).finally(() => sourceStream.destroy());
+        finished(output)
+          .finally(() => sourceStream.destroy())
+          .catch(noop);
       } else {
         sourceStream = snapshotSaveStream;
         snapPath = `@${SNAPSHOT_SAVE_FD}`;
@@ -477,14 +494,13 @@ export async function xsnap(options) {
         // ensuring that any previous save stream usage has ended. However we
         // must start the flow before receiving the command's response or the
         // xsnap process would block on a full pipe, causing an IPC deadlock.
-        batonKit.promise.then(maybePipe);
+        batonKit.promise.then(maybePipe, noop);
       }
 
       const cleanup = () => {
         if (cleaned) return;
         cleaned = true;
         sourceStream.unpipe(output);
-        // eslint-disable-next-line no-use-before-define
         output.off('data', onData);
         output.end();
       };
@@ -522,7 +538,6 @@ export async function xsnap(options) {
 
       yield* output;
     } finally {
-      // eslint-disable-next-line @jessie.js/no-nested-await
       await done;
       (piped && snapshotReadSize === snapshotSize) ||
         Fail`Snapshot size does not match. saved=${q(snapshotSize)}, read=${q(
@@ -530,6 +545,7 @@ export async function xsnap(options) {
         )}`;
     }
   }
+  harden(makeSnapshotInternal);
 
   /**
    * @param {string} [description]

package/src/xsrepl.js

@@ -1,21 +1,18 @@
-/* global process */
+/* eslint-env node */
 /* We make exceptions for test code. This is a test utility. */
-/* eslint-disable @jessie.js/no-nested-await */
 /* eslint no-await-in-loop: ["off"] */
 
 import '@endo/init';
 
-/**
- * @template T
- * @typedef {import('./defer').Deferred<T>} Deferred
- */
 import * as childProcess from 'child_process';
 import fs from 'fs';
 import { tmpName } from 'tmp';
 import * as os from 'os';
 import * as readline from 'readline';
+import { makePromiseKit } from '@endo/promise-kit';
 import { xsnap } from './xsnap.js';
-import { defer } from './defer.js';
+
+/** @import {PromiseKit} from '@endo/promise-kit' */
 
 const decoder = new TextDecoder();
 
@@ -66,7 +63,9 @@ async function main() {
    * @returns {Promise<string>}
    */
   function ask(prompt) {
-    const { promise, resolve } = /** @type {Deferred<string>} */ (defer());
+    const { promise, resolve } = /** @type {PromiseKit<string>} */ (
+      makePromiseKit()
+    );
     rl.question(prompt, resolve);
     return promise;
   }
@@ -97,4 +96,4 @@ async function main() {
   return vat.close();
 }
 
-main();
+await main().catch(err => console.log(err));

package/xsnap-native/xsnap/makefiles/lin/makefile

@@ -0,0 +1,10 @@
+.NOTPARALLEL:
+
+all: debug release
+
+debug:
+	make -f xsnap.mk
+
+release:
+	make GOAL=release -f xsnap.mk
+	

package/xsnap-native/xsnap/makefiles/lin/xsnap-worker.mk

@@ -0,0 +1,156 @@
+% : %.c
+%.o : %.c
+
+GOAL ?= debug
+NAME = xsnap-worker
+ifneq ($(VERBOSE),1)
+MAKEFLAGS += --silent
+endif
+
+EXTRA_DEPS =
+
+# MODDABLE = $(CURDIR)/../../moddable
+BUILD_DIR = $(CURDIR)/../../build
+TLS_DIR = $(CURDIR)/../../sources
+
+# BUILD_DIR = $(MODDABLE)/build
+# TLS_DIR = ../../sources
+
+XS_DIR = $(MODDABLE)/xs
+
+BIN_DIR = $(BUILD_DIR)/bin/lin/$(GOAL)
+INC_DIR = $(XS_DIR)/includes
+PLT_DIR = $(XS_DIR)/platforms
+SRC_DIR = $(XS_DIR)/sources
+TMP_DIR = $(BUILD_DIR)/tmp/lin/$(GOAL)/$(NAME)
+
+C_OPTIONS = \
+	-fno-common \
+	-DINCLUDE_XSPLATFORM \
+	-DXSPLATFORM=\"xsnapPlatform.h\" \
+	-DXSNAP_VERSION=\"$(XSNAP_VERSION)\" \
+	-DXSNAP_TEST_RECORD=0 \
+	-DmxLockdown=1 \
+	-DmxMetering=1 \
+	-DmxDebug=1 \
+	-UmxInstrument \
+	-DmxNoConsole=1 \
+	-DmxBoundsCheck=1 \
+	-DmxParse=1 \
+	-DmxRun=1 \
+	-DmxSloppy=1 \
+	-DmxSnapshot=1 \
+	-DmxRegExpUnicodePropertyEscapes=1 \
+	-DmxStringNormalize=1 \
+	-DmxMinusZero=1 \
+	-I$(INC_DIR) \
+	-I$(PLT_DIR) \
+	-I$(SRC_DIR) \
+	-I$(TLS_DIR) \
+	-I$(TMP_DIR)
+C_OPTIONS += \
+	-Wno-misleading-indentation \
+	-Wno-implicit-fallthrough
+ifeq ($(GOAL),debug)
+	C_OPTIONS += -g -O0 -Wall -Wextra -Wno-missing-field-initializers -Wno-unused-parameter
+else
+	C_OPTIONS += -O3
+endif
+
+LIBRARIES = -ldl -lm -lpthread
+
+ifeq ($(XSNAP_RANDOM_INIT),1)
+	LIBRARIES += -lbsd
+	C_OPTIONS += -DmxSnapshotRandomInit
+endif
+
+LINK_OPTIONS = -rdynamic
+
+OBJECTS = \
+	$(TMP_DIR)/xsAll.o \
+	$(TMP_DIR)/xsAPI.o \
+	$(TMP_DIR)/xsArguments.o \
+	$(TMP_DIR)/xsArray.o \
+	$(TMP_DIR)/xsAtomics.o \
+	$(TMP_DIR)/xsBigInt.o \
+	$(TMP_DIR)/xsBoolean.o \
+	$(TMP_DIR)/xsCode.o \
+	$(TMP_DIR)/xsCommon.o \
+	$(TMP_DIR)/xsDataView.o \
+	$(TMP_DIR)/xsDate.o \
+	$(TMP_DIR)/xsDebug.o \
+	$(TMP_DIR)/xsDefaults.o \
+	$(TMP_DIR)/xsError.o \
+	$(TMP_DIR)/xsFunction.o \
+	$(TMP_DIR)/xsGenerator.o \
+	$(TMP_DIR)/xsGlobal.o \
+	$(TMP_DIR)/xsJSON.o \
+	$(TMP_DIR)/xsLexical.o \
+	$(TMP_DIR)/xsLockdown.o \
+	$(TMP_DIR)/xsMapSet.o \
+	$(TMP_DIR)/xsMarshall.o \
+	$(TMP_DIR)/xsMath.o \
+	$(TMP_DIR)/xsMemory.o \
+	$(TMP_DIR)/xsModule.o \
+	$(TMP_DIR)/xsNumber.o \
+	$(TMP_DIR)/xsObject.o \
+	$(TMP_DIR)/xsPlatforms.o \
+	$(TMP_DIR)/xsProfile.o \
+	$(TMP_DIR)/xsPromise.o \
+	$(TMP_DIR)/xsProperty.o \
+	$(TMP_DIR)/xsProxy.o \
+	$(TMP_DIR)/xsRegExp.o \
+	$(TMP_DIR)/xsRun.o \
+	$(TMP_DIR)/xsScope.o \
+	$(TMP_DIR)/xsScript.o \
+	$(TMP_DIR)/xsSnapshot.o \
+	$(TMP_DIR)/xsSourceMap.o \
+	$(TMP_DIR)/xsString.o \
+	$(TMP_DIR)/xsSymbol.o \
+	$(TMP_DIR)/xsSyntaxical.o \
+	$(TMP_DIR)/xsTree.o \
+	$(TMP_DIR)/xsType.o \
+	$(TMP_DIR)/xsdtoa.o \
+	$(TMP_DIR)/xsre.o \
+	$(TMP_DIR)/xsmc.o \
+	$(TMP_DIR)/textdecoder.o \
+	$(TMP_DIR)/textencoder.o \
+	$(TMP_DIR)/modBase64.o \
+	$(TMP_DIR)/xsnapPlatform.o \
+	$(TMP_DIR)/xsnap-worker.o
+
+VPATH += $(SRC_DIR) $(TLS_DIR)
+VPATH += $(MODDABLE)/modules/data/text/decoder
+VPATH += $(MODDABLE)/modules/data/text/encoder
+VPATH += $(MODDABLE)/modules/data/base64
+
+build: $(TMP_DIR) $(BIN_DIR) $(BIN_DIR)/$(NAME)
+
+$(TMP_DIR):
+	mkdir -p $(TMP_DIR)
+
+$(BIN_DIR):
+	mkdir -p $(BIN_DIR)
+
+$(BIN_DIR)/$(NAME): $(OBJECTS)
+	@echo "#" $(NAME) $(GOAL) ": cc" $(@F)
+	$(CC) $(LINK_OPTIONS) $(OBJECTS) $(LIBRARIES) -o $@
+
+$(OBJECTS): $(TLS_DIR)/xsnap.h
+$(OBJECTS): $(TLS_DIR)/xsnapPlatform.h
+$(OBJECTS): $(PLT_DIR)/xsPlatform.h
+$(OBJECTS): $(SRC_DIR)/xsCommon.h
+$(OBJECTS): $(SRC_DIR)/xsAll.h
+$(OBJECTS): $(SRC_DIR)/xsScript.h
+$(OBJECTS): $(SRC_DIR)/xsSnapshot.h
+$(OBJECTS): $(INC_DIR)/xs.h
+$(OBJECTS): $(EXTRA_DEPS)
+$(TMP_DIR)/%.o: %.c
+	@echo "#" $(NAME) $(GOAL) ": cc" $(<F)
+	$(CC) $< $(C_OPTIONS) -c -o $@
+
+clean:
+	rm -rf $(BUILD_DIR)/bin/lin/debug/$(NAME)
+	rm -rf $(BUILD_DIR)/bin/lin/release/$(NAME)
+	rm -rf $(BUILD_DIR)/tmp/lin/debug/$(NAME)
+	rm -rf $(BUILD_DIR)/tmp/lin/release/$(NAME)