@endo/compartment-mapper 0.8.0 → 0.8.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@endo/compartment-mapper",
-  "version": "0.8.0",
+  "version": "0.8.1",
   "description": "The compartment mapper assembles Node applications in a sandbox",
   "keywords": [
     "node",
@@ -41,14 +41,14 @@
     "test": "ava"
   },
   "dependencies": {
-    "@endo/cjs-module-analyzer": "^0.2.28",
-    "@endo/static-module-record": "^0.7.15",
-    "@endo/zip": "^0.2.28",
-    "ses": "^0.18.0"
+    "@endo/cjs-module-analyzer": "^0.2.29",
+    "@endo/static-module-record": "^0.7.16",
+    "@endo/zip": "^0.2.29",
+    "ses": "^0.18.1"
   },
   "devDependencies": {
-    "@endo/eslint-config": "^0.5.1",
-    "ava": "^3.12.1",
+    "@endo/eslint-config": "^0.5.2",
+    "ava": "^5.1.0",
     "babel-eslint": "^10.0.3",
     "c8": "^7.7.3",
     "eslint": "^7.32.0",
@@ -57,7 +57,7 @@
     "eslint-plugin-eslint-comments": "^3.1.2",
     "eslint-plugin-import": "^2.26.0",
     "eslint-plugin-prettier": "^3.4.1",
-    "prettier": "^1.19.1",
+    "prettier": "^2.8.0",
     "typescript": "~4.8.4"
   },
   "files": [
@@ -76,6 +76,7 @@
     ]
   },
   "prettier": {
+    "arrowParens": "avoid",
     "trailingComma": "all",
     "singleQuote": true,
     "overrides": [
@@ -93,5 +94,5 @@
     ],
     "timeout": "2m"
   },
-  "gitHead": "da16a94856482e36296b7cae16d715aa63344928"
+  "gitHead": "ab8d64ae6fc9c628a2d1c02d16bf9ef249f5c8dc"
 }

package/src/archive.js

@@ -252,7 +252,10 @@ const digestLocation = async (powers, moduleLocation, options) => {
     moduleTransforms,
     modules: exitModules = {},
     dev = false,
+    tags = new Set(),
     captureSourceLocation = undefined,
+    searchSuffixes = undefined,
+    commonDependencies = undefined,
   } = options || {};
   const { read, computeSha512 } = unpackReadPowers(powers);
   const {
@@ -262,8 +265,6 @@ const digestLocation = async (powers, moduleLocation, options) => {
     moduleSpecifier,
   } = await search(read, moduleLocation);
 
-  /** @type {Set<string>} */
-  const tags = new Set();
   tags.add('endo');
   tags.add('import');
   tags.add('default');
@@ -278,7 +279,7 @@ const digestLocation = async (powers, moduleLocation, options) => {
     tags,
     packageDescriptor,
     moduleSpecifier,
-    { dev },
+    { dev, commonDependencies },
   );
 
   const {
@@ -296,6 +297,7 @@ const digestLocation = async (powers, moduleLocation, options) => {
     compartments,
     exitModules,
     computeSha512,
+    searchSuffixes,
   );
 
   // Induce importHook to record all the necessary modules to import the given module specifier.

package/src/bundle-cjs.js

@@ -0,0 +1,55 @@
+/** quotes strings */
+const q = JSON.stringify;
+
+const exportsCellRecord = exportsList =>
+  ''.concat(
+    ...exportsList.map(
+      exportName => `\
+      ${exportName}: cell(${q(exportName)}),
+`,
+    ),
+  );
+
+// This function is serialized and references variables from its destination scope.
+const runtime = function wrapCjsFunctor(num) {
+  /* eslint-disable no-undef */
+  return ({ imports = {} }) => {
+    const cModule = Object.freeze(
+      Object.defineProperty({}, 'exports', cells[num].default),
+    );
+    // TODO: specifier not found handling
+    const requireImpl = specifier => cells[imports[specifier]].default.get();
+    functors[num](Object.freeze(requireImpl), cModule.exports, cModule);
+    Object.keys(cells[num])
+      .filter(k => k !== 'default' && k !== '*')
+      .map(k => cells[num][k].set(cModule.exports[k]));
+  };
+  /* eslint-enable no-undef */
+}.toString();
+
+export default {
+  runtime,
+  getBundlerKit({
+    index,
+    indexedImports,
+    record: { cjsFunctor, exports: exportsList = {} },
+  }) {
+    const importsMap = JSON.stringify(indexedImports);
+
+    return {
+      getFunctor: () => `\
+// === functors[${index}] ===
+${cjsFunctor},
+`,
+      getCells: () => `\
+    {
+${exportsCellRecord(exportsList)}\
+    },
+`,
+      getReexportsWiring: () => '',
+      getFunctorCall: () => `\
+  wrapCjsFunctor(${index})({imports: ${importsMap}});
+`,
+    };
+  },
+};

package/src/bundle-mjs.js

@@ -0,0 +1,120 @@
+/** quotes strings */
+const q = JSON.stringify;
+
+const exportsCellRecord = exportMap =>
+  ''.concat(
+    ...Object.keys(exportMap).map(
+      exportName => `\
+      ${exportName}: cell(${q(exportName)}),
+`,
+    ),
+  );
+
+const importsCellSetter = (exportMap, index) =>
+  ''.concat(
+    ...Object.entries(exportMap).map(
+      ([exportName, [importName]]) => `\
+      ${importName}: cells[${index}].${exportName}.set,
+`,
+    ),
+  );
+
+const adaptReexport = reexportMap => {
+  if (!reexportMap) {
+    return {};
+  }
+  const ret = Object.fromEntries(
+    Object.values(reexportMap)
+      .flat()
+      .map(([local, exported]) => [exported, [local]]),
+  );
+  return ret;
+};
+
+export const runtime = `\
+function observeImports(map, importName, importIndex) {
+  for (const [name, observers] of map.get(importName)) {
+    const cell = cells[importIndex][name];
+    if (cell === undefined) {
+      throw new ReferenceError(\`Cannot import name \${name}\`);
+    }
+    for (const observer of observers) {
+      cell.observe(observer);
+    }
+  }
+}
+`;
+
+export default {
+  runtime,
+  getBundlerKit({
+    index,
+    indexedImports,
+    record: {
+      __syncModuleProgram__,
+      __fixedExportMap__ = {},
+      __liveExportMap__ = {},
+      __reexportMap__ = {},
+      reexports,
+    },
+  }) {
+    return {
+      getFunctor: () => `\
+// === functors[${index}] ===
+${__syncModuleProgram__},
+`,
+      getCells: () => `\
+    {
+${exportsCellRecord(__fixedExportMap__)}${exportsCellRecord(
+        __liveExportMap__,
+      )}${exportsCellRecord(adaptReexport(__reexportMap__))}\
+    },
+`,
+      getReexportsWiring: () => {
+        const mappings = reexports.map(
+          importSpecifier => `\
+  Object.defineProperties(cells[${index}], Object.getOwnPropertyDescriptors(cells[${indexedImports[importSpecifier]}]));
+`,
+        );
+        // Create references for export name as newname
+        const namedReexportsToProcess = Object.entries(__reexportMap__);
+        if (namedReexportsToProcess.length > 0) {
+          mappings.push(`
+  Object.defineProperties(cells[${index}], {${namedReexportsToProcess.map(
+            ([specifier, renames]) => {
+              return renames.map(
+                ([localName, exportedName]) =>
+                  `${q(exportedName)}: { value: cells[${
+                    indexedImports[specifier]
+                  }][${q(localName)}] }`,
+              );
+            },
+          )} });
+`);
+        }
+        return mappings.join('');
+      },
+      getFunctorCall: () => `\
+  functors[${index}]({
+    imports(entries) {
+      const map = new Map(entries);
+  ${''.concat(
+    ...Object.entries(indexedImports).map(
+      ([importName, importIndex]) => `\
+    observeImports(map, ${q(importName)}, ${importIndex});
+  `,
+    ),
+  )}\
+  },
+    liveVar: {
+  ${importsCellSetter(__liveExportMap__, index)}\
+  },
+    onceVar: {
+${importsCellSetter(__fixedExportMap__, index)}\
+    },
+    importMeta: {},
+  });
+`,
+    };
+  },
+};

package/src/bundle.js

@@ -24,10 +24,10 @@ import parserArchiveCjs from './parse-archive-cjs.js';
 import parserArchiveMjs from './parse-archive-mjs.js';
 import { parseLocatedJson } from './json.js';
 
-const textEncoder = new TextEncoder();
+import mjsSupport from './bundle-mjs.js';
+import cjsSupport from './bundle-cjs.js';
 
-/** quotes strings */
-const q = JSON.stringify;
+const textEncoder = new TextEncoder();
 
 /** @type {Record<string, ParserImplementation>} */
 const parserForLanguage = {
@@ -70,13 +70,16 @@ const sortedModules = (
 
     const resolve = compartmentResolvers[compartmentName];
     const source = compartmentSources[compartmentName][moduleSpecifier];
-    if (source) {
-      const { record, parser } = source;
+    if (source !== undefined) {
+      const { record, parser, deferredError } = source;
+      if (deferredError) {
+        throw new Error(
+          `Cannot bundle: encountered deferredError ${deferredError}`,
+        );
+      }
       if (record) {
-        const {
-          imports = [],
-          reexports = [],
-        } = /** @type {PrecompiledStaticModuleInterface} */ (record);
+        const { imports = [], reexports = [] } =
+          /** @type {PrecompiledStaticModuleInterface} */ (record);
         const resolvedImports = Object.create(null);
         for (const importSpecifier of [...imports, ...reexports]) {
           const resolvedSpecifier = resolve(importSpecifier, moduleSpecifier);
@@ -124,15 +127,53 @@ const sortedModules = (
   return modules;
 };
 
+const implementationPerParser = {
+  'pre-mjs-json': mjsSupport,
+  'pre-cjs-json': cjsSupport,
+};
+
+function getRuntime(parser) {
+  return implementationPerParser[parser]
+    ? implementationPerParser[parser].runtime
+    : `/*unknown parser:${parser}*/`;
+}
+
+function getBundlerKitForModule(module) {
+  const parser = module.parser;
+  if (!implementationPerParser[parser]) {
+    const warning = `/*unknown parser:${parser}*/`;
+    // each item is a function to avoid creating more in-memory copies of the source text etc.
+    return {
+      getFunctor: () => `(()=>{${warning}})`,
+      getCells: `{${warning}}`,
+      getFunctorCall: warning,
+    };
+  }
+  const getBundlerKit = implementationPerParser[parser].getBundlerKit;
+  return getBundlerKit(module);
+}
+
 /**
  * @param {ReadFn} read
  * @param {string} moduleLocation
  * @param {Object} [options]
  * @param {ModuleTransforms} [options.moduleTransforms]
+ * @param {boolean} [options.dev]
+ * @param {Set<string>} [options.tags]
+ * @param {Array<string>} [options.searchSuffixes]
+ * @param {Object} [options.commonDependencies]
  * @returns {Promise<string>}
  */
 export const makeBundle = async (read, moduleLocation, options) => {
-  const { moduleTransforms } = options || {};
+  const {
+    moduleTransforms,
+    dev,
+    tags: tagsOption,
+    searchSuffixes,
+    commonDependencies,
+  } = options || {};
+  const tags = new Set(tagsOption);
+
   const {
     packageLocation,
     packageDescriptorText,
@@ -140,9 +181,6 @@ export const makeBundle = async (read, moduleLocation, options) => {
     moduleSpecifier,
   } = await search(read, moduleLocation);
 
-  /** @type {Set<string>} */
-  const tags = new Set();
-
   const packageDescriptor = parseLocatedJson(
     packageDescriptorText,
     packageDescriptorLocation,
@@ -153,6 +191,7 @@ export const makeBundle = async (read, moduleLocation, options) => {
     tags,
     packageDescriptor,
     moduleSpecifier,
+    { dev, commonDependencies },
   );
 
   const {
@@ -167,6 +206,9 @@ export const makeBundle = async (read, moduleLocation, options) => {
     packageLocation,
     sources,
     compartments,
+    undefined,
+    undefined,
+    searchSuffixes,
   );
 
   // Induce importHook to record all the necessary modules to import the given module specifier.
@@ -194,6 +236,7 @@ export const makeBundle = async (read, moduleLocation, options) => {
     module.index = index;
     modulesByKey[module.key] = module;
   }
+  const parsersInUse = new Set();
   for (const module of modules) {
     module.indexedImports = Object.fromEntries(
       Object.entries(module.resolvedImports).map(([importSpecifier, key]) => [
@@ -201,142 +244,52 @@ export const makeBundle = async (read, moduleLocation, options) => {
         modulesByKey[key].index,
       ]),
     );
+    parsersInUse.add(module.parser);
+    module.bundlerKit = getBundlerKitForModule(module);
   }
 
-  // Only support mjs format.
-  const problems = modules
-    .filter(module => module.parser !== 'pre-mjs-json')
-    .map(
-      ({ moduleSpecifier, compartmentName, parser }) =>
-        `module ${moduleSpecifier} in compartment ${compartmentName} in language ${parser}`,
-    );
-  if (problems.length) {
-    throw new Error(
-      `Can only bundle applications that only have ESM (.mjs-type) modules, got ${problems.join(
-        ', ',
-      )}`,
-    );
-  }
-
-  const exportsCellRecord = exportMap =>
-    ''.concat(
-      ...Object.keys(exportMap).map(
-        exportName => `\
-      ${exportName}: cell(${q(exportName)}),
-`,
-      ),
-    );
-  const importsCellSetter = (exportMap, index) =>
-    ''.concat(
-      ...Object.entries(exportMap).map(
-        ([exportName, [importName]]) => `\
-      ${importName}: cells[${index}].${exportName}.set,
-`,
-      ),
-    );
-
   const bundle = `\
 'use strict';
 (() => {
   const functors = [
-${''.concat(
-  ...modules.map(
-    ({ record: { __syncModuleProgram__ } }, i) =>
-      `\
-// === functors[${i}] ===
-${__syncModuleProgram__},
-`,
-  ),
-)}\
+${''.concat(...modules.map(m => m.bundlerKit.getFunctor()))}\
 ]; // functors end
 
-  function cell(name, value = undefined) {
+  const cell = (name, value = undefined) => {
     const observers = [];
-    function set(newValue) {
-      value = newValue;
-      for (const observe of observers) {
+    return Object.freeze({
+      get: Object.freeze(() => {
+        return value;
+      }),
+      set: Object.freeze((newValue) => {
+        value = newValue;
+        for (const observe of observers) {
+          observe(value);
+        }
+      }),
+      observe: Object.freeze((observe) => {
+        observers.push(observe);
         observe(value);
-      }
-    }
-    function get() {
-      return value;
-    }
-    function observe(observe) {
-      observers.push(observe);
-      observe(value);
-    }
-    return { get, set, observe, enumerable: true };
-  }
+      }),
+      enumerable: true,
+    });
+  };
 
   const cells = [
-${''.concat(
-  ...modules.map(
-    ({ record: { __fixedExportMap__, __liveExportMap__ } }) => `\
-    {
-${exportsCellRecord(__fixedExportMap__)}${exportsCellRecord(__liveExportMap__)}\
-    },
-`,
-  ),
-)}\
+${''.concat(...modules.map(m => m.bundlerKit.getCells()))}\
   ];
 
-${''.concat(
-  ...modules.flatMap(({ index, indexedImports, record: { reexports } }) =>
-    reexports.map(
-      importSpecifier => `\
-  Object.defineProperties(cells[${index}], Object.getOwnPropertyDescriptors(cells[${indexedImports[importSpecifier]}]));
-`,
-    ),
-  ),
-)}\
+${''.concat(...modules.map(m => m.bundlerKit.getReexportsWiring()))}\
 
-  const namespaces = cells.map(cells => Object.create(null, cells));
+  const namespaces = cells.map(cells => Object.freeze(Object.create(null, cells)));
 
   for (let index = 0; index < namespaces.length; index += 1) {
     cells[index]['*'] = cell('*', namespaces[index]);
   }
 
-  function observeImports(map, importName, importIndex) {
-    for (const [name, observers] of map.get(importName)) {
-      const cell = cells[importIndex][name];
-      if (cell === undefined) {
-        throw new ReferenceError(\`Cannot import name \${name}\`);
-      }
-      for (const observer of observers) {
-        cell.observe(observer);
-      }
-    }
-  }
+${''.concat(...Array.from(parsersInUse).map(parser => getRuntime(parser)))}
 
-${''.concat(
-  ...modules.map(
-    ({
-      index,
-      indexedImports,
-      record: { __liveExportMap__, __fixedExportMap__ },
-    }) => `\
-  functors[${index}]({
-    imports(entries) {
-      const map = new Map(entries);
-${''.concat(
-  ...Object.entries(indexedImports).map(
-    ([importName, importIndex]) => `\
-      observeImports(map, ${q(importName)}, ${importIndex});
-`,
-  ),
-)}\
-    },
-    liveVar: {
-${importsCellSetter(__liveExportMap__, index)}\
-    },
-    onceVar: {
-${importsCellSetter(__fixedExportMap__, index)}\
-    },
-    importMeta: {},
-  });
-`,
-  ),
-)}\
+${''.concat(...modules.map(m => m.bundlerKit.getFunctorCall()))}\
 
   return cells[cells.length - 1]['*'].get();
 })();

package/src/compartment-map.js

@@ -19,7 +19,7 @@ const moduleLanguages = [
 
 /** @type {(a: string, b: string) => number} */
 // eslint-disable-next-line no-nested-ternary
-export const stringCompare = (a, b) => ((a === b ? 0 : a < b ? -1 : 1));
+export const stringCompare = (a, b) => (a === b ? 0 : a < b ? -1 : 1);
 
 /**
  * @param {number} length
@@ -195,14 +195,8 @@ const assertModule = (allegedModule, path, url) => {
     `${path} must be an object, got ${allegedModule} in ${q(url)}`,
   );
 
-  const {
-    compartment,
-    module,
-    location,
-    parser,
-    exit,
-    deferredError,
-  } = moduleDescriptor;
+  const { compartment, module, location, parser, exit, deferredError } =
+    moduleDescriptor;
   if (compartment !== undefined || module !== undefined) {
     assertCompartmentModule(moduleDescriptor, path, url);
   } else if (location !== undefined || parser !== undefined) {
@@ -374,16 +368,8 @@ const assertCompartment = (allegedCompartment, path, url) => {
     `${path} must be an object, got ${allegedCompartment} in ${q(url)}`,
   );
 
-  const {
-    location,
-    name,
-    label,
-    parsers,
-    types,
-    scopes,
-    modules,
-    ...extra
-  } = compartment;
+  const { location, name, label, parsers, types, scopes, modules, ...extra } =
+    compartment;
 
   assertEmptyObject(
     extra,

package/src/import-archive.js

@@ -96,6 +96,13 @@ const makeArchiveImportHookMaker = (
     const importHook = async moduleSpecifier => {
       // per-module:
       const module = modules[moduleSpecifier];
+      if (module === undefined) {
+        throw new Error(
+          `Cannot find module ${q(moduleSpecifier)} in package ${q(
+            packageLocation,
+          )} in archive ${q(archiveLocation)}`,
+        );
+      }
       if (module.deferredError !== undefined) {
         return postponeErrorToExecute(module.deferredError);
       }
@@ -132,10 +139,7 @@ const makeArchiveImportHookMaker = (
 
       let sourceLocation = `file:///${moduleLocation}`;
       if (packageName !== undefined) {
-        const base = packageName
-          .split('/')
-          .slice(-1)
-          .join('/');
+        const base = packageName.split('/').slice(-1).join('/');
         sourceLocation = `.../${join(base, moduleSpecifier)}`;
       }
       if (computeSourceLocation !== undefined) {