@bytecodealliance/jco 0.8.0 → 0.9.0

package/package.json

@@ -1,15 +1,31 @@
 {
   "name": "@bytecodealliance/jco",
+  "version": "0.9.0",
   "description": "JavaScript tooling for working with WebAssembly Components",
-  "version": "0.8.0",
-  "exports": "./api.mjs",
-  "types": "api.d.ts",
   "author": "Guy Bedford",
   "bin": {
-    "jco": "cli.mjs"
+    "jco": "src/jco.js"
   },
+  "exports": "./src/api.js",
+  "type": "module",
   "dependencies": {
-    "@bytecodealliance/preview2-shim": "0.0.9"
+    "@bytecodealliance/preview2-shim": "0.0.9",
+    "binaryen": "^111.0.0",
+    "chalk-template": "^0.4.0",
+    "commander": "^9.4.1",
+    "mkdirp": "^1.0.4",
+    "ora": "^6.1.2",
+    "terser": "^5.16.1"
+  },
+  "devDependencies": {
+    "@bytecodealliance/componentize-js": "0.0.7",
+    "@types/node": "^18.11.17",
+    "@typescript-eslint/eslint-plugin": "^5.41.0",
+    "@typescript-eslint/parser": "^5.41.0",
+    "eslint": "^8.30.0",
+    "mocha": "^10.2.0",
+    "terser": "^5.16.1",
+    "typescript": "^4.3.2"
   },
   "repository": {
     "type": "git",
@@ -24,5 +40,19 @@
   "bugs": {
     "url": "https://github.com/bytecodealliance/jco/issues"
   },
-  "homepage": "https://github.com/bytecodealliance/jco#readme"
+  "homepage": "https://github.com/bytecodealliance/jco#readme",
+  "scripts": {
+    "build": "cargo build --workspace --target wasm32-wasi --release && cargo run",
+    "build:types:preview2-shim": "PREVIEW2_SHIM_TYPES=* cargo build -p jco",
+    "lint": "eslint -c eslintrc.cjs lib/**/*.js packages/*/lib/**/*.js",
+    "test": "mocha -u tdd test/test.js --timeout 120000"
+  },
+  "files": [
+    "lib",
+    "src",
+    "obj"
+  ],
+  "workspaces": [
+    "packages/preview2-shim"
+  ]
 }

package/src/api.js

@@ -0,0 +1,45 @@
+export { optimizeComponent as opt } from './cmd/opt.js';
+export { transpileComponent as transpile } from './cmd/transpile.js';
+import { $init, print as printFn, parse as parseFn, componentWit as componentWitFn, componentNew as componentNewFn, componentEmbed as componentEmbedFn, metadataAdd as metadataAddFn, metadataShow as metadataShowFn } from "../obj/wasm-tools.js";
+
+/** @type {import('../obj/wasm-tools.js').print} */
+export async function print (binary) {
+  await $init;
+  return printFn(binary);
+}
+/** @type {import('../obj/wasm-tools.js').parse} */
+export async function parse (wat) {
+  await $init;
+  return parseFn(wat);
+}
+/** @type {import('../obj/wasm-tools.js').componentWit} */
+export async function componentWit (binary) {
+  await $init;
+  return componentWitFn(binary);
+}
+/** @type {import('../obj/wasm-tools.js').componentNew} */
+export async function componentNew (binary, adapters) {
+  await $init;
+  return componentNewFn(binary, adapters);
+}
+/** @type {import('../obj/wasm-tools.js').componentEmbed} */
+export async function componentEmbed (embedOpts) {
+  await $init;
+  return componentEmbedFn(embedOpts);
+}
+/** @type {import('../obj/wasm-tools.js').metadataAdd} */
+export async function metadataAdd () {
+  await $init;
+  return metadataAddFn(binary, metadata);
+}
+/** @type {import('../obj/wasm-tools.js').metadataShow} */
+export async function metadataShow (binary) {
+  await $init;
+  return metadataShowFn(binary);
+}
+export function preview1AdapterCommandPath () {
+  return new URL('../lib/wasi_snapshot_preview1.command.wasm', import.meta.url);
+}
+export function preview1AdapterReactorPath () {
+  return new URL('../lib/wasi_snapshot_preview1.reactor.wasm', import.meta.url);
+}

package/src/cmd/componentize.js

@@ -0,0 +1,21 @@
+import { readFile, writeFile } from 'node:fs/promises';
+import { resolve } from 'node:path';
+import c from 'chalk-template';
+
+export async function componentize (jsSource, opts) {
+  let componentizeFn;
+  try {
+    ({ componentize: componentizeFn } = await eval('import("@bytecodealliance/componentize-js")'));
+  } catch (e) {
+    if (e?.code === 'ERR_MODULE_NOT_FOUND' && e?.message?.includes('\'@bytecodealliance/componentize-js\''))
+      throw new Error(`componentize-js must first be installed separately via "npm install @bytecodealliance/componentize-js".`);
+    throw e;
+  }
+  const source = await readFile(jsSource, 'utf8');
+  const { component, imports } = await componentizeFn(source, {
+    witPath: resolve(opts.wit),
+    worldName: opts.worldName
+  });
+  await writeFile(opts.out, component);
+  console.log(c`{green OK} Successfully written {bold ${opts.out}} with imports (${imports.join(', ')}).`);
+}

package/src/cmd/opt.js

@@ -0,0 +1,151 @@
+import { $init, metadataShow, print } from '../../obj/wasm-tools.js';
+import { writeFile } from 'fs/promises';
+import { fileURLToPath } from 'url';
+import c from 'chalk-template';
+import { readFile, sizeStr, fixedDigitDisplay, table, spawnIOTmp, setShowSpinner, getShowSpinner } from '../common.js';
+import ora from 'ora';
+
+let WASM_OPT;
+try {
+  WASM_OPT = fileURLToPath(new URL('../../node_modules/binaryen/bin/wasm-opt', import.meta.url));
+} catch (e) {
+  WASM_OPT = new URL('../../node_modules/binaryen/bin/wasm-opt', import.meta.url);
+}
+
+export async function opt (componentPath, opts, program) {
+  await $init;
+  const varIdx = program.parent.rawArgs.indexOf('--');
+  if (varIdx !== -1)
+    opts.optArgs = program.parent.rawArgs.slice(varIdx + 1);
+  const componentBytes = await readFile(componentPath);
+
+  if (!opts.quiet) setShowSpinner(true);
+  const optPromise = optimizeComponent(componentBytes, opts);
+  const { component, compressionInfo } = await optPromise;
+
+  await writeFile(opts.output, component);
+
+  let totalBeforeBytes = 0, totalAfterBytes = 0;
+
+  if (!opts.quiet)
+    console.log(c`
+{bold Optimized WebAssembly Component Internal Core Modules:}
+
+${table([...compressionInfo.map(({ beforeBytes, afterBytes }, i) => {
+  totalBeforeBytes += beforeBytes;
+  totalAfterBytes += afterBytes;
+  return [
+    ` - Core Module ${i + 1}:  `,
+    sizeStr(beforeBytes),
+    ' -> ',
+    c`{cyan ${sizeStr(afterBytes)}} `,
+    `(${fixedDigitDisplay(afterBytes / beforeBytes * 100, 2)}%)`
+  ];
+}), ['', '', '', '', ''], [
+  ` = Total:  `,
+  `${sizeStr(totalBeforeBytes)}`,
+  ` => `,
+  c`{cyan ${sizeStr(totalAfterBytes)}} `,
+  `(${fixedDigitDisplay(totalAfterBytes / totalBeforeBytes * 100, 2)}%)`
+]], [,,,,'right'])}`);
+}
+
+/**
+ * 
+ * @param {Uint8Array} componentBytes 
+ * @param {{ quiet: boolean, optArgs?: string[] }} options?
+ * @returns {Promise<{ component: Uint8Array, compressionInfo: { beforeBytes: number, afterBytes: number }[] >}
+ */
+export async function optimizeComponent (componentBytes, opts) {
+  await $init;
+  const showSpinner = getShowSpinner();
+  let spinner;
+  try {
+    const coreModules = metadataShow(componentBytes).slice(1, -1).map(({ range }) => range);
+
+    let completed = 0;
+    const spinnerText = () => c`{cyan ${completed} / ${coreModules.length}} Running Binaryen on WebAssembly Component Internal Core Modules \n`;
+    if (showSpinner) {
+      spinner = ora({
+        color: 'cyan',
+        spinner: 'bouncingBar'
+      }).start();
+      spinner.text = spinnerText();
+    }
+
+    const optimizedCoreModules = await Promise.all(coreModules.map(async ([coreModuleStart, coreModuleEnd]) => {
+      const optimized = wasmOpt(componentBytes.subarray(coreModuleStart, coreModuleEnd), opts?.optArgs);
+      if (spinner) {
+        completed++;
+        spinner.text = spinnerText();
+      }
+      return optimized;
+    }));
+
+    let outComponentBytes = new Uint8Array(componentBytes.byteLength);
+    let nextReadPos = 0, nextWritePos = 0;
+    for (let i = 0; i < coreModules.length; i++) {
+      const [coreModuleStart, coreModuleEnd] = coreModules[i];
+      const optimizedCoreModule = optimizedCoreModules[i];
+
+      let lebByteLen = 1;
+      while (componentBytes[coreModuleStart - 1 - lebByteLen] & 0x80) lebByteLen++;
+
+      // Write from the last read to the LEB byte start of the core module
+      outComponentBytes.set(componentBytes.subarray(nextReadPos, coreModuleStart - lebByteLen), nextWritePos);
+      nextWritePos += coreModuleStart - lebByteLen - nextReadPos;
+
+      // Write the new LEB bytes
+      let val = optimizedCoreModule.byteLength;
+      do {
+        const byte = val & 0x7F;
+        val >>>= 7;
+        outComponentBytes[nextWritePos++] = val === 0 ? byte : byte | 0x80;
+      } while (val !== 0);
+
+      // Write the core module
+      outComponentBytes.set(optimizedCoreModule, nextWritePos);
+      nextWritePos += optimizedCoreModule.byteLength;
+
+      nextReadPos = coreModuleEnd;
+    }
+
+    outComponentBytes.set(componentBytes.subarray(nextReadPos, componentBytes.byteLength), nextWritePos);
+    nextWritePos += componentBytes.byteLength - nextReadPos;
+    nextReadPos += componentBytes.byteLength - nextReadPos;
+
+    outComponentBytes = outComponentBytes.subarray(0, outComponentBytes.length + nextWritePos - nextReadPos);
+
+    // verify it still parses ok
+    try {
+      await print(outComponentBytes);
+    } catch (e) {
+      throw new Error(`Internal error performing optimization.\n${e.message}`)
+    }
+
+    return {
+      component: outComponentBytes,
+      compressionInfo: coreModules.map(([s, e], i) => ({ beforeBytes: e - s, afterBytes: optimizedCoreModules[i].byteLength }))
+    };
+  }
+  finally {
+    if (spinner)
+      spinner.stop();
+  }
+}
+
+/**
+ * @param {Uint8Array} source 
+ * @returns {Promise<Uint8Array>}
+ */
+async function wasmOpt (source, args = ['-O1', '--low-memory-unused', '--enable-bulk-memory']) {
+  try {
+    return await spawnIOTmp(WASM_OPT, source, [
+      ...args, '-o'
+    ]);
+  } catch (e) {
+    if (e.toString().includes('BasicBlock requested'))
+      return wasmOpt(source, args);
+    throw e;
+  }
+}

package/src/cmd/transpile.js

@@ -0,0 +1,366 @@
+import { $init, generate } from '../../obj/js-component-bindgen-component.js';
+import { writeFile } from 'fs/promises';
+import { mkdir } from 'fs/promises';
+import { dirname, extname, basename } from 'path';
+import c from 'chalk-template';
+import { readFile, sizeStr, table, spawnIOTmp, setShowSpinner, getShowSpinner } from '../common.js';
+import { optimizeComponent } from './opt.js';
+import { minify } from 'terser';
+import { fileURLToPath } from 'url';
+import ora from 'ora';
+
+export async function transpile (componentPath, opts, program) {
+  const varIdx = program.parent.rawArgs.indexOf('--');
+  if (varIdx !== -1)
+    opts.optArgs = program.parent.rawArgs.slice(varIdx + 1);
+  const component = await readFile(componentPath);
+
+  if (!opts.quiet)
+    setShowSpinner(true);
+  if (!opts.name)
+    opts.name = basename(componentPath.slice(0, -extname(componentPath).length || Infinity));
+  if (opts.map)
+    opts.map = Object.fromEntries(opts.map.map(mapping => mapping.split('=')));
+  const { files } = await transpileComponent(component, opts);
+
+  await Promise.all(Object.entries(files).map(async ([name, file]) => {
+    await mkdir(dirname(name), { recursive: true });
+    await writeFile(name, file);
+  }));
+
+  if (!opts.quiet)
+    console.log(c`
+{bold Transpiled JS Component Files:}
+
+${table(Object.entries(files).map(([name, source]) => [
+  c` - {italic ${name}}  `,
+  c`{black.italic ${sizeStr(source.length)}}`
+]))}`);
+}
+
+let WASM_2_JS;
+try {
+  WASM_2_JS = fileURLToPath(new URL('../../node_modules/binaryen/bin/wasm2js', import.meta.url));
+} catch {
+  WASM_2_JS = new URL('../../node_modules/binaryen/bin/wasm2js', import.meta.url);
+}
+
+/**
+ * @param {Uint8Array} source 
+ * @returns {Promise<Uint8Array>}
+ */
+async function wasm2Js (source) {
+  try {
+    return await spawnIOTmp(WASM_2_JS, source, ['-Oz', '-o']);
+  } catch (e) {
+    if (e.toString().includes('BasicBlock requested'))
+      return wasm2Js(source);
+    throw e;
+  }
+}
+
+/**
+ * 
+ * @param {Uint8Array} component 
+ * @param {{
+ *   name: string,
+ *   instantiation?: bool,
+ *   map?: Record<string, string>,
+ *   validLiftingOptimization?: bool,
+ *   noNodejsCompat?: bool,
+ *   tlaCompat?: bool,
+ *   base64Cutoff?: bool,
+ *   js?: bool,
+ *   minify?: bool,
+ *   optimize?: bool,
+ *   optArgs?: string[],
+ * }} opts 
+ * @returns {Promise<{ files: { [filename: string]: Uint8Array }, imports: string[], exports: [string, 'function' | 'instance'][] }>}
+ */
+export async function transpileComponent (component, opts = {}) {
+  await $init;
+  if (opts.noWasiShim) opts.wasiShim = false;
+
+  let spinner;
+  const showSpinner = getShowSpinner();
+  if (opts.optimize) {
+    if (showSpinner) setShowSpinner(true);
+    ({ component } = await optimizeComponent(component, opts));
+  }
+
+  if (opts.wasiShim !== false) {
+    opts.map = Object.assign({
+      'wasi:cli-base/*': '@bytecodealliance/preview2-shim/cli-base#*',
+      'wasi:filesystem/*': '@bytecodealliance/preview2-shim/filesystem#*',
+      'wasi:io/*': '@bytecodealliance/preview2-shim/io#*',
+      'wasi:random/*': '@bytecodealliance/preview2-shim/random#*',
+    }, opts.map || {});
+  }
+
+  let { files, imports, exports } = generate(component, {
+    name: opts.name ?? 'component',
+    map: Object.entries(opts.map ?? {}),
+    instantiation: opts.instantiation || opts.js,
+    validLiftingOptimization: opts.validLiftingOptimization ?? false,
+    noNodejsCompat: !(opts.nodejsCompat ?? true),
+    tlaCompat: opts.tlaCompat ?? false,
+    base64Cutoff: opts.js ? 0 : opts.base64Cutoff ?? 5000
+  });
+
+  let outDir = (opts.outDir ?? '').replace(/\\/g, '/');
+  if (!outDir.endsWith('/') && outDir !== '')
+    outDir += '/';
+  files = files.map(([name, source]) => [`${outDir}${name}`, source]);
+
+  const jsFile = files.find(([name]) => name.endsWith('.js'));
+
+  if (opts.js) {
+    const source = Buffer.from(jsFile[1]).toString('utf8')
+      // update imports manging to match emscripten asm
+      .replace(/exports(\d+)\['([^']+)']/g, (_, i, s) => `exports${i}['${asmMangle(s)}']`);
+    
+    const wasmFiles = files.filter(([name]) => name.endsWith('.wasm'));
+    files = files.filter(([name]) => !name.endsWith('.wasm'));
+
+    // TODO: imports as specifier list
+
+    let completed = 0;
+    const spinnerText = () => c`{cyan ${completed} / ${wasmFiles.length}} Running Binaryen wasm2js on Wasm core modules (this takes a while)...\n`;
+    if (showSpinner) {
+      spinner = ora({
+        color: 'cyan',
+        spinner: 'bouncingBar'
+      }).start();
+      spinner.text = spinnerText();
+    }
+
+    try {
+      const asmFiles = await Promise.all(wasmFiles.map(async ([, source]) => {
+        const output = (await wasm2Js(source)).toString('utf8');
+        if (spinner) {
+          completed++;
+          spinner.text = spinnerText();
+        }
+        return output;
+      }));
+
+      const asms = asmFiles.map((asm, i) =>`function asm${i}(imports) {
+  ${
+      // strip and replace the asm instantiation wrapper
+        asm
+        .replace(/import \* as [^ ]+ from '[^']*';/g, '')
+        .replace('function asmFunc(imports) {', '')
+        .replace(/export var ([^ ]+) = ([^. ]+)\.([^ ]+);/g, '')
+        .replace(/var retasmFunc = [\s\S]*$/, '')
+        .replace(/var memasmFunc = new ArrayBuffer\(0\);/g, '')
+        .replace('memory.grow = __wasm_memory_grow;', '')
+        .trim()
+      }`).join(',\n');
+
+      const outSource = `${
+        imports.map((impt, i) => `import * as import${i} from '${impt}';`).join('\n')}
+${source.replace('export async function instantiate', 'async function instantiate')}
+
+let ${exports.filter(([, ty]) => ty === 'function').map(([name]) => '_' + name).join(', ')};
+
+${exports.map(([name, ty]) => ty === 'function' ? `\nfunction ${asmMangle(name)} () {
+  return _${name}.apply(this, arguments);
+}` : `\nlet ${asmMangle(name)};`).join('\n')}
+
+const asmInit = [${asms}];
+
+${opts.tlaCompat ? 'export ' : ''}const $init = (async () => {
+  let idx = 0;
+  ({ ${exports.map(([name, ty]) => asmMangle(name) === name ? name : `'${name}': ${asmMangle(name)}`).join(',\n')} } = await instantiate(n => idx++, {
+${imports.map((impt, i) => `    '${impt}': import${i},`).join('\n')}
+  }, (i, imports) => ({ exports: asmInit[i](imports) })));
+})();${exports.length > 0 ? `\nexport { ${exports.map(([name]) => name === asmMangle(name) ? `${name}` : `${asmMangle(name)} as "${name}"`).join(', ')} }` : ''}
+${opts.tlaCompat ? '' : '\nawait $init;\n'}`;
+
+      jsFile[1] = Buffer.from(outSource);
+    }
+    finally {
+      if (spinner)
+        spinner.stop();
+    }
+  }
+
+  if (opts.minify) {
+    ({ code: jsFile[1] } = await minify(Buffer.from(jsFile[1]).toString('utf8'), {
+      module: true,
+      compress: {
+        ecma: 9,
+        unsafe: true
+      },
+      mangle: {
+        keep_classnames: true
+      }
+    }));
+  }
+
+  return { files: Object.fromEntries(files), imports, exports };
+}
+
+// emscripten asm mangles specifiers to be valid identifiers
+// for imports to match up we must do the same
+// See https://github.com/WebAssembly/binaryen/blob/main/src/asmjs/asmangle.cpp
+function asmMangle (name) {
+  if (name === '')
+    return '$';
+  
+  let mightBeKeyword = true;
+  let i = 1;
+  
+  // Names must start with a character, $ or _
+  switch (name[0]) {
+    case '0':
+    case '1':
+    case '2':
+    case '3':
+    case '4':
+    case '5':
+    case '6':
+    case '7':
+    case '8':
+    case '9': {
+      name = '$' + name;
+      i = 2;
+      // fallthrough
+    }
+    case '$':
+    case '_': {
+      mightBeKeyword = false;
+      break;
+    }
+    default: {
+      let chNum = name.charCodeAt(0);
+      if (!(chNum >= 97 && chNum <= 122) && !(chNum >= 65 && chNum <= 90)) {
+        name = '$' + name.substr(1);
+        mightBeKeyword = false;
+      }
+    }
+  }
+  
+  // Names must contain only characters, digits, $ or _
+  let len = name.length;
+  for (; i < len; ++i) {
+    switch (name[i]) {
+      case '0':
+      case '1':
+      case '2':
+      case '3':
+      case '4':
+      case '5':
+      case '6':
+      case '7':
+      case '8':
+      case '9':
+      case '$':
+      case '_': {
+        mightBeKeyword = false;
+        break;
+      }
+      default: {
+        let chNum = name.charCodeAt(i);
+        if (!(chNum >= 97 && chNum <= 122) && !(chNum >= 65 && chNum <= 90)) {
+          name = name.substr(0, i) + '_' + name.substr(i + 1);
+          mightBeKeyword = false;
+        }
+      }
+    }
+  }
+  
+  // Names must not collide with keywords
+  if (mightBeKeyword && len >= 2 && len <= 10) {
+    switch (name[0]) {
+      case 'a': {
+        if (name == "arguments")
+          return name + '_';
+        break;
+      }
+      case 'b': {
+        if (name == "break")
+          return name + '_';
+        break;
+      }
+      case 'c': {
+        if (name == "case" || name == "continue" || name == "catch" ||
+            name == "const" || name == "class")
+          return name + '_';
+        break;
+      }
+      case 'd': {
+        if (name == "do" || name == "default" || name == "debugger")
+          return name + '_';
+        break;
+      }
+      case 'e': {
+        if (name == "else" || name == "enum" || name == "eval" || // to be sure
+            name == "export" || name == "extends")
+          return name + '_';
+        break;
+      }
+      case 'f': {
+        if (name == "for" || name == "false" || name == "finally" ||
+            name == "function")
+          return name + '_';
+        break;
+      }
+      case 'i': {
+        if (name == "if" || name == "in" || name == "import" ||
+            name == "interface" || name == "implements" ||
+            name == "instanceof")
+          return name + '_';
+        break;
+      }
+      case 'l': {
+        if (name == "let")
+          return name + '_';
+        break;
+      }
+      case 'n': {
+        if (name == "new" || name == "null")
+          return name + '_';
+        break;
+      }
+      case 'p': {
+        if (name == "public" || name == "package" || name == "private" ||
+            name == "protected")
+          return name + '_';
+        break;
+      }
+      case 'r': {
+        if (name == "return")
+          return name + '_';
+        break;
+      }
+      case 's': {
+        if (name == "super" || name == "static" || name == "switch")
+          return name + '_';
+        break;
+      }
+      case 't': {
+        if (name == "try" || name == "this" || name == "true" ||
+            name == "throw" || name == "typeof")
+          return name + '_';
+        break;
+      }
+      case 'v': {
+        if (name == "var" || name == "void")
+          return name + '_';
+        break;
+      }
+      case 'w': {
+        if (name == "with" || name == "while")
+          return name + '_';
+        break;
+      }
+      case 'y': {
+        if (name == "yield")
+          return name + '_';
+        break;
+      }
+    }
+  }
+  return name;
+}