@yowasp/clang 0.0.0

package/README.md

@@ -0,0 +1,42 @@
+YoWASP Clang/LLD package
+========================
+
+This package provides a complete [Clang/LLD][] toolchain built for [WebAssembly][] and targeting WebAssembly as well. See the [overview of the YoWASP project][yowasp] for details.
+
+At the moment, this package only provides an API allowing to run Clang and LLD in a virtual filesystem; no binaries are provided.
+
+[llvm/clang]: https://llvm.org/
+[webassembly]: https://webassembly.org/
+[yowasp]: https://yowasp.github.io/
+
+
+API reference
+-------------
+
+This package provides one function:
+
+- `runLLVM`
+
+For more detail, see the documentation for [the JavaScript YoWASP runtime](https://github.com/YoWASP/runtime-js#api-reference).
+
+
+Versioning
+----------
+
+The version of this package is derived from the upstream LLVM package version in the `X.Y.Z-S-M` or `X.Y.Z-M` format, where the symbols are:
+
+1. `X`: LLVM major version
+2. `Y`: LLVM minor version
+3. `Z`: LLVM patch version
+4. `S`: `gitK` for unreleased LLVM snapshots, `rcN` for LLVM release candidates, not present for LLVM releases
+5. `M`: package build version; disambiguates different builds produced from the same LLVM source tree
+
+With this scheme, there is a direct correspondence between upstream versions and [SemVer][semver] NPM package versions.
+
+[semver]: https://semver.org/
+
+
+License
+-------
+
+This package is covered by the [MIT license](LICENSE.txt), which is the same as the [Boolector license](https://github.com/Boolector/boolector/blob/master/COPYING).

package/lib/api.d.ts

@@ -0,0 +1,47 @@
+export type Tree = {
+    [name: string]: Tree | string | Uint8Array
+};
+
+export type InputStream =
+    (byteLength: number) => Uint8Array | null;
+
+export type OutputStream =
+    (bytes: Uint8Array | null) => void;
+
+export type ProgressCallback =
+    (event: { source: object, totalLength: number, doneLength: number }) => void;
+
+export type RunOptions = {
+    stdin?:  InputStream  | null;
+    stdout?: OutputStream | null;
+    stderr?: OutputStream | null;
+    decodeASCII?: boolean;
+    synchronously?: boolean;
+    fetchProgress?: ProgressCallback;
+};
+
+export type Command =
+    (args?: string[], files?: Tree, options?: RunOptions) => Promise<Tree> | Tree | undefined;
+
+export class Exit extends Error {
+    code: number;
+    files: Tree;
+}
+
+//--------8<--------8<--------8<--------8<--------8<--------8<--------8<--------8<--------8<--------
+
+export const runLLVM: Command;
+
+export const commands: {
+    'addr2line': Command,
+    'size': Command,
+    'objdump': Command,
+    'objcopy': Command,
+    'strip': Command,
+    'c++filt': Command,
+    'ar': Command,
+    'ranlib': Command,
+    'wasm-ld': Command,
+    'clang': Command,
+    'clang++': Command,
+};

package/lib/api.js

@@ -0,0 +1,217 @@
+import { Application } from '@yowasp/runtime';
+import * as resources from '../gen/llvm-resources.js';
+import { instantiate } from '../gen/llvm.js';
+
+import { Exit } from '@yowasp/runtime';
+
+const llvm = new Application(resources, instantiate, 'yowasp-llvm');
+const runLLVM = llvm.run.bind(llvm);
+
+function subcommand(command, subcommandName) {
+    return function (args, files, options) {
+        if (args === undefined || args == null)
+            return command(args, files, options); // preload resources
+        return command([subcommandName, ...args], files, options);
+    }
+}
+
+// horrific ??? code. [insert standard disclaimer here]
+// 'it cant hurt me if im not looking at it'
+function runClang(args, files, options = {}) {
+    if (args === undefined || args === null)
+        return runLLVM(args, files, options); // preload resources
+
+    // We pattern-match output of `-###` plus `args` to understand which subprocesses to run.
+    // If `args` contains `-###` this is an explicit user request to display these subprocesess,
+    // and we should not interpret the output in any way.
+    if (args.includes("-###"))
+        return runLLVM(args, files, options);
+
+    // All of these options have more priority than `-###`, and we shouldn't interfere with them.
+    if (args.includes(`--version`) ||
+            args.includes(`-help`) ||
+            args.includes(`--help`) ||
+            args.includes(`--help-hidden`))
+        return runLLVM(args, files, options);
+
+    function writeStderr(output) {
+        if (options.stderr === undefined) {
+            console.log(output);
+        } else {
+            options.stderr(new TextEncoder().encode(output));
+            options.stderr(null);
+        }
+    }
+
+    /** See the `clang/lib/Driver/Job.cpp` file, `Command::Print()` subroutine, as well as
+      * the `llvm/lib/Support/Program.cpp` file, `sys::printArg()` subroutine.
+      * @param {string} line
+      * @return {string[]} */
+    function unquoteClangArgs(line) {
+        return Array.from(line.matchAll(/ (?:([^ "]+)|"((?:[^"\\$]|\\["\\$])+)")/g), (match) => {
+            if (match[1] !== undefined) {
+                return match[1];
+            } else if (match[2] !== undefined) {
+                return match[2].replaceAll(/\\["$\\]/g, (m) => m[1]);
+            }
+        });
+    }
+
+    let gen = (function* () {
+        const [arg0, ...argsRest] = args;
+
+        /** @type {Uint8Array[]} Output of `-###` */
+        const outputSubarrays = [];
+        function captureOutput(bytes) {
+            if (bytes !== null)
+                outputSubarrays.push(new Uint8Array(bytes));
+        }
+
+        /** @type {Exit | undefined} Outcome of running `-###` */
+        let hash3Error = undefined;
+        try {
+            yield runLLVM([arg0, "-###", ...argsRest], files, {
+                stdout: captureOutput,
+                stderr: captureOutput,
+                synchronously: options.synchronously,
+            });
+        } catch (err) {
+            hash3Error = err;
+        }
+
+        const outputArray = new Uint8Array(outputSubarrays.reduce((a, b) => a + b.length, 0));
+        let outputLength = 0;
+        for (const outputSubarray of outputSubarrays) {
+            outputArray.subarray(outputLength, outputLength + outputSubarray.length).set(outputSubarray);
+            outputLength += outputSubarray.length;
+        }
+        const output = new TextDecoder().decode(outputArray);
+
+        if (hash3Error !== undefined) {
+            // Something definitely went wrong, and the output contains no commands to execute,
+            // but probably has a human-readable explanation. We had to squish it all to stderr
+            // though, even though some of it might've been printed to stdout originally.
+            writeStderr(output);
+            throw hash3Error;
+        }
+
+        // horrific in-band signaling code. please do not hold me to account for writing this
+        let state = 0;
+        /** @type {string[][]} */
+        const commands = [];
+        for (const line of output.split("\n")) {
+            if (state === 0) { // header
+                if (!(line.startsWith("clang") ||
+                      line.startsWith("Target:") ||
+                      line.startsWith("Thread model:") ||
+                      line.startsWith("InstalledDir:") ||
+                      line.startsWith("Build config:"))) {
+                    state = 1;
+                }
+            }
+            if (state === 1) { // command lines
+                if (line === " (in-process)") {
+                    // Ignore; indicates clang would ordinarily invoke itself as a library for
+                    // the following command line. Since we do not have an ordinarily working
+                    // compiler driver (but rather a compiler driver²), all ordinarily in-process
+                    // invocations have to be performed with a separate `runLLVM` call.
+                } else if (line.startsWith(' "')) {
+                    commands.push(unquoteClangArgs(line));
+                } else if (line === "") {
+                    state = 2; // final command; success!
+                } else {
+                    state = 3; // unknown input; error!
+                }
+                continue;
+            }
+            if (state === 2) { // success
+                state = 3;
+            }
+            if (state === 3) { // error
+                break;
+            }
+        }
+        if (state !== 2) { // no valid `-###` command list in the output
+            // Who knows what went wrong? Could have been an odd combination of options, could
+            // have been an error (with a zero exit code, other exit codes are handled above).
+            // We can't interpret the output of `-###` if there is even any, so just display it.
+            writeStderr(output);
+        } else { // valid `-###` command list recognized
+            // Verbose? Print `-###` output. This will differ slightly from a desktop compiler,
+            // but is more in the spirit of the `-v` option.
+            if (args.includes('-v'))
+                writeStderr(output);
+            // Run the command list.
+            for (const command of commands) {
+                if (command[0] === "") {
+                    // Clang would normally run this command in-process, which is indicated by
+                    // an empty argument in the command list, followed by clang's argv[0] for
+                    // this command. This distinction doesn't matter for us.
+                    command.shift();
+                }
+                // If this command line fails, the `Exit` exception will bubble up its exit code
+                // and output tree.
+                try {
+                    files = yield runLLVM(command, files, options);
+                } catch (err) {
+                    if (err instanceof Exit)
+                        delete err.files.tmp;
+                    throw err;
+                }
+            }
+        }
+        delete files.tmp;
+        return files;
+    })();
+
+    let promise, resolve, reject;
+    function runNext(value) {
+        try {
+            let done;
+            do {
+                ({ value, done } = gen.next(value));
+            } while (!(value instanceof Promise) && !done);
+            if (done) {
+                if (resolve) resolve(value);
+                else return value;
+            }
+            if (!promise) promise = new Promise((_resolve, _reject) =>
+                (resolve = _resolve, reject = _reject));
+            value.then(
+                nextVal => done ? resolve() : runNext(nextVal),
+                error => { // give the coroutine a first chance to handle the error.
+                    // we have SEH at home!
+                    try { ({ value, done } = gen.throw(error)); }
+                    catch (e) { reject(e); }
+                });
+        }
+        catch (e) {
+            if (reject) reject(e);
+            else throw e;
+        }
+    }
+    const maybeSyncReturn = runNext(null);
+    return promise || maybeSyncReturn;
+}
+
+export { runLLVM, runClang };
+export const commands = {
+    // LLVM tools
+    'addr2line': subcommand(runLLVM, 'addr2line'), // actually `symbolizer`
+    'ar': subcommand(runLLVM, 'ar'),
+    'c++filt': subcommand(runLLVM, 'c++filt'),
+    'dwarfdump': subcommand(runLLVM, 'dwarfdump'),
+    'nm': subcommand(runLLVM, 'nm'),
+    'objcopy': subcommand(runLLVM, 'objcopy'),
+    'objdump': subcommand(runLLVM, 'objdump'),
+    'readobj': subcommand(runLLVM, 'readobj'),
+    'ranlib': subcommand(runLLVM, 'ranlib'), // actually `ar`
+    'size': subcommand(runLLVM, 'size'),
+    'strip': subcommand(runLLVM, 'strip'), // actually `objcopy`
+    'symbolizer': subcommand(runLLVM, 'symbolizer'),
+    // Compiler and linker
+    'wasm-ld': subcommand(runLLVM, 'wasm-ld'),
+    'clang': subcommand(runClang, 'clang'),
+    'clang++': subcommand(runClang, 'clang++'),
+};
+export const version = VERSION;

package/package-in.json

@@ -0,0 +1,38 @@
+{
+    "name": "@yowasp/clang",
+    "version": "__VERSION__",
+    "description": "LLVM/Clang/LLD toolchain targeting WebAssembly",
+    "author": "Catherine <whitequark@whitequark.org>",
+    "license": "ISC",
+    "homepage": "https://yowasp.org/",
+    "repository": {
+        "type": "git",
+        "url": "git+https://github.com/YoWASP/clang.git"
+    },
+    "bugs": {
+        "url": "https://github.com/YoWASP/clang/issues"
+    },
+    "type": "module",
+    "files": [
+        "lib/api.d.ts",
+        "gen/bundle.js",
+        "gen/*-resources.tar",
+        "gen/*.wasm"
+    ],
+    "exports": {
+        "types": "./lib/api.d.ts",
+        "default": "./gen/bundle.js"
+    },
+    "types": "./lib/api.d.ts",
+    "devDependencies": {
+        "@bytecodealliance/jco": "1.15.1",
+        "@yowasp/runtime": "11.0.67",
+        "esbuild": "^0.25.11"
+    },
+    "scripts": {
+        "transpile": "jco new ../wasi-prefix/usr/bin/llvm.wasm --wasi-command --output llvm.wasm && jco transpile llvm.wasm --instantiation async --no-typescript --no-namespaced-exports --map 'wasi:io/*=runtime#io' --map 'wasi:cli/*=runtime#cli' --map 'wasi:clocks/*=runtime#*' --map 'wasi:filesystem/*=runtime#fs' --map 'wasi:random/*=runtime#random' --out-dir gen/",
+        "pack": "yowasp-pack-resources gen/llvm-resources.js gen ../wasi-prefix/usr usr",
+        "build": "esbuild --bundle lib/api.js --outfile=gen/bundle.js --format=esm --platform=node",
+        "all": "npm run transpile && npm run pack && npm run build"
+    }
+}

package/package.json

@@ -0,0 +1,15 @@
+{
+  "name": "@yowasp/clang",
+  "version": "0.0.0",
+  "description": "LLVM/Clang/LLD toolchain targeting WebAssembly",
+  "author": "Catherine <whitequark@whitequark.org>",
+  "license": "ISC",
+  "homepage": "https://yowasp.org/",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/YoWASP/clang.git"
+  },
+  "bugs": {
+    "url": "https://github.com/YoWASP/clang/issues"
+  }
+}

package/prepare.py

@@ -0,0 +1,34 @@
+import re
+import json
+import subprocess
+
+
+llvm_version_raw = subprocess.check_output([
+    "git", "-C", "../llvm-src", "describe", "--tags", "HEAD"
+], encoding="utf-8").strip()
+print(llvm_version_raw)
+
+git_rev_list_raw = subprocess.check_output([
+    "git", "rev-list", "HEAD"
+], encoding="utf-8").split()
+
+llvm_version = re.match(r"^llvmorg-(\d+)(?:\.(\d+)\.(\d+)(-rc\d+)?|-init-(.+)-g)?", llvm_version_raw)
+llvm_major   = int(llvm_version[1])
+llvm_minor   = int(llvm_version[2] or "0")
+llvm_patch   = int(llvm_version[3] or "0")
+if not llvm_version[5]:
+    llvm_suffix = llvm_version[4] or ""
+else:
+    llvm_suffix = f"-git{llvm_version[5]}"
+
+distance = len(git_rev_list_raw) - 1
+
+version = f"{llvm_major}.{llvm_minor}.{llvm_patch}{llvm_suffix}-{distance}"
+print(f"version {version}")
+
+with open("package-in.json", "rt") as f:
+    package_json = json.load(f)
+package_json["version"] = version
+package_json["scripts"]["build"] += f" --define:VERSION=\\\"{version}\\\""
+with open("package.json", "wt") as f:
+    json.dump(package_json, f, indent=2)

package/test/test_api.js

@@ -0,0 +1,17 @@
+const { runClang, commands } = await import('@yowasp/clang');
+
+await commands.clang(["--version"]);
+
+const { meowC } = await runClang(["clang", "test.c", "-o", "meowC"],
+    {"test.c": `#include <stdio.h>\nint main() { puts("meow"); }`});
+const { meowCXX } = await runClang(["clang++", "test.cc", "-o", "meowCXX"],
+    {"test.cc": `#include <iostream>\nint main() { std::cout << "meow++" << std::endl; }`});
+
+const { WASI } = await import('node:wasi');
+for (const meow of [meowC, meowCXX]) {
+    const meowdule = await WebAssembly.compile(meow);
+    const wasi = new WASI({ version: 'preview1' });
+    const instance = await WebAssembly.instantiate(meowdule,
+        {wasi_snapshot_preview1: wasi.wasiImport});
+    wasi.start(instance);
+}