@thi.ng/wasm-api 0.5.0 → 0.8.0

package/api.d.ts

@@ -1,5 +1,6 @@
-import type { Fn, Fn2 } from "@thi.ng/api";
+import type { BigType, FloatType, Fn, Fn2 } from "@thi.ng/api";
 import type { WasmBridge } from "./bridge.js";
+export declare const PKG_NAME = "@thi.ng/wasm-api";
 export declare type BigIntArray = bigint[] | BigInt64Array | BigUint64Array;
 /**
  * Common interface for WASM/JS child APIs which will be used in combination
@@ -60,12 +61,47 @@ export interface WasmExports {
      * Implementation specific function to free a previously allocated chunk of
      * of WASM memory (allocated via {@link WasmExports._wasm_allocate}).
      *
+     * @param addr
+     * @param numBytes
+     */
+    _wasm_free(addr: number, numBytes: number): void;
+}
+export interface IWasmMemoryAccess {
+    i8: Int8Array;
+    u8: Uint8Array;
+    i16: Int16Array;
+    u16: Uint16Array;
+    i32: Int32Array;
+    u32: Uint32Array;
+    i64: BigInt64Array;
+    u64: BigUint64Array;
+    f32: Float32Array;
+    f64: Float64Array;
+    /**
+     * Reads UTF-8 encoded string from given address and optional byte length.
+     * The default length is 0, which will be interpreted as a zero-terminated
+     * string. Returns string.
+     *
+     * @param addr
+     * @param len
+     */
+    getString(addr: number, len?: number): string;
+    /**
+     * Encodes given string as UTF-8 and writes it to WASM memory starting at
+     * `addr`. By default the string will be zero-terminated and only `maxBytes`
+     * will be written. Returns the number of bytes written.
+     *
      * @remarks
-     * In the supplied Zig bindings (/zig/core.zig) this is a no-op (currently).
+     * An error will be thrown if the encoded string doesn't fully fit into the
+     * designated memory region (also note that there might need to be space for
+     * the additional sentinel/termination byte).
      *
+     * @param str
      * @param addr
+     * @param maxBytes
+     * @param terminate
      */
-    _wasm_free(addr: number): void;
+    setString(str: string, addr: number, maxBytes: number, terminate?: boolean): number;
 }
 /**
  * Core API of WASM imports defined by the {@link WasmBridge}. The same
@@ -100,4 +136,182 @@ export interface CoreAPI extends WebAssembly.ModuleImports {
     _printStr0: (addr: number) => void;
     _printStr: (addr: number, len: number) => void;
 }
+export interface WasmTypeBase {
+    /**
+     * Base address in linear WASM memory.
+     */
+    readonly __base: number;
+    /**
+     * Obtain as byte buffer
+     */
+    readonly __bytes: Uint8Array;
+}
+export interface WasmType<T> {
+    readonly align: number;
+    readonly size: number;
+    instance: Fn<number, T>;
+}
+export declare type WasmTypeConstructor<T> = Fn<IWasmMemoryAccess, WasmType<T>>;
+export declare type WasmInt = "i8" | "i16" | "i32" | "i64";
+export declare type WasmUint = "u8" | "u16" | "u32" | "u64";
+export declare type WasmFloat = FloatType;
+export declare type WasmPrim = WasmInt | WasmUint | WasmFloat;
+export declare type WasmPrim32 = Exclude<WasmPrim, BigType>;
+export declare type TypeColl = Record<string, TopLevelType>;
+export interface TypeInfo {
+    /**
+     * Auto-computed size (in bytes)
+     *
+     * @internal
+     */
+    __size?: number;
+    /**
+     * Auto-computed offset (in bytes) in parent struct
+     *
+     * @internal
+     */
+    __offset?: number;
+    /**
+     * Auto-computed alignment (in bytes)
+     *
+     * @internal
+     */
+    __align?: number;
+}
+export interface TopLevelType extends TypeInfo {
+    /**
+     * Type name
+     */
+    name: string;
+    /**
+     * Optional (multi-line) docstring for this type
+     */
+    doc?: string;
+    /**
+     * Type / kind
+     */
+    type: "struct" | "enum";
+}
+export interface Struct extends TopLevelType {
+    type: "struct";
+    /**
+     * List of struct fields (might be re-ordered if {@link Struct.auto} is
+     * enabled).
+     */
+    fields: StructField[];
+    /**
+     * If true, struct fields will be re-ordered in descending order based on
+     * their {@link TypeInfo.__align} size. This might result in overall smaller
+     * structs due to minimizing inter-field padding.
+     *
+     * @defaultValue false
+     */
+    auto?: boolean;
+}
+export interface StructField extends TypeInfo {
+    /**
+     * Field name (prefix: "__" is reserved)
+     */
+    name: string;
+    /**
+     * Field docstring (can be multiline, will be formatted)
+     */
+    doc?: string;
+    /**
+     * Field type tag/qualifier (note: `slice` & `vec` are only supported by Zig
+     * & TS).
+     *
+     * @remarks
+     * - Array & vector fields are statically sized (using
+     *   {@link StructField.len})
+     * - Pointers are emitted as single-value pointers (where this distinction
+     *   exist), i.e. even if they're pointing to multiple values, there's no
+     *   explicit length encoded/available
+     * - Zig slices are essentially a pointer w/ associated length
+     * - Zig vectors will be processed using SIMD (if enabled in WASM target)
+     *   and therefore will have stricter (larger) alignment requirements.
+     *
+     * @defaultValue "scalar"
+     */
+    tag?: "scalar" | "array" | "ptr" | "slice" | "vec";
+    /**
+     * Field base type. If not a {@link WasmPrim}, `string` or `opaque`, the
+     * value is interpreted as another type name in the {@link TypeColl}.
+     *
+     * @remarks
+     * Please see {@link CodeGenOpts.stringType} and consult package readme for
+     * further details re: string handling.
+     *
+     * TODO `opaque` currently unsupported.
+     */
+    type: WasmPrim | "string" | "opaque" | string;
+    /**
+     * TODO currently unsupported & ignored!
+     */
+    sentinel?: number;
+    /**
+     * Array or vector length (see {@link StructField.tag})
+     */
+    len?: number;
+    /**
+     * TODO currently unsupported & ignored!
+     */
+    default?: any;
+}
+export interface Enum extends TopLevelType {
+    type: "enum";
+    /**
+     * No i64/u64 support, due to Typescript not supporting bigint enum values
+     */
+    tag: Exclude<WasmPrim32, FloatType>;
+    /**
+     * List of possible values/IDs. Use {@link EnumValue}s for more detailed
+     * config.
+     */
+    values: (string | EnumValue)[];
+}
+export interface EnumValue {
+    /**
+     * Enum value name/ID
+     */
+    name: string;
+    /**
+     * Optional associated numeric value
+     */
+    value?: number;
+    /**
+     * Optional docstring for this value
+     */
+    doc?: string;
+}
+export interface ICodeGen {
+    /**
+     * Optional prelude source, to be prepended before any generated type defs.
+     */
+    pre?: string;
+    /**
+     * Optional source code to be appended after any generated type defs.
+     */
+    post?: string;
+    /**
+     * Docstring codegen
+     */
+    doc: (doc: string, indent: string, acc: string[], topLevel?: boolean) => void;
+    /**
+     * Codegen for enum types.
+     */
+    enum: (type: Enum, types: TypeColl, acc: string[]) => void;
+    /**
+     * Codegen for struct types.
+     */
+    struct: (type: Struct, types: TypeColl, acc: string[]) => void;
+}
+/**
+ * WASM usize type. Assuming wasm32 until wasm64 surfaces, then need an option.
+ */
+export declare const USIZE = "u32";
+/**
+ * Byte size of {@link USIZE}.
+ */
+export declare const USIZE_SIZE = 4;
 //# sourceMappingURL=api.d.ts.map

package/api.js

@@ -1 +1,9 @@
-export {};
+export const PKG_NAME = "@thi.ng/wasm-api";
+/**
+ * WASM usize type. Assuming wasm32 until wasm64 surfaces, then need an option.
+ */
+export const USIZE = "u32";
+/**
+ * Byte size of {@link USIZE}.
+ */
+export const USIZE_SIZE = 4;

package/bin/wasm-api

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+# https://stackoverflow.com/a/246128/294515
+SOURCE="${BASH_SOURCE[0]}"
+while [ -h "$SOURCE" ]; do
+  DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )"
+  SOURCE="$(readlink "$SOURCE")"
+  [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE"
+done
+DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )"
+
+/usr/bin/env node "$DIR/../cli.js" "$DIR" "$@"

package/bridge.d.ts

@@ -1,7 +1,7 @@
 /// <reference types="node" />
 import type { NumericArray } from "@thi.ng/api";
 import type { ILogger } from "@thi.ng/logger";
-import type { BigIntArray, CoreAPI, IWasmAPI, WasmExports } from "./api.js";
+import type { BigIntArray, CoreAPI, IWasmAPI, WasmExports, IWasmMemoryAccess } from "./api.js";
 export declare const OutOfMemoryError: {
     new (msg?: string | undefined): {
         name: string;
@@ -28,7 +28,7 @@ export declare const OutOfMemoryError: {
  * 64bit integers are handled via JS `BigInt` and hence require the host env to
  * support it. No polyfill is provided.
  */
-export declare class WasmBridge<T extends WasmExports = WasmExports> {
+export declare class WasmBridge<T extends WasmExports = WasmExports> implements IWasmMemoryAccess {
     modules: Record<string, IWasmAPI<T>>;
     logger: ILogger;
     i8: Int8Array;
@@ -45,7 +45,7 @@ export declare class WasmBridge<T extends WasmExports = WasmExports> {
     utf8Encoder: TextEncoder;
     imports: WebAssembly.Imports;
     exports: T;
-    core: CoreAPI;
+    api: CoreAPI;
     constructor(modules?: Record<string, IWasmAPI<T>>, logger?: ILogger);
     /**
      * Instantiates WASM module from given `src` (and optional provided extra
@@ -95,7 +95,7 @@ export declare class WasmBridge<T extends WasmExports = WasmExports> {
      * bridge.getImports();
      * {
      *   // imports defined by the core API of the bridge itself
-     *   core: { ... },
+     *   wasmapi: { ... },
      *   // imports defined by the CustomAPI module
      *   custom: { ... }
      * }
@@ -121,15 +121,26 @@ export declare class WasmBridge<T extends WasmExports = WasmExports> {
      * Attempts to allocate `numBytes` using the exported WASM core API function
      * {@link WasmExports._wasm_allocate} (implementation specific) and returns
      * start address of the new memory block. If unsuccessful, throws an
-     * {@link OutOfMemoryError}.
+     * {@link OutOfMemoryError}. If `clear` is true, the allocated region will
+     * be zero-filled.
      *
      * @remarks
      * See {@link WasmExports._wasm_allocate} docs for further details.
      *
      * @param numBytes
+     * @param clear
      */
-    allocate(numBytes: number): number;
-    free(addr: number): void;
+    allocate(numBytes: number, clear?: boolean): number;
+    /**
+     * Frees a previous allocated memory region using the exported WASM core API
+     * function {@link WasmExports._wasm_free} (implementation specific). The
+     * `numBytes` value must be the same as previously given to
+     * {@link WasmBridge.allocate}.
+     *
+     * @param addr
+     * @param numBytes
+     */
+    free(addr: number, numBytes: number): void;
     getI8(addr: number): number;
     getU8(addr: number): number;
     getI16(addr: number): number;
@@ -170,7 +181,30 @@ export declare class WasmBridge<T extends WasmExports = WasmExports> {
     setU64Array(addr: number, buf: BigIntArray): this;
     setF32Array(addr: number, buf: NumericArray): this;
     setF64Array(addr: number, buf: NumericArray): this;
+    /**
+     * Reads UTF-8 encoded string from given address and optional byte length.
+     * The default length is 0, which will be interpreted as a zero-terminated
+     * string. Returns string.
+     *
+     * @param addr
+     * @param len
+     */
     getString(addr: number, len?: number): string;
+    /**
+     * Encodes given string as UTF-8 and writes it to WASM memory starting at
+     * `addr`. By default the string will be zero-terminated and only `maxBytes`
+     * will be written. Returns the number of bytes written.
+     *
+     * @remarks
+     * An error will be thrown if the encoded string doesn't fully fit into the
+     * designated memory region (also note that there might need to be space for
+     * the additional sentinel/termination byte).
+     *
+     * @param str
+     * @param addr
+     * @param maxBytes
+     * @param terminate
+     */
     setString(str: string, addr: number, maxBytes: number, terminate?: boolean): number;
     getElementById(addr: number, len?: number): HTMLElement;
 }

package/bridge.js

@@ -28,7 +28,7 @@ export class WasmBridge {
         this.utf8Encoder = new TextEncoder();
         const logN = (x) => this.logger.debug(x);
         const logA = (method) => (addr, len) => this.logger.debug(method(addr, len).join(", "));
-        this.core = {
+        this.api = {
             printI8: logN,
             printU8: logN,
             printU8Hex: (x) => this.logger.debug(`0x${U8(x)}`),
@@ -136,7 +136,7 @@ export class WasmBridge {
      * bridge.getImports();
      * {
      *   // imports defined by the core API of the bridge itself
-     *   core: { ... },
+     *   wasmapi: { ... },
      *   // imports defined by the CustomAPI module
      *   custom: { ... }
      * }
@@ -151,7 +151,7 @@ export class WasmBridge {
      */
     getImports() {
         if (!this.imports) {
-            this.imports = { core: this.core };
+            this.imports = { wasmapi: this.api };
             for (let id in this.modules) {
                 if (this.imports[id] !== undefined) {
                     illegalArgs(`attempt to redeclare API module ${id}`);
@@ -176,24 +176,36 @@ export class WasmBridge {
      * Attempts to allocate `numBytes` using the exported WASM core API function
      * {@link WasmExports._wasm_allocate} (implementation specific) and returns
      * start address of the new memory block. If unsuccessful, throws an
-     * {@link OutOfMemoryError}.
+     * {@link OutOfMemoryError}. If `clear` is true, the allocated region will
+     * be zero-filled.
      *
      * @remarks
      * See {@link WasmExports._wasm_allocate} docs for further details.
      *
      * @param numBytes
+     * @param clear
      */
-    allocate(numBytes) {
+    allocate(numBytes, clear = false) {
         const addr = this.exports._wasm_allocate(numBytes);
         if (!addr)
             throw new OutOfMemoryError(`unable to allocate: ${numBytes}`);
         this.logger.debug(`allocated ${numBytes} bytes @ 0x${U32(addr)}`);
         this.ensureMemory();
+        clear && this.u8.fill(0, addr, addr + numBytes);
         return addr;
     }
-    free(addr) {
-        this.logger.debug(`freeing memory @ 0x${U32(addr)}`);
-        this.exports._wasm_free(addr);
+    /**
+     * Frees a previous allocated memory region using the exported WASM core API
+     * function {@link WasmExports._wasm_free} (implementation specific). The
+     * `numBytes` value must be the same as previously given to
+     * {@link WasmBridge.allocate}.
+     *
+     * @param addr
+     * @param numBytes
+     */
+    free(addr, numBytes) {
+        this.logger.debug(`freeing memory @ 0x${U32(addr)} .. 0x${U32(addr + numBytes - 1)}`);
+        this.exports._wasm_free(addr, numBytes);
     }
     getI8(addr) {
         return this.i8[addr];
@@ -343,14 +355,38 @@ export class WasmBridge {
         this.f64.set(buf, addr >> 3);
         return this;
     }
+    /**
+     * Reads UTF-8 encoded string from given address and optional byte length.
+     * The default length is 0, which will be interpreted as a zero-terminated
+     * string. Returns string.
+     *
+     * @param addr
+     * @param len
+     */
     getString(addr, len = 0) {
+        this.ensureMemory();
         return this.utf8Decoder.decode(this.u8.subarray(addr, len > 0 ? addr + len : this.u8.indexOf(0, addr)));
     }
+    /**
+     * Encodes given string as UTF-8 and writes it to WASM memory starting at
+     * `addr`. By default the string will be zero-terminated and only `maxBytes`
+     * will be written. Returns the number of bytes written.
+     *
+     * @remarks
+     * An error will be thrown if the encoded string doesn't fully fit into the
+     * designated memory region (also note that there might need to be space for
+     * the additional sentinel/termination byte).
+     *
+     * @param str
+     * @param addr
+     * @param maxBytes
+     * @param terminate
+     */
     setString(str, addr, maxBytes, terminate = true) {
         maxBytes = Math.min(maxBytes, this.u8.length - addr);
         const len = this.utf8Encoder.encodeInto(str, this.u8.subarray(addr, addr + maxBytes)).written;
         if (len == null || len >= maxBytes + (terminate ? 0 : 1)) {
-            illegalArgs(`error writing string to 0x${U32(addr)}`);
+            illegalArgs(`error writing string to 0x${U32(addr)} (max. ${maxBytes} bytes, got at least ${str.length})`);
         }
         if (terminate) {
             this.u8[addr + len] = 0;

package/cli.d.ts

@@ -0,0 +1,5 @@
+export declare const INSTALL_DIR: string;
+export declare const PKG: any;
+export declare const APP_NAME: any;
+export declare const HEADER: string;
+//# sourceMappingURL=cli.d.ts.map

package/cli.js

@@ -0,0 +1,142 @@
+import { flag, oneOfMulti, parse, ParseError, string, strings, usage, } from "@thi.ng/args";
+import { isArray, isPlainObject } from "@thi.ng/checks";
+import { illegalArgs } from "@thi.ng/errors";
+import { readJSON, writeText } from "@thi.ng/file-io";
+import { ConsoleLogger } from "@thi.ng/logger";
+import { resolve } from "path";
+import { generateTypes } from "./codegen.js";
+import { TYPESCRIPT } from "./codegen/typescript.js";
+import { isWasmPrim, isWasmString } from "./codegen/utils.js";
+import { ZIG } from "./codegen/zig.js";
+const GENERATORS = { ts: TYPESCRIPT, zig: ZIG };
+const argOpts = {
+    config: string({
+        alias: "c",
+        hint: "FILE",
+        desc: "JSON config file with codegen options",
+    }),
+    debug: flag({ alias: "d", default: false, desc: "enable debug output" }),
+    dryRun: flag({
+        default: false,
+        desc: "enable dry run (don't overwrite files)",
+    }),
+    lang: oneOfMulti(Object.keys(GENERATORS), {
+        alias: "l",
+        desc: "target language",
+        default: ["ts", "zig"],
+        delim: ",",
+    }),
+    out: strings({ alias: "o", hint: "FILE", desc: "output file path" }),
+};
+export const INSTALL_DIR = resolve(`${process.argv[2]}/..`);
+export const PKG = readJSON(`${INSTALL_DIR}/package.json`);
+export const APP_NAME = PKG.name.split("/")[1];
+export const HEADER = `
+ █ █   █           │
+██ █               │
+ █ █ █ █   █ █ █ █ │ ${PKG.name} ${PKG.version}
+ █ █ █ █ █ █ █ █ █ │ Multi-language data bindings code generator
+                 █ │
+               █ █ │
+`;
+const usageOpts = {
+    lineWidth: process.stdout.columns,
+    prefix: `${HEADER}
+usage: ${APP_NAME} [OPTS] JSON-INPUT-FILE(S) ...
+       ${APP_NAME} --help
+
+`,
+    showGroupNames: true,
+    paramWidth: 32,
+};
+const showUsage = () => {
+    process.stderr.write(usage(argOpts, usageOpts));
+    process.exit(1);
+};
+const invalidSpec = (path, msg) => {
+    throw new Error(`invalid typedef: ${path}${msg ? ` (${msg})` : ""}`);
+};
+const addTypeSpec = (ctx, path, coll, spec) => {
+    if (!(spec.name && spec.type))
+        invalidSpec(path);
+    if (!(spec.type === "enum" || spec.type === "struct"))
+        invalidSpec(path, `${spec.name} type: ${spec.type}`);
+    if (coll[spec.name])
+        invalidSpec(path, `duplicate name: ${spec.name}`);
+    ctx.logger.debug(`registering ${spec.type}: ${spec.name}`);
+    coll[spec.name] = spec;
+    spec.__path = path;
+};
+const validateTypeRefs = (coll) => {
+    for (let spec of Object.values(coll)) {
+        if (spec.type !== "struct")
+            continue;
+        for (let f of spec.fields) {
+            if (!(isWasmPrim(f.type) || isWasmString(f.type) || coll[f.type])) {
+                invalidSpec(spec.__path, `structfield ${spec.name}.${f.name} of unknown type: ${f.type}`);
+            }
+        }
+    }
+};
+const parseTypeSpecs = (ctx, inputs) => {
+    const coll = {};
+    for (let path of inputs) {
+        try {
+            const spec = readJSON(resolve(path), ctx.logger);
+            if (isArray(spec)) {
+                for (let s of spec)
+                    addTypeSpec(ctx, path, coll, s);
+            }
+            else if (isPlainObject(spec)) {
+                addTypeSpec(ctx, path, coll, spec);
+            }
+            else {
+                invalidSpec(path);
+            }
+        }
+        catch (e) {
+            process.stderr.write(e.message);
+            process.exit(1);
+        }
+    }
+    validateTypeRefs(coll);
+    return coll;
+};
+const generateOutputs = ({ config, logger, opts }, coll) => {
+    for (let i = 0; i < opts.lang.length; i++) {
+        const lang = opts.lang[i];
+        logger.debug(`generating ${lang.toUpperCase()} output...`);
+        const src = generateTypes(coll, GENERATORS[lang](config[lang]), config.global);
+        if (opts.out) {
+            writeText(resolve(opts.out[i]), src, logger, opts.dryRun);
+        }
+        else {
+            process.stdout.write(src + "\n");
+        }
+    }
+};
+try {
+    const result = parse(argOpts, process.argv, { start: 3, usageOpts });
+    if (!result)
+        process.exit(1);
+    const { result: opts, rest } = result;
+    if (!rest.length)
+        showUsage();
+    if (opts.out && opts.lang.length != opts.out.length) {
+        illegalArgs(`expected ${opts.lang.length} outputs, but got ${opts.out.length}`);
+    }
+    const ctx = {
+        logger: new ConsoleLogger("wasm-api", opts.debug ? "DEBUG" : "INFO"),
+        config: {},
+        opts,
+    };
+    if (opts.config) {
+        ctx.config = readJSON(resolve(opts.config), ctx.logger);
+    }
+    generateOutputs(ctx, parseTypeSpecs(ctx, rest));
+}
+catch (e) {
+    if (!(e instanceof ParseError))
+        process.stderr.write(e.message);
+    process.exit(1);
+}

package/codegen/typescript.d.ts

@@ -0,0 +1,35 @@
+import { ICodeGen } from "../api.js";
+/**
+ * TypeScript code generator options.
+ */
+export interface TSOpts {
+    /**
+     * Indentation string
+     *
+     * @defaultValue "\t"
+     */
+    indent: string;
+    /**
+     * If true (default), forces uppercase enums
+     *
+     * @defaultValue true
+     */
+    uppercaseEnums: boolean;
+    /**
+     * Same as {@link CodeGenOpts.stringType}.
+     */
+    stringType: "slice" | "ptr";
+}
+/**
+ * TypeScript code generator. Call with options and then pass to
+ * {@link generateTypes} (see its docs for further usage).
+ *
+ * @remarks
+ * This codegen generates interface and enum definitions for a {@link TypeColl}
+ * given to {@link generateTypes}. For structs it will also generate memory
+ * mapped wrappers with fully typed accessors.
+ *
+ * @param opts
+ */
+export declare const TYPESCRIPT: (opts?: Partial<TSOpts>) => ICodeGen;
+//# sourceMappingURL=typescript.d.ts.map