@thi.ng/wasm-api 0.14.0 → 0.16.0

package/codegen.js

@@ -1,14 +1,21 @@
 import { SIZEOF } from "@thi.ng/api/typedarray";
-import { align } from "@thi.ng/binary/align";
-import { ceilPow2 } from "@thi.ng/binary/pow";
 import { compareByKey } from "@thi.ng/compare/keys";
 import { compareNumDesc } from "@thi.ng/compare/numeric";
 import { DEFAULT, defmulti } from "@thi.ng/defmulti/defmulti";
 import { illegalArgs } from "@thi.ng/errors/illegal-arguments";
-import { PKG_NAME, USIZE_SIZE, } from "./api.js";
-import { isNumeric, isWasmString } from "./codegen/utils.js";
+import { PKG_NAME, WASM32, } from "./api.js";
+import { selectAlignment } from "./codegen/align.js";
+import { isBigNumeric, isNumeric, isPointer, isPointerLike, isSlice, isStringSlice, isWasmString, } from "./codegen/utils.js";
+export const DEFAULT_CODEGEN_OPTS = {
+    debug: false,
+    header: true,
+    lineWidth: 80,
+    stringType: "slice",
+    target: WASM32,
+    uppercaseEnums: true,
+};
 const sizeOf = defmulti((x) => x.type, {}, {
-    [DEFAULT]: (field, types, opts) => {
+    [DEFAULT]: (field, types, align, opts) => {
         if (field.__size)
             return field.__size;
         if (field.pad != null) {
@@ -16,18 +23,20 @@ const sizeOf = defmulti((x) => x.type, {}, {
             return (field.__size = field.pad);
         }
         let size = 0;
-        if (field.tag === "ptr") {
-            size = USIZE_SIZE;
+        if (isPointer(field)) {
+            size = opts.target.usizeBytes;
         }
-        else if (field.tag === "slice") {
-            size = USIZE_SIZE * 2;
+        else if (isSlice(field)) {
+            size = opts.target.usizeBytes * 2;
         }
         else {
-            size = isNumeric(field.type)
-                ? SIZEOF[field.type]
-                : isWasmString(field.type)
-                    ? USIZE_SIZE * (opts.stringType === "slice" ? 2 : 1)
-                    : sizeOf(types[field.type], types, opts);
+            size =
+                isNumeric(field.type) || isBigNumeric(field.type)
+                    ? SIZEOF[field.type]
+                    : isWasmString(field.type)
+                        ? opts.target.usizeBytes *
+                            (isStringSlice(opts.stringType) ? 2 : 1)
+                        : sizeOf(types[field.type], types, align, opts);
             if (field.tag == "array" || field.tag === "vec") {
                 size *= field.len;
                 if (field.sentinel !== undefined && field.tag === "array") {
@@ -35,79 +44,95 @@ const sizeOf = defmulti((x) => x.type, {}, {
                 }
             }
         }
-        return (field.__size = align(size, field.__align));
+        return (field.__size = align.size(size, field.__align));
     },
     enum: (type) => {
         if (type.__size)
             return type.__size;
         return (type.__size = SIZEOF[type.tag]);
     },
-    struct: (type, types, opts) => {
+    struct: (type, types, align, opts) => {
         if (type.__size)
             return type.__size;
-        const struct = type;
-        let size = 0;
-        for (let f of struct.fields) {
-            size = align(size, f.__align);
-            f.__offset = size;
-            size += sizeOf(f, types, opts);
+        let offset = 0;
+        for (let f of type.fields) {
+            offset = align.offset(offset, f.__align);
+            f.__offset = offset;
+            offset += sizeOf(f, types, align, opts);
+        }
+        return (type.__size = align.size(offset, type.__align));
+    },
+    union: (type, types, align, opts) => {
+        if (type.__size)
+            return type.__size;
+        let maxSize = 0;
+        for (let f of type.fields) {
+            f.__offset = 0;
+            maxSize = Math.max(maxSize, sizeOf(f, types, align, opts));
         }
-        return (type.__size = align(size, type.__align));
+        return (type.__size = align.size(maxSize, type.__align));
     },
 });
 const alignOf = defmulti((x) => x.type, {}, {
-    [DEFAULT]: (field, types) => {
+    [DEFAULT]: (field, types, align, opts) => {
         if (field.__align)
             return field.__align;
+        if (field.type === "usize") {
+            field.type = opts.target.usize;
+        }
         if (field.pad)
             return (field.__align = 1);
-        let align = isNumeric(field.type)
-            ? SIZEOF[field.type]
-            : isWasmString(field.type)
-                ? USIZE_SIZE
-                : alignOf(types[field.type], types);
-        if (field.tag === "vec") {
-            align *= ceilPow2(field.len);
-        }
-        field.__align = align;
-        return align;
+        return (field.__align = isPointerLike(field)
+            ? align.align({ type: opts.target.usize })
+            : isNumeric(field.type) || isBigNumeric(field.type)
+                ? align.align(field)
+                : alignOf(types[field.type], types, selectAlignment(types[field.type]), opts));
     },
-    enum: (type) => {
+    enum: (type, _, align) => {
         const e = type;
         if (!e.tag)
             e.tag = "i32";
-        return (e.__align = SIZEOF[e.tag]);
+        return (e.__align = align.align({
+            type: e.tag,
+        }));
     },
-    struct: (type, types) => {
-        const struct = type;
-        let maxAlign = 0;
-        for (let f of struct.fields) {
-            maxAlign = Math.max(maxAlign, alignOf(f, types));
+    struct: (type, types, align, opts) => {
+        let maxAlign = 1;
+        for (let f of type.fields) {
+            maxAlign = Math.max(maxAlign, alignOf(f, types, align, opts));
+        }
+        return (type.__align = maxAlign);
+    },
+    union: (type, types, align, opts) => {
+        let maxAlign = 1;
+        for (let f of type.fields) {
+            maxAlign = Math.max(maxAlign, alignOf(f, types, align, opts));
         }
         return (type.__align = maxAlign);
     },
 });
 const prepareType = defmulti((x) => x.type, {}, {
-    [DEFAULT]: (x, types, opts) => {
+    [DEFAULT]: (x, types, alignImpl, opts) => {
         if (x.__align && x.__size)
             return;
-        alignOf(x, types);
-        sizeOf(x, types, opts);
+        alignOf(x, types, alignImpl, opts);
+        sizeOf(x, types, alignImpl, opts);
     },
-    struct: (x, types, opts) => {
+    struct: (x, types, align, opts) => {
         if (x.__align && x.__size)
             return;
         const struct = x;
-        alignOf(struct, types);
+        alignOf(struct, types, align, opts);
         if (struct.auto) {
             struct.fields.sort(compareByKey("__align", compareNumDesc));
         }
         for (let f of struct.fields) {
-            if (types[f.type]) {
-                prepareType(types[f.type], types, opts);
+            const type = types[f.type];
+            if (type) {
+                prepareType(type, types, selectAlignment(type), opts);
             }
         }
-        sizeOf(struct, types, opts);
+        sizeOf(struct, types, align, opts);
     },
 });
 /**
@@ -124,8 +149,9 @@ const prepareType = defmulti((x) => x.type, {}, {
  */
 export const prepareTypes = (types, opts) => {
     for (let id in types) {
-        prepareType(types[id], types, opts);
+        prepareType(types[id], types, selectAlignment(types[id]), opts);
     }
+    return types;
 };
 /**
  * Code generator main entry point. Takes an object of {@link TopLevelType}
@@ -144,10 +170,7 @@ export const prepareTypes = (types, opts) => {
  */
 export const generateTypes = (types, codegen, opts = {}) => {
     const $opts = {
-        header: true,
-        stringType: "slice",
-        uppercaseEnums: true,
-        lineWidth: 80,
+        ...DEFAULT_CODEGEN_OPTS,
         ...opts,
     };
     prepareTypes(types, $opts);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@thi.ng/wasm-api",
-  "version": "0.14.0",
+  "version": "0.16.0",
   "description": "Generic, modular, extensible API bridge, polyglot glue code and bindings code generators for hybrid JS & WebAssembly projects",
   "type": "module",
   "module": "./index.js",
@@ -33,30 +33,30 @@
     "doc:stats": "tools:module-stats",
     "pub": "yarn npm publish --access public",
     "test": "testament test",
-    "test:build-zig": "zig build-lib -O ReleaseSmall -target wasm32-freestanding -dynamic --strip --pkg-begin wasmapi include/wasmapi.zig --pkg-end test/custom.zig && wasm-dis -o custom.wast custom.wasm && cp custom.wasm test"
+    "test:build-zig": "zig build-lib -O ReleaseSmall -target wasm32-freestanding -dynamic --strip --pkg-begin wasmapi zig/wasmapi.zig --pkg-end test/custom.zig && wasm-dis -o custom.wast custom.wasm && cp custom.wasm test"
   },
   "dependencies": {
-    "@thi.ng/api": "^8.4.3",
-    "@thi.ng/args": "^2.2.5",
-    "@thi.ng/binary": "^3.3.6",
-    "@thi.ng/checks": "^3.3.0",
-    "@thi.ng/compare": "^2.1.13",
-    "@thi.ng/defmulti": "^2.1.18",
-    "@thi.ng/errors": "^2.2.2",
-    "@thi.ng/file-io": "^0.3.15",
-    "@thi.ng/hex": "^2.2.1",
-    "@thi.ng/idgen": "^2.1.15",
-    "@thi.ng/logger": "^1.4.1",
-    "@thi.ng/paths": "^5.1.18",
-    "@thi.ng/strings": "^3.3.14"
+    "@thi.ng/api": "^8.4.4",
+    "@thi.ng/args": "^2.2.7",
+    "@thi.ng/binary": "^3.3.8",
+    "@thi.ng/checks": "^3.3.2",
+    "@thi.ng/compare": "^2.1.14",
+    "@thi.ng/defmulti": "^2.1.19",
+    "@thi.ng/errors": "^2.2.3",
+    "@thi.ng/file-io": "^0.3.17",
+    "@thi.ng/hex": "^2.2.2",
+    "@thi.ng/idgen": "^2.1.16",
+    "@thi.ng/logger": "^1.4.2",
+    "@thi.ng/paths": "^5.1.20",
+    "@thi.ng/strings": "^3.3.15"
   },
   "devDependencies": {
-    "@microsoft/api-extractor": "^7.31.1",
-    "@thi.ng/testament": "^0.3.3",
+    "@microsoft/api-extractor": "^7.33.5",
+    "@thi.ng/testament": "^0.3.4",
     "rimraf": "^3.0.2",
     "tools": "^0.0.1",
-    "typedoc": "^0.22.17",
-    "typescript": "^4.8.3"
+    "typedoc": "^0.23.18",
+    "typescript": "^4.8.4"
   },
   "keywords": [
     "allocator",
@@ -96,7 +96,8 @@
     "*.d.ts",
     "bin",
     "codegen",
-    "include"
+    "include",
+    "zig"
   ],
   "exports": {
     ".": {
@@ -108,6 +109,9 @@
     "./bridge": {
       "default": "./bridge.js"
     },
+    "./codegen/align": {
+      "default": "./codegen/align.js"
+    },
     "./codegen/c11": {
       "default": "./codegen/c11.js"
     },
@@ -137,5 +141,5 @@
     "status": "alpha",
     "year": 2022
   },
-  "gitHead": "612b1b0bdf442473f04c2edac3012975a6ca28bb\n"
+  "gitHead": "41e59c7ad9bf24bb0230a5f60d05715e0fc1c1e6\n"
 }

package/string.d.ts

@@ -3,6 +3,10 @@ import type { IWasmMemoryAccess, ReadonlyWasmString } from "./api.js";
  * Memory mapped string wrapper for Zig-style UTF-8 encoded byte slices (aka
  * pointer & length pair). The actual JS string can be obtained via
  * {@link WasmStringSlice.deref} and mutated via {@link WasmStringSlice.set}.
+ *
+ * @remarks
+ * Currently only supports wasm32 target, need alt. solution for 64bit (possibly
+ * diff implementation) using bigint addresses (TODO)
  */
 export declare class WasmStringSlice implements ReadonlyWasmString {
     readonly mem: IWasmMemoryAccess;

package/string.js

@@ -3,6 +3,10 @@ import { unsupported } from "@thi.ng/errors/unsupported";
  * Memory mapped string wrapper for Zig-style UTF-8 encoded byte slices (aka
  * pointer & length pair). The actual JS string can be obtained via
  * {@link WasmStringSlice.deref} and mutated via {@link WasmStringSlice.set}.
+ *
+ * @remarks
+ * Currently only supports wasm32 target, need alt. solution for 64bit (possibly
+ * diff implementation) using bigint addresses (TODO)
  */
 export class WasmStringSlice {
     constructor(mem, base, isConst = true) {

package/zig/managed-index.zig

@@ -0,0 +1,223 @@
+const std = @import("std");
+const builtin = @import("builtin");
+const Allocator = std.mem.Allocator;
+
+/// `std.Arraylist` based data structure for associating and auto-indexing
+/// newly added items with numeric IDs and managing a pool of free IDs.
+/// Wraps `std.Arraylist` with a simple add()/remove()/get()/has() API and
+/// adds free slot tracking & re-use (depending on value type, potentially
+/// without any memory overhead). Free slots are stored as implicit linked
+/// list within the same array as the actual values/items.
+/// Important: The provided `.remove()` function MUST be used to remove
+/// unused items and mark their slot IDs for re-use.
+pub fn ManagedIndex(comptime T: type, comptime I: type) type {
+    const info = @typeInfo(I);
+    if (!(info == .Int and info.Int.bits > 0 and info.Int.signedness == std.builtin.Signedness.unsigned)) {
+        @compileError("require unsigned int as index type");
+    }
+    const Item = union(enum) {
+        /// Actual user defined value
+        value: T,
+        /// Free list slot ID. Index of the next avail free slot
+        id: ?I,
+    };
+
+    return struct {
+        /// backing array list
+        list: std.ArrayList(Item),
+        /// Index of the head of the implicitly stored list of free slots
+        /// Free slots in the array list use `Item.id` fields, each storing
+        /// the index of the next available free slot (or null)
+        /// The backing array list only grows if `freeID` is null
+        freeID: ?I = null,
+        /// Number of currently used items
+        used: usize = 0,
+
+        const Self = @This();
+
+        /// Initializes the underlying array list
+        pub fn init(allocator: Allocator) Self {
+            return .{
+                .list = std.ArrayList(Item).init(allocator),
+            };
+        }
+
+        /// De-initializes the underlying array list
+        pub fn deinit(self: *Self) void {
+            self.list.deinit();
+        }
+
+        /// Clears, frees & resets this list and its backing array list.
+        pub fn clear(self: *Self) void {
+            self.list.clearAndFree();
+            self.freeID = null;
+            self.used = 0;
+        }
+
+        /// Adds given `item` to the list and returns slot ID. If possible
+        /// re-uses existing free slots, else attempts to allocate a new slot.
+        pub fn add(self: *Self, item: T) Allocator.Error!I {
+            if (self.freeID) |id| {
+                self.freeID = self.list.items[id].id;
+                self.list.items[id] = .{ .value = item };
+                self.used += 1;
+                return @intCast(I, id);
+            } else {
+                const id = self.list.items.len;
+                try self.list.append(.{ .value = item });
+                self.used += 1;
+                return @intCast(I, id);
+            }
+        }
+
+        /// Adds given `id` to the internal free slot list and removes related
+        /// value/item
+        pub fn remove(self: *Self, id: I) void {
+            if (!self.has(id)) return;
+            self.list.items[id] = .{ .id = self.freeID };
+            self.freeID = id;
+            self.used -= 1;
+        }
+
+        /// Checks if given `id` is valid (skipped in unsafe release modes) and
+        /// returns item value or null if ID is invalid.
+        pub fn get(self: *const Self, id: I) ?T {
+            if (builtin.mode == .ReleaseFast or builtin.mode == .ReleaseSmall) {
+                return self.list.items[id].value;
+            }
+            return if (self.has(id)) self.list.items[id].value else null;
+        }
+
+        /// Returns true if given `id` is valid, i.e. is referring to a
+        /// currently used item (and isn't part of the list of free slots).
+        pub fn has(self: *const Self, id: I) bool {
+            if (id >= self.list.items.len) return false;
+            var freeID = self.freeID;
+            while (true) {
+                if (freeID) |fid| {
+                    if (fid == id) return false;
+                    freeID = self.list.items[fid].id;
+                } else {
+                    return true;
+                }
+            }
+        }
+
+        /// Iterator for currently used values (in ascending index order)
+        pub const ValueIterator = struct {
+            parent: *const Self,
+            i: usize = 0,
+
+            pub fn init(parent: *const Self) @This() {
+                return .{ .parent = parent };
+            }
+
+            pub fn next(self: *@This()) ?T {
+                const items = self.parent.list.items;
+                if (self.i >= items.len) return null;
+                while (self.i < items.len) {
+                    const val = switch (items[self.i]) {
+                        .value => |value| value,
+                        .id => null,
+                    };
+                    self.i += 1;
+                    if (val != null) return val;
+                }
+                return null;
+            }
+        };
+
+        /// Returns iterator of currently used values
+        pub fn values(self: *const Self) ValueIterator {
+            return ValueIterator.init(self);
+        }
+
+        /// Allocates a new slice containing only currently used values and
+        /// returns it. Caller owns returned memory.
+        pub fn valuesAsSlice(self: *Self, allocator: Allocator) Allocator.Error![]T {
+            var slice = try allocator.alloc(T, self.used);
+            var iter = self.values();
+            var i: usize = 0;
+            while (iter.next()) |value| {
+                slice[i] = value;
+                i += 1;
+            }
+            return slice;
+        }
+
+        /// Allocates a new slice of all currently free slot IDs and returns it.
+        /// Caller owns returned memory.
+        /// Internal/debug only.
+        fn freeIndicesAsSlice(self: *const Self, allocator: Allocator) Allocator.Error![]I {
+            var slice = try allocator.alloc(I, self.list.items.len - self.used);
+            var i: I = 0;
+            var freeID = self.freeID;
+            while (i < slice.len) {
+                if (freeID) |fid| {
+                    slice[i] = fid;
+                    i += 1;
+                    freeID = self.list.items[fid].id;
+                } else {
+                    break;
+                }
+            }
+            return slice;
+        }
+    };
+}
+
+test "ManagedIndex" {
+    const allocator = std.testing.allocator;
+    const expect = std.testing.expect;
+    const expectEqualSlices = std.testing.expectEqualSlices;
+
+    var list = ManagedIndex(f64, u8).init(allocator);
+    defer list.deinit();
+    try expect(try list.add(123) == 0);
+    try expect(try list.add(123) == 1);
+
+    try expect(list.has(0));
+    try expect(list.has(1));
+    try expect(list.used == 2);
+
+    list.remove(0);
+    try expect(!list.has(0));
+    try expect(!list.has(2));
+    try expect(list.used == 1);
+
+    try expect(try list.add(123) == 0);
+    try expect(try list.add(123) == 2);
+    try expect(list.has(2));
+    try expect(list.used == 3);
+
+    list.remove(1);
+    list.remove(2);
+
+    var ids = try list.freeIndicesAsSlice(allocator);
+    defer allocator.free(ids);
+    try expectEqualSlices(u8, &[_]u8{ 2, 1 }, ids);
+
+    try expect(try list.add(202) == 2);
+    list.remove(0);
+    try expect(try list.add(200) == 0);
+    try expect(try list.add(201) == 1);
+    try expect(list.freeID == null);
+
+    var iter = list.values();
+    try expect(if (iter.next()) |v| v == 200 else false);
+    try expect(if (iter.next()) |v| v == 201 else false);
+    try expect(if (iter.next()) |v| v == 202 else false);
+    try expect(iter.next() == null);
+
+    const slice = try list.valuesAsSlice(allocator);
+    defer allocator.free(slice);
+    try expectEqualSlices(f64, &[_]f64{ 200, 201, 202 }, slice);
+
+    list.remove(0);
+    list.remove(2);
+    list.remove(1);
+
+    var ids2 = try list.freeIndicesAsSlice(allocator);
+    defer allocator.free(ids2);
+    try expectEqualSlices(u8, &[_]u8{ 1, 2, 0 }, ids2);
+}

package/{include → zig}/wasmapi.zig

@@ -3,6 +3,8 @@
 const std = @import("std");
 const root = @import("root");
 
+pub const ManagedIndex = @import("managed-index.zig").ManagedIndex;
+
 /// Higher-order slice wrapper for extern struct use cases
 /// S = slice type
 /// P = pointer type