@mikrojs/native 0.17.0 → 0.17.1-pr-300.20260731003316

package/CMakeLists.txt

@@ -86,6 +86,7 @@ mikrojs_generate_bytecode(
     MODULE_PREFIX "mikro"
     SYMBOL_PREFIX "mikro"
     TARGET gen_bytecode
+    FREEZE_ATOMS
 )
 add_dependencies(mikrojs gen_bytecode)
 target_include_directories(mikrojs PRIVATE "${gen_bytecode_INCLUDE_DIR}")

package/cmake/mikrojs_bytecode.cmake

@@ -25,7 +25,11 @@ get_filename_component(_MIK_BC_CMAKE_DIR "${CMAKE_CURRENT_LIST_FILE}" DIRECTORY)
 set(_MIK_BC_SCRIPTS_DIR "${_MIK_BC_CMAKE_DIR}/../scripts" CACHE INTERNAL "")
 
 function(mikrojs_generate_bytecode)
-    cmake_parse_arguments(ARG "" "RUNTIME_DIR;MODULE_PREFIX;SYMBOL_PREFIX;TARGET;WORKING_DIRECTORY" "MODULES" ${ARGN})
+    # FREEZE_ATOMS: compile blobs against a generated frozen atom table so
+    # they load zero-copy (JS_READ_OBJ_INPLACE). Core runtime only: the
+    # device preloads exactly one table, so board/driver packages must
+    # stay classic (they load via the copy-and-relocate fallback).
+    cmake_parse_arguments(ARG "FREEZE_ATOMS" "RUNTIME_DIR;MODULE_PREFIX;SYMBOL_PREFIX;TARGET;WORKING_DIRECTORY" "MODULES" ${ARGN})
 
     # Defaults
     if(NOT ARG_TARGET)
@@ -86,7 +90,46 @@ function(mikrojs_generate_bytecode)
         WORKING_DIRECTORY ${ARG_WORKING_DIRECTORY}
     )
 
-    # Step 2: Compile each bundled JS file to a bytecode header with qjsc
+    # Steps 2a/2b (FREEZE_ATOMS only): pass-1 raw bytecode per module, then
+    # union the atom sections into the frozen atom table (frozen_atoms.bin
+    # for qjsc -A, <prefix>_frozen_atoms.h for the runtime preload).
+    # Module order fixes the table order.
+    set(_ATOMS_BIN "")
+    set(_ATOMS_HEADER "")
+    if(ARG_FREEZE_ATOMS)
+        set(_EXTRACT_SCRIPT "${_MIK_BC_SCRIPTS_DIR}/extract-atoms.js")
+        set(_PASS1_BLOBS "")
+        foreach(mod ${ARG_MODULES})
+            string(REGEX REPLACE "[^a-zA-Z0-9]" "_" _mod_safe "${mod}")
+            set(_blob "${_GEN_DIR}/pass1/${_mod_safe}.bjs")
+            add_custom_command(
+                OUTPUT ${_blob}
+                COMMAND ${CMAKE_COMMAND} -E make_directory ${_GEN_DIR}/pass1
+                COMMAND sh ${_COMPILE_SCRIPT}
+                    ${QJSC_EXECUTABLE}
+                    ${_BUNDLE_DIR}/${mod}.js
+                    ${_blob}
+                    "${ARG_MODULE_PREFIX}/${mod}"
+                    "unused"
+                DEPENDS ${_BUNDLE_DIR}/bundle.stamp ${_COMPILE_SCRIPT} ${QJSC_EXECUTABLE}
+                COMMENT "Bytecode pass 1: ${ARG_MODULE_PREFIX}/${mod}"
+            )
+            list(APPEND _PASS1_BLOBS ${_blob})
+        endforeach()
+
+        set(_ATOMS_BIN "${_GEN_DIR}/frozen_atoms.bin")
+        set(_ATOMS_HEADER "${_GEN_DIR}/${ARG_SYMBOL_PREFIX}_frozen_atoms.h")
+        add_custom_command(
+            OUTPUT ${_ATOMS_BIN} ${_ATOMS_HEADER}
+            COMMAND node ${_EXTRACT_SCRIPT} ${_GEN_DIR} ${ARG_SYMBOL_PREFIX} ${_PASS1_BLOBS}
+            DEPENDS ${_PASS1_BLOBS} ${_EXTRACT_SCRIPT}
+            COMMENT "Extracting frozen atom table"
+        )
+    endif()
+
+    # Step 2c: Compile each bundled JS file to a bytecode header with qjsc,
+    # against the frozen atom table so atom references are final ids
+    # (enables zero-copy loading via JS_READ_OBJ_INPLACE).
     set(_HEADERS "")
     foreach(mod ${ARG_MODULES})
         # Sanitize module name for C identifiers (replace non-alphanumeric with _)
@@ -101,11 +144,15 @@ function(mikrojs_generate_bytecode)
                 ${_header}
                 "${ARG_MODULE_PREFIX}/${mod}"
                 "${ARG_SYMBOL_PREFIX}_${_mod_safe}_bytecode"
-            DEPENDS ${_BUNDLE_DIR}/bundle.stamp ${_COMPILE_SCRIPT} ${QJSC_EXECUTABLE}
+                ${_ATOMS_BIN}
+            DEPENDS ${_BUNDLE_DIR}/bundle.stamp ${_COMPILE_SCRIPT} ${QJSC_EXECUTABLE} ${_ATOMS_BIN}
             COMMENT "Compiling bytecode: ${ARG_MODULE_PREFIX}/${mod}"
         )
         list(APPEND _HEADERS ${_header})
     endforeach()
+    if(_ATOMS_HEADER)
+        list(APPEND _HEADERS ${_ATOMS_HEADER})
+    endif()
 
     # Step 3: Generate builtins table header (includes + lookup table)
     set(_TABLE_HEADER "${_GEN_DIR}/${ARG_SYMBOL_PREFIX}_builtins_table.h")

package/include/mikrojs/cutils_wrap.h

@@ -36,6 +36,7 @@ typedef struct DynBuf {
 
 /* Wrapper functions defined in cutils_compat.c */
 int mik__dbuf_put(DynBuf *s, const void *data, size_t len);
+int mik__dbuf_claim(DynBuf *s, size_t len);
 int mik__dbuf_putc(DynBuf *s, uint8_t val);
 int mik__dbuf_putstr(DynBuf *s, const char *str);
 void mik__dbuf_free(DynBuf *s);
@@ -46,6 +47,7 @@ char *mik__js_pstrcat(char *buf, int buf_size, const char *s);
 
 /* Map original names to wrapper names */
 #define dbuf_put mik__dbuf_put
+#define dbuf_claim mik__dbuf_claim
 #define dbuf_putc mik__dbuf_putc
 #define dbuf_putstr mik__dbuf_putstr
 #define dbuf_free mik__dbuf_free

package/include/mikrojs/mikrojs.h

@@ -288,8 +288,12 @@ extern mik_module_desc_t* mik__module_registry_head;
  * Driver packages use MIK_REGISTER_BUILTIN() to embed their JS wrappers as
  * firmware builtins, resolved by the real npm package name (e.g. "@mikrojs/your-driver"). */
 typedef struct mik_ext_builtin_t {
-    const char* name;         /* module name, e.g. "@mikrojs/your-driver" */
-    const uint8_t* data;      /* compiled bytecode */
+    const char* name; /* module name, e.g. "@mikrojs/your-driver" */
+    /* Compiled bytecode. Must outlive every runtime that imports the
+     * module: the loader may keep instruction streams pointing into this
+     * buffer instead of copying them to the heap (JS_READ_OBJ_INPLACE).
+     * Use a static const array (flash/rodata), never a heap buffer. */
+    const uint8_t* data;
     uint32_t data_size;
     struct mik_ext_builtin_t* next;
 } mik_ext_builtin_t;

package/include/mikrojs/private.h

@@ -187,6 +187,9 @@ void mik__flush_unhandled_rejections(JSContext* ctx);
 /* Drop a promise from the pending-rejection queue without reporting it. */
 void mik__forget_rejection(JSContext* ctx, JSValue promise);
 JSModuleDef* mik__load_builtin(JSContext* ctx, const char* name);
+/* Preload the generated frozen atom table (must run right after
+ * JS_NewRuntime2, before any context/intrinsic creates atoms). */
+int mik__preload_frozen_atoms(JSRuntime* rt);
 int mik__load_file(JSContext* ctx, DynBuf* dbuf, const char* filename);
 void mik__resolve_fs_path(JSContext* ctx, const char* module_name, char* out, size_t out_size);
 int mik__resolve_fs_root(JSContext* ctx, const char* path, char* out, size_t out_size);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mikrojs/native",
-  "version": "0.17.0",
+  "version": "0.17.1-pr-300.20260731003316",
   "description": "Mikro.js C++ runtime library and Node.js native addon",
   "keywords": [
     "esp32",
@@ -86,14 +86,14 @@
     "cmake-js": "^8.0.0",
     "node-addon-api": "^8.7.0",
     "node-gyp-build": "^4.8.4",
-    "@mikrojs/quickjs": "0.17.0"
+    "@mikrojs/quickjs": "0.17.1-pr-300.20260731003316+67c29b9"
   },
   "devDependencies": {
     "@swc/core": "^1.15.30",
     "@types/node": "^24.12.2",
     "esbuild": "^0.28.0",
     "terser": "^5.46.2",
-    "@mikrojs/registry": "0.17.0"
+    "@mikrojs/registry": "0.17.1-pr-300.20260731003316+67c29b9"
   },
   "engines": {
     "node": ">=24.0.0"

package/scripts/compile-bytecode.sh

@@ -1,9 +1,12 @@
 #!/bin/sh
 # Compile a bundled JS module to a C bytecode header using qjsc.
 #
-# Usage: compile-bytecode.sh <qjsc> <input.js> <output.h> <module_name> <symbol_name>
+# Usage: compile-bytecode.sh <qjsc> <input.js> <output.h> <module_name> <symbol_name> [atoms.bin]
 #
 # Reads <input>.externals for external module declarations.
+# If <output> ends in .bjs, emits raw bytecode instead of a C header
+# (pass 1 of the frozen-atom pipeline; see extract-atoms.js).
+# If [atoms.bin] is given, compiles against that frozen atom table (-A).
 
 set -e
 
@@ -12,6 +15,7 @@ INPUT="$2"
 OUTPUT="$3"
 MODULE_NAME="$4"
 SYMBOL_NAME="$5"
+ATOMS_BIN="$6"
 
 EXTERNALS_FILE="${INPUT%.js}.externals"
 
@@ -34,5 +38,20 @@ fi
 # so any runtime module that reads `import.meta.*` (e.g. mikrojs/env
 # accessing import.meta.env) breaks at runtime. The ~1 KB heap saving
 # from stripping debug isn't worth crippling a spec-standard module API.
-# shellcheck disable=SC2086
-"$QJSC" -m -s -N "$SYMBOL_NAME" -n "$MODULE_NAME" $M_FLAGS -o "$OUTPUT" "$INPUT"
+A_FLAGS=""
+if [ -n "$ATOMS_BIN" ]; then
+    A_FLAGS="-A $ATOMS_BIN"
+fi
+
+case "$OUTPUT" in
+*.bjs)
+    # Raw bytecode (frozen-atom pass 1). Must use the same flags as the
+    # header pass so the atom set matches, minus -N (no C symbol).
+    # shellcheck disable=SC2086
+    "$QJSC" -b -m -s -n "$MODULE_NAME" $M_FLAGS $A_FLAGS -o "$OUTPUT" "$INPUT"
+    ;;
+*)
+    # shellcheck disable=SC2086
+    "$QJSC" -m -s -N "$SYMBOL_NAME" -n "$MODULE_NAME" $M_FLAGS $A_FLAGS -o "$OUTPUT" "$INPUT"
+    ;;
+esac

package/scripts/extract-atoms.js

@@ -0,0 +1,144 @@
+#!/usr/bin/env node
+/* eslint-disable no-console */
+/**
+ * Build the frozen atom table from pass-1 raw bytecode blobs.
+ *
+ * Reads the atom section of each .bjs blob (compiled without a frozen
+ * table), unions the narrow plain-string atoms in first-seen order, and
+ * emits:
+ *   - frozen_atoms.bin — binary table for `qjsc -A` (u32le count, then
+ *     u32le len + bytes per atom)
+ *   - <symbol_prefix>_frozen_atoms.h — C arrays for JS_PreloadFrozenAtoms
+ *
+ * Wide-char and symbol atoms are skipped: they stay dynamic, and any
+ * function referencing them falls back to the copy-and-relocate load path.
+ *
+ * Usage: extract-atoms.js <gen_dir> <symbol_prefix> <blob1.bjs> [blob2.bjs ...]
+ */
+import {readFileSync, writeFileSync} from 'node:fs'
+import {join} from 'node:path'
+
+const JS_ATOM_TYPE_STRING = 1
+
+function readLeb128(buf, pos) {
+  let v = 0
+  let shift = 0
+  for (;;) {
+    const b = buf[pos++]
+    v |= (b & 0x7f) << shift
+    if ((b & 0x80) === 0) break
+    shift += 7
+  }
+  return [v >>> 0, pos]
+}
+
+const BC_VERSION = 26
+const BC_VERSION_FROZEN = BC_VERSION | 0x40
+
+/** Parse the atom section of a classic bytecode blob, returning the
+ * narrow plain-string atom names as Buffers. */
+function extractAtoms(blob, path) {
+  // Layout: u8 version, u32 checksum, leb128 atom count, entries.
+  // A misparse here degrades silently (empty table → classic blobs →
+  // green build with zero memory win), so reject anything unexpected.
+  if (blob[0] === BC_VERSION_FROZEN) {
+    throw new Error(`${path}: pass-1 blob is already frozen-format — pipeline ordering bug`)
+  }
+  if (blob[0] !== BC_VERSION) {
+    throw new Error(`${path}: unknown bytecode version ${blob[0]} (expected ${BC_VERSION})`)
+  }
+  const [count, afterCount] = readLeb128(blob, 5)
+  let pos = afterCount
+  const atoms = []
+  for (let i = 0; i < count; i++) {
+    const type = blob[pos++]
+    if (type === 0) {
+      pos += 4 // predefined atom id
+      continue
+    }
+    let lenWide
+    ;[lenWide, pos] = readLeb128(blob, pos)
+    const isWide = lenWide & 1
+    const len = lenWide >>> 1
+    const size = isWide ? len * 2 : len
+    if (type === JS_ATOM_TYPE_STRING && !isWide) {
+      atoms.push(blob.subarray(pos, pos + len))
+    }
+    pos += size
+  }
+  return atoms
+}
+
+function cEscape(buf) {
+  let out = ''
+  for (const b of buf) {
+    if (b >= 0x20 && b < 0x7f && b !== 0x22 && b !== 0x5c && b !== 0x3f) {
+      out += String.fromCharCode(b)
+    } else {
+      // Octal escapes are always 3 digits here, so a following digit
+      // cannot extend the escape (unlike \x).
+      out += '\\' + b.toString(8).padStart(3, '0')
+    }
+  }
+  return out
+}
+
+const [genDir, symbolPrefix, ...blobPaths] = process.argv.slice(2)
+if (!genDir || !symbolPrefix || blobPaths.length === 0) {
+  console.error('usage: extract-atoms.js <gen_dir> <symbol_prefix> <blob.bjs>...')
+  process.exit(1)
+}
+
+const seen = new Set()
+const names = []
+let totalBlobBytes = 0
+for (const path of blobPaths) {
+  const blob = readFileSync(path)
+  totalBlobBytes += blob.length
+  for (const atom of extractAtoms(blob, path)) {
+    const key = atom.toString('latin1')
+    if (!seen.has(key)) {
+      seen.add(key)
+      names.push(atom)
+    }
+  }
+}
+
+// Real builtin bundles always introduce atoms; an empty table from
+// non-trivial input means the parse silently missed the atom section.
+if (names.length === 0 && totalBlobBytes > 1024) {
+  console.error(
+    `extract-atoms: parsed ${blobPaths.length} blobs (${totalBlobBytes} bytes) but found zero atoms — refusing to emit an empty frozen table`,
+  )
+  process.exit(1)
+}
+
+// Binary table for qjsc -A
+const chunks = [Buffer.alloc(4)]
+chunks[0].writeUInt32LE(names.length, 0)
+for (const name of names) {
+  const lenBuf = Buffer.alloc(4)
+  lenBuf.writeUInt32LE(name.length, 0)
+  chunks.push(lenBuf, name)
+}
+writeFileSync(join(genDir, 'frozen_atoms.bin'), Buffer.concat(chunks))
+
+// C header for JS_PreloadFrozenAtoms
+let h = '/* Auto-generated by extract-atoms.js — do not edit */\n'
+if (names.length === 0) {
+  h += `static const char* const ${symbolPrefix}_frozen_atom_names[] = {NULL};\n`
+  h += `static const uint32_t ${symbolPrefix}_frozen_atom_lens[] = {0};\n`
+} else {
+  h += `static const char* const ${symbolPrefix}_frozen_atom_names[] = {\n`
+  for (const name of names) {
+    h += `    "${cEscape(name)}",\n`
+  }
+  h += '};\n'
+  h += `static const uint32_t ${symbolPrefix}_frozen_atom_lens[] = {\n    `
+  h += names.map((n) => n.length).join(', ')
+  h += ',\n};\n'
+}
+h += `enum { ${symbolPrefix}_frozen_atom_count = ${names.length} };\n`
+writeFileSync(join(genDir, `${symbolPrefix}_frozen_atoms.h`), h)
+
+console.log(`extract-atoms: ${names.length} frozen atoms from ${blobPaths.length} blobs`)

package/src/builtins.cpp

@@ -17,6 +17,19 @@ typedef struct {
 #include "gen/mikro_builtins_table.h"
 #define builtins mikro_builtins
 
+/* Frozen atom table generated alongside the builtin bytecode: preloading it
+ * right after runtime creation gives builtin blobs final atom ids, letting
+ * their instruction streams load zero-copy from flash/rodata. */
+#include "gen/mikro_frozen_atoms.h"
+
+int mik__preload_frozen_atoms(JSRuntime* rt) {
+    if (mikro_frozen_atom_count == 0) {
+        return 0;
+    }
+    return JS_PreloadFrozenAtoms(rt, mikro_frozen_atom_names, mikro_frozen_atom_lens,
+                                 mikro_frozen_atom_count);
+}
+
 /* External builtins registered by board/driver packages via MIK_REGISTER_BUILTIN() */
 // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
 mik_ext_builtin_t* mik__ext_builtin_head = nullptr;
@@ -29,7 +42,10 @@ static JSModuleDef* mik__deserialize_builtin(JSContext* ctx, const char* name,
     platform->log(MIK_LOG_DEBUG, "mikrojs", "Loading builtin '%s' (%u bytes, bc_version=%u, data_ptr=%p)", name,
              data_size, data_size > 0 ? data[0] : 0, (void*)data);
 
-    JSValue obj = JS_ReadObject(ctx, data, data_size, JS_READ_OBJ_BYTECODE);
+    /* Builtin bytecode lives in flash/rodata for the runtime's lifetime, so
+     * qualifying instruction streams stay in place instead of being copied
+     * to the JS heap (falls back per function for non-final atom refs). */
+    JSValue obj = JS_ReadObject(ctx, data, data_size, JS_READ_OBJ_BYTECODE | JS_READ_OBJ_INPLACE);
 
     if (JS_IsException(obj)) {
         /* Peek the exception for the log, then re-throw it so the caller

package/src/cutils_compat.c

@@ -9,6 +9,10 @@ int mik__dbuf_put(DynBuf *s, const void *data, size_t len) {
     return dbuf_put(s, data, len);
 }
 
+int mik__dbuf_claim(DynBuf *s, size_t len) {
+    return dbuf_claim(s, len);
+}
+
 int mik__dbuf_putc(DynBuf *s, uint8_t val) {
     return dbuf_putc(s, val);
 }

package/src/mikrojs.cpp

@@ -286,6 +286,12 @@ MIKRuntime* MIK_NewRuntimeInternal(MIKRunOptions* options) {
     CHECK_NOT_NULL(rt);
     mik_rt->rt = rt;
 
+    /* Builtin blobs are compiled against this table; a mismatch would make
+     * every builtin import fail, and failure here can only mean atoms were
+     * created before this point (a programming error). Must run before any
+     * context/intrinsic setup creates atoms. */
+    CHECK(mik__preload_frozen_atoms(rt) == 0);
+
     /* Seed Math.random() with real entropy.
      * On ESP32 the system clock starts near epoch 0 on every boot, producing
      * the same seed. Use platform RNG to inject entropy. */
@@ -850,6 +856,17 @@ int mik__load_file(JSContext* ctx, DynBuf* dbuf, const char* filename) {
         return -1;
     }
 
+    /* Reserve the exact file size up front: DynBuf grows 1.5x from zero, so
+     * letting it grow incrementally peaks at ~2.5x the file size mid-copy.
+     * +4 covers the NUL/wrapper suffix callers append after loading. */
+    struct stat st;
+    if (fstat(fd, &st) == 0 && st.st_size > 0) {
+        if (dbuf_claim(dbuf, (size_t)st.st_size + 4)) {
+            close(fd);
+            return -1;
+        }
+    }
+
     uint8_t buf[512];
     ssize_t n;
     while ((n = read(fd, buf, sizeof(buf))) > 0) {