koffi 2.2.2-beta.6 → 2.2.3-beta.1

package/ChangeLog.md

@@ -2,6 +2,20 @@
 
 ## History
 
+### Koffi 2.2.2
+
+**Main fixes:**
+
+- Support transparent [asynchronous callbacks](callbacks.md#asynchronous-callbacks)
+- Expand from a maximum of 16+16 to 1024 callbacks running in parallel
+
+**Other fixes:**
+
+- Fix bundler support by removing shebang from index.js
+- Fix bugs when loading Koffi multiples times in same process (context aware module)
+- Check N-API version when module is loaded
+- Optimize callback unregistration
+
 ### Koffi 2.2.1
 
 **Main fixes:**

package/doc/callbacks.md

@@ -172,9 +172,16 @@ console.log(array); // Prints ['123', 'bar', 'foo', 'foobar']
 
 *New in Koffi 2.2.2*
 
-In Koffi, [asynchronous native calls](functions.md#asynchronous-calls) happen on a secondary thread. However, JS execution is inherently single-threaded, callbacks must run on the main thread.
+JS execution is inherently single-threaded, so JS callbacks must run on the main thread. There are two ways you may want to call a callback function from another thread:
 
-Koffi deals with this by running the JS callback function in the Node.js event loop. This means the callback cannot run while the engine is busy running synchronous code.
+- Call the callback from an asynchronous FFI call (e.g. `waitpid.async`)
+- Inside a synchronous FFI call, pass the callback to another thread
+
+In both cases, Koffi will queue the call back to JS to run on the main thread, as soon as the JS event loop has a chance to run (for example when you await a promise).
+
+```{warning}
+Be careful, you can easily get into a deadlock situation if you call a callback from a secondary thread and your main thread never lets the JS event loop run (for example, if the main thread waits for the secondary thread to finish something itself).
+```
 
 ## Handling of exceptions
 

package/package.json

@@ -1,7 +1,7 @@
 {
     "name": "koffi",
-    "version": "2.2.2-beta.6",
-    "stable": "2.2.1",
+    "version": "2.2.3-beta.1",
+    "stable": "2.2.2",
     "description": "Fast and simple C FFI (foreign function interface) for Node.js",
     "keywords": [
         "foreign",

package/src/koffi/src/abi_x64_win.cc

@@ -18,6 +18,14 @@
 #include "call.hh"
 #include "util.hh"
 
+#ifndef NOMINMAX
+    #define NOMINMAX
+#endif
+#ifndef WIN32_LEAN_AND_MEAN
+    #define WIN32_LEAN_AND_MEAN
+#endif
+#include <windows.h>
+
 #include <napi.h>
 
 namespace RG {
@@ -216,6 +224,16 @@ bool CallData::Prepare(const FunctionInfo *func, const Napi::CallbackInfo &info)
 
 void CallData::Execute(const FunctionInfo *func)
 {
+    NT_TIB *tib = (NT_TIB *)__readgsqword(0x30);
+
+    // Adjust TIB stack limits so SEH works correctly
+    RG_DEFER_C(base = tib->StackBase, limit = tib->StackLimit) {
+        tib->StackBase = base;
+        tib->StackLimit = limit;
+    };
+    tib->StackBase = mem->stack0.end();
+    tib->StackLimit = mem->stack0.ptr;
+
 #define PERFORM_CALL(Suffix) \
         ([&]() { \
             auto ret = (func->forward_fp ? ForwardCallX ## Suffix(func->func, new_sp, &old_sp) \
@@ -317,6 +335,16 @@ void CallData::Relay(Size idx, uint8_t *own_sp, uint8_t *caller_sp, bool async,
     if (RG_UNLIKELY(env.IsExceptionPending()))
         return;
 
+    NT_TIB *tib = (NT_TIB *)__readgsqword(0x30);
+
+    // Restore real thread stack limits
+    RG_DEFER_C(base = tib->StackBase, limit = tib->StackLimit) {
+        tib->StackBase = base;
+        tib->StackLimit = limit;
+    };
+    tib->StackBase = instance->main_stack_base;
+    tib->StackLimit = instance->main_stack_limit;
+
     const TrampolineInfo &trampoline = shared.trampolines[idx];
 
     const FunctionInfo *proto = trampoline.proto;

package/src/koffi/src/abi_x86.cc

@@ -18,6 +18,16 @@
 #include "call.hh"
 #include "util.hh"
 
+#ifdef _WIN32
+    #ifndef NOMINMAX
+        #define NOMINMAX
+    #endif
+    #ifndef WIN32_LEAN_AND_MEAN
+        #define WIN32_LEAN_AND_MEAN
+    #endif
+    #include <windows.h>
+#endif
+
 #include <napi.h>
 
 namespace RG {
@@ -297,6 +307,18 @@ bool CallData::Prepare(const FunctionInfo *func, const Napi::CallbackInfo &info)
 
 void CallData::Execute(const FunctionInfo *func)
 {
+#ifdef _WIN32
+    NT_TIB *tib = (NT_TIB *)__readfsdword(0x18);
+
+    // Adjust TIB stack limits so SEH works correctly
+    RG_DEFER_C(base = tib->StackBase, limit = tib->StackLimit) {
+        tib->StackBase = base;
+        tib->StackLimit = limit;
+    };
+    tib->StackBase = mem->stack0.end();
+    tib->StackLimit = mem->stack0.ptr;
+#endif
+
 #define PERFORM_CALL(Suffix) \
         ([&]() { \
             auto ret = (func->fast ? ForwardCallR ## Suffix(func->func, new_sp, &old_sp) \
@@ -394,11 +416,23 @@ Napi::Value CallData::Complete(const FunctionInfo *func)
     RG_UNREACHABLE();
 }
 
-void CallData::Relay(Size idx, uint8_t *own_sp, uint8_t *caller_sp, bool async, BackRegisters *out_reg)
+void CallData::Relay(Size idx, uint8_t *, uint8_t *caller_sp, bool async, BackRegisters *out_reg)
 {
     if (RG_UNLIKELY(env.IsExceptionPending()))
         return;
 
+#ifdef _WIN32
+    NT_TIB *tib = (NT_TIB *)__readfsdword(0x18);
+
+    // Restore real thread stack limits
+    RG_DEFER_C(base = tib->StackBase, limit = tib->StackLimit) {
+        tib->StackBase = base;
+        tib->StackLimit = limit;
+    };
+    tib->StackBase = instance->main_stack_base;
+    tib->StackLimit = instance->main_stack_limit;
+#endif
+
     const TrampolineInfo &trampoline = shared.trampolines[idx];
 
     const FunctionInfo *proto = trampoline.proto;

package/src/koffi/src/call.cc

@@ -81,6 +81,9 @@ CallData::~CallData()
 void CallData::RelaySafe(Size idx, uint8_t *own_sp, uint8_t *caller_sp, BackRegisters *out_reg)
 {
     if (std::this_thread::get_id() != instance->main_thread_id) {
+        // JS/V8 is single-threaded, and runs on main_thread_id. Forward the call
+        // to the JS event loop.
+
         RelayContext ctx;
 
         ctx.call = this;

package/src/koffi/src/ffi.cc

@@ -1030,6 +1030,9 @@ static InstanceMemory *AllocateMemory(InstanceData *instance, Size stack_size, S
     mem->stack.len -= 16;
 #endif
 
+    // Keep real stack limits intact, in case we need them
+    mem->stack0 = mem->stack;
+
     mem->heap.len = heap_size;
 #ifdef _WIN32
     mem->heap.ptr = (uint8_t *)VirtualAlloc(nullptr, mem->heap.len, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE);
@@ -1065,9 +1068,6 @@ static Napi::Value TranslateNormalCall(const Napi::CallbackInfo &info)
     InstanceMemory *mem = instance->memories[0];
     CallData call(env, instance, mem);
 
-    RG_DEFER_C(prev_call = exec_call) { exec_call = prev_call; };
-    exec_call = &call;
-
     if (!RG_UNLIKELY(call.Prepare(func, info)))
         return env.Null();
 
@@ -1113,7 +1113,7 @@ static Napi::Value TranslateVariadicCall(const Napi::CallbackInfo &info)
     for (Size i = func.parameters.len; i < (Size)info.Length(); i += 2) {
         ParameterInfo param = {};
 
-        param.type = ResolveType(info[i], &param.directions);
+        param.type = ResolveType(info[(uint32_t)i], &param.directions);
 
         if (RG_UNLIKELY(!param.type))
             return env.Null();
@@ -1893,6 +1893,9 @@ extern "C" void RelayCallback(Size idx, uint8_t *own_sp, uint8_t *caller_sp, Bac
     if (RG_LIKELY(exec_call)) {
         exec_call->RelaySafe(idx, own_sp, caller_sp, out_reg);
     } else {
+        // This happens if the callback pointer is called from a different thread
+        // than the one that runs the FFI call (sync or async).
+
         TrampolineInfo *trampoline = &shared.trampolines[idx];
 
         Napi::Env env = trampoline->func.Env();
@@ -1929,6 +1932,18 @@ static InstanceData *CreateInstance(Napi::Env env)
     }
     napi_unref_threadsafe_function(env, instance->broker);
 
+#if defined(_WIN32) && (defined(__x86_64__) || defined(_M_AMD64))
+    NT_TIB *tib = (NT_TIB *)__readgsqword(0x30);
+
+    instance->main_stack_base = tib->StackBase;
+    instance->main_stack_limit = tib->StackLimit;
+#elif defined(_WIN32) && (defined(__i386__) || defined(_M_IX86))
+    NT_TIB *tib = (NT_TIB *)__readfsdword(0x18);
+
+    instance->main_stack_base = tib->StackBase;
+    instance->main_stack_limit = tib->StackLimit;
+#endif
+
     err_guard.Disable();
     return instance;
 }

package/src/koffi/src/ffi.hh

@@ -225,6 +225,7 @@ struct InstanceMemory {
     ~InstanceMemory();
 
     Span<uint8_t> stack;
+    Span<uint8_t> stack0;
     Span<uint8_t> heap;
 
     uint16_t generation; // Can wrap without risk
@@ -254,6 +255,11 @@ struct InstanceData {
     std::thread::id main_thread_id;
     napi_threadsafe_function broker = nullptr;
 
+#ifdef _WIN32
+    void *main_stack_base;
+    void *main_stack_limit;
+#endif
+
     HashMap<void *, int16_t> trampolines_map;
 
     BlockAllocator str_alloc;

package/vendor/miniz/ChangeLog.md

@@ -1,5 +1,9 @@
 ## Changelog
 
+### 3.0.2
+
+ - Fix buffer overrun in mz_utf8z_to_widechar on Windows
+
 ### 3.0.1
 
  - Fix compilation error with MINIZ_USE_UNALIGNED_LOADS_AND_STORES=1

package/vendor/miniz/miniz.c

@@ -3068,7 +3068,7 @@ static WCHAR* mz_utf8z_to_widechar(const char* str)
 {
   int reqChars = MultiByteToWideChar(CP_UTF8, 0, str, -1, NULL, 0);
   WCHAR* wStr = (WCHAR*)malloc(reqChars * sizeof(WCHAR));
-  MultiByteToWideChar(CP_UTF8, 0, str, -1, wStr, sizeof(WCHAR) * reqChars);
+  MultiByteToWideChar(CP_UTF8, 0, str, -1, wStr, reqChars);
   return wStr;
 }
 

package/vendor/miniz/miniz.h

@@ -275,10 +275,10 @@ enum
     MZ_DEFAULT_COMPRESSION = -1
 };
 
-#define MZ_VERSION "11.0.1"
-#define MZ_VERNUM 0xB001
+#define MZ_VERSION "11.0.2"
+#define MZ_VERNUM 0xB002
 #define MZ_VER_MAJOR 11
-#define MZ_VER_MINOR 1
+#define MZ_VER_MINOR 2
 #define MZ_VER_REVISION 0
 #define MZ_VER_SUBREVISION 0