koffi 2.5.19 → 2.5.21-beta.1

package/CHANGELOG.md

@@ -4,6 +4,11 @@
 
 ### Koffi 2.5
 
+#### Koffi 2.5.20 (2023-08-31)
+
+- Fix possible crash with async registered callbacks introduced in Koffi 2.5.19
+- Various documentation fixes and improvements
+
 #### Koffi 2.5.19 (2023-08-29)
 
 - Create thread-safe function broker lazily
@@ -332,7 +337,7 @@ Pre-built binaries don't work correctly in Koffi 2.5.13 to 2.5.15, skip those ve
 **Main changes:**
 
 - Add [koffi.as()](polymorphism.md#input-polymorphism) to support polymorphic APIs based on `void *` parameters
-- Add [endian-sensitive integer types](input.md#endian-sensitive-types): `intX_le_t`, `intX_be_t`, `uintX_le_t`, `uintX_be_t`
+- Add [endian-sensitive integer types](input.md#endian-sensitive-integers): `intX_le_t`, `intX_be_t`, `uintX_le_t`, `uintX_be_t`
 - Accept typed arrays for `void *` parameters
 - Introduce `koffi.opaque()` to replace `koffi.handle()` (which remains supported until Koffi 3.0)
 - Support JS Array and TypedArray to fill struct and array pointer members

package/doc/callbacks.md

@@ -88,7 +88,7 @@ console.log(ret);
 
 Use registered callbacks when the function needs to be called at a later time (e.g. log handler, event handler, `fopencookie/funopen`). Call `koffi.register(func, type)` to register a callback function, with two arguments: the JS function, and the callback type.
 
-When you are done, call `koffi.unregister()` (with the value returned by `koffi.register()`) to release the slot. A maximum of 1024 callbacks can exist at the same time. Failure to do so will leak the slot, and subsequent registrations may fail (with an exception) once all slots are used.
+When you are done, call `koffi.unregister()` (with the value returned by `koffi.register()`) to release the slot. A maximum of 8192 callbacks can exist at the same time. Failure to do so will leak the slot, and subsequent registrations may fail (with an exception) once all slots are used.
 
 The example below shows how to register and unregister delayed callbacks.
 

package/doc/contribute.md

@@ -14,7 +14,7 @@ Start by cloning the repository with [Git](https://git-scm.com/):
 
 ```sh
 git clone https://github.com/Koromix/rygel
-cd rygel/src/koffi
+cd rygel
 ```
 
 As said before, this is a monorepository containg multiple projects, hence the name.
@@ -30,8 +30,8 @@ First, make sure the following dependencies are met:
 Once this is done, run this command _from the test or the benchmark directory_ (depending on what you want to build):
 
 ```sh
-cd koffi/test # or cd koffi/benchmark
-node ../../cnoke/cnoke.js
+cd src/koffi
+node ../cnoke/cnoke.js
 ```
 
 ### Other platforms
@@ -46,11 +46,31 @@ Make sure the following dependencies are met:
 Once this is done, run this command _from the test or the benchmark directory_ (depending on what you want to build):
 
 ```sh
-cd koffi/test # or cd koffi/benchmark
+cd src/koffi
+node ../cnoke/cnoke.js
+```
+
+## Run tests
+
+### On your machine
+
+Once Koffi is built, you can build the tests and run them with the following commands:
+
+```sh
+cd src/koffi/test
 node ../../cnoke/cnoke.js
+
+node sync.js # Run synchronous unit tests
+node async.js # Run asynchronous unit tests
+node callbacks.js # Run callback unit tests
+node union.js # Run union unit tests
+node win32.js # Run Windows-specific unit tests (only on Windows)
+
+node sqlite.js # Run SQLite integration tests
+node raylib.js # Run Raylib integration tests
 ```
 
-## Running tests
+### On virtual machines
 
 Koffi is tested on multiple architectures using emulated (accelerated when possible) QEMU machines. First, you need to install qemu packages, such as `qemu-system` (or even `qemu-system-gui`) on Ubuntu.
 
@@ -59,7 +79,7 @@ These machines are not included directly in this repository (for license and siz
 For example, if you want to run the tests on Debian ARM64, run the following commands:
 
 ```sh
-cd rygel/src/koffi/tools/
+cd src/koffi/tools/
 wget -q -O- https://koromix.dev/files/machines/qemu_debian_arm64.tar.zst | zstd -d | tar xv
 sha256sum -c --ignore-missing registry/sha256sum.txt
 ```
@@ -107,6 +127,18 @@ Each machine is configured to run a VNC server available locally, which you can
 node qemu.js info debian_x64
 ```
 
+## Making a release
+
+First, change the version numbers in `package.json`. Then publish a new release with the following commands:
+
+```sh
+node qemu.js test
+node qemu.js dist
+
+cd build/dist
+npm publish
+```
+
 ## Code style
 
 Koffi is programmed in a mix of C++ and assembly code (architecture-specific code). It uses [node-addon-api](https://github.com/nodejs/node-addon-api) (C++ N-API wrapper) to interact with Node.js.

package/doc/input.md

@@ -56,26 +56,26 @@ let struct1 = koffi.struct({ dummy: 'long' });
 let struct2 = koffi.struct({ dummy: koffi.types.long });
 ```
 
-### Endian-sensitive types
+### Endian-sensitive integers
 
 *New in Koffi 2.1*
 
 Koffi defines a bunch of endian-sensitive types, which can be used when dealing with binary data (network payloads, binary file formats, etc.).
 
-JS type          | C type                 | Bytes | Signedness | Endianness
----------------- | ---------------------- | ----- | ---------- | -------------
-Number (integer) | int16_le, int16_le_t   | 2     | Signed     | Little Endian
-Number (integer) | int16_be, int16_be_t   | 2     | Signed     | Big Endian
-Number (integer) | uint16_le, uint16_le_t | 2     | Unsigned   | Little Endian
-Number (integer) | uint16_be, uint16_be_t | 2     | Unsigned   | Big Endian
-Number (integer) | int32_le, int32_le_t   | 4     | Signed     | Little Endian
-Number (integer) | int32_be, int32_be_t   | 4     | Signed     | Big Endian
-Number (integer) | uint32_le, uint32_le_t | 4     | Unsigned   | Little Endian
-Number (integer) | uint32_be, uint32_be_t | 4     | Unsigned   | Big Endian
-Number (integer) | int64_le, int64_le_t   | 8     | Signed     | Little Endian
-Number (integer) | int64_be, int64_be_t   | 8     | Signed     | Big Endian
-Number (integer) | uint64_le, uint64_le_t | 8     | Unsigned   | Little Endian
-Number (integer) | uint64_be, uint64_be_t | 8     | Unsigned   | Big Endian
+C type                 | Bytes | Signedness | Endianness
+---------------------- | ----- | ---------- | -------------
+int16_le, int16_le_t   | 2     | Signed     | Little Endian
+int16_be, int16_be_t   | 2     | Signed     | Big Endian
+uint16_le, uint16_le_t | 2     | Unsigned   | Little Endian
+uint16_be, uint16_be_t | 2     | Unsigned   | Big Endian
+int32_le, int32_le_t   | 4     | Signed     | Little Endian
+int32_be, int32_be_t   | 4     | Signed     | Big Endian
+uint32_le, uint32_le_t | 4     | Unsigned   | Little Endian
+uint32_be, uint32_be_t | 4     | Unsigned   | Big Endian
+int64_le, int64_le_t   | 8     | Signed     | Little Endian
+int64_be, int64_be_t   | 8     | Signed     | Big Endian
+uint64_le, uint64_le_t | 8     | Unsigned   | Little Endian
+uint64_be, uint64_be_t | 8     | Unsigned   | Big Endian
 
 ## Struct types
 

package/doc/misc.md

@@ -106,9 +106,9 @@ You can use `koffi.stats()` to get a few statistics related to Koffi.
 
 *New in Koffi 2.3.14*
 
-You can use `koffi.errno()` to the current errno value, and `koffi.errno(value)` to change it.
+You can use `koffi.errno()` to get the current errno value, and `koffi.errno(value)` to change it.
 
-The standard POSIX error codes are available in `koffi.os.errno`, as in the example below:
+The standard POSIX error codes are available in `koffi.os.errno`, as shown below:
 
 ```js
 const assert = require('assert');
@@ -138,5 +138,5 @@ You can use `koffi.reset()` to clear some Koffi internal state such as:
 This function is mainly intended for test code, when you execute the same code over and over and you need to reuse type names.
 
 ```{warning}
-Trying to use a function or a type defined before the reset is undefined behavior and will likely lead to a crash!
+Trying to use a function or a type that was initially defined before the reset is undefined behavior and will likely lead to a crash!
 ```

package/package.json

@@ -1,7 +1,7 @@
 {
     "name": "koffi",
-    "version": "2.5.19",
-    "stable": "2.5.19",
+    "version": "2.5.21-beta.1",
+    "stable": "2.5.20",
     "description": "Fast and simple C FFI (foreign function interface) for Node.js",
     "keywords": [
         "foreign",

package/src/index.js

@@ -378,8 +378,8 @@ var require_package = __commonJS({
   "build/dist/src/koffi/package.json"(exports2, module2) {
     module2.exports = {
       name: "koffi",
-      version: "2.5.19",
-      stable: "2.5.19",
+      version: "2.5.21-beta.1",
+      stable: "2.5.20",
       description: "Fast and simple C FFI (foreign function interface) for Node.js",
       keywords: [
         "foreign",
@@ -413,6 +413,7 @@ var require_package = __commonJS({
         chalk: "^4.1.2",
         esbuild: "^0.19.2",
         "ffi-napi": "^4.0.3",
+        "ffi-rs": "^1.0.12",
         minimatch: "^5.0.1",
         "node-ssh": "^12.0.3",
         raylib: "^0.9.2",

package/src/koffi/CMakeLists.txt

@@ -66,6 +66,7 @@ set(KOFFI_SRC
     src/ffi.cc
     src/parser.cc
     src/util.cc
+    src/win32.cc
     ../../src/core/libcc/libcc.cc
 )
 if(CMAKE_SIZEOF_VOID_P EQUAL 8)

package/src/koffi/src/call.cc

@@ -1236,16 +1236,8 @@ void CallData::PopOutArguments()
 
 void *CallData::ReserveTrampoline(const FunctionInfo *proto, Napi::Function func)
 {
-    if (!instance->broker) {
-        if (napi_create_threadsafe_function(env, nullptr, nullptr,
-                                            Napi::String::New(env, "Koffi Async Callback Broker"),
-                                            0, 1, nullptr, nullptr, nullptr,
-                                            CallData::RelayAsync, &instance->broker) != napi_ok) {
-            LogError("Failed to create async callback broker");
-            return nullptr;
-        }
-        napi_unref_threadsafe_function(env, instance->broker);
-    }
+    if (!InitAsyncBroker(env, instance)) [[unlikely]]
+        return nullptr;
 
     int16_t idx;
     {

package/src/koffi/src/ffi.cc

@@ -1568,6 +1568,32 @@ static Napi::Value UnloadLibrary(const Napi::CallbackInfo &info)
     return env.Undefined();
 }
 
+#ifdef _WIN32
+static HANDLE LoadWindowsLibrary(Napi::Env env, const char16_t *filename)
+{
+    HANDLE module = LoadLibraryW((LPCWSTR)filename);
+
+    if (!module) {
+        if (GetLastError() == ERROR_BAD_EXE_FORMAT) {
+            int process = GetSelfMachine();
+            int dll = GetDllMachine(filename);
+
+            if (process >= 0 && dll >= 0 && dll != process) {
+                ThrowError<Napi::Error>(env, "Cannot load '%1' DLL in '%2' process",
+                                        WindowsMachineNames.FindValue(dll, "Unknown"),
+                                        WindowsMachineNames.FindValue(process, "Unknown"));
+                return nullptr;
+            }
+        }
+
+        ThrowError<Napi::Error>(env, "Failed to load shared library: %1", GetWin32ErrorString());
+        return nullptr;
+    }
+
+    return module;
+}
+#endif
+
 static Napi::Value LoadSharedLibrary(const Napi::CallbackInfo &info)
 {
     Napi::Env env = info.Env();
@@ -1592,12 +1618,10 @@ static Napi::Value LoadSharedLibrary(const Napi::CallbackInfo &info)
 #ifdef _WIN32
     if (info[0].IsString()) {
         std::u16string filename = info[0].As<Napi::String>();
-        module = LoadLibraryW((LPCWSTR)filename.c_str());
+        module = LoadWindowsLibrary(env, filename.c_str());
 
-        if (!module) {
-            ThrowError<Napi::Error>(env, "Failed to load shared library: %1", GetWin32ErrorString());
+        if (!module)
             return env.Null();
-        }
     } else {
         module = GetModuleHandle(nullptr);
         RG_ASSERT(module);
@@ -1656,6 +1680,9 @@ static Napi::Value RegisterCallback(const Napi::CallbackInfo &info)
     Napi::Env env = info.Env();
     InstanceData *instance = env.GetInstanceData<InstanceData>();
 
+    if (!InitAsyncBroker(env, instance)) [[unlikely]]
+        return env.Null();
+
     bool has_recv = (info.Length() >= 3 && info[1].IsFunction());
 
     if (info.Length() < 2u + has_recv) {
@@ -1937,6 +1964,22 @@ InstanceMemory::~InstanceMemory()
 #endif
 }
 
+bool InitAsyncBroker(Napi::Env env, InstanceData *instance)
+{
+    if (!instance->broker) {
+        if (napi_create_threadsafe_function(env, nullptr, nullptr,
+                                            Napi::String::New(env, "Koffi Async Callback Broker"),
+                                            0, 1, nullptr, nullptr, nullptr,
+                                            CallData::RelayAsync, &instance->broker) != napi_ok) {
+            LogError("Failed to create async callback broker");
+            return false;
+        }
+        napi_unref_threadsafe_function(env, instance->broker);
+    }
+
+    return true;
+}
+
 static void RegisterPrimitiveType(Napi::Env env, Napi::Object map, std::initializer_list<const char *> names,
                                   PrimitiveKind primitive, int32_t size, int16_t align, const char *ref = nullptr)
 {

package/src/koffi/src/ffi.hh

@@ -341,4 +341,6 @@ Napi::Value TranslateNormalCall(const Napi::CallbackInfo &info);
 Napi::Value TranslateVariadicCall(const Napi::CallbackInfo &info);
 Napi::Value TranslateAsyncCall(const Napi::CallbackInfo &info);
 
+bool InitAsyncBroker(Napi::Env env, InstanceData *instance);
+
 }

package/src/koffi/src/win32.cc

@@ -0,0 +1,155 @@
+// Copyright 2023 Niels Martignène <niels.martignene@protonmail.com>
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy of
+// this software and associated documentation files (the “Software”), to deal in 
+// the Software without restriction, including without limitation the rights to use,
+// copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
+// Software, and to permit persons to whom the Software is furnished to do so,
+// subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND,
+// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+// OTHER DEALINGS IN THE SOFTWARE.
+
+#ifdef _WIN32
+
+#include "win32.hh"
+
+#ifndef NOMINMAX
+    #define NOMINMAX
+#endif
+#ifndef WIN32_LEAN_AND_MEAN
+    #define WIN32_LEAN_AND_MEAN
+#endif
+#include <windows.h>
+#include <ntsecapi.h>
+#include <processthreadsapi.h>
+
+namespace RG {
+
+const HashMap<int, const char *> WindowsMachineNames = {
+    { 0x184, "Alpha AXP, 32-bit" },
+    { 0x284, "Alpha 64" },
+    { 0x1d3, "Matsushita AM33" },
+    { 0x8664, "AMD x64" },
+    { 0x1c0, "ARM little endian" },
+    { 0xaa64, "ARM64 little endian" },
+    { 0x1c4, "ARM Thumb-2 little endian" },
+    { 0x284, "AXP 64" },
+    { 0xebc, "EFI byte code" },
+    { 0x14c, "Intel 386+" },
+    { 0x200, "Intel Itanium" },
+    { 0x6232, "LoongArch 32-bit" },
+    { 0x6264, "LoongArch 64-bit" },
+    { 0x9041, "Mitsubishi M32R little endian" },
+    { 0x266, "MIPS16" },
+    { 0x366, "MIPS with FPU" },
+    { 0x466, "MIPS16 with FPU" },
+    { 0x1f0, "Power PC little endian" },
+    { 0x1f1, "Power PC with FP support" },
+    { 0x166, "MIPS little endian" },
+    { 0x5032, "RISC-V 32-bit" },
+    { 0x5064, "RISC-V 64-bit" },
+    { 0x5128, "RISC-V 128-bit" },
+    { 0x1a2, "Hitachi SH3" },
+    { 0x1a3, "Hitachi SH3 DSP" },
+    { 0x1a6, "Hitachi SH4" },
+    { 0x1a8, "Hitachi SH5" },
+    { 0x1c2, "Thumb" },
+    { 0x169, "MIPS little-endian WCE v2" }
+};
+
+// Fails silently on purpose
+static bool ReadAt(HANDLE h, int32_t offset, void *buf, int len)
+{
+    OVERLAPPED ov = {};
+    DWORD read;
+
+    ov.Offset = offset & 0x7FFFFFFFu;
+
+    if (offset < 0)
+        return false;
+    if (!ReadFile(h, buf, (DWORD)len, &read, &ov))
+        return false;
+    if (read != len)
+        return false;
+
+    return true;
+}
+
+static int GetFileMachine(HANDLE h, bool check_dll)
+{
+    PE_DOS_HEADER dos = {};
+    PE_NT_HEADERS nt = {};
+
+    if (!ReadAt(h, 0, &dos, RG_SIZE(dos)))
+        goto generic;
+    if (!ReadAt(h, dos.e_lfanew, &nt, RG_SIZE(nt)))
+        goto generic;
+
+    if (dos.e_magic != 0x5A4D) // MZ
+        goto generic;
+    if (nt.Signature != 0x00004550) // PE\0\0
+        goto generic;
+    if (check_dll && !(nt.FileHeader.Characteristics & IMAGE_FILE_DLL))
+        goto generic;
+
+    return (int)nt.FileHeader.Machine;
+
+generic:
+    LogError("Invalid or forbidden %1 file: %2", check_dll ? "DLL" : "executable", GetWin32ErrorString());
+    return -1;
+}
+
+int GetSelfMachine()
+{
+    const char *filename = GetApplicationExecutable();
+
+    HANDLE h;
+    if (IsWin32Utf8()) {
+        h = CreateFileA(filename, GENERIC_READ,
+                        FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
+                        nullptr, OPEN_EXISTING, 0, nullptr);
+    } else {
+        wchar_t filename_w[4096];
+        if (ConvertUtf8ToWin32Wide(filename, filename_w) < 0)
+            return -1;
+
+        h = CreateFileW(filename_w, GENERIC_READ,
+                        FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
+                        nullptr, OPEN_EXISTING, 0, nullptr);
+    }
+    if (h == INVALID_HANDLE_VALUE) {
+        LogError("Cannot open '%1': %2", filename, GetWin32ErrorString());
+        return -1;
+    }
+    RG_DEFER { CloseHandle(h); };
+
+    return GetFileMachine(h, false);
+}
+
+int GetDllMachine(const char16_t *filename)
+{
+    HANDLE h = CreateFileW((LPCWSTR)filename, GENERIC_READ,
+                           FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
+                           nullptr, OPEN_EXISTING, 0, nullptr);
+    if (h == INVALID_HANDLE_VALUE) {
+        LogError("Cannot open '%1': %2", filename, GetWin32ErrorString());
+        return -1;
+    }
+    RG_DEFER { CloseHandle(h); };
+
+    return GetFileMachine(h, true);
+}
+
+}
+
+#endif

package/src/koffi/src/win32.hh

@@ -27,6 +27,45 @@
 
 namespace RG {
 
+struct PE_DOS_HEADER {
+    uint16_t e_magic;
+    uint16_t e_cblp;
+    uint16_t e_cp;
+    uint16_t e_crlc;
+    uint16_t e_cparhdr;
+    uint16_t e_minalloc;
+    uint16_t e_maxalloc;
+    uint16_t e_ss;
+    uint16_t e_sp;
+    uint16_t e_csum;
+    uint16_t e_ip;
+    uint16_t e_cs;
+    uint16_t e_lfarlc;
+    uint16_t e_ovno;
+    uint16_t e_res[4];
+    uint16_t e_oemid;
+    uint16_t e_oeminfo;
+    uint16_t e_res2[10];
+    uint32_t e_lfanew;
+};
+
+struct PE_FILE_HEADER {
+    uint16_t Machine;
+    uint16_t NumberOfSections;
+    uint32_t TimeDateStamp;
+    uint32_t PointerToSymbolTable;
+    uint32_t NumberOfSymbols;
+    uint16_t SizeOfOptionalHeader;
+    uint16_t Characteristics;
+};
+
+struct PE_NT_HEADERS {
+    uint32_t Signature;
+    PE_FILE_HEADER FileHeader;
+
+    // ... OptionalHeader;
+};
+
 #if _WIN64
 
 struct TEB {
@@ -70,4 +109,9 @@ static inline TEB *GetTEB()
     return teb;
 }
 
+extern const HashMap<int, const char *> WindowsMachineNames;
+
+int GetSelfMachine();
+int GetDllMachine(const char16_t *filename);
+
 }