koffi 2.15.0 → 2.16.0-beta.1

package/src/koffi/src/call.hh

@@ -77,8 +77,10 @@ public:
 #if defined(UNITY_BUILD)
     #if defined(_MSC_VER)
         #define INLINE_IF_UNITY __forceinline
-    #else
+    #elif defined(__clang__)
         #define INLINE_IF_UNITY __attribute__((always_inline)) inline
+    #else
+        #define INLINE_IF_UNITY
     #endif
 #else
     #define INLINE_IF_UNITY
@@ -116,7 +118,7 @@ public:
 
 private:
     template <typename T>
-    bool AllocStack(Size size, Size align, T **out_ptr);
+    T *AllocStack(Size size);
     template <typename T = uint8_t>
     T *AllocHeap(Size size, Size align);
 
@@ -126,15 +128,15 @@ private:
 };
 
 template <typename T>
-inline bool CallData::AllocStack(Size size, Size align, T **out_ptr)
+inline T *CallData::AllocStack(Size size)
 {
-    uint8_t *ptr = AlignDown(mem->stack.end() - size, align);
+    uint8_t *ptr = AlignDown(mem->stack.end() - size, 16);
     Size delta = mem->stack.end() - ptr;
 
     // Keep 512 bytes for redzone (required in some ABIs)
     if (mem->stack.len - 512 < delta) [[unlikely]] {
         ThrowError<Napi::Error>(env, "FFI call is taking up too much memory");
-        return false;
+        return nullptr;
     }
 
 #if defined(K_DEBUG)
@@ -143,8 +145,7 @@ inline bool CallData::AllocStack(Size size, Size align, T **out_ptr)
 
     mem->stack.len -= delta;
 
-    *out_ptr = (T *)ptr;
-    return true;
+    return (T *)ptr;
 }
 
 template <typename T>
@@ -176,6 +177,6 @@ inline T *CallData::AllocHeap(Size size, Size align)
     }
 }
 
-void *GetTrampoline(int16_t idx, const FunctionInfo *proto);
+void *GetTrampoline(int16_t idx);
 
 }

package/src/koffi/src/ffi.cc

@@ -138,6 +138,20 @@ static Napi::Value GetSetConfig(const Napi::CallbackInfo &info)
             } else if (key == "max_type_size") {
                 if (!ChangeSize(key.c_str(), value, 32, Mebibytes(512), &new_config.max_type_size))
                     return env.Null();
+            } else if (key == "fast_pointers") {
+                if (!value.IsBoolean()) {
+                    ThrowError<Napi::TypeError>(env, "Unexpected %1 value for '%2', expected boolean", GetValueType(instance, value), key.c_str());
+                    return env.Null();
+                }
+
+                new_config.fast_pointers = value.As<Napi::Boolean>();
+            } else if (key == "fast_callbacks") {
+                if (!value.IsBoolean()) {
+                    ThrowError<Napi::TypeError>(env, "Unexpected %1 value for '%2', expected boolean", GetValueType(instance, value), key.c_str());
+                    return env.Null();
+                }
+
+                new_config.fast_callbacks = value.As<Napi::Boolean>();
             } else {
                 ThrowError<Napi::Error>(env, "Unexpected config member '%1'", key.c_str());
                 return env.Null();
@@ -162,6 +176,8 @@ static Napi::Value GetSetConfig(const Napi::CallbackInfo &info)
     obj.Set("resident_async_pools", instance->config.resident_async_pools);
     obj.Set("max_async_calls", instance->config.resident_async_pools + instance->config.max_temporaries);
     obj.Set("max_type_size", instance->config.max_type_size);
+    obj.Set("fast_pointers", instance->config.fast_pointers);
+    obj.Set("fast_callbacks", instance->config.fast_callbacks);
 
     return obj;
 }
@@ -790,13 +806,10 @@ static Napi::Value CreateDisposableType(const Napi::CallbackInfo &info)
             InstanceData *instance = env.GetInstanceData<InstanceData>();
             const Napi::FunctionReference &ref = type->dispose_ref;
 
-            Napi::External<void> external = Napi::External<void>::New(env, (void *)ptr);
-            SetValueTag(external, type->ref.marker);
+            Napi::Value p = WrapPointer(env, type->ref.type, (void *)ptr);
 
-            Napi::Value self = env.Null();
-            napi_value args[] = {
-                external
-            };
+            napi_value self = env.Null();
+            napi_value args[] = { p };
 
             ref.Call(self, K_LEN(args), args);
             instance->stats.disposed++;
@@ -842,21 +855,12 @@ static inline bool GetExternalPointer(Napi::Env env, Napi::Value value, void **o
 {
     InstanceData *instance = env.GetInstanceData<InstanceData>();
 
-    if (IsNullOrUndefined(value)) {
-        *out_ptr = 0;
-        return true;
-    } else if (value.IsExternal() && !CheckValueTag(value, &TypeInfoMarker) &&
-                                     !CheckValueTag(value, &CastMarker) &&
-                                     !CheckValueTag(value, &MagicUnionMarker)) {
-        Napi::External<void> external = value.As<Napi::External<void>>();
-        void *ptr = external.Data();
-
-        *out_ptr = ptr;
-        return true;
-    } else {
-        ThrowError<Napi::TypeError>(env, "Unexpected %1 value for ptr, expected external pointer", GetValueType(instance, value));
+    if (!TryPointer(value, out_ptr)) {
+        ThrowError<Napi::TypeError>(env, "Unexpected %1 value for ptr, expected pointer", GetValueType(instance, value));
         return false;
     }
+
+    return true;
 }
 
 static Napi::Value CallAlloc(const Napi::CallbackInfo &info)
@@ -902,10 +906,7 @@ static Napi::Value CallAlloc(const Napi::CallbackInfo &info)
         return env.Null();
     }
 
-    Napi::External<void> external = Napi::External<void>::New(env, ptr);
-    SetValueTag(external, type);
-
-    return external;
+    return WrapPointer(env, type, ptr);
 }
 
 static Napi::Value CallFree(const Napi::CallbackInfo &info)
@@ -1167,6 +1168,14 @@ static Napi::Value CreateFunctionType(const Napi::CallbackInfo &info)
     if (!AnalyseFunction(env, instance, func))
         return env.Null();
 
+    // Branchless push loop
+    for (const ParameterInfo &param: func->parameters) {
+        func->primitives.Append(param.type->primitive);
+    }
+    if (!func->variadic) {
+        func->primitives.Append(PrimitiveKind::Prototype);
+    }
+
     // Adjust parameter offsets for koffi.call()
     for (ParameterInfo &param: func->parameters) {
         param.offset += 2;
@@ -1517,62 +1526,105 @@ static Napi::Value TranslateVariadicCall(const FunctionInfo *func, void *native,
     Napi::Env env = info.Env();
     InstanceData *instance = env.GetInstanceData<InstanceData>();
 
-    FunctionInfo copy;
-    memcpy((void *)&copy, func, K_SIZE(*func));
-    copy.lib = nullptr;
+    FunctionInfo *variadic = nullptr;
+    K_DEFER_N(err_guard) { delete variadic; };
 
-    // This makes variadic calls non-reentrant
-    K_DEFER_C(len = copy.parameters.len) {
-        copy.parameters.RemoveFrom(len);
-        copy.parameters.Leak();
-    };
+    // Try cached function
+    {
+        FunctionInfo *prev = instance->variadic_func;
 
-    if (info.Length() < (uint32_t)copy.required_parameters) [[unlikely]] {
-        ThrowError<Napi::TypeError>(env, "Expected %1 arguments or more, got %2", copy.parameters.len, info.Length());
-        return env.Null();
-    }
-    if ((info.Length() - copy.required_parameters) % 2) [[unlikely]] {
-        ThrowError<Napi::Error>(env, "Missing value argument for variadic call");
-        return env.Null();
+        if (prev && prev->native == native) {
+            Size specified = (info.Length() - prev->required_parameters);
+            Size processed = (prev->parameters.len - prev->required_parameters) * 2;
+
+            if (specified == processed) {
+                bool match = true;
+
+                for (Size i = prev->required_parameters, j = prev->required_parameters; i < (Size)info.Length(); i += 2, j++) {
+                    int directions;
+                    const TypeInfo *type = ResolveType(info[(uint32_t)i], &directions);
+
+                    if (type != prev->parameters[j].type || directions != prev->parameters[j].directions) [[unlikely]] {
+                        match = false;
+                        break;
+                    }
+                }
+
+                if (match) [[likely]] {
+                    variadic = prev;
+
+                    // If an error happens it'll get destroyed, so don't keep it around
+                    instance->variadic_func = nullptr;
+                }
+            }
+        }
     }
 
-    for (Size i = copy.required_parameters; i < (Size)info.Length(); i += 2) {
-        ParameterInfo param = {};
+    if (!variadic) {
+        variadic = new FunctionInfo();
 
-        param.type = ResolveType(info[(uint32_t)i], &param.directions);
+        memcpy((void *)variadic, func, K_SIZE(*func));
+        memset((void *)&variadic->parameters, 0, K_SIZE(variadic->parameters));
+        memset((void *)&variadic->primitives, 0, K_SIZE(variadic->primitives));
 
-        if (!param.type) [[unlikely]]
-            return env.Null();
-        if (!CanPassType(param.type, param.directions)) [[unlikely]] {
-            ThrowError<Napi::TypeError>(env, "Type %1 cannot be used as a parameter", param.type->name);
+        variadic->parameters = func->parameters;
+        variadic->primitives = func->primitives;
+        variadic->lib = nullptr;
+
+        if (info.Length() < (uint32_t)variadic->required_parameters) [[unlikely]] {
+            ThrowError<Napi::TypeError>(env, "Expected %1 arguments or more, got %2", variadic->parameters.len, info.Length());
             return env.Null();
         }
-        if (copy.parameters.len >= MaxParameters) [[unlikely]] {
-            ThrowError<Napi::TypeError>(env, "Functions cannot have more than %1 parameters", MaxParameters);
+        if ((info.Length() - variadic->required_parameters) % 2) [[unlikely]] {
+            ThrowError<Napi::Error>(env, "Missing value argument for variadic call");
             return env.Null();
         }
-        if ((param.directions & 2) && ++copy.out_parameters >= MaxParameters) [[unlikely]] {
-            ThrowError<Napi::TypeError>(env, "Functions cannot have more than %1 output parameters", MaxParameters);
-            return env.Null();
+
+        for (Size i = variadic->required_parameters; i < (Size)info.Length(); i += 2) {
+            ParameterInfo param = {};
+
+            param.type = ResolveType(info[(uint32_t)i], &param.directions);
+
+            if (!param.type) [[unlikely]]
+                return env.Null();
+            if (!CanPassType(param.type, param.directions)) [[unlikely]] {
+                ThrowError<Napi::TypeError>(env, "Type %1 cannot be used as a parameter", param.type->name);
+                return env.Null();
+            }
+            if (variadic->parameters.len >= MaxParameters) [[unlikely]] {
+                ThrowError<Napi::TypeError>(env, "Functions cannot have more than %1 parameters", MaxParameters);
+                return env.Null();
+            }
+            if ((param.directions & 2) && ++variadic->out_parameters >= MaxParameters) [[unlikely]] {
+                ThrowError<Napi::TypeError>(env, "Functions cannot have more than %1 output parameters", MaxParameters);
+                return env.Null();
+            }
+
+            param.variadic = true;
+            param.offset = (int8_t)(i + 1);
+
+            variadic->parameters.Append(param);
         }
 
-        param.variadic = true;
-        param.offset = (int8_t)(i + 1);
+        if (!AnalyseFunction(env, instance, variadic)) [[unlikely]]
+            return env.Null();
 
-        copy.parameters.Append(param);
+        // Branchless push loop
+        for (Size i = func->parameters.len; i < variadic->parameters.len; i++) {
+            const ParameterInfo &param = variadic->parameters[i];
+            variadic->primitives.Append(param.type->primitive);
+        }
+        variadic->primitives.Append(PrimitiveKind::Prototype);
     }
 
-    if (!AnalyseFunction(env, instance, &copy)) [[unlikely]]
-        return env.Null();
-
     InstanceMemory *mem = instance->memories[0];
     CallData call(env, instance, mem);
 
-    if (!call.Prepare(&copy, info)) [[unlikely]]
+    if (!call.Prepare(variadic, info)) [[unlikely]]
         return env.Null();
 
     if (instance->debug) {
-        call.DumpForward(&copy);
+        call.DumpForward(variadic);
     }
 
     // Execute call
@@ -1580,10 +1632,17 @@ static Napi::Value TranslateVariadicCall(const FunctionInfo *func, void *native,
         K_DEFER_C(prev_call = exec_call) { exec_call = prev_call; };
         exec_call = &call;
 
-        call.Execute(&copy, native);
+        call.Execute(variadic, native);
     }
 
-    return call.Complete(&copy);
+    if (variadic != instance->variadic_func) {
+        err_guard.Disable();
+
+        delete instance->variadic_func;
+        instance->variadic_func = variadic;
+    }
+
+    return call.Complete(variadic);
 }
 
 Napi::Value TranslateVariadicCall(const Napi::CallbackInfo &info)
@@ -1640,11 +1699,8 @@ void AsyncCall::OnOK()
 
     Napi::FunctionReference &callback = Callback();
 
-    Napi::Value self = env.Null();
-    napi_value args[] = {
-        env.Null(),
-        call.Complete(func)
-    };
+    napi_value self = env.Null();
+    napi_value args[] = { env.Null(), call.Complete(func) };
 
     callback.Call(self, K_LEN(args), args);
 }
@@ -1755,9 +1811,16 @@ static Napi::Value FindLibraryFunction(const Napi::CallbackInfo &info)
 
     if (!AnalyseFunction(env, instance, func))
         return env.Null();
+
+    // Branchless push loop
+    for (const ParameterInfo &param: func->parameters) {
+        func->primitives.Append(param.type->primitive);
+    }
     if (func->variadic) {
-        // Minimize reallocations
         func->parameters.Grow(32);
+        func->primitives.Grow(32);
+    } else {
+        func->primitives.Append(PrimitiveKind::Prototype);
     }
 
 #if defined(_WIN32)
@@ -1822,10 +1885,7 @@ static Napi::Value FindSymbol(const Napi::CallbackInfo &info)
         return env.Null();
     }
 
-    Napi::External<void> external = Napi::External<void>::New(env, ptr);
-    SetValueTag(external, &type);
-
-    return external;
+    return WrapPointer(env, type, ptr);
 }
 
 static Napi::Value UnloadLibrary(const Napi::CallbackInfo &info)
@@ -1992,15 +2052,12 @@ static Napi::Value RegisterCallback(const Napi::CallbackInfo &info)
     }
     trampoline->generation = -1;
 
-    void *ptr = GetTrampoline(idx, type->ref.proto);
-
-    Napi::External<void> external = Napi::External<void>::New(env, ptr);
-    SetValueTag(external, type->ref.marker);
+    void *ptr = GetTrampoline(idx);
 
     // Cache index for fast unregistration
     instance->trampolines_map.Set(ptr, idx);
 
-    return external;
+    return WrapCallback(env, type->ref.type, ptr);
 }
 
 static Napi::Value UnregisterCallback(const Napi::CallbackInfo &info)
@@ -2012,14 +2069,13 @@ static Napi::Value UnregisterCallback(const Napi::CallbackInfo &info)
         ThrowError<Napi::TypeError>(env, "Expected 1 argument, got %1", info.Length());
         return env.Null();
     }
-    if (!info[0].IsExternal()) {
+
+    void *ptr;
+    if (!TryPointer(info[0], &ptr)) {
         ThrowError<Napi::TypeError>(env, "Unexpected %1 value for id, expected registered callback", GetValueType(instance, info[0]));
         return env.Null();
     }
 
-    Napi::External<void> external = info[0].As<Napi::External<void>>();
-    void *ptr = external.Data();
-
     int16_t idx;
     {
         int16_t *it = instance->trampolines_map.Find(ptr);
@@ -2375,10 +2431,8 @@ static void RegisterPrimitiveType(Napi::Env env, Napi::Object map, std::initiali
     }
 
     if (ref) {
-        const TypeInfo *marker = instance->types_map.FindValue(ref, nullptr);
-        K_ASSERT(marker);
-
-        type->ref.marker = marker;
+        type->ref.type = instance->types_map.FindValue(ref, nullptr);
+        K_ASSERT(type->ref.type);
     }
 
     Napi::Value wrapper = WrapType(env, type);
@@ -2582,7 +2636,8 @@ static Napi::Object InitModule(Napi::Env env, Napi::Object exports)
         instance->str16_type = instance->types_map.FindValue("char16_t *", nullptr);
         instance->str32_type = instance->types_map.FindValue("char32_t *", nullptr);
 
-        instance->active_symbol = Napi::Symbol::New(env, "active");
+        Napi::Symbol symbol = Napi::Symbol::New(env, "active");
+        instance->active_symbol.Reset(symbol, 1);
 
         instance->base_types_count = instance->types.count;
     }
@@ -2592,10 +2647,7 @@ static Napi::Object InitModule(Napi::Env env, Napi::Object exports)
         Napi::Object node = Napi::Object::New(env);
         exports.Set("node", node);
 
-        Napi::External<void> external = Napi::External<void>::New(env, (napi_env)env);
-        SetValueTag(external, instance->void_type);
-
-        node.Set("env", external);
+        node.Set("env", WrapPointer(env, instance->void_type, (napi_env)env));
 
         node.Set("poll", Napi::Function::New(env, &Poll, "poll"));
         node.Set("PollHandle", PollHandle::Define(env));
@@ -2608,6 +2660,8 @@ static Napi::Object InitModule(Napi::Env env, Napi::Object exports)
 
 InstanceData::~InstanceData()
 {
+    delete variadic_func;
+
     for (InstanceMemory *mem: memories) {
         delete mem;
     }

package/src/koffi/src/ffi.hh

@@ -22,64 +22,12 @@ static const Size MaxParameters = 64;
 static const Size MaxTrampolines = 8192;
 
 enum class PrimitiveKind {
-    // Void is explictly not first so that it is not 0, for reasons related to N-API type tags.
-    // Look at TypeInfo definition for more information!
-    Bool,
-    Void,
-    Int8,
-    UInt8,
-    Int16,
-    Int16S,  // Keep behind Int16
-    UInt16,
-    UInt16S, // Keep behind UInt16
-    Int32,
-    Int32S,  // Keep behind Int32
-    UInt32,
-    UInt32S, // Keep behind UInt32
-    Int64,
-    Int64S,  // Keep behind Int64
-    UInt64,
-    UInt64S, // Keep behind UInt64
-    String,
-    String16,
-    String32,
-    Pointer,
-    Record,
-    Union,
-    Array,
-    Float32,
-    Float64,
-    Prototype,
-    Callback
+    #define PRIMITIVE(Name) Name,
+    #include "primitives.inc"
 };
 static const char *const PrimitiveKindNames[] = {
-    "Bool",
-    "Void",
-    "Int8",
-    "UInt8",
-    "Int16",
-    "Int16S",
-    "UInt16",
-    "UInt16S",
-    "Int32",
-    "Int32S",
-    "UInt32",
-    "UInt32S",
-    "Int64",
-    "Int64S",
-    "UInt64",
-    "UInt64S",
-    "String",
-    "String16",
-    "String32",
-    "Pointer",
-    "Record",
-    "Union",
-    "Array",
-    "Float32",
-    "Float64",
-    "Prototype",
-    "Callback"
+    #define PRIMITIVE(Name) K_STRINGIFY(Name),
+    #include "primitives.inc"
 };
 
 struct TypeInfo;
@@ -109,7 +57,7 @@ static const char *const ArrayHintNames[] = {
 struct TypeInfo {
     const char *name;
 
-    // Make sure primitie ends up as the upper N-API tag value when we cast TypeInfo pointers to
+    // Make sure primitive ends up as the upper N-API tag value when we cast TypeInfo pointers to
     // napi_type_tag pointers. Yes, I want to do this. We don't do strict aliasing here.
     // N.B. Some node versions don't like when one of the two tag values is 0, so make sure
     // this does not happen! It would happen if primitive is 0 and size is 0. To avoid this
@@ -124,7 +72,6 @@ struct TypeInfo {
 
     HeapArray<RecordMember> members; // Record only
     union {
-        const void *marker;
         const TypeInfo *type; // Pointer or array
         const FunctionInfo *proto; // Callback only
     } ref;
@@ -223,6 +170,7 @@ struct FunctionInfo {
 
     ParameterInfo ret;
     HeapArray<ParameterInfo> parameters;
+    HeapArray<PrimitiveKind> primitives;
     int8_t required_parameters;
     int8_t out_parameters;
     bool variadic;
@@ -274,12 +222,15 @@ struct InstanceData {
     const TypeInfo *str16_type;
     const TypeInfo *str32_type;
 
-    Napi::Symbol active_symbol;
+    Napi::Reference<Napi::Symbol> active_symbol;
 
     std::mutex mem_mutex;
     LocalArray<InstanceMemory *, 17> memories;
     int temporaries = 0;
 
+    // Variadic cache
+    FunctionInfo *variadic_func = nullptr;
+
     std::thread::id main_thread_id;
     napi_threadsafe_function broker = nullptr;
 
@@ -305,6 +256,9 @@ struct InstanceData {
         int resident_async_pools = DefaultResidentAsyncPools;
         int max_temporaries = DefaultMaxAsyncCalls - DefaultResidentAsyncPools;
         Size max_type_size = DefaultMaxTypeSize;
+
+        bool fast_pointers = false;
+        bool fast_callbacks = false;
     } config;
 
     struct {

package/src/koffi/src/primitives.inc

@@ -0,0 +1,39 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+// SPDX-FileCopyrightText: 2025 Niels Martignène <niels.martignene@protonmail.com>
+
+#if !defined(PRIMITIVE)
+    #error Please define PRIMITIVE() before including primitives.inc
+#endif
+
+// Void is explictly not first so that it is not 0, for reasons related to N-API type tags.
+// Look at TypeInfo definition for more information!
+
+PRIMITIVE(Bool)
+PRIMITIVE(Void)
+PRIMITIVE(Int8)
+PRIMITIVE(UInt8)
+PRIMITIVE(Int16)
+PRIMITIVE(Int16S)  // Keep behind Int16
+PRIMITIVE(UInt16)
+PRIMITIVE(UInt16S) // Keep behind UInt16
+PRIMITIVE(Int32)
+PRIMITIVE(Int32S)  // Keep behind Int32
+PRIMITIVE(UInt32)
+PRIMITIVE(UInt32S) // Keep behind UInt32
+PRIMITIVE(Int64)
+PRIMITIVE(Int64S)  // Keep behind Int64
+PRIMITIVE(UInt64)
+PRIMITIVE(UInt64S) // Keep behind UInt64
+PRIMITIVE(String)
+PRIMITIVE(String16)
+PRIMITIVE(String32)
+PRIMITIVE(Pointer)
+PRIMITIVE(Record)
+PRIMITIVE(Union)
+PRIMITIVE(Array)
+PRIMITIVE(Float32)
+PRIMITIVE(Float64)
+PRIMITIVE(Callback)
+PRIMITIVE(Prototype)
+
+#undef PRIMITIVE