koffi 2.2.2-beta.5 → 2.2.2

package/src/koffi/src/call.cc

@@ -53,19 +53,20 @@ CallData::~CallData()
     mem->stack = old_stack_mem;
     mem->heap = old_heap_mem;
 
-    if (used_trampolines) {
+    if (used_trampolines.len) {
         std::lock_guard<std::mutex> lock(shared.mutex);
 
-        for (Size i = 0; i < MaxTrampolines; i++) {
-            if (used_trampolines & (1u << i)) {
-                TrampolineInfo *trampoline = &shared.trampolines[i];
+        for (Size i = used_trampolines.len - 1; i >= 0; i--) {
+            int16_t idx = used_trampolines[i];
+            TrampolineInfo *trampoline = &shared.trampolines[idx];
 
-                trampoline->func.Reset();
-                trampoline->recv.Reset();
-            }
-        }
+            RG_ASSERT(!trampoline->func.IsEmpty());
+
+            trampoline->func.Reset();
+            trampoline->recv.Reset();
 
-        shared.temp_trampolines ^= used_trampolines;
+            shared.available.Append(idx);
+        }
     }
 
     instance->temporaries -= mem->temporary;
@@ -80,6 +81,9 @@ CallData::~CallData()
 void CallData::RelaySafe(Size idx, uint8_t *own_sp, uint8_t *caller_sp, BackRegisters *out_reg)
 {
     if (std::this_thread::get_id() != instance->main_thread_id) {
+        // JS/V8 is single-threaded, and runs on main_thread_id. Forward the call
+        // to the JS event loop.
+
         RelayContext ctx;
 
         ctx.call = this;
@@ -1025,19 +1029,21 @@ void CallData::PopOutArguments()
 
 void *CallData::ReserveTrampoline(const FunctionInfo *proto, Napi::Function func)
 {
-    int idx;
+    int16_t idx;
     {
         std::lock_guard<std::mutex> lock(shared.mutex);
 
-        idx = CountTrailingZeros(~shared.temp_trampolines);
-
-        if (RG_UNLIKELY(idx >= MaxTrampolines)) {
-            ThrowError<Napi::Error>(env, "Too many temporary callbacks are in use (max = %1)", MaxTrampolines);
+        if (RG_UNLIKELY(!shared.available.len)) {
+            ThrowError<Napi::Error>(env, "Too many callbacks are in use (max = %1)", MaxTrampolines);
+            return env.Null();
+        }
+        if (RG_UNLIKELY(!used_trampolines.Available())) {
+            ThrowError<Napi::Error>(env, "This call uses too many temporary callbacks (max = %1)", RG_LEN(used_trampolines.data));
             return env.Null();
         }
 
-        shared.temp_trampolines |= 1u << idx;
-        used_trampolines |= 1u << idx;
+        idx = shared.available.data[--shared.available.len];
+        used_trampolines.Append(idx);
     }
 
     TrampolineInfo *trampoline = &shared.trampolines[idx];
@@ -1048,6 +1054,7 @@ void *CallData::ReserveTrampoline(const FunctionInfo *proto, Napi::Function func
     trampoline->generation = (int32_t)mem->generation;
 
     void *ptr = GetTrampoline(idx, proto);
+
     return ptr;
 }
 

package/src/koffi/src/call.hh

@@ -41,8 +41,7 @@ class alignas(8) CallData {
     Span<uint8_t> old_stack_mem;
     Span<uint8_t> old_heap_mem;
 
-    uint32_t used_trampolines = 0;
-
+    LocalArray<int16_t, 16> used_trampolines;
     LocalArray<OutArgument, MaxOutParameters> out_arguments;
 
     uint8_t *new_sp;
@@ -112,7 +111,6 @@ private:
 
     void *ReserveTrampoline(const FunctionInfo *proto, Napi::Function func);
 };
-RG_STATIC_ASSERT(MaxTrampolines <= 32);
 
 template <typename T>
 inline bool CallData::AllocStack(Size size, Size align, T **out_ptr)
@@ -165,6 +163,6 @@ inline T *CallData::AllocHeap(Size size, Size align)
     }
 }
 
-void *GetTrampoline(Size idx, const FunctionInfo *proto);
+void *GetTrampoline(int16_t idx, const FunctionInfo *proto);
 
 }

package/src/koffi/src/ffi.cc

@@ -1001,6 +1001,8 @@ static Napi::Value GetTypeDefinition(const Napi::CallbackInfo &info)
 
 static InstanceMemory *AllocateMemory(InstanceData *instance, Size stack_size, Size heap_size)
 {
+    std::lock_guard<std::mutex> lock(instance->memories_mutex);
+
     for (Size i = 1; i < instance->memories.len; i++) {
         InstanceMemory *mem = instance->memories[i];
 
@@ -1063,9 +1065,6 @@ static Napi::Value TranslateNormalCall(const Napi::CallbackInfo &info)
     InstanceMemory *mem = instance->memories[0];
     CallData call(env, instance, mem);
 
-    RG_DEFER_C(prev_call = exec_call) { exec_call = prev_call; };
-    exec_call = &call;
-
     if (!RG_UNLIKELY(call.Prepare(func, info)))
         return env.Null();
 
@@ -1111,7 +1110,7 @@ static Napi::Value TranslateVariadicCall(const Napi::CallbackInfo &info)
     for (Size i = func.parameters.len; i < (Size)info.Length(); i += 2) {
         ParameterInfo param = {};
 
-        param.type = ResolveType(info[i], &param.directions);
+        param.type = ResolveType(info[(uint32_t)i], &param.directions);
 
         if (RG_UNLIKELY(!param.type))
             return env.Null();
@@ -1436,19 +1435,16 @@ static Napi::Value RegisterCallback(const Napi::CallbackInfo &info)
         return env.Null();
     }
 
-    int idx;
+    int16_t idx;
     {
         std::lock_guard<std::mutex> lock(shared.mutex);
 
-        idx = CountTrailingZeros(~shared.registered_trampolines);
-
-        if (RG_UNLIKELY(idx >= MaxTrampolines)) {
-            ThrowError<Napi::Error>(env, "Too many registered callbacks are in use (max = %1)", MaxTrampolines);
+        if (RG_UNLIKELY(!shared.available.len)) {
+            ThrowError<Napi::Error>(env, "Too many callbacks are in use (max = %1)", MaxTrampolines);
             return env.Null();
         }
 
-        shared.registered_trampolines |= 1u << idx;
-        idx += MaxTrampolines;
+        idx = shared.available.data[--shared.available.len];
     }
 
     TrampolineInfo *trampoline = &shared.trampolines[idx];
@@ -1467,6 +1463,9 @@ static Napi::Value RegisterCallback(const Napi::CallbackInfo &info)
     Napi::External<void> external = Napi::External<void>::New(env, ptr);
     SetValueTag(instance, external, type->ref.marker);
 
+    // Cache index for fast unregistration
+    instance->trampolines_map.Set(ptr, idx);
+
     return external;
 }
 
@@ -1487,31 +1486,33 @@ static Napi::Value UnregisterCallback(const Napi::CallbackInfo &info)
     Napi::External<void> external = info[0].As<Napi::External<void>>();
     void *ptr = external.Data();
 
-    // Release shared trampoline safely
+    int16_t idx;
     {
-        std::lock_guard<std::mutex> lock(shared.mutex);
+        int16_t *it = instance->trampolines_map.Find(ptr);
 
-        for (Size i = 0; i < MaxTrampolines; i++) {
-            Size idx = i + MaxTrampolines;
+        if (RG_UNLIKELY(!it)) {
+            ThrowError<Napi::Error>(env, "Could not find matching registered callback");
+            return env.Null();
+        }
 
-            if (!(shared.registered_trampolines & (1u << i)))
-                continue;
+        idx = *it;
+        instance->trampolines_map.Remove(it);
+    }
 
-            TrampolineInfo *trampoline = &shared.trampolines[idx];
+    // Release shared trampoline safely
+    {
+        std::lock_guard<std::mutex> lock(shared.mutex);
 
-            if (GetTrampoline(idx, trampoline->proto) == ptr) {
-                shared.registered_trampolines &= ~(1u << i);
+        TrampolineInfo *trampoline = &shared.trampolines[idx];
+        RG_ASSERT(!trampoline->func.IsEmpty());
 
-                trampoline->func.Reset();
-                trampoline->recv.Reset();
+        trampoline->func.Reset();
+        trampoline->recv.Reset();
 
-                return env.Undefined();
-            }
-        }
+        shared.available.Append(idx);
     }
 
-    ThrowError<Napi::Error>(env, "Could not find matching registered callback");
-    return env.Null();
+    return env.Undefined();
 }
 
 LibraryHolder::~LibraryHolder()
@@ -1709,18 +1710,16 @@ InstanceData::~InstanceData()
         delete mem;
     }
 
-    // Clean-up leftover registered trampolines
+    // Clean-up leftover trampoline references
     {
         std::lock_guard<std::mutex> lock(shared.mutex);
 
-        for (Size i = 0; i < MaxTrampolines; i++) {
-            Size idx = i + MaxTrampolines;
+        for (int16_t idx = 0; idx < MaxTrampolines; idx++) {
+            TrampolineInfo *trampoline = &shared.trampolines[idx];
 
-            if (!(shared.registered_trampolines & (1u << i)))
+            if (trampoline->func.IsEmpty())
                 continue;
 
-            TrampolineInfo *trampoline = &shared.trampolines[idx];
-
             if (trampoline->func.Env().GetInstanceData<InstanceData>() == this) {
                 trampoline->func.Reset();
                 trampoline->recv.Reset();
@@ -1891,6 +1890,9 @@ extern "C" void RelayCallback(Size idx, uint8_t *own_sp, uint8_t *caller_sp, Bac
     if (RG_LIKELY(exec_call)) {
         exec_call->RelaySafe(idx, own_sp, caller_sp, out_reg);
     } else {
+        // This happens if the callback pointer is called from a different thread
+        // than the one that runs the FFI call (sync or async).
+
         TrampolineInfo *trampoline = &shared.trampolines[idx];
 
         Napi::Env env = trampoline->func.Env();

package/src/koffi/src/ffi.hh

@@ -30,7 +30,7 @@ static const Size DefaultMaxTypeSize = Mebibytes(64);
 static const int MaxAsyncCalls = 256;
 static const Size MaxParameters = 32;
 static const Size MaxOutParameters = 16;
-static const Size MaxTrampolines = 16;
+static const Size MaxTrampolines = 1024;
 
 extern const int TypeInfoMarker;
 extern const int CastMarker;
@@ -247,12 +247,15 @@ struct InstanceData {
     const TypeInfo *char_type;
     const TypeInfo *char16_type;
 
+    std::mutex memories_mutex;
     LocalArray<InstanceMemory *, 9> memories;
     int temporaries = 0;
 
     std::thread::id main_thread_id;
     napi_threadsafe_function broker = nullptr;
 
+    HashMap<void *, int16_t> trampolines_map;
+
     BlockAllocator str_alloc;
 
     Size sync_stack_size = DefaultSyncStackSize;
@@ -266,7 +269,6 @@ struct InstanceData {
 RG_STATIC_ASSERT(DefaultResidentAsyncPools <= RG_LEN(InstanceData::memories.data) - 1);
 RG_STATIC_ASSERT(DefaultMaxAsyncCalls >= DefaultResidentAsyncPools);
 RG_STATIC_ASSERT(MaxAsyncCalls >= DefaultMaxAsyncCalls);
-RG_STATIC_ASSERT(MaxTrampolines <= 16);
 
 struct TrampolineInfo {
     const FunctionInfo *proto;
@@ -279,10 +281,19 @@ struct TrampolineInfo {
 struct SharedData {
     std::mutex mutex;
 
-    TrampolineInfo trampolines[MaxTrampolines * 2];
-    uint32_t temp_trampolines = 0;
-    uint32_t registered_trampolines = 0;
+    TrampolineInfo trampolines[MaxTrampolines];
+    LocalArray<int16_t, MaxTrampolines> available;
+
+    SharedData()
+    {
+        available.len = MaxTrampolines;
+
+        for (int16_t i = 0; i < MaxTrampolines; i++) {
+            available[i] = i;
+        }
+    }
 };
+RG_STATIC_ASSERT(MaxTrampolines <= INT16_MAX);
 
 extern SharedData shared;
 

package/src/koffi/test/callbacks.js

@@ -75,8 +75,7 @@ async function test() {
     const ApplyStruct = lib.func('int ApplyStruct(int x, StructCallbacks callbacks)');
     const SetIndirect = lib.func('void SetIndirect(IntCallback *func)');
     const CallIndirect = lib.func('int CallIndirect(int x)');
-    const CallIndirectThreaded = lib.func('int CallIndirectThreaded(int x)');
-    const CallDirectThreaded = lib.func('int CallDirectThreaded(IntCallback *func, int x)');
+    const CallThreaded = lib.func('int CallThreaded(IntCallback *func, int x)');
     const MakeVectors = lib.func('int MakeVectors(int len, VectorCallback *func)');
     const CallQSort = lib.func('void CallQSort(_Inout_ void *base, size_t nmemb, size_t size, SortCallback *cb)');
     const CallMeChar = lib.func('int CallMeChar(CharCallback *func)');
@@ -263,16 +262,18 @@ async function test() {
         assert.equal(ret, 97 + 2 * 98);
     }
 
-    // Use temporary callback from secondary thread
-    for (let i = 0; i < 128; i++)
-        assert.equal(await util.promisify(CallDirectThreaded.async)(x => -x - 2, 27), -29);
-
-    // Use registered callback from secondary thread
-    for (let i = 0; i < 128; i++) {
+    // Use callbacks from secondary threads
+    for (let i = 0; i < 1024; i++) {
         let cb = koffi.register(x => -x - 2, koffi.pointer(IntCallback));
-
         SetIndirect(cb);
-        assert.equal(await util.promisify(CallIndirectThreaded.async)(27), -29);
+
+        let results = await Promise.all([
+            util.promisify(CallThreaded.async)(x => -x - 4, 27),
+            util.promisify(CallThreaded.async)(null, 27)
+        ]);
+
+        assert.equal(results[0], -31);
+        assert.equal(results[1], -29);
 
         koffi.unregister(cb);
     }

package/src/koffi/test/misc.c

@@ -681,17 +681,27 @@ EXPORT int CallIndirect(int x)
 
 #ifdef _WIN32
 
-static DWORD WINAPI CallIndirectThreadedFunc(void *udata)
+typedef struct CallContext {
+    IntCallback *callback;
+    int *ptr;
+} CallContext;
+
+static DWORD WINAPI CallThreadedFunc(void *udata)
 {
-    int *ptr = (int *)udata;
-    *ptr = callback(*ptr);
+    CallContext *ctx = (CallContext *)udata;
+    *ctx->ptr = ctx->callback(*ctx->ptr);
 
     return 0;
 }
 
-EXPORT int CallIndirectThreaded(int x)
+EXPORT int CallThreaded(IntCallback *func, int x)
 {
-    HANDLE h = CreateThread(NULL, 0, CallIndirectThreadedFunc, &x, 0, NULL);
+    CallContext ctx;
+
+    ctx.callback = func ? func : callback;
+    ctx.ptr = &x;
+
+    HANDLE h = CreateThread(NULL, 0, CallThreadedFunc, &ctx, 0, NULL);
     if (!h) {
         perror("CreateThread");
         exit(1);
@@ -705,19 +715,28 @@ EXPORT int CallIndirectThreaded(int x)
 
 #else
 
-static void *CallIndirectThreadedFunc(void *udata)
+typedef struct CallContext {
+    IntCallback *callback;
+    int *ptr;
+} CallContext;
+
+static void *CallThreadedFunc(void *udata)
 {
-    int *ptr = (int *)udata;
-    *ptr = callback(*ptr);
+    CallContext *ctx = (CallContext *)udata;
+    *ctx->ptr = ctx->callback(*ctx->ptr);
 
     return NULL;
 }
 
-EXPORT int CallIndirectThreaded(int x)
+EXPORT int CallThreaded(IntCallback *func, int x)
 {
-    pthread_t thread;
+    CallContext ctx;
 
-    if (pthread_create(&thread, NULL, CallIndirectThreadedFunc, &x)) {
+    ctx.callback = func ? func : callback;
+    ctx.ptr = &x;
+
+    pthread_t thread;
+    if (pthread_create(&thread, NULL, CallThreadedFunc, &ctx)) {
         perror("pthread_create");
         exit(1);
     }
@@ -729,12 +748,6 @@ EXPORT int CallIndirectThreaded(int x)
 
 #endif
 
-EXPORT int CallDirectThreaded(IntCallback *func, int x)
-{
-    callback = func;
-    return CallIndirectThreaded(x);
-}
-
 EXPORT void ReverseBytes(void *p, int len)
 {
     uint8_t *bytes = (uint8_t *)p;

package/src/koffi/{qemu → tools}/qemu.js

@@ -342,7 +342,7 @@ async function prepare() {
                 log(machine, `${suite} > Status`, chalk.bold.red(`[manual]`));
                 needed = true;
             } else {
-                log(machine, `${suite} > Status`, chalk.bold.red(`[build]`));
+                log(machine, `${suite} > Status`, chalk.bold.red(`[missing]`));
                 needed = true;
             }
         }
@@ -494,6 +494,7 @@ async function prepare() {
         fs.renameSync(dist_dir + '/src/koffi/LICENSE.txt', dist_dir + '/LICENSE.txt');
         fs.renameSync(dist_dir + '/src/koffi/ChangeLog.md', dist_dir + '/ChangeLog.md');
         fs.renameSync(dist_dir + '/web/koffi.dev', dist_dir + '/doc');
+        fs.rmdirSync(dist_dir + '/web');
     }
 
     return dist_dir;
@@ -514,6 +515,8 @@ function snapshot() {
         } else if (parts[0] == 'src') {
             return parts[1] == null || parts[1] == 'cnoke' ||
                                        parts[1] == 'koffi';
+        } else if (parts[0] == 'tools') {
+            return parts[1] == null || parts[1] != 'qemu';
         } else if (parts[0] == 'vendor') {
             return parts[1] == null || parts[1] == 'brotli' ||
                                        parts[1] == 'dragonbox' ||

package/src/koffi/tools/write_trampolines.py

@@ -0,0 +1,138 @@
+#!/usr/bin/env python3
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see https://www.gnu.org/licenses/.
+
+import argparse
+import os
+
+def write_asm_trampolines(filename, comment_char, n,
+                          fmt_export, fmt_export_x, fmt_proc, fmt_proc_x, end = None):
+    with open(filename) as f:
+        lines = f.readlines()
+
+    with open(filename, 'w') as f:
+        for line in lines:
+            if line.rstrip() == comment_char + ' Trampolines':
+                break
+            f.write(line)
+
+        print(comment_char + ' Trampolines', file = f)
+        print(comment_char + ' ----------------------------', file = f)
+        print('', file = f)
+
+        for i in range(0, n):
+            print(fmt_export.format(i), file = f)
+        print('', file = f)
+        for i in range(0, n):
+            print(fmt_export_x.format(i), file = f)
+        print('', file = f)
+
+        for i in range(0, n):
+            print(fmt_proc.format(i), file = f)
+        print('', file = f)
+        for i in range(0, n):
+            print(fmt_proc_x.format(i), file = f)
+
+        if end is not None:
+            print('', file = f)
+            print(end, file = f)
+
+def write_cxx_trampolines(filename, n):
+    with open(filename) as f:
+        lines = f.readlines()
+
+    with open(filename, 'w') as f:
+        for line in lines:
+            if not line.startswith('//'):
+                break
+            f.write(line)
+
+        print('', file = f)
+        for i in range(0, n):
+            print('extern "C" int Trampoline{0}; extern "C" int TrampolineX{0};'.format(i), file = f)
+
+        print('', file = f)
+        print('static void *const Trampolines[][2] = {', file = f)
+        for i in range(0, n):
+            if i + 1 < n:
+                print('    {{ &Trampoline{0}, &TrampolineX{0} }},'.format(i), file = f)
+            else:
+                print('    {{ &Trampoline{0}, &TrampolineX{0} }}'.format(i), file = f)
+        print('};', file = f)
+        print('RG_STATIC_ASSERT(RG_LEN(Trampolines) == MaxTrampolines);', file = f)
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description = 'Generate static trampolines')
+    parser.add_argument('n', metavar = 'n', type = int, help = 'Number of trampolines')
+    args = parser.parse_args()
+
+    src_dir = os.path.dirname(__file__) + '/../src'
+
+    write_asm_trampolines(src_dir + '/abi_arm32_fwd.S', '#', args.n,
+        '.global Trampoline{0}',
+        '.global TrampolineX{0}',
+        'Trampoline{0}:\n    trampoline {0}',
+        'TrampolineX{0}:\n    trampoline_vec {0}'
+    )
+
+    write_asm_trampolines(src_dir + '/abi_arm64_fwd.asm', ';', args.n,
+        '    EXPORT Trampoline{0}',
+        '    EXPORT TrampolineX{0}',
+        'Trampoline{0} PROC\n    trampoline {0}\n    ENDP',
+        'TrampolineX{0} PROC\n    trampoline_vec {0}\n    ENDP',
+        '    END'
+    )
+    write_asm_trampolines(src_dir + '/abi_arm64_fwd.S', '#', args.n,
+        '.global SYMBOL(Trampoline{0})',
+        '.global SYMBOL(TrampolineX{0})',
+        'SYMBOL(Trampoline{0}):\n    trampoline {0}',
+        'SYMBOL(TrampolineX{0}):\n    trampoline_vec {0}'
+    )
+
+    write_asm_trampolines(src_dir + '/abi_riscv64_fwd.S', '#', args.n,
+        '.global Trampoline{0}',
+        '.global TrampolineX{0}',
+        'Trampoline{0}:\n    trampoline {0}',
+        'TrampolineX{0}:\n    trampoline_vec {0}'
+    )
+
+    write_asm_trampolines(src_dir + '/abi_x64_sysv_fwd.S', '#', args.n,
+        '.global SYMBOL(Trampoline{0})',
+        '.global SYMBOL(TrampolineX{0})',
+        'SYMBOL(Trampoline{0}):\n    trampoline {0}',
+        'SYMBOL(TrampolineX{0}):\n    trampoline_xmm {0}'
+    )
+    write_asm_trampolines(src_dir + '/abi_x64_win_fwd.asm', ';', args.n,
+        'public Trampoline{0}',
+        'public TrampolineX{0}',
+        'Trampoline{0} proc frame\n    trampoline {0}\nTrampoline{0} endp',
+        'TrampolineX{0} proc frame\n    trampoline_xmm {0}\nTrampolineX{0} endp',
+        'end'
+    )
+
+    write_asm_trampolines(src_dir + '/abi_x86_fwd.asm', ';', args.n,
+        'public Trampoline{0}',
+        'public TrampolineX{0}',
+        'Trampoline{0} proc\n    trampoline {0}\nTrampoline{0} endp',
+        'TrampolineX{0} proc\n    trampoline_x87 {0}\nTrampolineX{0} endp',
+        'end'
+    )
+    write_asm_trampolines(src_dir + '/abi_x86_fwd.S', '#', args.n,
+        '.global Trampoline{0}',
+        '.global TrampolineX{0}',
+        'Trampoline{0}:\n    trampoline {0}',
+        'TrampolineX{0}:\n    trampoline_x87 {0}'
+    )
+
+    write_cxx_trampolines(src_dir + '/abi_trampolines.inc', args.n)