koffi 1.1.2 → 1.1.5

package/src/abi_riscv64_fwd.S

@@ -0,0 +1,129 @@
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see https://www.gnu.org/licenses/.
+
+// These three are the same, but they differ (in the C side) by their return type.
+// Unlike the three next functions, these ones don't forward FA argument registers.
+.global ForwardCallGG
+.global ForwardCallF
+.global ForwardCallDG
+.global ForwardCallGD
+.global ForwardCallDD
+
+// The X variants are slightly slower, and are used when FA arguments must be forwarded.
+.global ForwardCallXGG
+.global ForwardCallXF
+.global ForwardCallXDG
+.global ForwardCallXGD
+.global ForwardCallXDD
+
+// Copy function pointer to t0, in order to save it through argument forwarding.
+// Save SP in s1, and use carefully assembled stack provided by caller.
+.macro prologue
+    addi sp, sp, -16
+    mv t0, a0
+    sd ra, 0(sp)
+    sd s1, 8(sp)
+    mv s1, sp
+    addi sp, a1, 128
+.endm
+
+// Call native function.
+// Once done, restore normal stack pointer and return.
+// The return value is passed untouched through registers.
+.macro epilogue
+    jalr t0
+    mv sp, s1
+    ld ra, 0(sp)
+    ld s1, 8(sp)
+    addi sp, sp, 16
+    ret
+.endm
+
+// Prepare general purpose argument registers from array passed by caller.
+.macro forward_int
+    ld a7, 120(a1)
+    ld a6, 112(a1)
+    ld a5, 104(a1)
+    ld a4, 96(a1)
+    ld a3, 88(a1)
+    ld a2, 80(a1)
+    ld a0, 64(a1)
+    ld a1, 72(a1)
+.endm
+
+// Prepare vector argument registers from array passed by caller.
+.macro forward_vec
+    fld fa7, 56(a1)
+    fld fa6, 48(a1)
+    fld fa5, 40(a1)
+    fld fa4, 32(a1)
+    fld fa3, 24(a1)
+    fld fa2, 16(a1)
+    fld fa1, 8(a1)
+    fld fa0, 0(a1)
+.endm
+
+ForwardCallGG:
+    prologue
+    forward_int
+    epilogue
+
+ForwardCallF:
+    prologue
+    forward_int
+    epilogue
+
+ForwardCallDG:
+    prologue
+    forward_int
+    epilogue
+
+ForwardCallGD:
+    prologue
+    forward_int
+    epilogue
+
+ForwardCallDD:
+    prologue
+    forward_int
+    epilogue
+
+ForwardCallXGG:
+    prologue
+    forward_vec
+    forward_int
+    epilogue
+
+ForwardCallXF:
+    prologue
+    forward_vec
+    forward_int
+    epilogue
+
+ForwardCallXDG:
+    prologue
+    forward_vec
+    forward_int
+    epilogue
+
+ForwardCallXGD:
+    prologue
+    forward_vec
+    forward_int
+    epilogue
+
+ForwardCallXDD:
+    prologue
+    forward_vec
+    forward_int
+    epilogue

package/src/abi_x64_sysv.cc

@@ -11,7 +11,7 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program. If not, see https://www.gnu.org/licenses/.
 
-#if defined(__x86_64__) && !defined(_WIN64)
+#if defined(__x86_64__) && !defined(_WIN32)
 
 #include "vendor/libcc/libcc.hh"
 #include "ffi.hh"
@@ -165,7 +165,7 @@ static void AnalyseParameter(ParameterInfo *param, int gpr_avail, int xmm_avail)
         xmm_count += (cls == RegisterClass::SSE);
     }
 
-    if (gpr_count <= gpr_avail && xmm_count <= xmm_avail){
+    if (gpr_count <= gpr_avail && xmm_count <= xmm_avail) {
         param->gpr_count = (int8_t)gpr_count;
         param->xmm_count = (int8_t)xmm_count;
         param->gpr_first = (classes[0] == RegisterClass::Integer);
@@ -420,8 +420,7 @@ bool CallData::Prepare(const Napi::CallbackInfo &info)
         }
     }
 
-    stack = MakeSpan(mem->stack.end(), old_stack_mem.end() - mem->stack.end());
-    heap = MakeSpan(old_heap_mem.ptr, mem->heap.ptr - old_heap_mem.ptr);
+    sp = mem->stack.end();
 
     return true;
 }
@@ -430,8 +429,8 @@ void CallData::Execute()
 {
 #define PERFORM_CALL(Suffix) \
         ([&]() { \
-            auto ret = (func->forward_fp ? ForwardCallX ## Suffix(func->func, stack.ptr) \
-                                         : ForwardCall ## Suffix(func->func, stack.ptr)); \
+            auto ret = (func->forward_fp ? ForwardCallX ## Suffix(func->func, sp) \
+                                         : ForwardCall ## Suffix(func->func, sp)); \
             return ret; \
         })()
 
@@ -453,16 +452,16 @@ void CallData::Execute()
         case PrimitiveKind::Record: {
             if (func->ret.gpr_first && !func->ret.xmm_count) {
                 RaxRdxRet ret = PERFORM_CALL(GG);
-                memcpy_safe(&result.buf, &ret, RG_SIZE(ret));
+                memcpy(&result.buf, &ret, RG_SIZE(ret));
             } else if (func->ret.gpr_first) {
                 RaxXmm0Ret ret = PERFORM_CALL(GD);
-                memcpy_safe(&result.buf, &ret, RG_SIZE(ret));
+                memcpy(&result.buf, &ret, RG_SIZE(ret));
             } else if (func->ret.xmm_count == 2) {
                 Xmm0Xmm1Ret ret = PERFORM_CALL(DD);
-                memcpy_safe(&result.buf, &ret, RG_SIZE(ret));
+                memcpy(&result.buf, &ret, RG_SIZE(ret));
             } else {
                 Xmm0RaxRet ret = PERFORM_CALL(DG);
-                memcpy_safe(&result.buf, &ret, RG_SIZE(ret));
+                memcpy(&result.buf, &ret, RG_SIZE(ret));
             }
         } break;
         case PrimitiveKind::Array: { RG_UNREACHABLE(); } break;

package/src/abi_x64_sysv_fwd.S

@@ -7,7 +7,7 @@
 // but WITHOUT ANY WARRANTY; without even the implied warranty of
 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 // GNU Affero General Public License for more details.
-
+//
 // You should have received a copy of the GNU Affero General Public License
 // along with this program. If not, see https://www.gnu.org/licenses/.
 
@@ -17,6 +17,8 @@
     #define SYMBOL(Symbol) Symbol
 #endif
 
+// Forward
+
 // These five are the same, but they differ (in the C side) by their return type.
 // Unlike the five next functions, these ones don't forward XMM argument registers.
 .global SYMBOL(ForwardCallGG)
@@ -144,3 +146,113 @@ SYMBOL(ForwardCallXDD):
     forward_int
     movb $8, %al
     epilogue
+
+// Callbacks
+
+.global SYMBOL(CallBack0)
+.global SYMBOL(CallBack1)
+.global SYMBOL(CallBack2)
+.global SYMBOL(CallBack3)
+.global SYMBOL(CallBack4)
+.global SYMBOL(CallBack5)
+.global SYMBOL(CallBack6)
+.global SYMBOL(CallBack7)
+.global SYMBOL(CallBackX0)
+.global SYMBOL(CallBackX1)
+.global SYMBOL(CallBackX2)
+.global SYMBOL(CallBackX3)
+.global SYMBOL(CallBackX4)
+.global SYMBOL(CallBackX5)
+.global SYMBOL(CallBackX6)
+.global SYMBOL(CallBackX7)
+.global SYMBOL(RelayCallBack)
+
+.macro jumper id
+    .cfi_startproc
+    .cfi_def_cfa rsp, 8
+    endbr64
+    subq $152, %rsp
+    .cfi_def_cfa rsp, 160
+    movq %rdi, 0(%rsp)
+    movq %rsi, 8(%rsp)
+    movq %rdx, 16(%rsp)
+    movq %rcx, 24(%rsp)
+    movq %r8, 32(%rsp)
+    movq %r9, 40(%rsp)
+    leaq 112(%rsp), %rdi
+    movq $\id, %rsi
+    movq %rsp, %rdx
+    call SYMBOL(RelayCallBack)
+    movq 8(%rax), %rdx
+    movq 0(%rax), %rax
+    addq $152, %rsp
+    ret
+    .cfi_endproc
+.endm
+
+.macro jumperx id
+    .cfi_startproc
+    .cfi_def_cfa rsp, 8
+    endbr64
+    subq $152, %rsp
+    .cfi_def_cfa rsp, 160
+    movq %rdi, 0(%rsp)
+    movq %rsi, 8(%rsp)
+    movq %rdx, 16(%rsp)
+    movq %rcx, 24(%rsp)
+    movq %r8, 32(%rsp)
+    movq %r9, 40(%rsp)
+    movsd %xmm0, 48(%rsp)
+    movsd %xmm1, 56(%rsp)
+    movsd %xmm2, 64(%rsp)
+    movsd %xmm3, 72(%rsp)
+    movsd %xmm4, 80(%rsp)
+    movsd %xmm5, 88(%rsp)
+    movsd %xmm6, 96(%rsp)
+    movsd %xmm7, 104(%rsp)
+    leaq 112(%rsp), %rdi
+    movq $\id, %rsi
+    movq %rsp, %rdx
+    call SYMBOL(RelayCallBack)
+    movsd 16(%rax), %xmm0
+    movsd 24(%rax), %xmm1
+    movq 8(%rax), %rdx
+    movq 0(%rax), %rax
+    addq $152, %rsp
+    ret
+    .cfi_endproc
+.endm
+
+SYMBOL(CallBack0):
+    jumper 0
+SYMBOL(CallBack1):
+    jumper 1
+SYMBOL(CallBack2):
+    jumper 2
+SYMBOL(CallBack3):
+    jumper 3
+SYMBOL(CallBack4):
+    jumper 4
+SYMBOL(CallBack5):
+    jumper 5
+SYMBOL(CallBack6):
+    jumper 6
+SYMBOL(CallBack7):
+    jumper 7
+
+SYMBOL(CallBackX0):
+    jumperx 0
+SYMBOL(CallBackX1):
+    jumperx 1
+SYMBOL(CallBackX2):
+    jumperx 2
+SYMBOL(CallBackX3):
+    jumperx 3
+SYMBOL(CallBackX4):
+    jumperx 4
+SYMBOL(CallBackX5):
+    jumperx 5
+SYMBOL(CallBackX6):
+    jumperx 6
+SYMBOL(CallBackX7):
+    jumperx 7

package/src/abi_x64_win.cc

@@ -11,7 +11,7 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program. If not, see https://www.gnu.org/licenses/.
 
-#ifdef _WIN64
+#if defined(_WIN32) && (defined(__x86_64__) || defined(_M_AMD64))
 
 #include "vendor/libcc/libcc.hh"
 #include "ffi.hh"
@@ -41,9 +41,7 @@ bool AnalyseFunction(InstanceData *, FunctionInfo *func)
 
     for (ParameterInfo &param: func->parameters) {
         param.regular = IsRegular(param.type->size);
-
-        func->forward_fp |= (param.type->primitive == PrimitiveKind::Float32 ||
-                             param.type->primitive == PrimitiveKind::Float64);
+        func->forward_fp |= IsFloat(param.type);
     }
 
     func->args_size = AlignLen(8 * std::max((Size)4, func->parameters.len + !func->ret.regular), 16);
@@ -204,8 +202,7 @@ bool CallData::Prepare(const Napi::CallbackInfo &info)
         }
     }
 
-    stack = MakeSpan(mem->stack.end(), old_stack_mem.end() - mem->stack.end());
-    heap = MakeSpan(old_heap_mem.ptr, mem->heap.ptr - old_heap_mem.ptr);
+    sp = mem->stack.end();
 
     return true;
 }
@@ -214,8 +211,8 @@ void CallData::Execute()
 {
 #define PERFORM_CALL(Suffix) \
         ([&]() { \
-            auto ret = (func->forward_fp ? ForwardCallX ## Suffix(func->func, stack.ptr) \
-                                         : ForwardCall ## Suffix(func->func, stack.ptr)); \
+            auto ret = (func->forward_fp ? ForwardCallX ## Suffix(func->func, sp) \
+                                         : ForwardCall ## Suffix(func->func, sp)); \
             return ret; \
         })()
 

package/src/abi_x86.cc

@@ -37,7 +37,9 @@ static inline bool IsRegular(Size size)
 
 bool AnalyseFunction(InstanceData *instance, FunctionInfo *func)
 {
-    int fast = (func->convention == CallConvention::Fastcall) ? 2 : 0;
+    int fast = (func->convention == CallConvention::Fastcall) ? 2 :
+               (func->convention == CallConvention::Thiscall) ? 1 : 0;
+    func->fast = fast;
 
     if (func->ret.type->primitive != PrimitiveKind::Record) {
         func->ret.trivial = true;
@@ -77,6 +79,10 @@ bool AnalyseFunction(InstanceData *instance, FunctionInfo *func)
             func->decorated_name = Fmt(&instance->str_alloc, "@%1@%2", func->name, params_size).ptr;
             func->args_size += 16;
         } break;
+        case CallConvention::Thiscall: {
+            RG_ASSERT(!func->variadic);
+            func->args_size += 16;
+        } break;
     }
 
     return true;
@@ -90,7 +96,7 @@ bool CallData::Prepare(const Napi::CallbackInfo &info)
     // Pass return value in register or through memory
     if (RG_UNLIKELY(!AllocStack(func->args_size, 16, &args_ptr)))
         return false;
-    if (func->convention == CallConvention::Fastcall) {
+    if (func->fast) {
         fast_ptr = args_ptr;
         args_ptr += 4;
     }
@@ -249,8 +255,7 @@ bool CallData::Prepare(const Napi::CallbackInfo &info)
         }
     }
 
-    stack = MakeSpan(mem->stack.end(), old_stack_mem.end() - mem->stack.end());
-    heap = MakeSpan(old_heap_mem.ptr, mem->heap.ptr - old_heap_mem.ptr);
+    sp = mem->stack.end();
 
     return true;
 }
@@ -259,8 +264,8 @@ void CallData::Execute()
 {
 #define PERFORM_CALL(Suffix) \
         ([&]() { \
-            auto ret = (func->convention == CallConvention::Fastcall ? ForwardCallR ## Suffix(func->func, stack.ptr) \
-                                                                     : ForwardCall ## Suffix(func->func, stack.ptr)); \
+            auto ret = (func->fast ? ForwardCallR ## Suffix(func->func, sp) \
+                                   : ForwardCall ## Suffix(func->func, sp)); \
             return ret; \
         })()
 

package/src/call.cc

@@ -21,7 +21,7 @@
 namespace RG {
 
 CallData::CallData(Napi::Env env, InstanceData *instance, const FunctionInfo *func, InstanceMemory *mem)
-    : env(env), instance(instance), func(func), debug(instance->debug),
+    : env(env), instance(instance), func(func),
       mem(mem), old_stack_mem(mem->stack), old_heap_mem(mem->heap)
 {
     mem->depth++;
@@ -565,13 +565,11 @@ void CallData::PopObject(Napi::Object obj, const uint8_t *src, const TypeInfo *t
                 obj.Set(member.name, value);
             } break;
             case PrimitiveKind::Float32: {
-                float f;
-                memcpy(&f, src, 4);
+                float f = *(float *)src;
                 obj.Set(member.name, Napi::Number::New(env, (double)f));
             } break;
             case PrimitiveKind::Float64: {
-                double d;
-                memcpy(&d, src, 8);
+                double d = *(double *)src;
                 obj.Set(member.name, Napi::Number::New(env, d));
             } break;
         }
@@ -762,19 +760,6 @@ Napi::Value CallData::PopArray(const uint8_t *src, const TypeInfo *type, int16_t
     RG_UNREACHABLE();
 }
 
-Napi::Value CallData::Run(const Napi::CallbackInfo &info)
-{
-    if (!RG_UNLIKELY(Prepare(info)))
-        return env.Null();
-
-    if (debug) {
-        DumpDebug();
-    }
-    Execute();
-
-    return Complete();
-}
-
 static void DumpMemory(const char *type, Span<const uint8_t> bytes)
 {
     if (bytes.len) {
@@ -805,6 +790,9 @@ void CallData::DumpDebug() const
     }
     PrintLn(stderr, "Return: %1 (%2)", func->ret.type->name, FmtMemSize(func->ret.type->size));
 
+    Span<const uint8_t> stack = MakeSpan(mem->stack.end(), old_stack_mem.end() - mem->stack.end());
+    Span<const uint8_t> heap = MakeSpan(old_heap_mem.ptr, mem->heap.ptr - old_heap_mem.ptr);
+
     DumpMemory("Stack", stack);
     DumpMemory("Heap", heap);
 }

package/src/call.hh

@@ -34,16 +34,12 @@ class CallData {
     InstanceData *instance;
     const FunctionInfo *func;
 
-    bool debug;
-
     InstanceMemory *mem;
     Span<uint8_t> old_stack_mem;
     Span<uint8_t> old_heap_mem;
 
     LocalArray<OutObject, MaxOutParameters> out_objects;
-
-    Span<uint8_t> heap;
-    Span<uint8_t> stack;
+    uint8_t *sp;
 
     union {
         uint32_t u32;
@@ -65,15 +61,13 @@ public:
     void Execute();
     Napi::Value Complete();
 
-    Napi::Value Run(const Napi::CallbackInfo &info);
-
     void DumpDebug() const;
 
 private:
     template <typename T = void>
-    bool AllocStack(Size size, Size align, T **out_ptr = nullptr);
+    bool AllocStack(Size size, Size align, T **out_ptr);
     template <typename T = void>
-    bool AllocHeap(Size size, Size align, T **out_ptr = nullptr);
+    bool AllocHeap(Size size, Size align, T **out_ptr);
 
     const char *PushString(const Napi::Value &value);
     const char16_t *PushString16(const Napi::Value &value);
@@ -86,7 +80,7 @@ private:
 };
 
 template <typename T>
-bool CallData::AllocStack(Size size, Size align, T **out_ptr)
+inline bool CallData::AllocStack(Size size, Size align, T **out_ptr)
 {
     uint8_t *ptr = AlignDown(mem->stack.end() - size, align);
     Size delta = mem->stack.end() - ptr;
@@ -96,20 +90,18 @@ bool CallData::AllocStack(Size size, Size align, T **out_ptr)
         return false;
     }
 
-    if (debug) {
-        memset(ptr, 0, delta);
-    }
+#ifdef RG_DEBUG
+    memset(ptr, 0, delta);
+#endif
 
     mem->stack.len -= delta;
 
-    if (out_ptr) {
-        *out_ptr = (T *)ptr;
-    }
+    *out_ptr = (T *)ptr;
     return true;
 }
 
 template <typename T>
-bool CallData::AllocHeap(Size size, Size align, T **out_ptr)
+inline bool CallData::AllocHeap(Size size, Size align, T **out_ptr)
 {
     uint8_t *ptr = AlignUp(mem->heap.ptr, align);
     Size delta = size + (ptr - mem->heap.ptr);
@@ -119,16 +111,14 @@ bool CallData::AllocHeap(Size size, Size align, T **out_ptr)
         return false;
     }
 
-    if (debug) {
-        memset(mem->heap.ptr, 0, (size_t)delta);
-    }
+#ifdef RG_DEBUG
+    memset(mem->heap.ptr, 0, (size_t)delta);
+#endif
 
     mem->heap.ptr += delta;
     mem->heap.len -= delta;
 
-    if (out_ptr) {
-        *out_ptr = (T *)ptr;
-    }
+    *out_ptr = (T *)ptr;
     return true;
 }