koffi 0.9.4 → 0.9.7

package/src/call_arm32_fwd.S

@@ -0,0 +1,108 @@
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see https://www.gnu.org/licenses/.
+
+.syntax unified
+
+// These three are the same, but they differ (in the C side) by their return type.
+// Unlike the three next functions, these ones don't forward XMM argument registers.
+.global ForwardCallGG
+.global ForwardCallF
+.global ForwardCallDDDD
+
+// The X variants are slightly slower, and are used when XMM arguments must be forwarded.
+.global ForwardCallXGG
+.global ForwardCallXF
+.global ForwardCallXDDDD
+
+// Copy function pointer to r12, in order to save it through argument forwarding.
+// Save RSP in fp (non-volatile), and use carefully assembled stack provided by caller.
+.macro prologue
+    .cfi_startproc
+    push {fp, lr}
+    .cfi_def_cfa sp, 8
+    .cfi_offset 11, 4
+    .cfi_offset 14, 8
+    mov fp, sp
+    .cfi_def_cfa fp, 8
+    mov r12, r0
+    mov sp, r1
+    add sp, sp, #80
+.endm
+
+// Call native function.
+// Once done, restore normal stack pointer and return.
+// The return value is passed untouched through r0, r1, and or FP registers
+.macro epilogue
+    blx r12
+    mov sp, fp
+    .cfi_def_cfa sp, 8
+    pop {fp, lr}
+    .cfi_def_cfa sp, 0
+    .cfi_restore 11
+    .cfi_restore 14
+    bx lr
+    .cfi_endproc
+.endm
+
+// Prepare general purpose argument registers from array passed by caller.
+.macro forward_int
+    ldr r3, [r1, 76]
+    ldr r2, [r1, 72]
+    ldr r0, [r1, 64]
+    ldr r1, [r1, 68]
+.endm
+
+// Prepare vector argument registers from array passed by caller.
+.macro forward_vec
+    vldr d7, [r1, 56]
+    vldr d6, [r1, 48]
+    vldr d5, [r1, 40]
+    vldr d4, [r1, 32]
+    vldr d3, [r1, 24]
+    vldr d2, [r1, 16]
+    vldr d1, [r1, 8]
+    vldr d0, [r1, 0]
+.endm
+
+ForwardCallGG:
+    prologue
+    forward_int
+    epilogue
+
+ForwardCallF:
+    prologue
+    forward_int
+    epilogue
+
+ForwardCallDDDD:
+    prologue
+    forward_int
+    epilogue
+
+ForwardCallXGG:
+    prologue
+    forward_vec
+    forward_int
+    epilogue
+
+ForwardCallXF:
+    prologue
+    forward_vec
+    forward_int
+    epilogue
+
+ForwardCallXDDDD:
+    prologue
+    forward_vec
+    forward_int
+    epilogue

package/src/call_arm64.cc

@@ -60,12 +60,14 @@ static bool IsHFA(const TypeInfo *type)
     return true;
 }
 
-bool AnalyseFunction(FunctionInfo *func)
+bool AnalyseFunction(InstanceData *, FunctionInfo *func)
 {
     if (IsHFA(func->ret.type)) {
         func->ret.vec_count = func->ret.type->members.len;
     } else if (func->ret.type->size <= 16) {
         func->ret.gpr_count = (func->ret.type->size + 7) / 8;
+    } else {
+        func->ret.use_memory = true;
     }
 
     int gpr_avail = 8;
@@ -104,22 +106,34 @@ bool AnalyseFunction(FunctionInfo *func)
                 if (IsHFA(param.type)) {
                     int vec_count = (int)param.type->members.len;
 
-                    param.vec_count = std::min(vec_avail, vec_count);
-                    vec_avail -= vec_count;
+                    if (vec_count <= vec_avail) {
+                        param.vec_count = vec_count;
+                        vec_avail -= vec_count;
+                    } else {
+                        vec_avail = 0;
+                    }
                 } else if (param.type->size <= 16) {
                     int gpr_count = (param.type->size + 7) / 8;
 
-                    param.gpr_count = std::min(gpr_avail, gpr_count);
-                    gpr_avail -= gpr_count;
+                    if (gpr_count <= gpr_avail) {
+                        param.gpr_count = gpr_count;
+                        gpr_avail -= gpr_count;
+                    } else {
+                        gpr_avail = 0;
+                    }
                 } else if (gpr_avail) {
                     // Big types (more than 16 bytes) are replaced by a pointer
-                    param.gpr_count = 1;
-                    gpr_avail -= 1;
+                    if (gpr_avail) {
+                        param.gpr_count = 1;
+                        gpr_avail -= 1;
+                    }
+                    param.use_memory = true;
                 }
             } break;
         }
     }
 
+    func->args_size = 16 * func->parameters.len;
     func->forward_fp = (vec_avail < 8);
 
     return true;
@@ -143,14 +157,14 @@ static bool PushHFA(const Napi::Object &obj, const TypeInfo *type, uint8_t *dest
                 return false;
             }
 
-            *(float *)dest = CopyNodeNumber<float>(value);
+            *(float *)dest = CopyNumber<float>(value);
         } else if (member.type->primitive == PrimitiveKind::Float64) {
             if (!value.IsNumber() && !value.IsBigInt()) {
                 ThrowError<Napi::TypeError>(env, "Unexpected value %1 for member '%2', expected number", GetValueType(instance, value), member.name);
                 return false;
             }
 
-            *(double *)dest = CopyNodeNumber<double>(value);
+            *(double *)dest = CopyNumber<double>(value);
         } else {
             RG_UNREACHABLE();
         }
@@ -188,11 +202,9 @@ Napi::Value TranslateCall(const Napi::CallbackInfo &info)
 {
     Napi::Env env = info.Env();
     InstanceData *instance = env.GetInstanceData<InstanceData>();
-
     FunctionInfo *func = (FunctionInfo *)info.Data();
-    LibraryData *lib = func->lib.get();
 
-    RG_DEFER { lib->tmp_alloc.ReleaseAll(); };
+    CallData call(env, instance, func);
 
     // Sanity checks
     if (info.Length() < (uint32_t)func->parameters.len) {
@@ -200,43 +212,24 @@ Napi::Value TranslateCall(const Napi::CallbackInfo &info)
         return env.Null();
     }
 
-    // Stack pointer and register
-    uint8_t *top_ptr = lib->stack.end();
-    uint8_t *scratch_ptr = top_ptr - func->scratch_size;
     uint8_t *return_ptr = nullptr;
     uint8_t *args_ptr = nullptr;
-    uint64_t *gpr_ptr = nullptr, *vec_ptr = nullptr;
-    uint8_t *sp_ptr = nullptr;
+    uint64_t *gpr_ptr = nullptr;
+    uint64_t *vec_ptr = nullptr;
 
     // Return through registers unless it's too big
-    if (!func->ret.type->size || func->ret.vec_count || func->ret.gpr_count) {
-        args_ptr = scratch_ptr - AlignLen(8 * func->parameters.len, 16);
-        vec_ptr = (uint64_t *)args_ptr - 8;
-        gpr_ptr = vec_ptr - 9;
-        sp_ptr = (uint8_t *)(gpr_ptr - 7);
-
-#ifdef RG_DEBUG
-        memset(sp_ptr, 0, top_ptr - sp_ptr);
-#endif
-    } else {
-        return_ptr = scratch_ptr - AlignLen(func->ret.type->size, 16);
-
-        args_ptr = return_ptr - AlignLen(8 * func->parameters.len, 16);
-        vec_ptr = (uint64_t *)args_ptr - 8;
-        gpr_ptr = vec_ptr - 9;
-        sp_ptr = (uint8_t *)(gpr_ptr - 7);
-
-#ifdef RG_DEBUG
-        memset(sp_ptr, 0, top_ptr - sp_ptr);
-#endif
-
+    if (RG_UNLIKELY(!call.AllocStack(func->args_size, 16, &args_ptr)))
+        return env.Null();
+    if (RG_UNLIKELY(!call.AllocStack(8 * 8, 8, &vec_ptr)))
+        return env.Null();
+    if (RG_UNLIKELY(!call.AllocStack(9 * 8, 8, &gpr_ptr)))
+        return env.Null();
+    if (func->ret.use_memory) {
+        if (RG_UNLIKELY(!call.AllocHeap(func->ret.type->size, 16, &return_ptr)))
+            return env.Null();
         gpr_ptr[8] = (uint64_t)return_ptr;
     }
 
-    RG_ASSERT(AlignUp(lib->stack.ptr, 16) == lib->stack.ptr);
-    RG_ASSERT(AlignUp(lib->stack.end(), 16) == lib->stack.end());
-    RG_ASSERT(AlignUp(sp_ptr, 16) == sp_ptr);
-
     // Push arguments
     for (Size i = 0; i < func->parameters.len; i++) {
         const ParameterInfo &param = func->parameters[i];
@@ -272,7 +265,7 @@ Napi::Value TranslateCall(const Napi::CallbackInfo &info)
                     return env.Null();
                 }
 
-                int64_t v = CopyNodeNumber<int64_t>(value);
+                int64_t v = CopyNumber<int64_t>(value);
 
                 if (RG_LIKELY(param.gpr_count)) {
                     *(gpr_ptr++) = (uint64_t)v;
@@ -288,7 +281,7 @@ Napi::Value TranslateCall(const Napi::CallbackInfo &info)
                     return env.Null();
                 }
 
-                float f = CopyNodeNumber<float>(value);
+                float f = CopyNumber<float>(value);
 
                 if (RG_LIKELY(param.vec_count)) {
                     memcpy(vec_ptr++, &f, 4);
@@ -304,7 +297,7 @@ Napi::Value TranslateCall(const Napi::CallbackInfo &info)
                     return env.Null();
                 }
 
-                double d = CopyNodeNumber<double>(value);
+                double d = CopyNumber<double>(value);
 
                 if (RG_LIKELY(param.vec_count)) {
                     memcpy(vec_ptr++, &d, 8);
@@ -320,7 +313,9 @@ Napi::Value TranslateCall(const Napi::CallbackInfo &info)
                     return env.Null();
                 }
 
-                const char *str = CopyNodeString(value, &lib->tmp_alloc);
+                const char *str = call.PushString(value);
+                if (RG_UNLIKELY(!str))
+                    return env.Null();
 
                 if (RG_LIKELY(param.gpr_count)) {
                     *(gpr_ptr++) = (uint64_t)str;
@@ -359,22 +354,25 @@ Napi::Value TranslateCall(const Napi::CallbackInfo &info)
                     if (!PushHFA(obj, param.type, (uint8_t *)vec_ptr))
                         return env.Null();
                     vec_ptr += param.vec_count;
-                } else if (param.type->size <= 16) {
+                } else if (!param.use_memory) {
                     if (param.gpr_count) {
                         RG_ASSERT(param.type->align <= 8);
 
-                        if (!PushObject(obj, param.type, &lib->tmp_alloc, (uint8_t *)gpr_ptr))
+                        if (!call.PushObject(obj, param.type, (uint8_t *)gpr_ptr))
                             return env.Null();
                         gpr_ptr += param.gpr_count;
                     } else if (param.type->size) {
-                        args_ptr = AlignUp(args_ptr, param.type->align);
-                        if (!PushObject(obj, param.type, &lib->tmp_alloc, args_ptr))
+                        int16_t align = (param.type->align <= 4) ? 4 : 8;
+
+                        args_ptr = AlignUp(args_ptr, align);
+                        if (!call.PushObject(obj, param.type, args_ptr))
                             return env.Null();
                         args_ptr += AlignLen(param.type->size, 8);
                     }
                 } else {
-                    uint8_t *ptr = scratch_ptr;
-                    scratch_ptr += AlignLen(param.type->size, 16);
+                    uint8_t *ptr;
+                    if (RG_UNLIKELY(!call.AllocHeap(param.type->size, 16, &ptr)))
+                        return env.Null();
 
                     if (param.gpr_count) {
                         RG_ASSERT(param.gpr_count == 1);
@@ -387,18 +385,20 @@ Napi::Value TranslateCall(const Napi::CallbackInfo &info)
                         args_ptr += 8;
                     }
 
-                    if (!PushObject(obj, param.type, &lib->tmp_alloc, ptr))
+                    if (!call.PushObject(obj, param.type, ptr))
                         return env.Null();
                 }
             } break;
         }
     }
 
-    // DumpStack(func, MakeSpan(sp_ptr, top_ptr - sp_ptr));
+    if (instance->debug) {
+        call.DumpDebug();
+    }
 
 #define PERFORM_CALL(Suffix) \
-        (func->forward_fp ? ForwardCallX ## Suffix(func->func, sp_ptr) \
-                          : ForwardCall ## Suffix(func->func, sp_ptr))
+        (func->forward_fp ? ForwardCallX ## Suffix(func->func, call.GetSP()) \
+                          : ForwardCall ## Suffix(func->func, call.GetSP()))
 
     // Execute and convert return value
     switch (func->ret.type->primitive) {
@@ -429,7 +429,7 @@ Napi::Value TranslateCall(const Napi::CallbackInfo &info)
                 RG_ASSERT(return_ptr);
 
                 X0X1Ret ret = PERFORM_CALL(GG);
-                RG_ASSERT(ret.x0 = (uint64_t)return_ptr);
+                RG_ASSERT(ret.x0 == (uint64_t)return_ptr);
 
                 Napi::Object obj = PopObject(env, return_ptr, func->ret.type);
                 return obj;

package/src/call_arm64_fwd.S

@@ -11,8 +11,8 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program. If not, see https://www.gnu.org/licenses/.
 
-// These five are the same, but they differ (in the C side) by their return type.
-// Unlike the five next functions, these ones don't forward XMM argument registers.
+// These three are the same, but they differ (in the C side) by their return type.
+// Unlike the three next functions, these ones don't forward XMM argument registers.
 .global ForwardCallGG
 .global ForwardCallF
 .global ForwardCallDDDD
@@ -23,20 +23,17 @@
 .global ForwardCallXDDDD
 
 // Copy function pointer to r9, in order to save it through argument forwarding.
-// Save RSP in r19 (non-volatile), and use carefully assembled stack provided by caller.
+// Save RSP in r29 (non-volatile), and use carefully assembled stack provided by caller.
 .macro prologue
     .cfi_startproc
     stp x29, x30, [sp, -16]!
     .cfi_def_cfa sp, 16
     .cfi_offset 29, 16
     .cfi_offset 30, 8
-    str x19, [sp, -16]!
-    .cfi_def_cfa sp, 32
+    mov x29, sp
+    .cfi_def_cfa x29, 16
     mov x9, x0
-    mov x19, sp
-    .cfi_def_cfa x19, 32
-    mov sp, x1
-    add sp, sp, #192
+    add sp, x1, #136
 .endm
 
 // Call native function.
@@ -44,9 +41,7 @@
 // The return value is passed untouched through r0, r1, v0 and/or v1.
 .macro epilogue
     blr x9
-    mov sp, x19
-    .cfi_def_cfa sp, 32
-    ldr x19, [sp], 16
+    mov sp, x29
     .cfi_def_cfa sp, 16
     ldp x29, x30, [sp], 16
     .cfi_def_cfa sp, 0
@@ -58,27 +53,27 @@
 
 // Prepare general purpose argument registers from array passed by caller.
 .macro forward_int
-    ldr x8, [x1, 120]
-    ldr x7, [x1, 112]
-    ldr x6, [x1, 104]
-    ldr x5, [x1, 96]
-    ldr x4, [x1, 88]
-    ldr x3, [x1, 80]
-    ldr x2, [x1, 72]
-    ldr x0, [x1, 56]
-    ldr x1, [x1, 64]
+    ldr x8, [x1, 64]
+    ldr x7, [x1, 56]
+    ldr x6, [x1, 48]
+    ldr x5, [x1, 40]
+    ldr x4, [x1, 32]
+    ldr x3, [x1, 24]
+    ldr x2, [x1, 16]
+    ldr x0, [x1, 0]
+    ldr x1, [x1, 8]
 .endm
 
 // Prepare vector argument registers from array passed by caller.
 .macro forward_vec
-    ldr d7, [x1, 184]
-    ldr d6, [x1, 176]
-    ldr d5, [x1, 168]
-    ldr d4, [x1, 160]
-    ldr d3, [x1, 152]
-    ldr d2, [x1, 144]
-    ldr d1, [x1, 136]
-    ldr d0, [x1, 128]
+    ldr d7, [x1, 128]
+    ldr d6, [x1, 120]
+    ldr d5, [x1, 112]
+    ldr d4, [x1, 104]
+    ldr d3, [x1, 96]
+    ldr d2, [x1, 88]
+    ldr d1, [x1, 80]
+    ldr d0, [x1, 72]
 .endm
 
 ForwardCallGG:

package/src/call_x64_sysv.cc

@@ -159,7 +159,7 @@ static void AnalyseParameter(ParameterInfo *param, int gpr_avail, int xmm_avail)
     }
 }
 
-bool AnalyseFunction(FunctionInfo *func)
+bool AnalyseFunction(InstanceData *, FunctionInfo *func)
 {
     AnalyseParameter(&func->ret, 2, 2);
 
@@ -171,6 +171,8 @@ bool AnalyseFunction(FunctionInfo *func)
 
         gpr_avail -= param.gpr_count;
         xmm_avail -= param.xmm_count;
+
+        func->args_size += AlignLen(param.type->size, 16);
     }
 
     func->forward_fp = (xmm_avail < 8);
@@ -182,11 +184,9 @@ Napi::Value TranslateCall(const Napi::CallbackInfo &info)
 {
     Napi::Env env = info.Env();
     InstanceData *instance = env.GetInstanceData<InstanceData>();
-
     FunctionInfo *func = (FunctionInfo *)info.Data();
-    LibraryData *lib = func->lib.get();
 
-    RG_DEFER { lib->tmp_alloc.ReleaseAll(); };
+    CallData call(env, instance, func);
 
     // Sanity checks
     if (info.Length() < (uint32_t)func->parameters.len) {
@@ -194,42 +194,24 @@ Napi::Value TranslateCall(const Napi::CallbackInfo &info)
         return env.Null();
     }
 
-    // Stack pointer and register
-    uint8_t *top_ptr = lib->stack.end();
     uint8_t *return_ptr = nullptr;
     uint8_t *args_ptr = nullptr;
-    uint64_t *gpr_ptr = nullptr, *xmm_ptr = nullptr;
-    uint8_t *sp_ptr = nullptr;
+    uint64_t *gpr_ptr = nullptr;
+    uint64_t *xmm_ptr = nullptr;
 
     // Return through registers unless it's too big
-    if (!func->ret.use_memory) {
-        args_ptr = top_ptr - func->scratch_size;
-        xmm_ptr = (uint64_t *)args_ptr - 8;
-        gpr_ptr = xmm_ptr - 6;
-        sp_ptr = (uint8_t *)gpr_ptr;
-
-#ifdef RG_DEBUG
-        memset(sp_ptr, 0, top_ptr - sp_ptr);
-#endif
-    } else {
-        return_ptr = top_ptr - AlignLen(func->ret.type->size, 16);
-
-        args_ptr = return_ptr - func->scratch_size;
-        xmm_ptr = (uint64_t *)args_ptr - 8;
-        gpr_ptr = xmm_ptr - 6;
-        sp_ptr = (uint8_t *)gpr_ptr;
-
-#ifdef RG_DEBUG
-        memset(sp_ptr, 0, top_ptr - sp_ptr);
-#endif
-
-        *(gpr_ptr++) = (uint64_t)return_ptr;
+    if (RG_UNLIKELY(!call.AllocStack(func->args_size, 16, &args_ptr)))
+        return env.Null();
+    if (RG_UNLIKELY(!call.AllocStack(8 * 8, 8, &xmm_ptr)))
+        return env.Null();
+    if (RG_UNLIKELY(!call.AllocStack(6 * 8, 8, &gpr_ptr)))
+        return env.Null();
+    if (func->ret.use_memory) {
+        if (RG_UNLIKELY(!call.AllocHeap(func->ret.type->size, 16, &return_ptr)))
+            return env.Null();
+        *(uint8_t **)(gpr_ptr++) = return_ptr;
     }
 
-    RG_ASSERT(AlignUp(lib->stack.ptr, 16) == lib->stack.ptr);
-    RG_ASSERT(AlignUp(lib->stack.end(), 16) == lib->stack.end());
-    RG_ASSERT(AlignUp(args_ptr, 16) == args_ptr);
-
     // Push arguments
     for (Size i = 0; i < func->parameters.len; i++) {
         const ParameterInfo &param = func->parameters[i];
@@ -265,7 +247,7 @@ Napi::Value TranslateCall(const Napi::CallbackInfo &info)
                     return env.Null();
                 }
 
-                int64_t v = CopyNodeNumber<int64_t>(value);
+                int64_t v = CopyNumber<int64_t>(value);
 
                 if (RG_LIKELY(param.gpr_count)) {
                     *(gpr_ptr++) = (uint64_t)v;
@@ -281,7 +263,7 @@ Napi::Value TranslateCall(const Napi::CallbackInfo &info)
                     return env.Null();
                 }
 
-                float f = CopyNodeNumber<float>(value);
+                float f = CopyNumber<float>(value);
 
                 if (RG_LIKELY(param.xmm_count)) {
                     memcpy(xmm_ptr++, &f, 4);
@@ -297,7 +279,7 @@ Napi::Value TranslateCall(const Napi::CallbackInfo &info)
                     return env.Null();
                 }
 
-                double d = CopyNodeNumber<double>(value);
+                double d = CopyNumber<double>(value);
 
                 if (RG_LIKELY(param.xmm_count)) {
                     memcpy(xmm_ptr++, &d, 8);
@@ -313,7 +295,9 @@ Napi::Value TranslateCall(const Napi::CallbackInfo &info)
                     return env.Null();
                 }
 
-                const char *str = CopyNodeString(value, &lib->tmp_alloc);
+                const char *str = call.PushString(value);
+                if (RG_UNLIKELY(!str))
+                    return env.Null();
 
                 if (RG_LIKELY(param.gpr_count)) {
                     *(gpr_ptr++) = (uint64_t)str;
@@ -352,7 +336,7 @@ Napi::Value TranslateCall(const Napi::CallbackInfo &info)
                     RG_ASSERT(param.type->size <= 16);
 
                     uint64_t buf[2] = {};
-                    if (!PushObject(obj, param.type, &lib->tmp_alloc, (uint8_t *)buf))
+                    if (!call.PushObject(obj, param.type, (uint8_t *)buf))
                         return env.Null();
 
                     if (param.gpr_first) {
@@ -376,7 +360,7 @@ Napi::Value TranslateCall(const Napi::CallbackInfo &info)
                     }
                 } else if (param.use_memory) {
                     args_ptr = AlignUp(args_ptr, param.type->align);
-                    if (!PushObject(obj, param.type, &lib->tmp_alloc, args_ptr))
+                    if (!call.PushObject(obj, param.type, args_ptr))
                         return env.Null();
                     args_ptr += param.type->size;
                 }
@@ -384,11 +368,13 @@ Napi::Value TranslateCall(const Napi::CallbackInfo &info)
         }
     }
 
-    // DumpStack(func, MakeSpan(sp_ptr, top_ptr - sp_ptr));
+    if (instance->debug) {
+        call.DumpDebug();
+    }
 
 #define PERFORM_CALL(Suffix) \
-        (func->forward_fp ? ForwardCallX ## Suffix(func->func, sp_ptr) \
-                          : ForwardCall ## Suffix(func->func, sp_ptr))
+        (func->forward_fp ? ForwardCallX ## Suffix(func->func, call.GetSP()) \
+                          : ForwardCall ## Suffix(func->func, call.GetSP()))
 
     // Execute and convert return value
     switch (func->ret.type->primitive) {