koffi 1.1.5 → 1.2.0-alpha.3

package/src/abi_x64_sysv.cc

@@ -46,17 +46,67 @@ struct Xmm0Xmm1Ret {
     double xmm1;
 };
 
-extern "C" RaxRdxRet ForwardCallGG(const void *func, uint8_t *sp);
-extern "C" float ForwardCallF(const void *func, uint8_t *sp);
-extern "C" Xmm0RaxRet ForwardCallDG(const void *func, uint8_t *sp);
-extern "C" RaxXmm0Ret ForwardCallGD(const void *func, uint8_t *sp);
-extern "C" Xmm0Xmm1Ret ForwardCallDD(const void *func, uint8_t *sp);
-
-extern "C" RaxRdxRet ForwardCallXGG(const void *func, uint8_t *sp);
-extern "C" float ForwardCallXF(const void *func, uint8_t *sp);
-extern "C" Xmm0RaxRet ForwardCallXDG(const void *func, uint8_t *sp);
-extern "C" RaxXmm0Ret ForwardCallXGD(const void *func, uint8_t *sp);
-extern "C" Xmm0Xmm1Ret ForwardCallXDD(const void *func, uint8_t *sp);
+struct BackRegisters {
+    uint64_t rax;
+    uint64_t rdx;
+    double xmm0;
+    double xmm1;
+};
+
+extern "C" RaxRdxRet ForwardCallGG(const void *func, uint8_t *sp, uint8_t **out_old_sp);
+extern "C" float ForwardCallF(const void *func, uint8_t *sp, uint8_t **out_old_sp);
+extern "C" Xmm0RaxRet ForwardCallDG(const void *func, uint8_t *sp, uint8_t **out_old_sp);
+extern "C" RaxXmm0Ret ForwardCallGD(const void *func, uint8_t *sp, uint8_t **out_old_sp);
+extern "C" Xmm0Xmm1Ret ForwardCallDD(const void *func, uint8_t *sp, uint8_t **out_old_sp);
+
+extern "C" RaxRdxRet ForwardCallXGG(const void *func, uint8_t *sp, uint8_t **out_old_sp);
+extern "C" float ForwardCallXF(const void *func, uint8_t *sp, uint8_t **out_old_sp);
+extern "C" Xmm0RaxRet ForwardCallXDG(const void *func, uint8_t *sp, uint8_t **out_old_sp);
+extern "C" RaxXmm0Ret ForwardCallXGD(const void *func, uint8_t *sp, uint8_t **out_old_sp);
+extern "C" Xmm0Xmm1Ret ForwardCallXDD(const void *func, uint8_t *sp, uint8_t **out_old_sp);
+
+extern "C" int Trampoline0; extern "C" int TrampolineX0;
+extern "C" int Trampoline1; extern "C" int TrampolineX1;
+extern "C" int Trampoline2; extern "C" int TrampolineX2;
+extern "C" int Trampoline3; extern "C" int TrampolineX3;
+extern "C" int Trampoline4; extern "C" int TrampolineX4;
+extern "C" int Trampoline5; extern "C" int TrampolineX5;
+extern "C" int Trampoline6; extern "C" int TrampolineX6;
+extern "C" int Trampoline7; extern "C" int TrampolineX7;
+extern "C" int Trampoline8; extern "C" int TrampolineX8;
+extern "C" int Trampoline9; extern "C" int TrampolineX9;
+extern "C" int Trampoline10; extern "C" int TrampolineX10;
+extern "C" int Trampoline11; extern "C" int TrampolineX11;
+extern "C" int Trampoline12; extern "C" int TrampolineX12;
+extern "C" int Trampoline13; extern "C" int TrampolineX13;
+extern "C" int Trampoline14; extern "C" int TrampolineX14;
+extern "C" int Trampoline15; extern "C" int TrampolineX15;
+
+extern "C" napi_value CallSwitchStack(Napi::Function *func, size_t argc, napi_value *argv,
+                                      uint8_t *old_sp, Span<uint8_t> *new_stack,
+                                      napi_value (*call)(Napi::Function *func, size_t argc, napi_value *argv));
+
+static void *const Trampolines[][2] = {
+    { &Trampoline0, &TrampolineX0 },
+    { &Trampoline1, &TrampolineX1 },
+    { &Trampoline2, &TrampolineX2 },
+    { &Trampoline3, &TrampolineX3 },
+    { &Trampoline4, &TrampolineX4 },
+    { &Trampoline5, &TrampolineX5 },
+    { &Trampoline6, &TrampolineX6 },
+    { &Trampoline7, &TrampolineX7 },
+    { &Trampoline8, &TrampolineX8 },
+    { &Trampoline9, &TrampolineX9 },
+    { &Trampoline10, &TrampolineX10 },
+    { &Trampoline11, &TrampolineX11 },
+    { &Trampoline12, &TrampolineX12 },
+    { &Trampoline13, &TrampolineX13 },
+    { &Trampoline14, &TrampolineX14 },
+    { &Trampoline15, &TrampolineX15 }
+};
+RG_STATIC_ASSERT(RG_LEN(Trampolines) == MaxTrampolines);
+
+static RG_THREAD_LOCAL CallData *exec_call;
 
 static inline RegisterClass MergeClasses(RegisterClass cls1, RegisterClass cls2)
 {
@@ -94,7 +144,8 @@ static Size ClassifyType(const TypeInfo *type, Size offset, Span<RegisterClass>
         case PrimitiveKind::UInt64:
         case PrimitiveKind::String:
         case PrimitiveKind::String16:
-        case PrimitiveKind::Pointer: {
+        case PrimitiveKind::Pointer:
+        case PrimitiveKind::Callback: {
             classes[0] = MergeClasses(classes[0], RegisterClass::Integer);
             return 1;
         } break;
@@ -197,7 +248,7 @@ bool AnalyseFunction(InstanceData *, FunctionInfo *func)
 
 bool CallData::Prepare(const Napi::CallbackInfo &info)
 {
-    uint8_t *args_ptr = nullptr;
+    uint64_t *args_ptr = nullptr;
     uint64_t *gpr_ptr = nullptr;
     uint64_t *xmm_ptr = nullptr;
 
@@ -231,13 +282,7 @@ bool CallData::Prepare(const Napi::CallbackInfo &info)
                 }
 
                 bool b = value.As<Napi::Boolean>();
-
-                if (RG_LIKELY(param.gpr_count)) {
-                    *(gpr_ptr++) = (uint64_t)b;
-                } else {
-                    *args_ptr = (uint8_t)b;
-                    args_ptr += 8;
-                }
+                *((param.gpr_count ? gpr_ptr : args_ptr)++) = (uint64_t)b;
             } break;
             case PrimitiveKind::Int8:
             case PrimitiveKind::UInt8:
@@ -245,22 +290,23 @@ bool CallData::Prepare(const Napi::CallbackInfo &info)
             case PrimitiveKind::UInt16:
             case PrimitiveKind::Int32:
             case PrimitiveKind::UInt32:
-            case PrimitiveKind::Int64:
-            case PrimitiveKind::UInt64: {
+            case PrimitiveKind::Int64: {
                 if (RG_UNLIKELY(!value.IsNumber() && !value.IsBigInt())) {
                     ThrowError<Napi::TypeError>(env, "Unexpected %1 value for argument %2, expected number", GetValueType(instance, value), i + 1);
                     return false;
                 }
 
                 int64_t v = CopyNumber<int64_t>(value);
-
-                if (RG_LIKELY(param.gpr_count)) {
-                    *(gpr_ptr++) = (uint64_t)v;
-                } else {
-                    args_ptr = AlignUp(args_ptr, param.type->align);
-                    memcpy(args_ptr, &v, param.type->size); // Little Endian
-                    args_ptr += 8;
+                *(int64_t *)((param.gpr_count ? gpr_ptr : args_ptr)++) = v;
+            } break;
+            case PrimitiveKind::UInt64: {
+                if (RG_UNLIKELY(!value.IsNumber() && !value.IsBigInt())) {
+                    ThrowError<Napi::TypeError>(env, "Unexpected %1 value for argument %2, expected number", GetValueType(instance, value), i + 1);
+                    return false;
                 }
+
+                uint64_t v = CopyNumber<uint64_t>(value);
+                *((param.gpr_count ? gpr_ptr : args_ptr)++) = v;
             } break;
             case PrimitiveKind::String: {
                 const char *str;
@@ -275,13 +321,7 @@ bool CallData::Prepare(const Napi::CallbackInfo &info)
                     return false;
                 }
 
-                if (RG_LIKELY(param.gpr_count)) {
-                    *(gpr_ptr++) = (uint64_t)str;
-                } else {
-                    args_ptr = AlignUp(args_ptr, 8);
-                    *(uint64_t *)args_ptr = (uint64_t)str;
-                    args_ptr += 8;
-                }
+                *(const char **)((param.gpr_count ? gpr_ptr : args_ptr)++) = str;
             } break;
             case PrimitiveKind::String16: {
                 const char16_t *str16;
@@ -296,13 +336,7 @@ bool CallData::Prepare(const Napi::CallbackInfo &info)
                     return false;
                 }
 
-                if (RG_LIKELY(param.gpr_count)) {
-                    *(gpr_ptr++) = (uint64_t)str16;
-                } else {
-                    args_ptr = AlignUp(args_ptr, 8);
-                    *(uint64_t *)args_ptr = (uint64_t)str16;
-                    args_ptr += 8;
-                }
+                *(const char16_t **)((param.gpr_count ? gpr_ptr : args_ptr)++) = str16;
             } break;
             case PrimitiveKind::Pointer: {
                 uint8_t *ptr;
@@ -335,13 +369,7 @@ bool CallData::Prepare(const Napi::CallbackInfo &info)
                     return false;
                 }
 
-                if (RG_LIKELY(param.gpr_count)) {
-                    *(gpr_ptr++) = (uint64_t)ptr;
-                } else {
-                    args_ptr = AlignUp(args_ptr, 8);
-                    *(uint64_t *)args_ptr = (uint64_t)ptr;
-                    args_ptr += 8;
-                }
+                *(void **)((param.gpr_count ? gpr_ptr : args_ptr)++) = ptr;
             } break;
             case PrimitiveKind::Record: {
                 if (RG_UNLIKELY(!IsObject(value))) {
@@ -357,10 +385,9 @@ bool CallData::Prepare(const Napi::CallbackInfo &info)
                     uint64_t buf[2] = {};
                     if (!PushObject(obj, param.type, (uint8_t *)buf))
                         return false;
+                    uint64_t *ptr = buf;
 
                     if (param.gpr_first) {
-                        uint64_t *ptr = buf;
-
                         *(gpr_ptr++) = *(ptr++);
                         if (param.gpr_count == 2) {
                             *(gpr_ptr++) = *(ptr++);
@@ -368,8 +395,6 @@ bool CallData::Prepare(const Napi::CallbackInfo &info)
                             *(xmm_ptr++) = *(ptr++);
                         }
                     } else {
-                        uint64_t *ptr = buf;
-
                         *(xmm_ptr++) = *(ptr++);
                         if (param.xmm_count == 2) {
                             *(xmm_ptr++) = *(ptr++);
@@ -379,9 +404,9 @@ bool CallData::Prepare(const Napi::CallbackInfo &info)
                     }
                 } else if (param.use_memory) {
                     args_ptr = AlignUp(args_ptr, param.type->align);
-                    if (!PushObject(obj, param.type, args_ptr))
+                    if (!PushObject(obj, param.type, (uint8_t *)args_ptr))
                         return false;
-                    args_ptr += AlignLen(param.type->size, 8);
+                    args_ptr += (param.type->size + 7) / 8;
                 }
             } break;
             case PrimitiveKind::Array: { RG_UNREACHABLE(); } break;
@@ -392,14 +417,7 @@ bool CallData::Prepare(const Napi::CallbackInfo &info)
                 }
 
                 float f = CopyNumber<float>(value);
-
-                if (RG_LIKELY(param.xmm_count)) {
-                    memcpy(xmm_ptr++, &f, 4);
-                } else {
-                    args_ptr = AlignUp(args_ptr, 4);
-                    memcpy(args_ptr, &f, 4);
-                    args_ptr += 8;
-                }
+                *(float *)((param.xmm_count ? xmm_ptr : args_ptr)++) = f;
             } break;
             case PrimitiveKind::Float64: {
                 if (RG_UNLIKELY(!value.IsNumber() && !value.IsBigInt())) {
@@ -408,29 +426,46 @@ bool CallData::Prepare(const Napi::CallbackInfo &info)
                 }
 
                 double d = CopyNumber<double>(value);
+                *(double *)((param.xmm_count ? xmm_ptr : args_ptr)++) = d;
+            } break;
+            case PrimitiveKind::Callback: {
+                void *ptr;
+
+                if (value.IsFunction()) {
+                    Napi::Function func = value.As<Napi::Function>();
 
-                if (RG_LIKELY(param.xmm_count)) {
-                    memcpy(xmm_ptr++, &d, 8);
+                    Size idx = ReserveTrampoline(param.type->proto, func);
+                    if (RG_UNLIKELY(idx < 0))
+                        return false;
+
+                    ptr = GetTrampoline(idx, param.type->proto);
+                } else if (CheckValueTag(instance, value, param.type)) {
+                    ptr = value.As<Napi::External<void>>().Data();
+                } else if (IsNullOrUndefined(value)) {
+                    ptr = nullptr;
                 } else {
-                    args_ptr = AlignUp(args_ptr, 8);
-                    memcpy(args_ptr, &d, 8);
-                    args_ptr += 8;
+                    ThrowError<Napi::TypeError>(env, "Unexpected %1 value for argument %2, expected %3", GetValueType(instance, value), i + 1, param.type->name);
+                    return false;
                 }
+
+                *(void **)((param.gpr_count ? gpr_ptr : args_ptr)++) = ptr;
             } break;
         }
     }
 
-    sp = mem->stack.end();
+    new_sp = mem->stack.end();
 
     return true;
 }
 
 void CallData::Execute()
 {
+    exec_call = this;
+
 #define PERFORM_CALL(Suffix) \
         ([&]() { \
-            auto ret = (func->forward_fp ? ForwardCallX ## Suffix(func->func, sp) \
-                                         : ForwardCall ## Suffix(func->func, sp)); \
+            auto ret = (func->forward_fp ? ForwardCallX ## Suffix(func->func, new_sp, &old_sp) \
+                                         : ForwardCall ## Suffix(func->func, new_sp, &old_sp)); \
             return ret; \
         })()
 
@@ -448,7 +483,8 @@ void CallData::Execute()
         case PrimitiveKind::UInt64:
         case PrimitiveKind::String:
         case PrimitiveKind::String16:
-        case PrimitiveKind::Pointer: { result.u64 = PERFORM_CALL(GG).rax; } break;
+        case PrimitiveKind::Pointer:
+        case PrimitiveKind::Callback: { result.u64 = PERFORM_CALL(GG).rax; } break;
         case PrimitiveKind::Record: {
             if (func->ret.gpr_first && !func->ret.xmm_count) {
                 RaxRdxRet ret = PERFORM_CALL(GG);
@@ -492,11 +528,16 @@ Napi::Value CallData::Complete()
         case PrimitiveKind::UInt64: return Napi::BigInt::New(env, result.u64);
         case PrimitiveKind::String: return Napi::String::New(env, (const char *)result.ptr);
         case PrimitiveKind::String16: return Napi::String::New(env, (const char16_t *)result.ptr);
-        case PrimitiveKind::Pointer: {
-            Napi::External<void> external = Napi::External<void>::New(env, result.ptr);
-            SetValueTag(instance, external, func->ret.type);
+        case PrimitiveKind::Pointer:
+        case PrimitiveKind::Callback: {
+            if (result.ptr) {
+                Napi::External<void> external = Napi::External<void>::New(env, result.ptr);
+                SetValueTag(instance, external, func->ret.type);
 
-            return external;
+                return external;
+            } else {
+                return env.Null();
+            }
         } break;
         case PrimitiveKind::Record: {
             const uint8_t *ptr = return_ptr ? (const uint8_t *)return_ptr
@@ -513,6 +554,337 @@ Napi::Value CallData::Complete()
     RG_UNREACHABLE();
 }
 
+void CallData::Relay(Size idx, uint8_t *own_sp, uint8_t *caller_sp, BackRegisters *out_reg)
+{
+    const FunctionInfo *proto = instance->trampolines[idx].proto;
+    Napi::Function func = instance->trampolines[idx].func;
+
+    // Allow reuse of static trampoline
+    instance->free_trampolines |= 1u << idx;
+    used_trampolines &= ~(1u << idx);
+
+    uint64_t *gpr_ptr = (uint64_t *)own_sp;
+    uint64_t *xmm_ptr = gpr_ptr + 6;
+    uint64_t *args_ptr = (uint64_t *)caller_sp;
+
+    uint8_t *return_ptr = proto->ret.use_memory ? (uint8_t *)gpr_ptr[0] : nullptr;
+    gpr_ptr += proto->ret.use_memory;
+
+    LocalArray<napi_value, MaxParameters> arguments;
+
+    // Convert to JS arguments
+    for (Size i = 0; i < proto->parameters.len; i++) {
+        const ParameterInfo &param = proto->parameters[i];
+        RG_ASSERT(param.directions >= 1 && param.directions <= 3);
+
+        switch (param.type->primitive) {
+            case PrimitiveKind::Void: { RG_UNREACHABLE(); } break;
+
+            case PrimitiveKind::Bool: {
+                bool b = *(bool *)((param.gpr_count ? gpr_ptr : args_ptr)++);
+
+                Napi::Value arg = Napi::Boolean::New(env, b);
+                arguments.Append(arg);
+            } break;
+            case PrimitiveKind::Int8: {
+                double d = (double)*(int8_t *)((param.gpr_count ? gpr_ptr : args_ptr)++);
+
+                Napi::Value arg = Napi::Number::New(env, d);
+                arguments.Append(arg);
+            } break;
+            case PrimitiveKind::UInt8: {
+                double d = (double)*(uint8_t *)((param.gpr_count ? gpr_ptr : args_ptr)++);
+
+                Napi::Value arg = Napi::Number::New(env, d);
+                arguments.Append(arg);
+            } break;
+            case PrimitiveKind::Int16: {
+                double d = (double)*(int16_t *)((param.gpr_count ? gpr_ptr : args_ptr)++);
+
+                Napi::Value arg = Napi::Number::New(env, d);
+                arguments.Append(arg);
+            } break;
+            case PrimitiveKind::UInt16: {
+                double d = (double)*(uint16_t *)((param.gpr_count ? gpr_ptr : args_ptr)++);
+
+                Napi::Value arg = Napi::Number::New(env, d);
+                arguments.Append(arg);
+            } break;
+            case PrimitiveKind::Int32: {
+                double d = (double)*(int32_t *)((param.gpr_count ? gpr_ptr : args_ptr)++);
+
+                Napi::Value arg = Napi::Number::New(env, d);
+                arguments.Append(arg);
+            } break;
+            case PrimitiveKind::UInt32: {
+                double d = (double)*(int32_t *)((param.gpr_count ? gpr_ptr : args_ptr)++);
+
+                Napi::Value arg = Napi::Number::New(env, d);
+                arguments.Append(arg);
+            } break;
+            case PrimitiveKind::Int64: {
+                int64_t v = *(int64_t *)((param.gpr_count ? gpr_ptr : args_ptr)++);
+
+                Napi::Value arg = Napi::BigInt::New(env, v);
+                arguments.Append(arg);
+            } break;
+            case PrimitiveKind::UInt64: {
+                uint64_t v = *(uint64_t *)((param.gpr_count ? gpr_ptr : args_ptr)++);
+
+                Napi::Value arg = Napi::BigInt::New(env, v);
+                arguments.Append(arg);
+            } break;
+            case PrimitiveKind::String: {
+                const char *str = *(const char **)((param.gpr_count ? gpr_ptr : args_ptr)++);
+
+                Napi::Value arg = Napi::String::New(env, str);
+                arguments.Append(arg);
+            } break;
+            case PrimitiveKind::String16: {
+                const char16_t *str16 = *(const char16_t **)((param.gpr_count ? gpr_ptr : args_ptr)++);
+
+                Napi::Value arg = Napi::String::New(env, str16);
+                arguments.Append(arg);
+            } break;
+            case PrimitiveKind::Pointer:
+            case PrimitiveKind::Callback: {
+                void *ptr2 = *(void **)((param.gpr_count ? gpr_ptr : args_ptr)++);
+
+                if (ptr2) {
+                    Napi::External<void> external = Napi::External<void>::New(env, ptr2);
+                    SetValueTag(instance, external, param.type);
+
+                    arguments.Append(external);
+                } else {
+                    arguments.Append(env.Null());
+                }
+            } break;
+            case PrimitiveKind::Record: {
+                if (param.gpr_count || param.xmm_count) {
+                    RG_ASSERT(param.type->size <= 16);
+
+                    uint64_t buf[2] = {};
+                    uint64_t *ptr = buf;
+
+                    if (param.gpr_first) {
+                        *(ptr++) = *(gpr_ptr++);
+                        if (param.gpr_count == 2) {
+                            *(ptr++) = *(gpr_ptr++);
+                        } else if (param.xmm_count == 1) {
+                            *(ptr++) = *(xmm_ptr++);
+                        }
+                    } else {
+                        *(ptr++) = *(xmm_ptr++);
+                        if (param.xmm_count == 2) {
+                            *(ptr++) = *(xmm_ptr++);
+                        } else if (param.gpr_count == 1) {
+                            *(ptr++) = *(gpr_ptr++);
+                        }
+                    }
+
+                    Napi::Object obj = PopObject((const uint8_t *)buf, param.type);
+                    arguments.Append(obj);
+                } else if (param.use_memory) {
+                    args_ptr = AlignUp(args_ptr, param.type->align);
+
+                    Napi::Object obj = PopObject((const uint8_t *)args_ptr, param.type);
+                    arguments.Append(obj);
+
+                    args_ptr += (param.type->size + 7) / 8;
+                }
+            } break;
+            case PrimitiveKind::Array: { RG_UNREACHABLE(); } break;
+            case PrimitiveKind::Float32: {
+                float f = *(float *)((param.xmm_count ? xmm_ptr : args_ptr)++);
+
+                Napi::Value arg = Napi::Number::New(env, (double)f);
+                arguments.Append(arg);
+            } break;
+            case PrimitiveKind::Float64: {
+                double d = *(double *)((param.xmm_count ? xmm_ptr : args_ptr)++);
+
+                Napi::Value arg = Napi::Number::New(env, d);
+                arguments.Append(arg);
+            } break;
+        }
+    }
+
+    const TypeInfo *type = proto->ret.type;
+
+    // Make the call
+    napi_value ret = CallSwitchStack(&func, (size_t)arguments.len, arguments.data, old_sp, &mem->stack,
+                                     [](Napi::Function *func, size_t argc, napi_value *argv) { return (napi_value)func->Call(argc, argv); });
+    Napi::Value value(env, ret);
+
+    // Convert the result
+    switch (type->primitive) {
+        case PrimitiveKind::Void: {} break;
+        case PrimitiveKind::Bool: {
+            if (RG_UNLIKELY(!value.IsBoolean())) {
+                ThrowError<Napi::TypeError>(env, "Unexpected %1 value for return value, expected boolean", GetValueType(instance, value));
+                return;
+            }
+
+            bool b = value.As<Napi::Boolean>();
+            out_reg->rax = (uint64_t)b;
+        } break;
+        case PrimitiveKind::Int8:
+        case PrimitiveKind::UInt8:
+        case PrimitiveKind::Int16:
+        case PrimitiveKind::UInt16:
+        case PrimitiveKind::Int32:
+        case PrimitiveKind::UInt32:
+        case PrimitiveKind::Int64:
+        case PrimitiveKind::UInt64: {
+            if (RG_UNLIKELY(!value.IsNumber() && !value.IsBigInt())) {
+                ThrowError<Napi::TypeError>(env, "Unexpected %1 value for return value, expected number", GetValueType(instance, value));
+                return;
+            }
+
+            int64_t v = CopyNumber<int64_t>(value);
+            out_reg->rax = (uint64_t)v;
+        } break;
+        case PrimitiveKind::String: {
+            const char *str;
+            if (RG_LIKELY(value.IsString())) {
+                str = PushString(value);
+                if (RG_UNLIKELY(!str))
+                    return;
+            } else if (IsNullOrUndefined(value)) {
+                str = nullptr;
+            } else {
+                ThrowError<Napi::TypeError>(env, "Unexpected %1 value for return value, expected string", GetValueType(instance, value));
+                return;
+            }
+
+            out_reg->rax = (uint64_t)str;
+        } break;
+        case PrimitiveKind::String16: {
+            const char16_t *str16;
+            if (RG_LIKELY(value.IsString())) {
+                str16 = PushString16(value);
+                if (RG_UNLIKELY(!str16))
+                    return;
+            } else if (IsNullOrUndefined(value)) {
+                str16 = nullptr;
+            } else {
+                ThrowError<Napi::TypeError>(env, "Unexpected %1 value for return value, expected string", GetValueType(instance, value));
+                return;
+            }
+
+            out_reg->rax = (uint64_t)str16;
+        } break;
+        case PrimitiveKind::Pointer: {
+            uint8_t *ptr;
+
+            if (CheckValueTag(instance, value, type)) {
+                ptr = value.As<Napi::External<uint8_t>>().Data();
+            } else if (IsObject(value) && type->ref->primitive == PrimitiveKind::Record) {
+                Napi::Object obj = value.As<Napi::Object>();
+
+                if (RG_UNLIKELY(!AllocHeap(type->ref->size, 16, &ptr)))
+                    return;
+
+                if (!PushObject(obj, type->ref, ptr))
+                    return;
+            } else if (IsNullOrUndefined(value)) {
+                ptr = nullptr;
+            } else {
+                ThrowError<Napi::TypeError>(env, "Unexpected %1 value for return value, expected %2", GetValueType(instance, value), type->name);
+                return;
+            }
+
+            out_reg->rax = (uint64_t)ptr;
+        } break;
+        case PrimitiveKind::Record: {
+            if (RG_UNLIKELY(!IsObject(value))) {
+                ThrowError<Napi::TypeError>(env, "Unexpected %1 value for return value, expected object", GetValueType(instance, value));
+                return;
+            }
+
+            Napi::Object obj = value.As<Napi::Object>();
+
+            if (return_ptr) {
+                if (!PushObject(obj, type, return_ptr))
+                    return;
+                out_reg->rax = (uint64_t)return_ptr;
+            } else {
+                RG_ASSERT(type->size <= 16);
+
+                uint8_t buf[16] = {};
+                if (!PushObject(obj, type, buf))
+                    return;
+
+                if (proto->ret.gpr_first && !proto->ret.xmm_count) {
+                    memcpy(&out_reg->rax, buf + 0, 8);
+                    memcpy(&out_reg->rdx, buf + 8, 8);
+                } else if (proto->ret.gpr_first) {
+                    memcpy(&out_reg->rax, buf + 0, 8);
+                    memcpy(&out_reg->xmm0, buf + 8, 8);
+                } else if (proto->ret.xmm_count == 2) {
+                    memcpy(&out_reg->xmm0, buf + 0, 8);
+                    memcpy(&out_reg->xmm1, buf + 8, 8);
+                } else {
+                    memcpy(&out_reg->xmm0, buf + 0, 8);
+                    memcpy(&out_reg->rax, buf + 8, 8);
+                }
+            }
+        } break;
+        case PrimitiveKind::Array: { RG_UNREACHABLE(); } break;
+        case PrimitiveKind::Float32: {
+            if (RG_UNLIKELY(!value.IsNumber() && !value.IsBigInt())) {
+                ThrowError<Napi::TypeError>(env, "Unexpected %1 value for return value, expected number", GetValueType(instance, value));
+                return;
+            }
+
+            float f = CopyNumber<float>(value);
+            memcpy(&out_reg->xmm0, &f, 4);
+        } break;
+        case PrimitiveKind::Float64: {
+            if (RG_UNLIKELY(!value.IsNumber() && !value.IsBigInt())) {
+                ThrowError<Napi::TypeError>(env, "Unexpected %1 value for return value, expected number", GetValueType(instance, value));
+                return;
+            }
+
+            double d = CopyNumber<double>(value);
+            out_reg->xmm0 = d;
+        } break;
+        case PrimitiveKind::Callback: {
+            void *ptr;
+
+            if (value.IsFunction()) {
+                Napi::Function func = value.As<Napi::Function>();
+
+                Size idx = ReserveTrampoline(type->proto, func);
+                if (RG_UNLIKELY(idx < 0))
+                    return;
+
+                ptr = GetTrampoline(idx, type->proto);
+            } else if (CheckValueTag(instance, value, type)) {
+                ptr = value.As<Napi::External<uint8_t>>().Data();
+            } else if (IsNullOrUndefined(value)) {
+                ptr = nullptr;
+            } else {
+                ThrowError<Napi::TypeError>(env, "Unexpected %1 value for return value, expected %2", GetValueType(instance, value), type->name);
+                return;
+            }
+
+            out_reg->rax = (uint64_t)ptr;
+        } break;
+    }
+}
+
+void *GetTrampoline(Size idx, const FunctionInfo *proto)
+{
+    bool xmm = proto->forward_fp || IsFloat(proto->ret.type);
+    return Trampolines[idx][xmm];
+}
+
+extern "C" void RelayCallBack(Size idx, uint8_t *own_sp, uint8_t *caller_sp, BackRegisters *out_reg)
+{
+    exec_call->Relay(idx, own_sp, caller_sp, out_reg);
+}
+
 }
 
 #endif