ffi 1.15.5 → 1.17.0

data/ext/ffi_c/libffi/src/aarch64/ffi.c

@@ -30,6 +30,7 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.  */
 #ifdef _WIN32
 #include <windows.h> /* FlushInstructionCache */
 #endif
+#include <tramp.h>
 
 /* Force FFI_TYPE_LONGDOUBLE to be different than FFI_TYPE_DOUBLE;
    all further uses in this file will refer to the 128-bit type.  */
@@ -247,13 +248,18 @@ is_vfp_type (const ffi_type *ty)
    state.
 
    The terse state variable names match the names used in the AARCH64
-   PCS. */
+   PCS.
+
+   The struct area is allocated downwards from the top of the argument
+   area.  It is used to hold copies of structures passed by value that are
+   bigger than 16 bytes.  */
 
 struct arg_state
 {
   unsigned ngrn;                /* Next general-purpose register number. */
   unsigned nsrn;                /* Next vector register number. */
   size_t nsaa;                  /* Next stack offset. */
+  size_t next_struct_area;	/* Place to allocate big structs. */
 
 #if defined (__APPLE__)
   unsigned allocating_variadic;
@@ -262,11 +268,12 @@ struct arg_state
 
 /* Initialize a procedure call argument marshalling state.  */
 static void
-arg_init (struct arg_state *state)
+arg_init (struct arg_state *state, size_t size)
 {
   state->ngrn = 0;
   state->nsrn = 0;
   state->nsaa = 0;
+  state->next_struct_area = size;
 #if defined (__APPLE__)
   state->allocating_variadic = 0;
 #endif
@@ -288,37 +295,83 @@ allocate_to_stack (struct arg_state *state, void *stack,
   if (alignment < 8)
     alignment = 8;
 #endif
-    
+
   nsaa = FFI_ALIGN (nsaa, alignment);
   state->nsaa = nsaa + size;
 
   return (char *)stack + nsaa;
 }
 
+/* Allocate and copy a structure that is passed by value on the stack and
+   return a pointer to it.  */
+static void *
+allocate_and_copy_struct_to_stack (struct arg_state *state, void *stack,
+				   size_t alignment, size_t size, void *value)
+{
+  size_t dest = state->next_struct_area - size;
+
+  /* Round down to the natural alignment of the value.  */
+  dest = FFI_ALIGN_DOWN (dest, alignment);
+  state->next_struct_area = dest;
+
+  return memcpy ((char *) stack + dest, value, size);
+}
+
 static ffi_arg
 extend_integer_type (void *source, int type)
 {
   switch (type)
     {
     case FFI_TYPE_UINT8:
-      return *(UINT8 *) source;
+      {
+        UINT8 u8;
+        memcpy (&u8, source, sizeof (u8));
+        return u8;
+      }
     case FFI_TYPE_SINT8:
-      return *(SINT8 *) source;
+      {
+        SINT8 s8;
+        memcpy (&s8, source, sizeof (s8));
+        return s8;
+      }
     case FFI_TYPE_UINT16:
-      return *(UINT16 *) source;
+      {
+        UINT16 u16;
+        memcpy (&u16, source, sizeof (u16));
+        return u16;
+      }
     case FFI_TYPE_SINT16:
-      return *(SINT16 *) source;
+      {
+        SINT16 s16;
+        memcpy (&s16, source, sizeof (s16));
+        return s16;
+      }
     case FFI_TYPE_UINT32:
-      return *(UINT32 *) source;
+      {
+        UINT32 u32;
+        memcpy (&u32, source, sizeof (u32));
+        return u32;
+      }
     case FFI_TYPE_INT:
     case FFI_TYPE_SINT32:
-      return *(SINT32 *) source;
+      {
+        SINT32 s32;
+        memcpy (&s32, source, sizeof (s32));
+        return s32;
+      }
     case FFI_TYPE_UINT64:
     case FFI_TYPE_SINT64:
-      return *(UINT64 *) source;
-      break;
+      {
+        UINT64 u64;
+        memcpy (&u64, source, sizeof (u64));
+        return u64;
+      }
     case FFI_TYPE_POINTER:
-      return *(uintptr_t *) source;
+      {
+        uintptr_t uptr;
+        memcpy (&uptr, source, sizeof (uptr));
+        return uptr;
+      }
     default:
       abort();
     }
@@ -333,51 +386,64 @@ extend_hfa_type (void *dest, void *src, int h)
   ssize_t f = h - AARCH64_RET_S4;
   void *x0;
 
+#define BTI_J "hint #36"
   asm volatile (
 	"adr	%0, 0f\n"
 "	add	%0, %0, %1\n"
 "	br	%0\n"
-"0:	ldp	s16, s17, [%3]\n"	/* S4 */
+"0:	"BTI_J"\n"			/* S4 */
+"	ldp	s16, s17, [%3]\n"
 "	ldp	s18, s19, [%3, #8]\n"
 "	b	4f\n"
-"	ldp	s16, s17, [%3]\n"	/* S3 */
+"	"BTI_J"\n"			/* S3 */
+"	ldp	s16, s17, [%3]\n"
 "	ldr	s18, [%3, #8]\n"
 "	b	3f\n"
-"	ldp	s16, s17, [%3]\n"	/* S2 */
+"	"BTI_J"\n"			/* S2 */
+"	ldp	s16, s17, [%3]\n"
 "	b	2f\n"
 "	nop\n"
-"	ldr	s16, [%3]\n"		/* S1 */
+"	"BTI_J"\n"			/* S1 */
+"	ldr	s16, [%3]\n"
 "	b	1f\n"
 "	nop\n"
-"	ldp	d16, d17, [%3]\n"	/* D4 */
+"	"BTI_J"\n"			/* D4 */
+"	ldp	d16, d17, [%3]\n"
 "	ldp	d18, d19, [%3, #16]\n"
 "	b	4f\n"
-"	ldp	d16, d17, [%3]\n"	/* D3 */
+"	"BTI_J"\n"			/* D3 */
+"	ldp	d16, d17, [%3]\n"
 "	ldr	d18, [%3, #16]\n"
 "	b	3f\n"
-"	ldp	d16, d17, [%3]\n"	/* D2 */
+"	"BTI_J"\n"			/* D2 */
+"	ldp	d16, d17, [%3]\n"
 "	b	2f\n"
 "	nop\n"
-"	ldr	d16, [%3]\n"		/* D1 */
+"	"BTI_J"\n"			/* D1 */
+"	ldr	d16, [%3]\n"
 "	b	1f\n"
 "	nop\n"
-"	ldp	q16, q17, [%3]\n"	/* Q4 */
+"	"BTI_J"\n"			/* Q4 */
+"	ldp	q16, q17, [%3]\n"
 "	ldp	q18, q19, [%3, #32]\n"
 "	b	4f\n"
-"	ldp	q16, q17, [%3]\n"	/* Q3 */
+"	"BTI_J"\n"			/* Q3 */
+"	ldp	q16, q17, [%3]\n"
 "	ldr	q18, [%3, #32]\n"
 "	b	3f\n"
-"	ldp	q16, q17, [%3]\n"	/* Q2 */
+"	"BTI_J"\n"			/* Q2 */
+"	ldp	q16, q17, [%3]\n"
 "	b	2f\n"
 "	nop\n"
-"	ldr	q16, [%3]\n"		/* Q1 */
+"	"BTI_J"\n"			/* Q1 */
+"	ldr	q16, [%3]\n"
 "	b	1f\n"
 "4:	str	q19, [%2, #48]\n"
 "3:	str	q18, [%2, #32]\n"
 "2:	str	q17, [%2, #16]\n"
 "1:	str	q16, [%2]"
     : "=&r"(x0)
-    : "r"(f * 12), "r"(dest), "r"(src)
+    : "r"(f * 16), "r"(dest), "r"(src)
     : "memory", "v16", "v17", "v18", "v19");
 }
 #endif
@@ -616,19 +682,21 @@ ffi_call_int (ffi_cif *cif, void (*fn)(void), void *orig_rvalue,
   else if (flags & AARCH64_RET_NEED_COPY)
     rsize = 16;
 
-  /* Allocate consectutive stack for everything we'll need.  */
-  context = alloca (sizeof(struct call_context) + stack_bytes + 32 + rsize);
+  /* Allocate consectutive stack for everything we'll need.
+     The frame uses 40 bytes for: lr, fp, rvalue, flags, sp */
+  context = alloca (sizeof(struct call_context) + stack_bytes + 40 + rsize);
   stack = context + 1;
   frame = (void*)((uintptr_t)stack + (uintptr_t)stack_bytes);
-  rvalue = (rsize ? (void*)((uintptr_t)frame + 32) : orig_rvalue);
+  rvalue = (rsize ? (void*)((uintptr_t)frame + 40) : orig_rvalue);
 
-  arg_init (&state);
+  arg_init (&state, stack_bytes);
   for (i = 0, nargs = cif->nargs; i < nargs; i++)
     {
       ffi_type *ty = cif->arg_types[i];
       size_t s = ty->size;
       void *a = avalue[i];
       int h, t;
+      void *dest;
 
       t = ty->type;
       switch (t)
@@ -676,8 +744,6 @@ ffi_call_int (ffi_cif *cif, void (*fn)(void), void *orig_rvalue,
 	case FFI_TYPE_STRUCT:
 	case FFI_TYPE_COMPLEX:
 	  {
-	    void *dest;
-
 	    h = is_vfp_type (ty);
 	    if (h)
 	      {
@@ -710,9 +776,12 @@ ffi_call_int (ffi_cif *cif, void (*fn)(void), void *orig_rvalue,
 	    else if (s > 16)
 	      {
 		/* If the argument is a composite type that is larger than 16
-		   bytes, then the argument has been copied to memory, and
+		   bytes, then the argument is copied to memory, and
 		   the argument is replaced by a pointer to the copy.  */
-		a = &avalue[i];
+		dest = allocate_and_copy_struct_to_stack (&state, stack,
+							  ty->alignment, s,
+							  avalue[i]);
+		a = &dest;
 		t = FFI_TYPE_POINTER;
 		s = sizeof (void *);
 		goto do_pointer;
@@ -769,6 +838,8 @@ ffi_call (ffi_cif *cif, void (*fn) (void), void *rvalue, void **avalue)
   ffi_call_int (cif, fn, rvalue, avalue, NULL);
 }
 
+#if FFI_CLOSURES
+
 #ifdef FFI_GO_CLOSURES
 void
 ffi_call_go (ffi_cif *cif, void (*fn) (void), void *rvalue,
@@ -782,6 +853,10 @@ ffi_call_go (ffi_cif *cif, void (*fn) (void), void *rvalue,
 
 extern void ffi_closure_SYSV (void) FFI_HIDDEN;
 extern void ffi_closure_SYSV_V (void) FFI_HIDDEN;
+#if defined(FFI_EXEC_STATIC_TRAMP)
+extern void ffi_closure_SYSV_alt (void) FFI_HIDDEN;
+extern void ffi_closure_SYSV_V_alt (void) FFI_HIDDEN;
+#endif
 
 ffi_status
 ffi_prep_closure_loc (ffi_closure *closure,
@@ -790,25 +865,25 @@ ffi_prep_closure_loc (ffi_closure *closure,
                       void *user_data,
                       void *codeloc)
 {
-  if (cif->abi != FFI_SYSV)
+  if (cif->abi != FFI_SYSV && cif->abi != FFI_WIN64)
     return FFI_BAD_ABI;
 
   void (*start)(void);
-  
+
   if (cif->flags & AARCH64_FLAG_ARG_V)
     start = ffi_closure_SYSV_V;
   else
     start = ffi_closure_SYSV;
 
 #if FFI_EXEC_TRAMPOLINE_TABLE
-#ifdef __MACH__
-#ifdef HAVE_PTRAUTH
-  codeloc = ptrauth_strip (codeloc, ptrauth_key_asia);
-#endif
+# ifdef __MACH__
+#  ifdef HAVE_PTRAUTH
+  codeloc = ptrauth_auth_data(codeloc, ptrauth_key_function_pointer, 0);
+#  endif
   void **config = (void **)((uint8_t *)codeloc - PAGE_MAX_SIZE);
   config[0] = closure;
   config[1] = start;
-#endif
+# endif
 #else
   static const unsigned char trampoline[16] = {
     0x90, 0x00, 0x00, 0x58,	/* ldr	x16, tramp+16	*/
@@ -816,22 +891,39 @@ ffi_prep_closure_loc (ffi_closure *closure,
     0x00, 0x02, 0x1f, 0xd6	/* br	x16		*/
   };
   char *tramp = closure->tramp;
-  
+
+# if defined(FFI_EXEC_STATIC_TRAMP)
+  if (ffi_tramp_is_present(closure))
+    {
+      /* Initialize the static trampoline's parameters. */
+      if (start == ffi_closure_SYSV_V)
+          start = ffi_closure_SYSV_V_alt;
+      else
+          start = ffi_closure_SYSV_alt;
+      ffi_tramp_set_parms (closure->ftramp, start, closure);
+      goto out;
+    }
+# endif
+
+  /* Initialize the dynamic trampoline. */
   memcpy (tramp, trampoline, sizeof(trampoline));
-  
+
   *(UINT64 *)(tramp + 16) = (uintptr_t)start;
 
   ffi_clear_cache(tramp, tramp + FFI_TRAMPOLINE_SIZE);
 
   /* Also flush the cache for code mapping.  */
-#ifdef _WIN32
+# ifdef _WIN32
   // Not using dlmalloc.c for Windows ARM64 builds
   // so calling ffi_data_to_code_pointer() isn't necessary
   unsigned char *tramp_code = tramp;
-  #else
+# else
   unsigned char *tramp_code = ffi_data_to_code_pointer (tramp);
-  #endif
+# endif
   ffi_clear_cache (tramp_code, tramp_code + FFI_TRAMPOLINE_SIZE);
+# if defined(FFI_EXEC_STATIC_TRAMP)
+out:
+# endif
 #endif
 
   closure->cif = cif;
@@ -851,7 +943,7 @@ ffi_prep_go_closure (ffi_go_closure *closure, ffi_cif* cif,
 {
   void (*start)(void);
 
-  if (cif->abi != FFI_SYSV)
+  if (cif->abi != FFI_SYSV && cif->abi != FFI_WIN64)
     return FFI_BAD_ABI;
 
   if (cif->flags & AARCH64_FLAG_ARG_V)
@@ -891,10 +983,17 @@ ffi_closure_SYSV_inner (ffi_cif *cif,
 			void *stack, void *rvalue, void *struct_rvalue)
 {
   void **avalue = (void**) alloca (cif->nargs * sizeof (void*));
-  int i, h, nargs, flags;
+  int i, h, nargs, flags, isvariadic = 0;
   struct arg_state state;
 
-  arg_init (&state);
+  arg_init (&state, cif->bytes);
+
+  flags = cif->flags;
+  if (flags & AARCH64_FLAG_VARARG)
+  {
+    isvariadic = 1;
+    flags &= ~AARCH64_FLAG_VARARG;
+  }
 
   for (i = 0, nargs = cif->nargs; i < nargs; i++)
     {
@@ -930,14 +1029,13 @@ ffi_closure_SYSV_inner (ffi_cif *cif,
 	  if (h)
 	    {
 	      n = 4 - (h & 3);
-#ifdef _WIN32  /* for handling armasm calling convention */
-              if (cif->is_variadic)
+              if (cif->abi == FFI_WIN64 && isvariadic)
                 {
                   if (state.ngrn + n <= N_X_ARG_REG)
                     {
                       void *reg = &context->x[state.ngrn];
                       state.ngrn += (unsigned int)n;
-    
+
                       /* Eeek! We need a pointer to the structure, however the
                        homogeneous float elements are being passed in individual
                        registers, therefore for float and double the structure
@@ -957,7 +1055,6 @@ ffi_closure_SYSV_inner (ffi_cif *cif,
                 }
               else
                 {
-#endif  /* for handling armasm calling convention */
                   if (state.nsrn + n <= N_V_ARG_REG)
                     {
                       void *reg = &context->v[state.nsrn];
@@ -970,17 +1067,24 @@ ffi_closure_SYSV_inner (ffi_cif *cif,
                       avalue[i] = allocate_to_stack(&state, stack,
                                                    ty->alignment, s);
                     }
-#ifdef _WIN32  /* for handling armasm calling convention */
                 }
-#endif  /* for handling armasm calling convention */
             }
           else if (s > 16)
             {
               /* Replace Composite type of size greater than 16 with a
                   pointer.  */
+#ifdef __ILP32__
+             UINT64 avalue_tmp;
+             memcpy (&avalue_tmp,
+                  allocate_int_to_reg_or_stack (context, &state,
+                                               stack, sizeof (void *)),
+                  sizeof (UINT64));
+             avalue[i] = (void *)(UINT32)avalue_tmp;
+#else
               avalue[i] = *(void **)
               allocate_int_to_reg_or_stack (context, &state, stack,
                                          sizeof (void *));
+#endif
             }
           else
             {
@@ -1013,7 +1117,6 @@ ffi_closure_SYSV_inner (ffi_cif *cif,
 #endif
     }
 
-  flags = cif->flags;
   if (flags & AARCH64_RET_IN_MEM)
     rvalue = struct_rvalue;
 
@@ -1022,4 +1125,18 @@ ffi_closure_SYSV_inner (ffi_cif *cif,
   return flags;
 }
 
+#if defined(FFI_EXEC_STATIC_TRAMP)
+void *
+ffi_tramp_arch (size_t *tramp_size, size_t *map_size)
+{
+  extern void *trampoline_code_table;
+
+  *tramp_size = AARCH64_TRAMP_SIZE;
+  *map_size = AARCH64_TRAMP_MAP_SIZE;
+  return &trampoline_code_table;
+}
+#endif
+
+#endif /* FFI_CLOSURES */
+
 #endif /* (__aarch64__) || defined(__arm64__)|| defined (_M_ARM64)*/

data/ext/ffi_c/libffi/src/aarch64/internal.h

@@ -66,3 +66,35 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.  */
 #define N_X_ARG_REG		8
 #define N_V_ARG_REG		8
 #define CALL_CONTEXT_SIZE	(N_V_ARG_REG * 16 + N_X_ARG_REG * 8)
+
+#if defined(FFI_EXEC_STATIC_TRAMP)
+/*
+ * For the trampoline code table mapping, a mapping size of 16K is chosen to
+ * cover the base page sizes of 4K and 16K.
+ */
+#define AARCH64_TRAMP_MAP_SHIFT	14
+#define AARCH64_TRAMP_MAP_SIZE	(1 << AARCH64_TRAMP_MAP_SHIFT)
+#define AARCH64_TRAMP_SIZE	32
+
+#endif
+
+/* Helpers for writing assembly compatible with arm ptr auth */
+#ifdef LIBFFI_ASM
+
+#ifdef HAVE_PTRAUTH
+#define SIGN_LR pacibsp
+#define SIGN_LR_WITH_REG(x) pacib lr, x
+#define AUTH_LR_AND_RET retab
+#define AUTH_LR_WITH_REG(x) autib lr, x
+#define BRANCH_AND_LINK_TO_REG blraaz
+#define BRANCH_TO_REG braaz
+#else
+#define SIGN_LR
+#define SIGN_LR_WITH_REG(x)
+#define AUTH_LR_AND_RET ret
+#define AUTH_LR_WITH_REG(x)
+#define BRANCH_AND_LINK_TO_REG blr
+#define BRANCH_TO_REG br
+#endif
+
+#endif