ffi 1.16.3 → 1.17.0-java

data/ext/ffi_c/libffi/src/powerpc/ffi_linux64.c

@@ -1,1153 +0,0 @@
-/* -----------------------------------------------------------------------
-   ffi_linux64.c - Copyright (C) 2013 IBM
-                   Copyright (C) 2011 Anthony Green
-                   Copyright (C) 2011 Kyle Moffett
-                   Copyright (C) 2008 Red Hat, Inc
-                   Copyright (C) 2007, 2008 Free Software Foundation, Inc
-                   Copyright (c) 1998 Geoffrey Keating
-
-   PowerPC Foreign Function Interface
-
-   Permission is hereby granted, free of charge, to any person obtaining
-   a copy of this software and associated documentation files (the
-   ``Software''), to deal in the Software without restriction, including
-   without limitation the rights to use, copy, modify, merge, publish,
-   distribute, sublicense, and/or sell copies of the Software, and to
-   permit persons to whom the Software is furnished to do so, subject to
-   the following conditions:
-
-   The above copyright notice and this permission notice shall be included
-   in all copies or substantial portions of the Software.
-
-   THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND, EXPRESS
-   OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-   MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-   IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR
-   OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
-   ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
-   OTHER DEALINGS IN THE SOFTWARE.
-   ----------------------------------------------------------------------- */
-
-#include "ffi.h"
-
-#ifdef POWERPC64
-#include "ffi_common.h"
-#include "ffi_powerpc.h"
-
-
-/* About the LINUX64 ABI.  */
-enum {
-  NUM_GPR_ARG_REGISTERS64 = 8,
-  NUM_FPR_ARG_REGISTERS64 = 13,
-  NUM_VEC_ARG_REGISTERS64 = 12,
-};
-enum { ASM_NEEDS_REGISTERS64 = 4 };
-
-
-#if HAVE_LONG_DOUBLE_VARIANT && FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
-/* Adjust size of ffi_type_longdouble.  */
-void FFI_HIDDEN
-ffi_prep_types_linux64 (ffi_abi abi)
-{
-  if ((abi & (FFI_LINUX | FFI_LINUX_LONG_DOUBLE_128)) == FFI_LINUX)
-    {
-      ffi_type_longdouble.size = 8;
-      ffi_type_longdouble.alignment = 8;
-    }
-  else
-    {
-      ffi_type_longdouble.size = 16;
-      ffi_type_longdouble.alignment = 16;
-    }
-}
-#endif
-
-
-static unsigned int
-discover_homogeneous_aggregate (ffi_abi abi,
-                                const ffi_type *t,
-                                unsigned int *elnum)
-{
-  switch (t->type)
-    {
-#if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
-    case FFI_TYPE_LONGDOUBLE:
-      /* 64-bit long doubles are equivalent to doubles. */
-      if ((abi & FFI_LINUX_LONG_DOUBLE_128) == 0)
-        {
-          *elnum = 1;
-          return FFI_TYPE_DOUBLE;
-        }
-      /* IBM extended precision values use unaligned pairs
-         of FPRs, but according to the ABI must be considered
-         distinct from doubles. They are also limited to a
-         maximum of four members in a homogeneous aggregate. */
-      else if ((abi & FFI_LINUX_LONG_DOUBLE_IEEE128) == 0)
-        {
-          *elnum = 2;
-          return FFI_TYPE_LONGDOUBLE;
-        }
-      /* Fall through. */
-#endif
-    case FFI_TYPE_FLOAT:
-    case FFI_TYPE_DOUBLE:
-      *elnum = 1;
-      return (int) t->type;
-
-    case FFI_TYPE_STRUCT:;
-      {
-	unsigned int base_elt = 0, total_elnum = 0;
-	ffi_type **el = t->elements;
-	while (*el)
-	  {
-	    unsigned int el_elt, el_elnum = 0;
-	    el_elt = discover_homogeneous_aggregate (abi, *el, &el_elnum);
-	    if (el_elt == 0
-		|| (base_elt && base_elt != el_elt))
-	      return 0;
-	    base_elt = el_elt;
-	    total_elnum += el_elnum;
-#if _CALL_ELF == 2
-	    if (total_elnum > 8)
-	      return 0;
-#else
-	    if (total_elnum > 1)
-	      return 0;
-#endif
-	    el++;
-	  }
-	*elnum = total_elnum;
-	return base_elt;
-      }
-
-    default:
-      return 0;
-    }
-}
-
-
-/* Perform machine dependent cif processing */
-static ffi_status
-ffi_prep_cif_linux64_core (ffi_cif *cif)
-{
-  ffi_type **ptr;
-  unsigned bytes;
-  unsigned i, fparg_count = 0, intarg_count = 0, vecarg_count = 0;
-  unsigned flags = cif->flags;
-  unsigned elt, elnum, rtype;
-
-#if FFI_TYPE_LONGDOUBLE == FFI_TYPE_DOUBLE
-  /* If compiled without long double support... */
-  if ((cif->abi & FFI_LINUX_LONG_DOUBLE_128) != 0 ||
-      (cif->abi & FFI_LINUX_LONG_DOUBLE_IEEE128) != 0)
-    return FFI_BAD_ABI;
-#elif !defined(__VEC__)
-  /* If compiled without vector register support (used by assembly)... */
-  if ((cif->abi & FFI_LINUX_LONG_DOUBLE_IEEE128) != 0)
-    return FFI_BAD_ABI;
-#else
-  /* If the IEEE128 flag is set, but long double is only 64 bits wide... */
-  if ((cif->abi & FFI_LINUX_LONG_DOUBLE_128) == 0 &&
-      (cif->abi & FFI_LINUX_LONG_DOUBLE_IEEE128) != 0)
-    return FFI_BAD_ABI;
-#endif
-
-  /* The machine-independent calculation of cif->bytes doesn't work
-     for us.  Redo the calculation.  */
-#if _CALL_ELF == 2
-  /* Space for backchain, CR, LR, TOC and the asm's temp regs.  */
-  bytes = (4 + ASM_NEEDS_REGISTERS64) * sizeof (long);
-
-  /* Space for the general registers.  */
-  bytes += NUM_GPR_ARG_REGISTERS64 * sizeof (long);
-#else
-  /* Space for backchain, CR, LR, cc/ld doubleword, TOC and the asm's temp
-     regs.  */
-  bytes = (6 + ASM_NEEDS_REGISTERS64) * sizeof (long);
-
-  /* Space for the mandatory parm save area and general registers.  */
-  bytes += 2 * NUM_GPR_ARG_REGISTERS64 * sizeof (long);
-#endif
-
-  /* Return value handling.  */
-  rtype = cif->rtype->type;
-#if _CALL_ELF == 2
-homogeneous:
-#endif
-  switch (rtype)
-    {
-#if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
-    case FFI_TYPE_LONGDOUBLE:
-      if ((cif->abi & FFI_LINUX_LONG_DOUBLE_IEEE128) != 0)
-        {
-          flags |= FLAG_RETURNS_VEC;
-          break;
-        }
-      if ((cif->abi & FFI_LINUX_LONG_DOUBLE_128) != 0)
-	flags |= FLAG_RETURNS_128BITS;
-      /* Fall through.  */
-#endif
-    case FFI_TYPE_DOUBLE:
-      flags |= FLAG_RETURNS_64BITS;
-      /* Fall through.  */
-    case FFI_TYPE_FLOAT:
-      flags |= FLAG_RETURNS_FP;
-      break;
-
-    case FFI_TYPE_UINT128:
-      flags |= FLAG_RETURNS_128BITS;
-      /* Fall through.  */
-    case FFI_TYPE_UINT64:
-    case FFI_TYPE_SINT64:
-    case FFI_TYPE_POINTER:
-      flags |= FLAG_RETURNS_64BITS;
-      break;
-
-    case FFI_TYPE_STRUCT:
-#if _CALL_ELF == 2
-      elt = discover_homogeneous_aggregate (cif->abi, cif->rtype, &elnum);
-      if (elt)
-        {
-          flags |= FLAG_RETURNS_SMST;
-          rtype = elt;
-          goto homogeneous;
-        }
-      if (cif->rtype->size <= 16)
-        {
-          flags |= FLAG_RETURNS_SMST;
-          break;
-        }
-#endif
-      intarg_count++;
-      flags |= FLAG_RETVAL_REFERENCE;
-      /* Fall through.  */
-    case FFI_TYPE_VOID:
-      flags |= FLAG_RETURNS_NOTHING;
-      break;
-
-    default:
-      /* Returns 32-bit integer, or similar.  Nothing to do here.  */
-      break;
-    }
-
-  for (ptr = cif->arg_types, i = cif->nargs; i > 0; i--, ptr++)
-    {
-      unsigned int align;
-
-      switch ((*ptr)->type)
-	{
-#if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
-	case FFI_TYPE_LONGDOUBLE:
-          if ((cif->abi & FFI_LINUX_LONG_DOUBLE_IEEE128) != 0)
-            {
-              vecarg_count++;
-              /* Align to 16 bytes, plus the 16-byte argument. */
-              intarg_count = (intarg_count + 3) & ~0x1;
-              if (vecarg_count > NUM_VEC_ARG_REGISTERS64)
-                flags |= FLAG_ARG_NEEDS_PSAVE;
-              break;
-            }
-	  if ((cif->abi & FFI_LINUX_LONG_DOUBLE_128) != 0)
-	    {
-	      fparg_count++;
-	      intarg_count++;
-	    }
-	  /* Fall through.  */
-#endif
-	case FFI_TYPE_DOUBLE:
-	case FFI_TYPE_FLOAT:
-	  fparg_count++;
-	  intarg_count++;
-	  if (fparg_count > NUM_FPR_ARG_REGISTERS64)
-	    flags |= FLAG_ARG_NEEDS_PSAVE;
-	  break;
-
-	case FFI_TYPE_STRUCT:
-	  if ((cif->abi & FFI_LINUX_STRUCT_ALIGN) != 0)
-	    {
-	      align = (*ptr)->alignment;
-	      if (align > 16)
-		align = 16;
-	      align = align / 8;
-	      if (align > 1)
-		intarg_count = FFI_ALIGN (intarg_count, align);
-	    }
-	  intarg_count += ((*ptr)->size + 7) / 8;
-	  elt = discover_homogeneous_aggregate (cif->abi, *ptr, &elnum);
-#if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
-          if (elt == FFI_TYPE_LONGDOUBLE &&
-              (cif->abi & FFI_LINUX_LONG_DOUBLE_IEEE128) != 0)
-            {
-              vecarg_count += elnum;
-              if (vecarg_count > NUM_VEC_ARG_REGISTERS64)
-                flags |= FLAG_ARG_NEEDS_PSAVE;
-              break;
-            }
-	  else
-#endif
-	  if (elt)
-	    {
-	      fparg_count += elnum;
-	      if (fparg_count > NUM_FPR_ARG_REGISTERS64)
-		flags |= FLAG_ARG_NEEDS_PSAVE;
-	    }
-	  else
-	    {
-	      if (intarg_count > NUM_GPR_ARG_REGISTERS64)
-		flags |= FLAG_ARG_NEEDS_PSAVE;
-	    }
-	  break;
-
-	case FFI_TYPE_POINTER:
-	case FFI_TYPE_UINT64:
-	case FFI_TYPE_SINT64:
-	case FFI_TYPE_INT:
-	case FFI_TYPE_UINT32:
-	case FFI_TYPE_SINT32:
-	case FFI_TYPE_UINT16:
-	case FFI_TYPE_SINT16:
-	case FFI_TYPE_UINT8:
-	case FFI_TYPE_SINT8:
-	  /* Everything else is passed as a 8-byte word in a GPR, either
-	     the object itself or a pointer to it.  */
-	  intarg_count++;
-	  if (intarg_count > NUM_GPR_ARG_REGISTERS64)
-	    flags |= FLAG_ARG_NEEDS_PSAVE;
-	  break;
-	default:
-	  FFI_ASSERT (0);
-	}
-    }
-
-  if (fparg_count != 0)
-    flags |= FLAG_FP_ARGUMENTS;
-  if (intarg_count > 4)
-    flags |= FLAG_4_GPR_ARGUMENTS;
-  if (vecarg_count != 0)
-    flags |= FLAG_VEC_ARGUMENTS;
-
-  /* Space for the FPR registers, if needed.  */
-  if (fparg_count != 0)
-    bytes += NUM_FPR_ARG_REGISTERS64 * sizeof (double);
-  /* Space for the vector registers, if needed, aligned to 16 bytes. */
-  if (vecarg_count != 0) {
-    bytes = (bytes + 15) & ~0xF;
-    bytes += NUM_VEC_ARG_REGISTERS64 * sizeof (float128);
-  }
-
-  /* Stack space.  */
-#if _CALL_ELF == 2
-  if ((flags & FLAG_ARG_NEEDS_PSAVE) != 0)
-    bytes += intarg_count * sizeof (long);
-#else
-  if (intarg_count > NUM_GPR_ARG_REGISTERS64)
-    bytes += (intarg_count - NUM_GPR_ARG_REGISTERS64) * sizeof (long);
-#endif
-
-  /* The stack space allocated needs to be a multiple of 16 bytes.  */
-  bytes = (bytes + 15) & ~0xF;
-
-  cif->flags = flags;
-  cif->bytes = bytes;
-
-  return FFI_OK;
-}
-
-ffi_status FFI_HIDDEN
-ffi_prep_cif_linux64 (ffi_cif *cif)
-{
-  if ((cif->abi & FFI_LINUX) != 0)
-    cif->nfixedargs = cif->nargs;
-#if _CALL_ELF != 2
-  else if (cif->abi == FFI_COMPAT_LINUX64)
-    {
-      /* This call is from old code.  Don't touch cif->nfixedargs
-	 since old code will be using a smaller cif.  */
-      cif->flags |= FLAG_COMPAT;
-      /* Translate to new abi value.  */
-      cif->abi = FFI_LINUX | FFI_LINUX_LONG_DOUBLE_128;
-    }
-#endif
-  else
-    return FFI_BAD_ABI;
-  return ffi_prep_cif_linux64_core (cif);
-}
-
-ffi_status FFI_HIDDEN
-ffi_prep_cif_linux64_var (ffi_cif *cif,
-			  unsigned int nfixedargs,
-			  unsigned int ntotalargs MAYBE_UNUSED)
-{
-  if ((cif->abi & FFI_LINUX) != 0)
-    cif->nfixedargs = nfixedargs;
-#if _CALL_ELF != 2
-  else if (cif->abi == FFI_COMPAT_LINUX64)
-    {
-      /* This call is from old code.  Don't touch cif->nfixedargs
-	 since old code will be using a smaller cif.  */
-      cif->flags |= FLAG_COMPAT;
-      /* Translate to new abi value.  */
-      cif->abi = FFI_LINUX | FFI_LINUX_LONG_DOUBLE_128;
-    }
-#endif
-  else
-    return FFI_BAD_ABI;
-#if _CALL_ELF == 2
-  cif->flags |= FLAG_ARG_NEEDS_PSAVE;
-#endif
-  return ffi_prep_cif_linux64_core (cif);
-}
-
-
-/* ffi_prep_args64 is called by the assembly routine once stack space
-   has been allocated for the function's arguments.
-
-   The stack layout we want looks like this:
-
-   |   Ret addr from ffi_call_LINUX64	8bytes	|	higher addresses
-   |--------------------------------------------|
-   |   CR save area			8bytes	|
-   |--------------------------------------------|
-   |   Previous backchain pointer	8	|	stack pointer here
-   |--------------------------------------------|<+ <<<	on entry to
-   |   Saved r28-r31			4*8	| |	ffi_call_LINUX64
-   |--------------------------------------------| |
-   |   GPR registers r3-r10		8*8	| |
-   |--------------------------------------------| |
-   |   FPR registers f1-f13 (optional)	13*8	| |
-   |--------------------------------------------| |
-   |   VEC registers v2-v13 (optional)  12*16   | |
-   |--------------------------------------------| |
-   |   Parameter save area		        | |
-   |--------------------------------------------| |
-   |   TOC save area			8	| |
-   |--------------------------------------------| |	stack	|
-   |   Linker doubleword		8	| |	grows	|
-   |--------------------------------------------| |	down	V
-   |   Compiler doubleword		8	| |
-   |--------------------------------------------| |	lower addresses
-   |   Space for callee's LR		8	| |
-   |--------------------------------------------| |
-   |   CR save area			8	| |
-   |--------------------------------------------| |	stack pointer here
-   |   Current backchain pointer	8	|-/	during
-   |--------------------------------------------|   <<<	ffi_call_LINUX64
-
-*/
-
-void FFI_HIDDEN
-ffi_prep_args64 (extended_cif *ecif, unsigned long *const stack)
-{
-  const unsigned long bytes = ecif->cif->bytes;
-  const unsigned long flags = ecif->cif->flags;
-
-  typedef union
-  {
-    char *c;
-    unsigned long *ul;
-    float *f;
-    double *d;
-    float128 *f128;
-    size_t p;
-  } valp;
-
-  /* 'stacktop' points at the previous backchain pointer.  */
-  valp stacktop;
-
-  /* 'next_arg' points at the space for gpr3, and grows upwards as
-     we use GPR registers, then continues at rest.  */
-  valp gpr_base;
-  valp gpr_end;
-  valp rest;
-  valp next_arg;
-
-  /* 'fpr_base' points at the space for f1, and grows upwards as
-     we use FPR registers.  */
-  valp fpr_base;
-  unsigned int fparg_count;
-
-  /* 'vec_base' points at the space for v2, and grows upwards as
-     we use vector registers.  */
-  valp vec_base;
-  unsigned int vecarg_count;
-
-  unsigned int i, words, nargs, nfixedargs;
-  ffi_type **ptr;
-  double double_tmp;
-  union
-  {
-    void **v;
-    char **c;
-    signed char **sc;
-    unsigned char **uc;
-    signed short **ss;
-    unsigned short **us;
-    signed int **si;
-    unsigned int **ui;
-    unsigned long **ul;
-    float **f;
-    double **d;
-    float128 **f128;
-  } p_argv;
-  unsigned long gprvalue;
-  unsigned long align;
-
-  stacktop.c = (char *) stack + bytes;
-  gpr_base.ul = stacktop.ul - ASM_NEEDS_REGISTERS64 - NUM_GPR_ARG_REGISTERS64;
-  gpr_end.ul = gpr_base.ul + NUM_GPR_ARG_REGISTERS64;
-#if _CALL_ELF == 2
-  rest.ul = stack + 4 + NUM_GPR_ARG_REGISTERS64;
-#else
-  rest.ul = stack + 6 + NUM_GPR_ARG_REGISTERS64;
-#endif
-  fpr_base.d = gpr_base.d - NUM_FPR_ARG_REGISTERS64;
-  fparg_count = 0;
-  /* Place the vector args below the FPRs, if used, else the GPRs. */
-  if (ecif->cif->flags & FLAG_FP_ARGUMENTS)
-    vec_base.p = fpr_base.p & ~0xF;
-  else
-    vec_base.p = gpr_base.p;
-  vec_base.f128 -= NUM_VEC_ARG_REGISTERS64;
-  vecarg_count = 0;
-  next_arg.ul = gpr_base.ul;
-
-  /* Check that everything starts aligned properly.  */
-  FFI_ASSERT (((unsigned long) (char *) stack & 0xF) == 0);
-  FFI_ASSERT (((unsigned long) stacktop.c & 0xF) == 0);
-  FFI_ASSERT (((unsigned long) gpr_base.c & 0xF) == 0);
-  FFI_ASSERT (((unsigned long) gpr_end.c  & 0xF) == 0);
-  FFI_ASSERT (((unsigned long) vec_base.c & 0xF) == 0);
-  FFI_ASSERT ((bytes & 0xF) == 0);
-
-  /* Deal with return values that are actually pass-by-reference.  */
-  if (flags & FLAG_RETVAL_REFERENCE)
-    *next_arg.ul++ = (unsigned long) (char *) ecif->rvalue;
-
-  /* Now for the arguments.  */
-  p_argv.v = ecif->avalue;
-  nargs = ecif->cif->nargs;
-#if _CALL_ELF != 2
-  nfixedargs = (unsigned) -1;
-  if ((flags & FLAG_COMPAT) == 0)
-#endif
-    nfixedargs = ecif->cif->nfixedargs;
-  for (ptr = ecif->cif->arg_types, i = 0;
-       i < nargs;
-       i++, ptr++, p_argv.v++)
-    {
-      unsigned int elt, elnum;
-
-      switch ((*ptr)->type)
-	{
-#if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
-	case FFI_TYPE_LONGDOUBLE:
-          if ((ecif->cif->abi & FFI_LINUX_LONG_DOUBLE_IEEE128) != 0)
-            {
-              next_arg.p = FFI_ALIGN (next_arg.p, 16);
-              if (next_arg.ul == gpr_end.ul)
-                next_arg.ul = rest.ul;
-              if (vecarg_count < NUM_VEC_ARG_REGISTERS64 && i < nfixedargs)
-		memcpy (vec_base.f128++, *p_argv.f128, sizeof (float128));
-              else
-		memcpy (next_arg.f128, *p_argv.f128, sizeof (float128));
-              if (++next_arg.f128 == gpr_end.f128)
-                next_arg.f128 = rest.f128;
-              vecarg_count++;
-              FFI_ASSERT (__LDBL_MANT_DIG__ == 113);
-              FFI_ASSERT (flags & FLAG_VEC_ARGUMENTS);
-              break;
-            }
-	  if ((ecif->cif->abi & FFI_LINUX_LONG_DOUBLE_128) != 0)
-	    {
-	      double_tmp = (*p_argv.d)[0];
-	      if (fparg_count < NUM_FPR_ARG_REGISTERS64 && i < nfixedargs)
-		{
-		  *fpr_base.d++ = double_tmp;
-# if _CALL_ELF != 2
-		  if ((flags & FLAG_COMPAT) != 0)
-		    *next_arg.d = double_tmp;
-# endif
-		}
-	      else
-		*next_arg.d = double_tmp;
-	      if (++next_arg.ul == gpr_end.ul)
-		next_arg.ul = rest.ul;
-	      fparg_count++;
-	      double_tmp = (*p_argv.d)[1];
-	      if (fparg_count < NUM_FPR_ARG_REGISTERS64 && i < nfixedargs)
-		{
-		  *fpr_base.d++ = double_tmp;
-# if _CALL_ELF != 2
-		  if ((flags & FLAG_COMPAT) != 0)
-		    *next_arg.d = double_tmp;
-# endif
-		}
-	      else
-		*next_arg.d = double_tmp;
-	      if (++next_arg.ul == gpr_end.ul)
-		next_arg.ul = rest.ul;
-	      fparg_count++;
-	      FFI_ASSERT (__LDBL_MANT_DIG__ == 106);
-	      FFI_ASSERT (flags & FLAG_FP_ARGUMENTS);
-	      break;
-	    }
-	  /* Fall through.  */
-#endif
-	case FFI_TYPE_DOUBLE:
-#if _CALL_ELF != 2
-	do_double:
-#endif
-	  double_tmp = **p_argv.d;
-	  if (fparg_count < NUM_FPR_ARG_REGISTERS64 && i < nfixedargs)
-	    {
-	      *fpr_base.d++ = double_tmp;
-#if _CALL_ELF != 2
-	      if ((flags & FLAG_COMPAT) != 0)
-		*next_arg.d = double_tmp;
-#endif
-	    }
-	  else
-	    *next_arg.d = double_tmp;
-	  if (++next_arg.ul == gpr_end.ul)
-	    next_arg.ul = rest.ul;
-	  fparg_count++;
-	  FFI_ASSERT (flags & FLAG_FP_ARGUMENTS);
-	  break;
-
-	case FFI_TYPE_FLOAT:
-#if _CALL_ELF != 2
-	do_float:
-#endif
-	  double_tmp = **p_argv.f;
-	  if (fparg_count < NUM_FPR_ARG_REGISTERS64 && i < nfixedargs)
-	    {
-	      *fpr_base.d++ = double_tmp;
-#if _CALL_ELF != 2
-	      if ((flags & FLAG_COMPAT) != 0)
-		{
-# ifndef __LITTLE_ENDIAN__
-		  next_arg.f[1] = (float) double_tmp;
-# else
-		  next_arg.f[0] = (float) double_tmp;
-# endif
-		}
-#endif
-	    }
-	  else
-	    {
-# ifndef __LITTLE_ENDIAN__
-	      next_arg.f[1] = (float) double_tmp;
-# else
-	      next_arg.f[0] = (float) double_tmp;
-# endif
-	    }
-	  if (++next_arg.ul == gpr_end.ul)
-	    next_arg.ul = rest.ul;
-	  fparg_count++;
-	  FFI_ASSERT (flags & FLAG_FP_ARGUMENTS);
-	  break;
-
-	case FFI_TYPE_STRUCT:
-	  if ((ecif->cif->abi & FFI_LINUX_STRUCT_ALIGN) != 0)
-	    {
-	      align = (*ptr)->alignment;
-	      if (align > 16)
-		align = 16;
-	      if (align > 1)
-                {
-                  next_arg.p = FFI_ALIGN (next_arg.p, align);
-                  if (next_arg.ul == gpr_end.ul)
-                    next_arg.ul = rest.ul;
-                }
-	    }
-	  elt = discover_homogeneous_aggregate (ecif->cif->abi, *ptr, &elnum);
-	  if (elt)
-	    {
-#if _CALL_ELF == 2
-	      union {
-		void *v;
-		float *f;
-		double *d;
-		float128 *f128;
-	      } arg;
-
-	      arg.v = *p_argv.v;
-#if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
-              if (elt == FFI_TYPE_LONGDOUBLE &&
-                  (ecif->cif->abi & FFI_LINUX_LONG_DOUBLE_IEEE128) != 0)
-                {
-                  do
-                    {
-                      if (vecarg_count < NUM_VEC_ARG_REGISTERS64
-                          && i < nfixedargs)
-		        memcpy (vec_base.f128++, arg.f128++, sizeof (float128));
-                      else
-		        memcpy (next_arg.f128, arg.f128++, sizeof (float128));
-                      if (++next_arg.f128 == gpr_end.f128)
-                        next_arg.f128 = rest.f128;
-                      vecarg_count++;
-                    }
-                  while (--elnum != 0);
-                }
-              else
-#endif
-	      if (elt == FFI_TYPE_FLOAT)
-		{
-		  do
-		    {
-		      double_tmp = *arg.f++;
-		      if (fparg_count < NUM_FPR_ARG_REGISTERS64
-			  && i < nfixedargs)
-			*fpr_base.d++ = double_tmp;
-		      else
-			*next_arg.f = (float) double_tmp;
-		      if (++next_arg.f == gpr_end.f)
-			next_arg.f = rest.f;
-		      fparg_count++;
-		    }
-		  while (--elnum != 0);
-		  if ((next_arg.p & 7) != 0)
-                    if (++next_arg.f == gpr_end.f)
-                      next_arg.f = rest.f;
-		}
-	      else
-		do
-		  {
-		    double_tmp = *arg.d++;
-		    if (fparg_count < NUM_FPR_ARG_REGISTERS64 && i < nfixedargs)
-		      *fpr_base.d++ = double_tmp;
-		    else
-		      *next_arg.d = double_tmp;
-		    if (++next_arg.d == gpr_end.d)
-		      next_arg.d = rest.d;
-		    fparg_count++;
-		  }
-		while (--elnum != 0);
-#else
-	      if (elt == FFI_TYPE_FLOAT)
-		goto do_float;
-	      else
-		goto do_double;
-#endif
-	    }
-	  else
-	    {
-	      words = ((*ptr)->size + 7) / 8;
-	      if (next_arg.ul >= gpr_base.ul && next_arg.ul + words > gpr_end.ul)
-		{
-		  size_t first = gpr_end.c - next_arg.c;
-		  memcpy (next_arg.c, *p_argv.c, first);
-		  memcpy (rest.c, *p_argv.c + first, (*ptr)->size - first);
-		  next_arg.c = rest.c + words * 8 - first;
-		}
-	      else
-		{
-		  char *where = next_arg.c;
-
-#ifndef __LITTLE_ENDIAN__
-		  /* Structures with size less than eight bytes are passed
-		     left-padded.  */
-		  if ((*ptr)->size < 8)
-		    where += 8 - (*ptr)->size;
-#endif
-		  memcpy (where, *p_argv.c, (*ptr)->size);
-		  next_arg.ul += words;
-		  if (next_arg.ul == gpr_end.ul)
-		    next_arg.ul = rest.ul;
-		}
-	    }
-	  break;
-
-	case FFI_TYPE_UINT8:
-	  gprvalue = **p_argv.uc;
-	  goto putgpr;
-	case FFI_TYPE_SINT8:
-	  gprvalue = **p_argv.sc;
-	  goto putgpr;
-	case FFI_TYPE_UINT16:
-	  gprvalue = **p_argv.us;
-	  goto putgpr;
-	case FFI_TYPE_SINT16:
-	  gprvalue = **p_argv.ss;
-	  goto putgpr;
-	case FFI_TYPE_UINT32:
-	  gprvalue = **p_argv.ui;
-	  goto putgpr;
-	case FFI_TYPE_INT:
-	case FFI_TYPE_SINT32:
-	  gprvalue = **p_argv.si;
-	  goto putgpr;
-
-	case FFI_TYPE_UINT64:
-	case FFI_TYPE_SINT64:
-	case FFI_TYPE_POINTER:
-	  gprvalue = **p_argv.ul;
-	putgpr:
-	  *next_arg.ul++ = gprvalue;
-	  if (next_arg.ul == gpr_end.ul)
-	    next_arg.ul = rest.ul;
-	  break;
-	}
-    }
-
-  FFI_ASSERT (flags & FLAG_4_GPR_ARGUMENTS
-	      || (next_arg.ul >= gpr_base.ul
-		  && next_arg.ul <= gpr_base.ul + 4));
-}
-
-
-#if _CALL_ELF == 2
-#define MIN_CACHE_LINE_SIZE 8
-
-static void
-flush_icache (char *wraddr, char *xaddr, int size)
-{
-  int i;
-  for (i = 0; i < size; i += MIN_CACHE_LINE_SIZE)
-    __asm__ volatile ("icbi 0,%0;" "dcbf 0,%1;"
-		      : : "r" (xaddr + i), "r" (wraddr + i) : "memory");
-  __asm__ volatile ("icbi 0,%0;" "dcbf 0,%1;" "sync;" "isync;"
-		    : : "r"(xaddr + size - 1), "r"(wraddr + size - 1)
-		    : "memory");
-}
-#endif
-
-
-ffi_status FFI_HIDDEN
-ffi_prep_closure_loc_linux64 (ffi_closure *closure,
-			      ffi_cif *cif,
-			      void (*fun) (ffi_cif *, void *, void **, void *),
-			      void *user_data,
-			      void *codeloc)
-{
-#if _CALL_ELF == 2
-  unsigned int *tramp = (unsigned int *) &closure->tramp[0];
-
-  if (cif->abi < FFI_LINUX || cif->abi >= FFI_LAST_ABI)
-    return FFI_BAD_ABI;
-
-  tramp[0] = 0xe96c0018;	/* 0:	ld	11,2f-0b(12)	*/
-  tramp[1] = 0xe98c0010;	/*	ld	12,1f-0b(12)	*/
-  tramp[2] = 0x7d8903a6;	/*	mtctr	12		*/
-  tramp[3] = 0x4e800420;	/*	bctr			*/
-				/* 1:	.quad	function_addr	*/
-				/* 2:	.quad	context		*/
-  *(void **) &tramp[4] = (void *) ffi_closure_LINUX64;
-  *(void **) &tramp[6] = codeloc;
-  flush_icache ((char *) tramp, (char *) codeloc, 4 * 4);
-#else
-  void **tramp = (void **) &closure->tramp[0];
-
-  if (cif->abi < FFI_LINUX || cif->abi >= FFI_LAST_ABI)
-    return FFI_BAD_ABI;
-
-  /* Copy function address and TOC from ffi_closure_LINUX64 OPD.  */
-  memcpy (&tramp[0], (void **) ffi_closure_LINUX64, sizeof (void *));
-  tramp[1] = codeloc;
-  memcpy (&tramp[2], (void **) ffi_closure_LINUX64 + 1, sizeof (void *));
-#endif
-
-  closure->cif = cif;
-  closure->fun = fun;
-  closure->user_data = user_data;
-
-  return FFI_OK;
-}
-
-
-int FFI_HIDDEN
-ffi_closure_helper_LINUX64 (ffi_cif *cif,
-			    void (*fun) (ffi_cif *, void *, void **, void *),
-			    void *user_data,
-			    void *rvalue,
-			    unsigned long *pst,
-                            ffi_dblfl *pfr,
-                            float128 *pvec)
-{
-  /* rvalue is the pointer to space for return value in closure assembly */
-  /* pst is the pointer to parameter save area
-     (r3-r10 are stored into its first 8 slots by ffi_closure_LINUX64) */
-  /* pfr is the pointer to where f1-f13 are stored in ffi_closure_LINUX64 */
-  /* pvec is the pointer to where v2-v13 are stored in ffi_closure_LINUX64 */
-
-  void **avalue;
-  ffi_type **arg_types;
-  unsigned long i, avn, nfixedargs;
-  ffi_dblfl *end_pfr = pfr + NUM_FPR_ARG_REGISTERS64;
-  float128 *end_pvec = pvec + NUM_VEC_ARG_REGISTERS64;
-  unsigned long align;
-
-  avalue = alloca (cif->nargs * sizeof (void *));
-
-  /* Copy the caller's structure return value address so that the
-     closure returns the data directly to the caller.  */
-  if (cif->rtype->type == FFI_TYPE_STRUCT
-      && (cif->flags & FLAG_RETURNS_SMST) == 0)
-    {
-      rvalue = (void *) *pst;
-      pst++;
-    }
-
-  i = 0;
-  avn = cif->nargs;
-#if _CALL_ELF != 2
-  nfixedargs = (unsigned) -1;
-  if ((cif->flags & FLAG_COMPAT) == 0)
-#endif
-    nfixedargs = cif->nfixedargs;
-  arg_types = cif->arg_types;
-
-  /* Grab the addresses of the arguments from the stack frame.  */
-  while (i < avn)
-    {
-      unsigned int elt, elnum;
-
-      switch (arg_types[i]->type)
-	{
-	case FFI_TYPE_SINT8:
-	case FFI_TYPE_UINT8:
-#ifndef __LITTLE_ENDIAN__
-	  avalue[i] = (char *) pst + 7;
-	  pst++;
-	  break;
-#endif
-
-	case FFI_TYPE_SINT16:
-	case FFI_TYPE_UINT16:
-#ifndef __LITTLE_ENDIAN__
-	  avalue[i] = (char *) pst + 6;
-	  pst++;
-	  break;
-#endif
-
-	case FFI_TYPE_SINT32:
-	case FFI_TYPE_UINT32:
-#ifndef __LITTLE_ENDIAN__
-	  avalue[i] = (char *) pst + 4;
-	  pst++;
-	  break;
-#endif
-
-	case FFI_TYPE_SINT64:
-	case FFI_TYPE_UINT64:
-	case FFI_TYPE_POINTER:
-	  avalue[i] = pst;
-	  pst++;
-	  break;
-
-	case FFI_TYPE_STRUCT:
-	  if ((cif->abi & FFI_LINUX_STRUCT_ALIGN) != 0)
-	    {
-	      align = arg_types[i]->alignment;
-	      if (align > 16)
-		align = 16;
-	      if (align > 1)
-		pst = (unsigned long *) FFI_ALIGN ((size_t) pst, align);
-	    }
-	  elt = discover_homogeneous_aggregate (cif->abi, arg_types[i], &elnum);
-	  if (elt)
-	    {
-#if _CALL_ELF == 2
-	      union {
-		void *v;
-		unsigned long *ul;
-		float *f;
-		double *d;
-		float128 *f128;
-		size_t p;
-	      } to, from;
-
-	      /* Repackage the aggregate from its parts.  The
-		 aggregate size is not greater than the space taken by
-		 the registers so store back to the register/parameter
-		 save arrays.  */
-#if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
-              if (elt == FFI_TYPE_LONGDOUBLE &&
-                  (cif->abi & FFI_LINUX_LONG_DOUBLE_IEEE128) != 0)
-                {
-                  if (pvec + elnum <= end_pvec)
-                    to.v = pvec;
-                  else
-                    to.v = pst;
-                }
-              else
-#endif
-	      if (pfr + elnum <= end_pfr)
-		to.v = pfr;
-	      else
-		to.v = pst;
-
-	      avalue[i] = to.v;
-	      from.ul = pst;
-#if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
-              if (elt == FFI_TYPE_LONGDOUBLE &&
-                  (cif->abi & FFI_LINUX_LONG_DOUBLE_IEEE128) != 0)
-                {
-                  do
-                    {
-                      if (pvec < end_pvec && i < nfixedargs)
-		        memcpy (to.f128, pvec++, sizeof (float128));
-                      else
-		        memcpy (to.f128, from.f128, sizeof (float128));
-                      to.f128++;
-                      from.f128++;
-                    }
-                  while (--elnum != 0);
-                }
-              else
-#endif
-	      if (elt == FFI_TYPE_FLOAT)
-		{
-		  do
-		    {
-		      if (pfr < end_pfr && i < nfixedargs)
-			{
-			  *to.f = (float) pfr->d;
-			  pfr++;
-			}
-		      else
-			*to.f = *from.f;
-		      to.f++;
-		      from.f++;
-		    }
-		  while (--elnum != 0);
-		}
-	      else
-		{
-		  do
-		    {
-		      if (pfr < end_pfr && i < nfixedargs)
-			{
-			  *to.d = pfr->d;
-			  pfr++;
-			}
-		      else
-			*to.d = *from.d;
-		      to.d++;
-		      from.d++;
-		    }
-		  while (--elnum != 0);
-		}
-#else
-	      if (elt == FFI_TYPE_FLOAT)
-		goto do_float;
-	      else
-		goto do_double;
-#endif
-	    }
-	  else
-	    {
-#ifndef __LITTLE_ENDIAN__
-	      /* Structures with size less than eight bytes are passed
-		 left-padded.  */
-	      if (arg_types[i]->size < 8)
-		avalue[i] = (char *) pst + 8 - arg_types[i]->size;
-	      else
-#endif
-		avalue[i] = pst;
-	    }
-	  pst += (arg_types[i]->size + 7) / 8;
-	  break;
-
-#if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
-	case FFI_TYPE_LONGDOUBLE:
-          if ((cif->abi & FFI_LINUX_LONG_DOUBLE_IEEE128) != 0)
-            {
-              if (((unsigned long) pst & 0xF) != 0)
-                ++pst;
-              if (pvec < end_pvec && i < nfixedargs)
-                avalue[i] = pvec++;
-              else
-                avalue[i] = pst;
-              pst += 2;
-              break;
-            }
-          else if ((cif->abi & FFI_LINUX_LONG_DOUBLE_128) != 0)
-	    {
-	      if (pfr + 1 < end_pfr && i + 1 < nfixedargs)
-		{
-		  avalue[i] = pfr;
-		  pfr += 2;
-		}
-	      else
-		{
-		  if (pfr < end_pfr && i < nfixedargs)
-		    {
-		      /* Passed partly in f13 and partly on the stack.
-			 Move it all to the stack.  */
-		      *pst = *(unsigned long *) pfr;
-		      pfr++;
-		    }
-		  avalue[i] = pst;
-		}
-	      pst += 2;
-	      break;
-	    }
-	  /* Fall through.  */
-#endif
-	case FFI_TYPE_DOUBLE:
-#if _CALL_ELF != 2
-	do_double:
-#endif
-	  /* On the outgoing stack all values are aligned to 8 */
-	  /* there are 13 64bit floating point registers */
-
-	  if (pfr < end_pfr && i < nfixedargs)
-	    {
-	      avalue[i] = pfr;
-	      pfr++;
-	    }
-	  else
-	    avalue[i] = pst;
-	  pst++;
-	  break;
-
-	case FFI_TYPE_FLOAT:
-#if _CALL_ELF != 2
-	do_float:
-#endif
-	  if (pfr < end_pfr && i < nfixedargs)
-	    {
-	      /* Float values are stored as doubles in the
-		 ffi_closure_LINUX64 code.  Fix them here.  */
-	      pfr->f = (float) pfr->d;
-	      avalue[i] = pfr;
-	      pfr++;
-	    }
-	  else
-	    {
-#ifndef __LITTLE_ENDIAN__
-	      avalue[i] = (char *) pst + 4;
-#else
-	      avalue[i] = pst;
-#endif
-	    }
-	  pst++;
-	  break;
-
-	default:
-	  FFI_ASSERT (0);
-	}
-
-      i++;
-    }
-
-  (*fun) (cif, rvalue, avalue, user_data);
-
-  /* Tell ffi_closure_LINUX64 how to perform return type promotions.  */
-  if ((cif->flags & FLAG_RETURNS_SMST) != 0)
-    {
-      if ((cif->flags & (FLAG_RETURNS_FP | FLAG_RETURNS_VEC)) == 0)
-	return FFI_V2_TYPE_SMALL_STRUCT + cif->rtype->size - 1;
-      else if ((cif->flags & FLAG_RETURNS_VEC) != 0)
-        return FFI_V2_TYPE_VECTOR_HOMOG;
-      else if ((cif->flags & FLAG_RETURNS_64BITS) != 0)
-	return FFI_V2_TYPE_DOUBLE_HOMOG;
-      else
-	return FFI_V2_TYPE_FLOAT_HOMOG;
-    }
-  if ((cif->flags & FLAG_RETURNS_VEC) != 0)
-    return FFI_V2_TYPE_VECTOR;
-  return cif->rtype->type;
-}
-#endif