immunio 1.2.1 → 2.0.2

data/lua-hooks/ext/luajit/src/lj_asm_ppc.h

@@ -1,6 +1,6 @@
 /*
 ** PPC IR assembler (SSA IR -> machine code).
-** Copyright (C) 2005-2016 Mike Pall. See Copyright Notice in luajit.h
+** Copyright (C) 2005-2017 Mike Pall. See Copyright Notice in luajit.h
 */
 
 /* -- Register allocator extensions --------------------------------------- */
@@ -393,8 +393,7 @@ static void asm_tointg(ASMState *as, IRIns *ir, Reg left)
   emit_asi(as, PPCI_XORIS, RID_TMP, dest, 0x8000);
   emit_tai(as, PPCI_LWZ, dest, RID_SP, SPOFS_TMPLO);
   emit_lsptr(as, PPCI_LFS, (fbias & 31),
-	     (void *)lj_ir_k64_find(as->J, U64x(59800004,59800000)),
-	     RSET_GPR);
+	     (void *)&as->J->k32[LJ_K32_2P52_2P31], RSET_GPR);
   emit_fai(as, PPCI_STFD, tmp, RID_SP, SPOFS_TMP);
   emit_fb(as, PPCI_FCTIWZ, tmp, left);
 }
@@ -433,13 +432,11 @@ static void asm_conv(ASMState *as, IRIns *ir)
       Reg left = ra_alloc1(as, lref, allow);
       Reg hibias = ra_allock(as, 0x43300000, rset_clear(allow, left));
       Reg fbias = ra_scratch(as, rset_exclude(RSET_FPR, dest));
-      const float *kbias;
       if (irt_isfloat(ir->t)) emit_fb(as, PPCI_FRSP, dest, dest);
       emit_fab(as, PPCI_FSUB, dest, dest, fbias);
       emit_fai(as, PPCI_LFD, dest, RID_SP, SPOFS_TMP);
-      kbias = (const float *)lj_ir_k64_find(as->J, U64x(59800004,59800000));
-      if (st == IRT_U32) kbias++;
-      emit_lsptr(as, PPCI_LFS, (fbias & 31), (void *)kbias,
+      emit_lsptr(as, PPCI_LFS, (fbias & 31),
+		 &as->J->k32[st == IRT_U32 ? LJ_K32_2P52 : LJ_K32_2P52_2P31],
 		 rset_clear(allow, hibias));
       emit_tai(as, PPCI_STW, st == IRT_U32 ? left : RID_TMP,
 	       RID_SP, SPOFS_TMPLO);
@@ -472,8 +469,7 @@ static void asm_conv(ASMState *as, IRIns *ir)
 	emit_fb(as, PPCI_FCTIWZ, tmp, tmp);
 	emit_fab(as, PPCI_FSUB, tmp, left, tmp);
 	emit_lsptr(as, PPCI_LFS, (tmp & 31),
-		   (void *)lj_ir_k64_find(as->J, U64x(4f000000,00000000)),
-		   RSET_GPR);
+		   (void *)&as->J->k32[LJ_K32_2P31], RSET_GPR);
       } else {
 	emit_tai(as, PPCI_LWZ, dest, RID_SP, SPOFS_TMPLO);
 	emit_fai(as, PPCI_STFD, tmp, RID_SP, SPOFS_TMP);
@@ -717,7 +713,6 @@ static void asm_hrefk(ASMState *as, IRIns *ir)
 
 static void asm_uref(ASMState *as, IRIns *ir)
 {
-  /* NYI: Check that UREFO is still open and not aliasing a slot. */
   Reg dest = ra_dest(as, ir, RSET_GPR);
   if (irref_isk(ir->op1)) {
     GCfunc *fn = ir_kfunc(IR(ir->op1));
@@ -809,17 +804,23 @@ static PPCIns asm_fxstoreins(IRIns *ir)
 static void asm_fload(ASMState *as, IRIns *ir)
 {
   Reg dest = ra_dest(as, ir, RSET_GPR);
-  Reg idx = ra_alloc1(as, ir->op1, RSET_GPR);
   PPCIns pi = asm_fxloadins(ir);
+  Reg idx;
   int32_t ofs;
-  if (ir->op2 == IRFL_TAB_ARRAY) {
-    ofs = asm_fuseabase(as, ir->op1);
-    if (ofs) {  /* Turn the t->array load into an add for colocated arrays. */
-      emit_tai(as, PPCI_ADDI, dest, idx, ofs);
-      return;
+  if (ir->op1 == REF_NIL) {
+    idx = RID_JGL;
+    ofs = (ir->op2 << 2) - 32768;
+  } else {
+    idx = ra_alloc1(as, ir->op1, RSET_GPR);
+    if (ir->op2 == IRFL_TAB_ARRAY) {
+      ofs = asm_fuseabase(as, ir->op1);
+      if (ofs) {  /* Turn the t->array load into an add for colocated arrays. */
+	emit_tai(as, PPCI_ADDI, dest, idx, ofs);
+	return;
+      }
     }
+    ofs = field_ofs[ir->op2];
   }
-  ofs = field_ofs[ir->op2];
   lua_assert(!irt_isi8(ir->t));
   emit_tai(as, pi, dest, idx, ofs);
 }
@@ -975,7 +976,7 @@ static void asm_sload(ASMState *as, IRIns *ir)
 	emit_fab(as, PPCI_FSUB, dest, dest, fbias);
 	emit_fai(as, PPCI_LFD, dest, RID_SP, SPOFS_TMP);
 	emit_lsptr(as, PPCI_LFS, (fbias & 31),
-		   (void *)lj_ir_k64_find(as->J, U64x(59800004,59800000)),
+		   (void *)&as->J->k32[LJ_K32_2P52_2P31],
 		   rset_clear(allow, hibias));
 	emit_tai(as, PPCI_STW, tmp, RID_SP, SPOFS_TMPLO);
 	emit_tai(as, PPCI_STW, hibias, RID_SP, SPOFS_TMPHI);

data/lua-hooks/ext/luajit/src/lj_asm_x86.h

@@ -1,6 +1,6 @@
 /*
 ** x86/x64 IR assembler (SSA IR -> machine code).
-** Copyright (C) 2005-2016 Mike Pall. See Copyright Notice in luajit.h
+** Copyright (C) 2005-2017 Mike Pall. See Copyright Notice in luajit.h
 */
 
 /* -- Guard handling ------------------------------------------------------ */
@@ -21,12 +21,14 @@ static MCode *asm_exitstub_gen(ASMState *as, ExitNo group)
   }
   /* Push the high byte of the exitno for each exit stub group. */
   *mxp++ = XI_PUSHi8; *mxp++ = (MCode)((group*EXITSTUBS_PER_GROUP)>>8);
+#if !LJ_GC64
   /* Store DISPATCH at original stack slot 0. Account for the two push ops. */
   *mxp++ = XI_MOVmi;
   *mxp++ = MODRM(XM_OFS8, 0, RID_ESP);
   *mxp++ = MODRM(XM_SCALE1, RID_ESP, RID_ESP);
   *mxp++ = 2*sizeof(void *);
   *(int32_t *)mxp = ptr2addr(J2GG(as->J)->dispatch); mxp += 4;
+#endif
   /* Jump to exit handler which fills in the ExitState. */
   *mxp++ = XI_JMP; mxp += 4;
   *((int32_t *)(mxp-4)) = jmprel(mxp, (MCode *)(void *)lj_vm_exit_handler);
@@ -62,10 +64,14 @@ static void asm_guardcc(ASMState *as, int cc)
     target = p;
     cc ^= 1;
     if (as->realign) {
+      if (LJ_GC64 && LJ_UNLIKELY(as->mrm.base == RID_RIP))
+	as->mrm.ofs += 2;  /* Fixup RIP offset for pending fused load. */
       emit_sjcc(as, cc, target);
       return;
     }
   }
+  if (LJ_GC64 && LJ_UNLIKELY(as->mrm.base == RID_RIP))
+    as->mrm.ofs += 6;  /* Fixup RIP offset for pending fused load. */
   emit_jcc(as, cc, target);
 }
 
@@ -79,6 +85,15 @@ static int asm_isk32(ASMState *as, IRRef ref, int32_t *k)
 {
   if (irref_isk(ref)) {
     IRIns *ir = IR(ref);
+#if LJ_GC64
+    if (ir->o == IR_KNULL || !irt_is64(ir->t)) {
+      *k = ir->i;
+      return 1;
+    } else if (checki32((int64_t)ir_k64(ir)->u64)) {
+      *k = (int32_t)ir_k64(ir)->u64;
+      return 1;
+    }
+#else
     if (ir->o != IR_KINT64) {
       *k = ir->i;
       return 1;
@@ -86,6 +101,7 @@ static int asm_isk32(ASMState *as, IRRef ref, int32_t *k)
       *k = (int32_t)ir_kint64(ir)->u64;
       return 1;
     }
+#endif
   }
   return 0;
 }
@@ -185,9 +201,19 @@ static void asm_fuseahuref(ASMState *as, IRRef ref, RegSet allow)
       if (irref_isk(ir->op1)) {
 	GCfunc *fn = ir_kfunc(IR(ir->op1));
 	GCupval *uv = &gcref(fn->l.uvptr[(ir->op2 >> 8)])->uv;
+#if LJ_GC64
+	int64_t ofs = dispofs(as, &uv->tv);
+	if (checki32(ofs) && checki32(ofs+4)) {
+	  as->mrm.ofs = (int32_t)ofs;
+	  as->mrm.base = RID_DISPATCH;
+	  as->mrm.idx = RID_NONE;
+	  return;
+	}
+#else
 	as->mrm.ofs = ptr2addr(&uv->tv);
 	as->mrm.base = as->mrm.idx = RID_NONE;
 	return;
+#endif
       }
       break;
     default:
@@ -205,14 +231,40 @@ static void asm_fuseahuref(ASMState *as, IRRef ref, RegSet allow)
 static void asm_fusefref(ASMState *as, IRIns *ir, RegSet allow)
 {
   lua_assert(ir->o == IR_FLOAD || ir->o == IR_FREF);
-  as->mrm.ofs = field_ofs[ir->op2];
   as->mrm.idx = RID_NONE;
+  if (ir->op1 == REF_NIL) {
+#if LJ_GC64
+    as->mrm.ofs = (int32_t)(ir->op2 << 2) - GG_OFS(dispatch);
+    as->mrm.base = RID_DISPATCH;
+#else
+    as->mrm.ofs = (int32_t)(ir->op2 << 2) + ptr2addr(J2GG(as->J));
+    as->mrm.base = RID_NONE;
+#endif
+    return;
+  }
+  as->mrm.ofs = field_ofs[ir->op2];
   if (irref_isk(ir->op1)) {
-    as->mrm.ofs += IR(ir->op1)->i;
+    IRIns *op1 = IR(ir->op1);
+#if LJ_GC64
+    if (ir->op1 == REF_NIL) {
+      as->mrm.ofs -= GG_OFS(dispatch);
+      as->mrm.base = RID_DISPATCH;
+      return;
+    } else if (op1->o == IR_KPTR || op1->o == IR_KKPTR) {
+      intptr_t ofs = dispofs(as, ir_kptr(op1));
+      if (checki32(as->mrm.ofs + ofs)) {
+	as->mrm.ofs += (int32_t)ofs;
+	as->mrm.base = RID_DISPATCH;
+	return;
+      }
+    }
+#else
+    as->mrm.ofs += op1->i;
     as->mrm.base = RID_NONE;
-  } else {
-    as->mrm.base = (uint8_t)ra_alloc1(as, ir->op1, allow);
+    return;
+#endif
   }
+  as->mrm.base = (uint8_t)ra_alloc1(as, ir->op1, allow);
 }
 
 /* Fuse string reference into memory operand. */
@@ -223,7 +275,7 @@ static void asm_fusestrref(ASMState *as, IRIns *ir, RegSet allow)
   as->mrm.base = as->mrm.idx = RID_NONE;
   as->mrm.scale = XM_SCALE1;
   as->mrm.ofs = sizeof(GCstr);
-  if (irref_isk(ir->op1)) {
+  if (!LJ_GC64 && irref_isk(ir->op1)) {
     as->mrm.ofs += IR(ir->op1)->i;
   } else {
     Reg r = ra_alloc1(as, ir->op1, allow);
@@ -255,10 +307,20 @@ static void asm_fusexref(ASMState *as, IRRef ref, RegSet allow)
   IRIns *ir = IR(ref);
   as->mrm.idx = RID_NONE;
   if (ir->o == IR_KPTR || ir->o == IR_KKPTR) {
+#if LJ_GC64
+    intptr_t ofs = dispofs(as, ir_kptr(ir));
+    if (checki32(ofs)) {
+      as->mrm.ofs = (int32_t)ofs;
+      as->mrm.base = RID_DISPATCH;
+      return;
+    }
+  } if (0) {
+#else
     as->mrm.ofs = ir->i;
     as->mrm.base = RID_NONE;
   } else if (ir->o == IR_STRREF) {
     asm_fusestrref(as, ir, allow);
+#endif
   } else {
     as->mrm.ofs = 0;
     if (canfuse(as, ir) && ir->o == IR_ADD && ra_noreg(ir->r)) {
@@ -301,7 +363,45 @@ static void asm_fusexref(ASMState *as, IRRef ref, RegSet allow)
   }
 }
 
-/* Fuse load into memory operand. */
+/* Fuse load of 64 bit IR constant into memory operand. */
+static Reg asm_fuseloadk64(ASMState *as, IRIns *ir)
+{
+  const uint64_t *k = &ir_k64(ir)->u64;
+  if (!LJ_GC64 || checki32((intptr_t)k)) {
+    as->mrm.ofs = ptr2addr(k);
+    as->mrm.base = RID_NONE;
+#if LJ_GC64
+  } else if (checki32(dispofs(as, k))) {
+    as->mrm.ofs = (int32_t)dispofs(as, k);
+    as->mrm.base = RID_DISPATCH;
+  } else if (checki32(mcpofs(as, k)) && checki32(mcpofs(as, k+1)) &&
+	     checki32(mctopofs(as, k)) && checki32(mctopofs(as, k+1))) {
+    as->mrm.ofs = (int32_t)mcpofs(as, k);
+    as->mrm.base = RID_RIP;
+  } else {
+    if (ir->i) {
+      lua_assert(*k == *(uint64_t*)(as->mctop - ir->i));
+    } else {
+      while ((uintptr_t)as->mcbot & 7) *as->mcbot++ = XI_INT3;
+      *(uint64_t*)as->mcbot = *k;
+      ir->i = (int32_t)(as->mctop - as->mcbot);
+      as->mcbot += 8;
+      as->mclim = as->mcbot + MCLIM_REDZONE;
+    }
+    as->mrm.ofs = (int32_t)mcpofs(as, as->mctop - ir->i);
+    as->mrm.base = RID_RIP;
+#endif
+  }
+  as->mrm.idx = RID_NONE;
+  return RID_MRM;
+}
+
+/* Fuse load into memory operand.
+**
+** Important caveat: this may emit RIP-relative loads! So don't place any
+** code emitters between this function and the use of its result.
+** The only permitted exception is asm_guardcc().
+*/
 static Reg asm_fuseload(ASMState *as, IRRef ref, RegSet allow)
 {
   IRIns *ir = IR(ref);
@@ -320,26 +420,35 @@ static Reg asm_fuseload(ASMState *as, IRRef ref, RegSet allow)
   if (ir->o == IR_KNUM) {
     RegSet avail = as->freeset & ~as->modset & RSET_FPR;
     lua_assert(allow != RSET_EMPTY);
-    if (!(avail & (avail-1))) {  /* Fuse if less than two regs available. */
-      as->mrm.ofs = ptr2addr(ir_knum(ir));
-      as->mrm.base = as->mrm.idx = RID_NONE;
-      return RID_MRM;
-    }
-  } else if (ir->o == IR_KINT64) {
+    if (!(avail & (avail-1)))  /* Fuse if less than two regs available. */
+      return asm_fuseloadk64(as, ir);
+  } else if (ref == REF_BASE || ir->o == IR_KINT64) {
     RegSet avail = as->freeset & ~as->modset & RSET_GPR;
     lua_assert(allow != RSET_EMPTY);
     if (!(avail & (avail-1))) {  /* Fuse if less than two regs available. */
-      as->mrm.ofs = ptr2addr(ir_kint64(ir));
-      as->mrm.base = as->mrm.idx = RID_NONE;
-      return RID_MRM;
+      if (ref == REF_BASE) {
+#if LJ_GC64
+	as->mrm.ofs = (int32_t)dispofs(as, &J2G(as->J)->jit_base);
+	as->mrm.base = RID_DISPATCH;
+#else
+	as->mrm.ofs = ptr2addr(&J2G(as->J)->jit_base);
+	as->mrm.base = RID_NONE;
+#endif
+	as->mrm.idx = RID_NONE;
+	return RID_MRM;
+      } else {
+	return asm_fuseloadk64(as, ir);
+      }
     }
   } else if (mayfuse(as, ref)) {
     RegSet xallow = (allow & RSET_GPR) ? allow : RSET_GPR;
     if (ir->o == IR_SLOAD) {
       if (!(ir->op2 & (IRSLOAD_PARENT|IRSLOAD_CONVERT)) &&
-	  noconflict(as, ref, IR_RETF, 0)) {
+	  noconflict(as, ref, IR_RETF, 0) &&
+	  !(LJ_GC64 && irt_isaddr(ir->t))) {
 	as->mrm.base = (uint8_t)ra_alloc1(as, REF_BASE, xallow);
-	as->mrm.ofs = 8*((int32_t)ir->op1-1) + ((ir->op2&IRSLOAD_FRAME)?4:0);
+	as->mrm.ofs = 8*((int32_t)ir->op1-1-LJ_FR2) +
+		      (!LJ_FR2 && (ir->op2 & IRSLOAD_FRAME) ? 4 : 0);
 	as->mrm.idx = RID_NONE;
 	return RID_MRM;
       }
@@ -351,7 +460,8 @@ static Reg asm_fuseload(ASMState *as, IRRef ref, RegSet allow)
 	return RID_MRM;
       }
     } else if (ir->o == IR_ALOAD || ir->o == IR_HLOAD || ir->o == IR_ULOAD) {
-      if (noconflict(as, ref, ir->o + IRDELTA_L2S, 0)) {
+      if (noconflict(as, ref, ir->o + IRDELTA_L2S, 0) &&
+	  !(LJ_GC64 && irt_isaddr(ir->t))) {
 	asm_fuseahuref(as, ir->op1, xallow);
 	return RID_MRM;
       }
@@ -364,12 +474,16 @@ static Reg asm_fuseload(ASMState *as, IRRef ref, RegSet allow)
 	asm_fusexref(as, ir->op1, xallow);
 	return RID_MRM;
       }
-    } else if (ir->o == IR_VLOAD) {
+    } else if (ir->o == IR_VLOAD && !(LJ_GC64 && irt_isaddr(ir->t))) {
       asm_fuseahuref(as, ir->op1, xallow);
       return RID_MRM;
     }
   }
-  if (!(as->freeset & allow) && !irref_isk(ref) &&
+  if (ir->o == IR_FLOAD && ir->op1 == REF_NIL) {
+    asm_fusefref(as, ir, RSET_EMPTY);
+    return RID_MRM;
+  }
+  if (!(as->freeset & allow) && !emit_canremat(ref) &&
       (allow == RSET_EMPTY || ra_hasspill(ir->s) || iscrossref(as, ref)))
     goto fusespill;
   return ra_allocref(as, ref, allow);
@@ -485,8 +599,8 @@ static void asm_gencall(ASMState *as, const CCallInfo *ci, IRRef *args)
     if (r) {  /* Argument is in a register. */
       if (r < RID_MAX_GPR && ref < ASMREF_TMP1) {
 #if LJ_64
-	if (ir->o == IR_KINT64)
-	  emit_loadu64(as, r, ir_kint64(ir)->u64);
+	if (LJ_GC64 ? !(ir->o == IR_KINT || ir->o == IR_KNULL) : ir->o == IR_KINT64)
+	  emit_loadu64(as, r, ir_k64(ir)->u64);
 	else
 #endif
 	  emit_loadi(as, r, ir->i);
@@ -642,6 +756,9 @@ static void asm_callx(ASMState *as, IRIns *ir)
 static void asm_retf(ASMState *as, IRIns *ir)
 {
   Reg base = ra_alloc1(as, REF_BASE, RSET_GPR);
+#if LJ_FR2
+  Reg rpc = ra_scratch(as, rset_exclude(RSET_GPR, base));
+#endif
   void *pc = ir_kptr(IR(ir->op2));
   int32_t delta = 1+LJ_FR2+bc_a(*((const BCIns *)pc - 1));
   as->topslot -= (BCReg)delta;
@@ -650,7 +767,12 @@ static void asm_retf(ASMState *as, IRIns *ir)
   emit_setgl(as, base, jit_base);
   emit_addptr(as, base, -8*delta);
   asm_guardcc(as, CC_NE);
+#if LJ_FR2
+  emit_rmro(as, XO_CMP, rpc|REX_GC64, base, -8);
+  emit_loadu64(as, rpc, u64ptr(pc));
+#else
   emit_gmroi(as, XG_ARITHi(XOg_CMP), base, -4, ptr2addr(pc));
+#endif
 }
 
 /* -- Type conversions ---------------------------------------------------- */
@@ -674,8 +796,9 @@ static void asm_tobit(ASMState *as, IRIns *ir)
   Reg tmp = ra_noreg(IR(ir->op1)->r) ?
 	      ra_alloc1(as, ir->op1, RSET_FPR) :
 	      ra_scratch(as, RSET_FPR);
-  Reg right = asm_fuseload(as, ir->op2, rset_exclude(RSET_FPR, tmp));
+  Reg right;
   emit_rr(as, XO_MOVDto, tmp, dest);
+  right = asm_fuseload(as, ir->op2, rset_exclude(RSET_FPR, tmp));
   emit_mrm(as, XO_ADDSD, tmp, right);
   ra_left(as, tmp, ir->op1);
 }
@@ -696,13 +819,13 @@ static void asm_conv(ASMState *as, IRIns *ir)
       if (left == dest) return;  /* Avoid the XO_XORPS. */
     } else if (LJ_32 && st == IRT_U32) {  /* U32 to FP conversion on x86. */
       /* number = (2^52+2^51 .. u32) - (2^52+2^51) */
-      cTValue *k = lj_ir_k64_find(as->J, U64x(43380000,00000000));
+      cTValue *k = &as->J->k64[LJ_K64_TOBIT];
       Reg bias = ra_scratch(as, rset_exclude(RSET_FPR, dest));
       if (irt_isfloat(ir->t))
 	emit_rr(as, XO_CVTSD2SS, dest, dest);
       emit_rr(as, XO_SUBSD, dest, bias);  /* Subtract 2^52+2^51 bias. */
       emit_rr(as, XO_XORPS, dest, bias);  /* Merge bias and integer. */
-      emit_loadn(as, bias, k);
+      emit_rma(as, XO_MOVSD, bias, k);
       emit_mrm(as, XO_MOVD, dest, asm_fuseload(as, lref, RSET_GPR));
       return;
     } else {  /* Integer to FP conversion. */
@@ -711,7 +834,7 @@ static void asm_conv(ASMState *as, IRIns *ir)
 		 asm_fuseloadm(as, lref, RSET_GPR, st64);
       if (LJ_64 && st == IRT_U64) {
 	MCLabel l_end = emit_label(as);
-	const void *k = lj_ir_k64_find(as->J, U64x(43f00000,00000000));
+	cTValue *k = &as->J->k64[LJ_K64_2P64];
 	emit_rma(as, XO_ADDSD, dest, k);  /* Add 2^64 to compensate. */
 	emit_sjcc(as, CC_NS, l_end);
 	emit_rr(as, XO_TEST, left|REX_64, left);  /* Check if u64 >= 2^63. */
@@ -738,23 +861,20 @@ static void asm_conv(ASMState *as, IRIns *ir)
 	  emit_gri(as, XG_ARITHi(XOg_ADD), dest, (int32_t)0x80000000);
 	emit_rr(as, op, dest|REX_64, tmp);
 	if (st == IRT_NUM)
-	  emit_rma(as, XO_ADDSD, tmp, lj_ir_k64_find(as->J,
-		   LJ_64 ? U64x(c3f00000,00000000) : U64x(c1e00000,00000000)));
+	  emit_rma(as, XO_ADDSD, tmp, &as->J->k64[LJ_K64_M2P64_31]);
 	else
-	  emit_rma(as, XO_ADDSS, tmp, lj_ir_k64_find(as->J,
-		   LJ_64 ? U64x(00000000,df800000) : U64x(00000000,cf000000)));
+	  emit_rma(as, XO_ADDSS, tmp, &as->J->k32[LJ_K32_M2P64_31]);
 	emit_sjcc(as, CC_NS, l_end);
 	emit_rr(as, XO_TEST, dest|REX_64, dest);  /* Check if dest negative. */
 	emit_rr(as, op, dest|REX_64, tmp);
 	ra_left(as, tmp, lref);
       } else {
-	Reg left = asm_fuseload(as, lref, RSET_FPR);
 	if (LJ_64 && irt_isu32(ir->t))
 	  emit_rr(as, XO_MOV, dest, dest);  /* Zero hiword. */
 	emit_mrm(as, op,
 		 dest|((LJ_64 &&
 			(irt_is64(ir->t) || irt_isu32(ir->t))) ? REX_64 : 0),
-		 left);
+		 asm_fuseload(as, lref, RSET_FPR));
       }
     }
   } else if (st >= IRT_I8 && st <= IRT_U16) {  /* Extend to 32 bit integer. */
@@ -828,8 +948,7 @@ static void asm_conv_fp_int64(ASMState *as, IRIns *ir)
   if (((ir-1)->op2 & IRCONV_SRCMASK) == IRT_U64) {
     /* For inputs in [2^63,2^64-1] add 2^64 to compensate. */
     MCLabel l_end = emit_label(as);
-    emit_rma(as, XO_FADDq, XOg_FADDq,
-	     lj_ir_k64_find(as->J, U64x(43f00000,00000000)));
+    emit_rma(as, XO_FADDq, XOg_FADDq, &as->J->k64[LJ_K64_2P64]);
     emit_sjcc(as, CC_NS, l_end);
     emit_rr(as, XO_TEST, hi, hi);  /* Check if u64 >= 2^63. */
   } else {
@@ -869,8 +988,7 @@ static void asm_conv_int64_fp(ASMState *as, IRIns *ir)
       emit_rmro(as, XO_FISTTPq, XOg_FISTTPq, RID_ESP, 0);
     else
       emit_rmro(as, XO_FISTPq, XOg_FISTPq, RID_ESP, 0);
-    emit_rma(as, XO_FADDq, XOg_FADDq,
-	     lj_ir_k64_find(as->J, U64x(c3f00000,00000000)));
+    emit_rma(as, XO_FADDq, XOg_FADDq, &as->J->k64[LJ_K64_M2P64]);
     emit_sjcc(as, CC_NS, l_pop);
     emit_rr(as, XO_TEST, hi, hi);  /* Check if out-of-range (2^63). */
   }
@@ -934,6 +1052,25 @@ static void asm_tvptr(ASMState *as, Reg dest, IRRef ref)
       emit_rmro(as, XO_LEA, dest|REX_64, RID_ESP, ra_spill(as, ir));
   } else {
     /* Otherwise use g->tmptv to hold the TValue. */
+#if LJ_GC64
+    if (irref_isk(ref)) {
+      TValue k;
+      lj_ir_kvalue(as->J->L, &k, ir);
+      emit_movmroi(as, dest, 4, k.u32.hi);
+      emit_movmroi(as, dest, 0, k.u32.lo);
+    } else {
+      /* TODO: 64 bit store + 32 bit load-modify-store is suboptimal. */
+      Reg src = ra_alloc1(as, ref, rset_exclude(RSET_GPR, dest));
+      if (irt_is64(ir->t)) {
+	emit_u32(as, irt_toitype(ir->t) << 15);
+	emit_rmro(as, XO_ARITHi, XOg_OR, dest, 4);
+      } else {
+	/* Currently, no caller passes integers that might end up here. */
+	emit_movmroi(as, dest, 4, (irt_toitype(ir->t) << 15));
+      }
+      emit_movtomro(as, REX_64IR(ir, src), dest, 0);
+    }
+#else
     if (!irref_isk(ref)) {
       Reg src = ra_alloc1(as, ref, rset_exclude(RSET_GPR, dest));
       emit_movtomro(as, REX_64IR(ir, src), dest, 0);
@@ -942,6 +1079,7 @@ static void asm_tvptr(ASMState *as, Reg dest, IRRef ref)
     }
     if (!(LJ_64 && irt_islightud(ir->t)))
       emit_movmroi(as, dest, 4, irt_toitype(ir->t));
+#endif
     emit_loada(as, dest, &J2G(as->J)->tmptv);
   }
 }
@@ -951,9 +1089,9 @@ static void asm_aref(ASMState *as, IRIns *ir)
   Reg dest = ra_dest(as, ir, RSET_GPR);
   asm_fusearef(as, ir, RSET_GPR);
   if (!(as->mrm.idx == RID_NONE && as->mrm.ofs == 0))
-    emit_mrm(as, XO_LEA, dest, RID_MRM);
+    emit_mrm(as, XO_LEA, dest|REX_GC64, RID_MRM);
   else if (as->mrm.base != dest)
-    emit_rr(as, XO_MOV, dest, as->mrm.base);
+    emit_rr(as, XO_MOV, dest|REX_GC64, as->mrm.base);
 }
 
 /* Inlined hash lookup. Specialized for key type and for const keys.
@@ -980,7 +1118,7 @@ static void asm_href(ASMState *as, IRIns *ir, IROp merge)
   if (!isk) {
     rset_clear(allow, tab);
     key = ra_alloc1(as, ir->op2, irt_isnum(kt) ? RSET_FPR : allow);
-    if (!irt_isstr(kt))
+    if (LJ_GC64 || !irt_isstr(kt))
       tmp = ra_scratch(as, rset_exclude(allow, key));
   }
 
@@ -993,8 +1131,8 @@ static void asm_href(ASMState *as, IRIns *ir, IROp merge)
 
   /* Follow hash chain until the end. */
   l_loop = emit_sjcc_label(as, CC_NZ);
-  emit_rr(as, XO_TEST, dest, dest);
-  emit_rmro(as, XO_MOV, dest, dest, offsetof(Node, next));
+  emit_rr(as, XO_TEST, dest|REX_GC64, dest);
+  emit_rmro(as, XO_MOV, dest|REX_GC64, dest, offsetof(Node, next));
   l_next = emit_label(as);
 
   /* Type and value comparison. */
@@ -1015,7 +1153,7 @@ static void asm_href(ASMState *as, IRIns *ir, IROp merge)
       emit_rmro(as, XO_UCOMISD, key, dest, offsetof(Node, key.n));
       emit_sjcc(as, CC_AE, l_next);
       /* The type check avoids NaN penalties and complaints from Valgrind. */
-#if LJ_64
+#if LJ_64 && !LJ_GC64
       emit_u32(as, LJ_TISNUM);
       emit_rmro(as, XO_ARITHi, XOg_CMP, dest, offsetof(Node, key.it));
 #else
@@ -1023,10 +1161,28 @@ static void asm_href(ASMState *as, IRIns *ir, IROp merge)
       emit_rmro(as, XO_ARITHi8, XOg_CMP, dest, offsetof(Node, key.it));
 #endif
     }
-#if LJ_64
+#if LJ_64 && !LJ_GC64
   } else if (irt_islightud(kt)) {
     emit_rmro(as, XO_CMP, key|REX_64, dest, offsetof(Node, key.u64));
 #endif
+#if LJ_GC64
+  } else if (irt_isaddr(kt)) {
+    if (isk) {
+      TValue k;
+      k.u64 = ((uint64_t)irt_toitype(irkey->t) << 47) | irkey[1].tv.u64;
+      emit_gmroi(as, XG_ARITHi(XOg_CMP), dest, offsetof(Node, key.u32.lo),
+		 k.u32.lo);
+      emit_sjcc(as, CC_NE, l_next);
+      emit_gmroi(as, XG_ARITHi(XOg_CMP), dest, offsetof(Node, key.u32.hi),
+		 k.u32.hi);
+    } else {
+      emit_rmro(as, XO_CMP, tmp|REX_64, dest, offsetof(Node, key.u64));
+    }
+  } else {
+    lua_assert(irt_ispri(kt) && !irt_isnil(kt));
+    emit_u32(as, (irt_toitype(kt)<<15)|0x7fff);
+    emit_rmro(as, XO_ARITHi, XOg_CMP, dest, offsetof(Node, key.it));
+#else
   } else {
     if (!irt_ispri(kt)) {
       lua_assert(irt_isaddr(kt));
@@ -1040,16 +1196,23 @@ static void asm_href(ASMState *as, IRIns *ir, IROp merge)
     lua_assert(!irt_isnil(kt));
     emit_i8(as, irt_toitype(kt));
     emit_rmro(as, XO_ARITHi8, XOg_CMP, dest, offsetof(Node, key.it));
+#endif
   }
   emit_sfixup(as, l_loop);
   checkmclim(as);
+#if LJ_GC64
+  if (!isk && irt_isaddr(kt)) {
+    emit_rr(as, XO_OR, tmp|REX_64, key);
+    emit_loadu64(as, tmp, (uint64_t)irt_toitype(kt) << 47);
+  }
+#endif
 
   /* Load main position relative to tab->node into dest. */
   khash = isk ? ir_khash(irkey) : 1;
   if (khash == 0) {
-    emit_rmro(as, XO_MOV, dest, tab, offsetof(GCtab, node));
+    emit_rmro(as, XO_MOV, dest|REX_GC64, tab, offsetof(GCtab, node));
   } else {
-    emit_rmro(as, XO_ARITH(XOg_ADD), dest, tab, offsetof(GCtab, node));
+    emit_rmro(as, XO_ARITH(XOg_ADD), dest|REX_GC64, tab, offsetof(GCtab,node));
     if ((as->flags & JIT_F_PREFER_IMUL)) {
       emit_i8(as, sizeof(Node));
       emit_rr(as, XO_IMULi8, dest, dest);
@@ -1084,7 +1247,19 @@ static void asm_href(ASMState *as, IRIns *ir, IROp merge)
 #endif
       } else {
 	emit_rr(as, XO_MOV, tmp, key);
+#if LJ_GC64
+	checkmclim(as);
+	emit_gri(as, XG_ARITHi(XOg_XOR), dest, irt_toitype(kt) << 15);
+	if ((as->flags & JIT_F_BMI2)) {
+	  emit_i8(as, 32);
+	  emit_mrm(as, XV_RORX|VEX_64, dest, key);
+	} else {
+	  emit_shifti(as, XOg_SHR|REX_64, dest, 32);
+	  emit_rr(as, XO_MOV, dest|REX_64, key|REX_64);
+	}
+#else
 	emit_rmro(as, XO_LEA, dest, key, HASH_BIAS);
+#endif
       }
     }
   }
@@ -1104,11 +1279,11 @@ static void asm_hrefk(ASMState *as, IRIns *ir)
   if (ra_hasreg(dest)) {
     if (ofs != 0) {
       if (dest == node && !(as->flags & JIT_F_LEA_AGU))
-	emit_gri(as, XG_ARITHi(XOg_ADD), dest, ofs);
+	emit_gri(as, XG_ARITHi(XOg_ADD), dest|REX_GC64, ofs);
       else
-	emit_rmro(as, XO_LEA, dest, node, ofs);
+	emit_rmro(as, XO_LEA, dest|REX_GC64, node, ofs);
     } else if (dest != node) {
-      emit_rr(as, XO_MOV, dest, node);
+      emit_rr(as, XO_MOV, dest|REX_GC64, node);
     }
   }
   asm_guardcc(as, CC_NE);
@@ -1120,13 +1295,24 @@ static void asm_hrefk(ASMState *as, IRIns *ir)
     lua_assert(irt_isnum(irkey->t) || irt_isgcv(irkey->t));
     /* Assumes -0.0 is already canonicalized to +0.0. */
     emit_loadu64(as, key, irt_isnum(irkey->t) ? ir_knum(irkey)->u64 :
+#if LJ_GC64
+			  ((uint64_t)irt_toitype(irkey->t) << 47) |
+			  (uint64_t)ir_kgc(irkey));
+#else
 			  ((uint64_t)irt_toitype(irkey->t) << 32) |
 			  (uint64_t)(uint32_t)ptr2addr(ir_kgc(irkey)));
+#endif
   } else {
     lua_assert(!irt_isnil(irkey->t));
+#if LJ_GC64
+    emit_i32(as, (irt_toitype(irkey->t)<<15)|0x7fff);
+    emit_rmro(as, XO_ARITHi, XOg_CMP, node,
+	      ofs + (int32_t)offsetof(Node, key.it));
+#else
     emit_i8(as, irt_toitype(irkey->t));
     emit_rmro(as, XO_ARITHi8, XOg_CMP, node,
 	      ofs + (int32_t)offsetof(Node, key.it));
+#endif
   }
 #else
   l_exit = emit_label(as);
@@ -1157,25 +1343,25 @@ static void asm_hrefk(ASMState *as, IRIns *ir)
 
 static void asm_uref(ASMState *as, IRIns *ir)
 {
-  /* NYI: Check that UREFO is still open and not aliasing a slot. */
   Reg dest = ra_dest(as, ir, RSET_GPR);
   if (irref_isk(ir->op1)) {
     GCfunc *fn = ir_kfunc(IR(ir->op1));
     MRef *v = &gcref(fn->l.uvptr[(ir->op2 >> 8)])->uv.v;
-    emit_rma(as, XO_MOV, dest, v);
+    emit_rma(as, XO_MOV, dest|REX_GC64, v);
   } else {
     Reg uv = ra_scratch(as, RSET_GPR);
     Reg func = ra_alloc1(as, ir->op1, RSET_GPR);
     if (ir->o == IR_UREFC) {
-      emit_rmro(as, XO_LEA, dest, uv, offsetof(GCupval, tv));
+      emit_rmro(as, XO_LEA, dest|REX_GC64, uv, offsetof(GCupval, tv));
       asm_guardcc(as, CC_NE);
       emit_i8(as, 1);
       emit_rmro(as, XO_ARITHib, XOg_CMP, uv, offsetof(GCupval, closed));
     } else {
-      emit_rmro(as, XO_MOV, dest, uv, offsetof(GCupval, v));
+      emit_rmro(as, XO_MOV, dest|REX_GC64, uv, offsetof(GCupval, v));
     }
-    emit_rmro(as, XO_MOV, uv, func,
-	      (int32_t)offsetof(GCfuncL, uvptr) + 4*(int32_t)(ir->op2 >> 8));
+    emit_rmro(as, XO_MOV, uv|REX_GC64, func,
+	      (int32_t)offsetof(GCfuncL, uvptr) +
+	      (int32_t)sizeof(MRef) * (int32_t)(ir->op2 >> 8));
   }
 }
 
@@ -1193,9 +1379,9 @@ static void asm_strref(ASMState *as, IRIns *ir)
   if (as->mrm.base == RID_NONE)
     emit_loadi(as, dest, as->mrm.ofs);
   else if (as->mrm.base == dest && as->mrm.idx == RID_NONE)
-    emit_gri(as, XG_ARITHi(XOg_ADD), dest, as->mrm.ofs);
+    emit_gri(as, XG_ARITHi(XOg_ADD), dest|REX_GC64, as->mrm.ofs);
   else
-    emit_mrm(as, XO_LEA, dest, RID_MRM);
+    emit_mrm(as, XO_LEA, dest|REX_GC64, RID_MRM);
 }
 
 /* -- Loads and stores ---------------------------------------------------- */
@@ -1264,7 +1450,7 @@ static void asm_fxstore(ASMState *as, IRIns *ir)
     case IRT_I16: case IRT_U16: xo = XO_MOVtow; break;
     case IRT_NUM: xo = XO_MOVSDto; break;
     case IRT_FLOAT: xo = XO_MOVSSto; break;
-#if LJ_64
+#if LJ_64 && !LJ_GC64
     case IRT_LIGHTUD: lua_assert(0);  /* NYI: mask 64 bit lightuserdata. */
 #endif
     default:
@@ -1296,7 +1482,7 @@ static void asm_fxstore(ASMState *as, IRIns *ir)
 #define asm_fstore(as, ir)	asm_fxstore(as, ir)
 #define asm_xstore(as, ir)	asm_fxstore(as, ir)
 
-#if LJ_64
+#if LJ_64 && !LJ_GC64
 static Reg asm_load_lightud64(ASMState *as, IRIns *ir, int typecheck)
 {
   if (ra_used(ir) || typecheck) {
@@ -1318,9 +1504,12 @@ static Reg asm_load_lightud64(ASMState *as, IRIns *ir, int typecheck)
 
 static void asm_ahuvload(ASMState *as, IRIns *ir)
 {
+#if LJ_GC64
+  Reg tmp = RID_NONE;
+#endif
   lua_assert(irt_isnum(ir->t) || irt_ispri(ir->t) || irt_isaddr(ir->t) ||
 	     (LJ_DUALNUM && irt_isint(ir->t)));
-#if LJ_64
+#if LJ_64 && !LJ_GC64
   if (irt_islightud(ir->t)) {
     Reg dest = asm_load_lightud64(as, ir, 1);
     if (ra_hasreg(dest)) {
@@ -1334,20 +1523,64 @@ static void asm_ahuvload(ASMState *as, IRIns *ir)
     RegSet allow = irt_isnum(ir->t) ? RSET_FPR : RSET_GPR;
     Reg dest = ra_dest(as, ir, allow);
     asm_fuseahuref(as, ir->op1, RSET_GPR);
+#if LJ_GC64
+    if (irt_isaddr(ir->t)) {
+      emit_shifti(as, XOg_SHR|REX_64, dest, 17);
+      asm_guardcc(as, CC_NE);
+      emit_i8(as, irt_toitype(ir->t));
+      emit_rr(as, XO_ARITHi8, XOg_CMP, dest);
+      emit_i8(as, XI_O16);
+      if ((as->flags & JIT_F_BMI2)) {
+	emit_i8(as, 47);
+	emit_mrm(as, XV_RORX|VEX_64, dest, RID_MRM);
+      } else {
+	emit_shifti(as, XOg_ROR|REX_64, dest, 47);
+	emit_mrm(as, XO_MOV, dest|REX_64, RID_MRM);
+      }
+      return;
+    } else
+#endif
     emit_mrm(as, dest < RID_MAX_GPR ? XO_MOV : XO_MOVSD, dest, RID_MRM);
   } else {
-    asm_fuseahuref(as, ir->op1, RSET_GPR);
+    RegSet gpr = RSET_GPR;
+#if LJ_GC64
+    if (irt_isaddr(ir->t)) {
+      tmp = ra_scratch(as, RSET_GPR);
+      gpr = rset_exclude(gpr, tmp);
+    }
+#endif
+    asm_fuseahuref(as, ir->op1, gpr);
   }
   /* Always do the type check, even if the load result is unused. */
   as->mrm.ofs += 4;
   asm_guardcc(as, irt_isnum(ir->t) ? CC_AE : CC_NE);
   if (LJ_64 && irt_type(ir->t) >= IRT_NUM) {
     lua_assert(irt_isinteger(ir->t) || irt_isnum(ir->t));
+#if LJ_GC64
+    emit_u32(as, LJ_TISNUM << 15);
+#else
     emit_u32(as, LJ_TISNUM);
+#endif
+    emit_mrm(as, XO_ARITHi, XOg_CMP, RID_MRM);
+#if LJ_GC64
+  } else if (irt_isaddr(ir->t)) {
+    as->mrm.ofs -= 4;
+    emit_i8(as, irt_toitype(ir->t));
+    emit_mrm(as, XO_ARITHi8, XOg_CMP, tmp);
+    emit_shifti(as, XOg_SAR|REX_64, tmp, 47);
+    emit_mrm(as, XO_MOV, tmp|REX_64, RID_MRM);
+  } else if (irt_isnil(ir->t)) {
+    as->mrm.ofs -= 4;
+    emit_i8(as, -1);
+    emit_mrm(as, XO_ARITHi8, XOg_CMP|REX_64, RID_MRM);
+  } else {
+    emit_u32(as, (irt_toitype(ir->t) << 15) | 0x7fff);
     emit_mrm(as, XO_ARITHi, XOg_CMP, RID_MRM);
+#else
   } else {
     emit_i8(as, irt_toitype(ir->t));
     emit_mrm(as, XO_ARITHi8, XOg_CMP, RID_MRM);
+#endif
   }
 }
 
@@ -1359,11 +1592,27 @@ static void asm_ahustore(ASMState *as, IRIns *ir)
     Reg src = ra_alloc1(as, ir->op2, RSET_FPR);
     asm_fuseahuref(as, ir->op1, RSET_GPR);
     emit_mrm(as, XO_MOVSDto, src, RID_MRM);
-#if LJ_64
+#if LJ_64 && !LJ_GC64
   } else if (irt_islightud(ir->t)) {
     Reg src = ra_alloc1(as, ir->op2, RSET_GPR);
     asm_fuseahuref(as, ir->op1, rset_exclude(RSET_GPR, src));
     emit_mrm(as, XO_MOVto, src|REX_64, RID_MRM);
+#endif
+#if LJ_GC64
+  } else if (irref_isk(ir->op2)) {
+    TValue k;
+    lj_ir_kvalue(as->J->L, &k, IR(ir->op2));
+    asm_fuseahuref(as, ir->op1, RSET_GPR);
+    if (tvisnil(&k)) {
+      emit_i32(as, -1);
+      emit_mrm(as, XO_MOVmi, REX_64, RID_MRM);
+    } else {
+      emit_u32(as, k.u32.lo);
+      emit_mrm(as, XO_MOVmi, 0, RID_MRM);
+      as->mrm.ofs += 4;
+      emit_u32(as, k.u32.hi);
+      emit_mrm(as, XO_MOVmi, 0, RID_MRM);
+    }
 #endif
   } else {
     IRIns *irr = IR(ir->op2);
@@ -1375,6 +1624,17 @@ static void asm_ahustore(ASMState *as, IRIns *ir)
     }
     asm_fuseahuref(as, ir->op1, allow);
     if (ra_hasreg(src)) {
+#if LJ_GC64
+      if (!(LJ_DUALNUM && irt_isinteger(ir->t))) {
+	/* TODO: 64 bit store + 32 bit load-modify-store is suboptimal. */
+	as->mrm.ofs += 4;
+	emit_u32(as, irt_toitype(ir->t) << 15);
+	emit_mrm(as, XO_ARITHi, XOg_OR, RID_MRM);
+	as->mrm.ofs -= 4;
+	emit_mrm(as, XO_MOVto, src|REX_64, RID_MRM);
+	return;
+      }
+#endif
       emit_mrm(as, XO_MOVto, src, RID_MRM);
     } else if (!irt_ispri(irr->t)) {
       lua_assert(irt_isaddr(ir->t) || (LJ_DUALNUM && irt_isinteger(ir->t)));
@@ -1382,14 +1642,20 @@ static void asm_ahustore(ASMState *as, IRIns *ir)
       emit_mrm(as, XO_MOVmi, 0, RID_MRM);
     }
     as->mrm.ofs += 4;
+#if LJ_GC64
+    lua_assert(LJ_DUALNUM && irt_isinteger(ir->t));
+    emit_i32(as, LJ_TNUMX << 15);
+#else
     emit_i32(as, (int32_t)irt_toitype(ir->t));
+#endif
     emit_mrm(as, XO_MOVmi, 0, RID_MRM);
   }
 }
 
 static void asm_sload(ASMState *as, IRIns *ir)
 {
-  int32_t ofs = 8*((int32_t)ir->op1-1) + ((ir->op2 & IRSLOAD_FRAME) ? 4 : 0);
+  int32_t ofs = 8*((int32_t)ir->op1-1-LJ_FR2) +
+		(!LJ_FR2 && (ir->op2 & IRSLOAD_FRAME) ? 4 : 0);
   IRType1 t = ir->t;
   Reg base;
   lua_assert(!(ir->op2 & IRSLOAD_PARENT));  /* Handled by asm_head_side(). */
@@ -1402,7 +1668,7 @@ static void asm_sload(ASMState *as, IRIns *ir)
     base = ra_alloc1(as, REF_BASE, RSET_GPR);
     emit_rmro(as, XO_MOVSD, left, base, ofs);
     t.irt = IRT_NUM;  /* Continue with a regular number type check. */
-#if LJ_64
+#if LJ_64 && !LJ_GC64
   } else if (irt_islightud(t)) {
     Reg dest = asm_load_lightud64(as, ir, (ir->op2 & IRSLOAD_TYPECHECK));
     if (ra_hasreg(dest)) {
@@ -1420,6 +1686,36 @@ static void asm_sload(ASMState *as, IRIns *ir)
       t.irt = irt_isint(t) ? IRT_NUM : IRT_INT;  /* Check for original type. */
       emit_rmro(as, irt_isint(t) ? XO_CVTSI2SD : XO_CVTTSD2SI, dest, base, ofs);
     } else {
+#if LJ_GC64
+      if (irt_isaddr(t)) {
+	/* LJ_GC64 type check + tag removal without BMI2 and with BMI2:
+	**
+	**  mov r64, [addr]    rorx r64, [addr], 47
+	**  ror r64, 47
+	**  cmp r16, itype     cmp r16, itype
+	**  jne ->exit         jne ->exit
+	**  shr r64, 16        shr r64, 16
+	*/
+	emit_shifti(as, XOg_SHR|REX_64, dest, 17);
+	if ((ir->op2 & IRSLOAD_TYPECHECK)) {
+	  asm_guardcc(as, CC_NE);
+	  emit_i8(as, irt_toitype(t));
+	  emit_rr(as, XO_ARITHi8, XOg_CMP, dest);
+	  emit_i8(as, XI_O16);
+	}
+	if ((as->flags & JIT_F_BMI2)) {
+	  emit_i8(as, 47);
+	  emit_rmro(as, XV_RORX|VEX_64, dest, base, ofs);
+	} else {
+	  if ((ir->op2 & IRSLOAD_TYPECHECK))
+	    emit_shifti(as, XOg_ROR|REX_64, dest, 47);
+	  else
+	    emit_shifti(as, XOg_SHL|REX_64, dest, 17);
+	  emit_rmro(as, XO_MOV, dest|REX_64, base, ofs);
+	}
+	return;
+      } else
+#endif
       emit_rmro(as, irt_isnum(t) ? XO_MOVSD : XO_MOV, dest, base, ofs);
     }
   } else {
@@ -1432,11 +1728,42 @@ static void asm_sload(ASMState *as, IRIns *ir)
     asm_guardcc(as, irt_isnum(t) ? CC_AE : CC_NE);
     if (LJ_64 && irt_type(t) >= IRT_NUM) {
       lua_assert(irt_isinteger(t) || irt_isnum(t));
+#if LJ_GC64
+      emit_u32(as, LJ_TISNUM << 15);
+#else
       emit_u32(as, LJ_TISNUM);
+#endif
+      emit_rmro(as, XO_ARITHi, XOg_CMP, base, ofs+4);
+#if LJ_GC64
+    } else if (irt_isnil(t)) {
+      /* LJ_GC64 type check for nil:
+      **
+      **   cmp qword [addr], -1
+      **   jne ->exit
+      */
+      emit_i8(as, -1);
+      emit_rmro(as, XO_ARITHi8, XOg_CMP|REX_64, base, ofs);
+    } else if (irt_ispri(t)) {
+      emit_u32(as, (irt_toitype(t) << 15) | 0x7fff);
       emit_rmro(as, XO_ARITHi, XOg_CMP, base, ofs+4);
+    } else {
+      /* LJ_GC64 type check only:
+      **
+      **   mov r64, [addr]
+      **   sar r64, 47
+      **   cmp r32, itype
+      **   jne ->exit
+      */
+      Reg tmp = ra_scratch(as, rset_exclude(RSET_GPR, base));
+      emit_i8(as, irt_toitype(t));
+      emit_rr(as, XO_ARITHi8, XOg_CMP, tmp);
+      emit_shifti(as, XOg_SAR|REX_64, tmp, 47);
+      emit_rmro(as, XO_MOV, tmp|REX_64, base, ofs+4);
+#else
     } else {
       emit_i8(as, irt_toitype(t));
       emit_rmro(as, XO_ARITHi8, XOg_CMP, base, ofs+4);
+#endif
     }
   }
 }
@@ -1464,8 +1791,9 @@ static void asm_cnew(ASMState *as, IRIns *ir)
     Reg r64 = sz == 8 ? REX_64 : 0;
     if (irref_isk(ir->op2)) {
       IRIns *irk = IR(ir->op2);
-      uint64_t k = irk->o == IR_KINT64 ? ir_k64(irk)->u64 :
-					 (uint64_t)(uint32_t)irk->i;
+      uint64_t k = (irk->o == IR_KINT64 ||
+		    (LJ_GC64 && (irk->o == IR_KPTR || irk->o == IR_KKPTR))) ?
+		   ir_k64(irk)->u64 : (uint64_t)(uint32_t)irk->i;
       if (sz == 4 || checki32((int64_t)k)) {
 	emit_i32(as, (int32_t)k);
 	emit_rmro(as, XO_MOVmi, r64, RID_RET, sizeof(GCcdata));
@@ -1530,7 +1858,7 @@ static void asm_tbar(ASMState *as, IRIns *ir)
   Reg tab = ra_alloc1(as, ir->op1, RSET_GPR);
   Reg tmp = ra_scratch(as, rset_exclude(RSET_GPR, tab));
   MCLabel l_end = emit_label(as);
-  emit_movtomro(as, tmp, tab, offsetof(GCtab, gclist));
+  emit_movtomro(as, tmp|REX_GC64, tab, offsetof(GCtab, gclist));
   emit_setgl(as, tab, gc.grayagain);
   emit_getgl(as, tmp, gc.grayagain);
   emit_i8(as, ~LJ_GC_BLACK);
@@ -1956,7 +2284,7 @@ static void asm_bswap(ASMState *as, IRIns *ir)
 #define asm_bor(as, ir)		asm_intarith(as, ir, XOg_OR)
 #define asm_bxor(as, ir)	asm_intarith(as, ir, XOg_XOR)
 
-static void asm_bitshift(ASMState *as, IRIns *ir, x86Shift xs)
+static void asm_bitshift(ASMState *as, IRIns *ir, x86Shift xs, x86Op xv)
 {
   IRRef rref = ir->op2;
   IRIns *irr = IR(rref);
@@ -1965,11 +2293,27 @@ static void asm_bitshift(ASMState *as, IRIns *ir, x86Shift xs)
     int shift;
     dest = ra_dest(as, ir, RSET_GPR);
     shift = irr->i & (irt_is64(ir->t) ? 63 : 31);
+    if (!xv && shift && (as->flags & JIT_F_BMI2)) {
+      Reg left = asm_fuseloadm(as, ir->op1, RSET_GPR, irt_is64(ir->t));
+      if (left != dest) {  /* BMI2 rotate right by constant. */
+	emit_i8(as, xs == XOg_ROL ? -shift : shift);
+	emit_mrm(as, VEX_64IR(ir, XV_RORX), dest, left);
+	return;
+      }
+    }
     switch (shift) {
     case 0: break;
     case 1: emit_rr(as, XO_SHIFT1, REX_64IR(ir, xs), dest); break;
     default: emit_shifti(as, REX_64IR(ir, xs), dest, shift); break;
     }
+  } else if ((as->flags & JIT_F_BMI2) && xv) {	/* BMI2 variable shifts. */
+    Reg left, right;
+    dest = ra_dest(as, ir, RSET_GPR);
+    right = ra_alloc1(as, rref, RSET_GPR);
+    left = asm_fuseloadm(as, ir->op1, rset_exclude(RSET_GPR, right),
+			 irt_is64(ir->t));
+    emit_mrm(as, VEX_64IR(ir, xv) ^ (right << 19), dest, left);
+    return;
   } else {  /* Variable shifts implicitly use register cl (i.e. ecx). */
     Reg right;
     dest = ra_dest(as, ir, rset_exclude(RSET_GPR, RID_ECX));
@@ -1995,11 +2339,11 @@ static void asm_bitshift(ASMState *as, IRIns *ir, x86Shift xs)
   */
 }
 
-#define asm_bshl(as, ir)	asm_bitshift(as, ir, XOg_SHL)
-#define asm_bshr(as, ir)	asm_bitshift(as, ir, XOg_SHR)
-#define asm_bsar(as, ir)	asm_bitshift(as, ir, XOg_SAR)
-#define asm_brol(as, ir)	asm_bitshift(as, ir, XOg_ROL)
-#define asm_bror(as, ir)	asm_bitshift(as, ir, XOg_ROR)
+#define asm_bshl(as, ir)	asm_bitshift(as, ir, XOg_SHL, XV_SHLX)
+#define asm_bshr(as, ir)	asm_bitshift(as, ir, XOg_SHR, XV_SHRX)
+#define asm_bsar(as, ir)	asm_bitshift(as, ir, XOg_SAR, XV_SARX)
+#define asm_brol(as, ir)	asm_bitshift(as, ir, XOg_ROL, 0)
+#define asm_bror(as, ir)	asm_bitshift(as, ir, XOg_ROR, 0)
 
 /* -- Comparisons --------------------------------------------------------- */
 
@@ -2050,7 +2394,6 @@ static void asm_comp(ASMState *as, IRIns *ir)
       cc ^= (VCC_PS|(5<<4));  /* A <-> B, AE <-> BE, PS <-> none */
     }
     left = ra_alloc1(as, lref, RSET_FPR);
-    right = asm_fuseload(as, rref, rset_exclude(RSET_FPR, left));
     l_around = emit_label(as);
     asm_guardcc(as, cc >> 4);
     if (cc & VCC_P) {  /* Extra CC_P branch required? */
@@ -2067,6 +2410,7 @@ static void asm_comp(ASMState *as, IRIns *ir)
 	  emit_jcc(as, CC_P, as->mcp);
       }
     }
+    right = asm_fuseload(as, rref, rset_exclude(RSET_FPR, left));
     emit_mrm(as, XO_UCOMISD, left, right);
   } else {
     IRRef lref = ir->op1, rref = ir->op2;
@@ -2343,13 +2687,18 @@ static void asm_stack_check(ASMState *as, BCReg topslot,
     emit_rmro(as, XO_MOV, r|REX_64, RID_ESP, 0);
   else
     ra_modified(as, r);
-  emit_gri(as, XG_ARITHi(XOg_CMP), r, (int32_t)(8*topslot));
+  emit_gri(as, XG_ARITHi(XOg_CMP), r|REX_GC64, (int32_t)(8*topslot));
   if (ra_hasreg(pbase) && pbase != r)
-    emit_rr(as, XO_ARITH(XOg_SUB), r, pbase);
+    emit_rr(as, XO_ARITH(XOg_SUB), r|REX_GC64, pbase);
   else
+#if LJ_GC64
+    emit_rmro(as, XO_ARITH(XOg_SUB), r|REX_64, RID_DISPATCH,
+	      (int32_t)dispofs(as, &J2G(as->J)->jit_base));
+#else
     emit_rmro(as, XO_ARITH(XOg_SUB), r, RID_NONE,
 	      ptr2addr(&J2G(as->J)->jit_base));
-  emit_rmro(as, XO_MOV, r, r, offsetof(lua_State, maxstack));
+#endif
+  emit_rmro(as, XO_MOV, r|REX_GC64, r, offsetof(lua_State, maxstack));
   emit_getgl(as, r, cur_L);
   if (allow == RSET_EMPTY)  /* Spill temp. register. */
     emit_rmro(as, XO_MOVto, r|REX_64, RID_ESP, 0);
@@ -2359,13 +2708,15 @@ static void asm_stack_check(ASMState *as, BCReg topslot,
 static void asm_stack_restore(ASMState *as, SnapShot *snap)
 {
   SnapEntry *map = &as->T->snapmap[snap->mapofs];
-  SnapEntry *flinks = &as->T->snapmap[snap_nextofs(as->T, snap)-1];
+#if !LJ_FR2 || defined(LUA_USE_ASSERT)
+  SnapEntry *flinks = &as->T->snapmap[snap_nextofs(as->T, snap)-1-LJ_FR2];
+#endif
   MSize n, nent = snap->nent;
   /* Store the value of all modified slots to the Lua stack. */
   for (n = 0; n < nent; n++) {
     SnapEntry sn = map[n];
     BCReg s = snap_slot(sn);
-    int32_t ofs = 8*((int32_t)s-1);
+    int32_t ofs = 8*((int32_t)s-1-LJ_FR2);
     IRRef ref = snap_ref(sn);
     IRIns *ir = IR(ref);
     if ((sn & SNAP_NORESTORE))
@@ -2378,16 +2729,44 @@ static void asm_stack_restore(ASMState *as, SnapShot *snap)
 		 (LJ_DUALNUM && irt_isinteger(ir->t)));
       if (!irref_isk(ref)) {
 	Reg src = ra_alloc1(as, ref, rset_exclude(RSET_GPR, RID_BASE));
+#if LJ_GC64
+	if (irt_is64(ir->t)) {
+	  /* TODO: 64 bit store + 32 bit load-modify-store is suboptimal. */
+	  emit_u32(as, irt_toitype(ir->t) << 15);
+	  emit_rmro(as, XO_ARITHi, XOg_OR, RID_BASE, ofs+4);
+	} else if (LJ_DUALNUM && irt_isinteger(ir->t)) {
+	  emit_movmroi(as, RID_BASE, ofs+4, LJ_TISNUM << 15);
+	} else {
+	  emit_movmroi(as, RID_BASE, ofs+4, (irt_toitype(ir->t)<<15)|0x7fff);
+	}
+#endif
 	emit_movtomro(as, REX_64IR(ir, src), RID_BASE, ofs);
+#if LJ_GC64
+      } else {
+	TValue k;
+	lj_ir_kvalue(as->J->L, &k, ir);
+	if (tvisnil(&k)) {
+	  emit_i32(as, -1);
+	  emit_rmro(as, XO_MOVmi, REX_64, RID_BASE, ofs);
+	} else {
+	  emit_movmroi(as, RID_BASE, ofs+4, k.u32.hi);
+	  emit_movmroi(as, RID_BASE, ofs, k.u32.lo);
+	}
+#else
       } else if (!irt_ispri(ir->t)) {
 	emit_movmroi(as, RID_BASE, ofs, ir->i);
+#endif
       }
       if ((sn & (SNAP_CONT|SNAP_FRAME))) {
+#if !LJ_FR2
 	if (s != 0)  /* Do not overwrite link to previous frame. */
 	  emit_movmroi(as, RID_BASE, ofs+4, (int32_t)(*flinks--));
+#endif
+#if !LJ_GC64
       } else {
 	if (!(LJ_64 && irt_islightud(ir->t)))
 	  emit_movmroi(as, RID_BASE, ofs+4, irt_toitype(ir->t));
+#endif
       }
     }
     checkmclim(as);
@@ -2413,11 +2792,15 @@ static void asm_gc_check(ASMState *as)
   args[1] = ASMREF_TMP2;  /* MSize steps     */
   asm_gencall(as, ci, args);
   tmp = ra_releasetmp(as, ASMREF_TMP1);
+#if LJ_GC64
+  emit_rmro(as, XO_LEA, tmp|REX_64, RID_DISPATCH, GG_DISP2G);
+#else
   emit_loada(as, tmp, J2G(as->J));
+#endif
   emit_loadi(as, ra_releasetmp(as, ASMREF_TMP2), as->gcsteps);
   /* Jump around GC step if GC total < GC threshold. */
   emit_sjcc(as, CC_B, l_end);
-  emit_opgl(as, XO_ARITH(XOg_CMP), tmp, gc.threshold);
+  emit_opgl(as, XO_ARITH(XOg_CMP), tmp|REX_GC64, gc.threshold);
   emit_getgl(as, tmp, gc.total);
   as->gcsteps = 0;
   checkmclim(as);
@@ -2482,7 +2865,7 @@ static void asm_head_root_base(ASMState *as)
     if (rset_test(as->modset, r) || irt_ismarked(ir->t))
       ir->r = RID_INIT;  /* No inheritance for modified BASE register. */
     if (r != RID_BASE)
-      emit_rr(as, XO_MOV, r, RID_BASE);
+      emit_rr(as, XO_MOV, r|REX_GC64, RID_BASE);
   }
 }
 
@@ -2498,8 +2881,9 @@ static RegSet asm_head_side_base(ASMState *as, IRIns *irp, RegSet allow)
     if (irp->r == r) {
       rset_clear(allow, r);  /* Mark same BASE register as coalesced. */
     } else if (ra_hasreg(irp->r) && rset_test(as->freeset, irp->r)) {
+      /* Move from coalesced parent reg. */
       rset_clear(allow, irp->r);
-      emit_rr(as, XO_MOV, r, irp->r);  /* Move from coalesced parent reg. */
+      emit_rr(as, XO_MOV, r|REX_GC64, irp->r);
     } else {
       emit_getgl(as, r, jit_base);  /* Otherwise reload BASE. */
     }
@@ -2600,10 +2984,111 @@ static Reg asm_setup_call_slots(ASMState *as, IRIns *ir, const CCallInfo *ci)
 static void asm_setup_target(ASMState *as)
 {
   asm_exitstub_setup(as, as->T->nsnap);
+  as->mrm.base = 0;
 }
 
 /* -- Trace patching ------------------------------------------------------ */
 
+static const uint8_t map_op1[256] = {
+0x92,0x92,0x92,0x92,0x52,0x45,0x51,0x51,0x92,0x92,0x92,0x92,0x52,0x45,0x51,0x20,
+0x92,0x92,0x92,0x92,0x52,0x45,0x51,0x51,0x92,0x92,0x92,0x92,0x52,0x45,0x51,0x51,
+0x92,0x92,0x92,0x92,0x52,0x45,0x10,0x51,0x92,0x92,0x92,0x92,0x52,0x45,0x10,0x51,
+0x92,0x92,0x92,0x92,0x52,0x45,0x10,0x51,0x92,0x92,0x92,0x92,0x52,0x45,0x10,0x51,
+#if LJ_64
+0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x14,0x14,0x14,0x14,0x14,0x14,0x14,0x14,
+#else
+0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,
+#endif
+0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,
+0x51,0x51,0x92,0x92,0x10,0x10,0x12,0x11,0x45,0x86,0x52,0x93,0x51,0x51,0x51,0x51,
+0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,
+0x93,0x86,0x93,0x93,0x92,0x92,0x92,0x92,0x92,0x92,0x92,0x92,0x92,0x92,0x92,0x92,
+0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x47,0x51,0x51,0x51,0x51,0x51,
+#if LJ_64
+0x59,0x59,0x59,0x59,0x51,0x51,0x51,0x51,0x52,0x45,0x51,0x51,0x51,0x51,0x51,0x51,
+#else
+0x55,0x55,0x55,0x55,0x51,0x51,0x51,0x51,0x52,0x45,0x51,0x51,0x51,0x51,0x51,0x51,
+#endif
+0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x05,0x05,0x05,0x05,0x05,0x05,0x05,0x05,
+0x93,0x93,0x53,0x51,0x70,0x71,0x93,0x86,0x54,0x51,0x53,0x51,0x51,0x52,0x51,0x51,
+0x92,0x92,0x92,0x92,0x52,0x52,0x51,0x51,0x92,0x92,0x92,0x92,0x92,0x92,0x92,0x92,
+0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x45,0x45,0x47,0x52,0x51,0x51,0x51,0x51,
+0x10,0x51,0x10,0x10,0x51,0x51,0x63,0x66,0x51,0x51,0x51,0x51,0x51,0x51,0x92,0x92
+};
+
+static const uint8_t map_op2[256] = {
+0x93,0x93,0x93,0x93,0x52,0x52,0x52,0x52,0x52,0x52,0x51,0x52,0x51,0x93,0x52,0x94,
+0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,
+0x53,0x53,0x53,0x53,0x53,0x53,0x53,0x53,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,
+0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x34,0x51,0x35,0x51,0x51,0x51,0x51,0x51,
+0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,
+0x53,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,
+0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,
+0x94,0x54,0x54,0x54,0x93,0x93,0x93,0x52,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,
+0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,
+0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,
+0x52,0x52,0x52,0x93,0x94,0x93,0x51,0x51,0x52,0x52,0x52,0x93,0x94,0x93,0x93,0x93,
+0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x94,0x93,0x93,0x93,0x93,0x93,
+0x93,0x93,0x94,0x93,0x94,0x94,0x94,0x93,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,
+0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,
+0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,
+0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x52
+};
+
+static uint32_t asm_x86_inslen(const uint8_t* p)
+{
+  uint32_t result = 0;
+  uint32_t prefixes = 0;
+  uint32_t x = map_op1[*p];
+  for (;;) {
+    switch (x >> 4) {
+    case 0: return result + x + (prefixes & 4);
+    case 1: prefixes |= x; x = map_op1[*++p]; result++; break;
+    case 2: x = map_op2[*++p]; break;
+    case 3: p++; goto mrm;
+    case 4: result -= (prefixes & 2);  /* fallthrough */
+    case 5: return result + (x & 15);
+    case 6:  /* Group 3. */
+      if (p[1] & 0x38) x = 2;
+      else if ((prefixes & 2) && (x == 0x66)) x = 4;
+      goto mrm;
+    case 7: /* VEX c4/c5. */
+      if (LJ_32 && p[1] < 0xc0) {
+	x = 2;
+	goto mrm;
+      }
+      if (x == 0x70) {
+	x = *++p & 0x1f;
+	result++;
+	if (x >= 2) {
+	  p += 2;
+	  result += 2;
+	  goto mrm;
+	}
+      }
+      p++;
+      result++;
+      x = map_op2[*++p];
+      break;
+    case 8: result -= (prefixes & 2);  /* fallthrough */
+    case 9: mrm:  /* ModR/M and possibly SIB. */
+      result += (x & 15);
+      x = *++p;
+      switch (x >> 6) {
+      case 0: if ((x & 7) == 5) return result + 4; break;
+      case 1: result++; break;
+      case 2: result += 4; break;
+      case 3: return result;
+      }
+      if ((x & 7) == 4) {
+	result++;
+	if (x < 0x40 && (p[1] & 7) == 5) result += 4;
+      }
+      return result;
+    }
+  }
+}
+
 /* Patch exit jumps of existing machine code to a new target. */
 void lj_asm_patchexit(jit_State *J, GCtrace *T, ExitNo exitno, MCode *target)
 {
@@ -2612,22 +3097,23 @@ void lj_asm_patchexit(jit_State *J, GCtrace *T, ExitNo exitno, MCode *target)
   MSize len = T->szmcode;
   MCode *px = exitstub_addr(J, exitno) - 6;
   MCode *pe = p+len-6;
-  uint32_t stateaddr = u32ptr(&J2G(J)->vmstate);
+#if LJ_GC64
+  uint32_t statei = (uint32_t)(GG_OFS(g.vmstate) - GG_OFS(dispatch));
+#else
+  uint32_t statei = u32ptr(&J2G(J)->vmstate);
+#endif
   if (len > 5 && p[len-5] == XI_JMP && p+len-6 + *(int32_t *)(p+len-4) == px)
     *(int32_t *)(p+len-4) = jmprel(p+len, target);
   /* Do not patch parent exit for a stack check. Skip beyond vmstate update. */
-  for (; p < pe; p++)
-    if (*(uint32_t *)(p+(LJ_64 ? 3 : 2)) == stateaddr && p[0] == XI_MOVmi) {
-      p += LJ_64 ? 11 : 10;
+  for (; p < pe; p += asm_x86_inslen(p)) {
+    intptr_t ofs = LJ_GC64 ? (p[0] & 0xf0) == 0x40 : LJ_64;
+    if (*(uint32_t *)(p+2+ofs) == statei && p[ofs+LJ_GC64-LJ_64] == XI_MOVmi)
       break;
-    }
+  }
   lua_assert(p < pe);
-  for (; p < pe; p++) {
-    if ((*(uint16_t *)p & 0xf0ff) == 0x800f && p + *(int32_t *)(p+2) == px) {
+  for (; p < pe; p += asm_x86_inslen(p))
+    if ((*(uint16_t *)p & 0xf0ff) == 0x800f && p + *(int32_t *)(p+2) == px)
       *(int32_t *)(p+2) = jmprel(p+6, target);
-      p += 5;
-    }
-  }
   lj_mcode_sync(T->mcode, T->mcode + T->szmcode);
   lj_mcode_patch(J, mcarea, 1);
 }