immunio 0.15.4 → 0.16.0

data/lua-hooks/ext/luajit/src/lj_target_x86.h

@@ -0,0 +1,345 @@
+/*
+** Definitions for x86 and x64 CPUs.
+** Copyright (C) 2005-2015 Mike Pall. See Copyright Notice in luajit.h
+*/
+
+#ifndef _LJ_TARGET_X86_H
+#define _LJ_TARGET_X86_H
+
+/* -- Registers IDs ------------------------------------------------------- */
+
+#if LJ_64
+#define GPRDEF(_) \
+  _(EAX) _(ECX) _(EDX) _(EBX) _(ESP) _(EBP) _(ESI) _(EDI) \
+  _(R8D) _(R9D) _(R10D) _(R11D) _(R12D) _(R13D) _(R14D) _(R15D)
+#define FPRDEF(_) \
+  _(XMM0) _(XMM1) _(XMM2) _(XMM3) _(XMM4) _(XMM5) _(XMM6) _(XMM7) \
+  _(XMM8) _(XMM9) _(XMM10) _(XMM11) _(XMM12) _(XMM13) _(XMM14) _(XMM15)
+#else
+#define GPRDEF(_) \
+  _(EAX) _(ECX) _(EDX) _(EBX) _(ESP) _(EBP) _(ESI) _(EDI)
+#define FPRDEF(_) \
+  _(XMM0) _(XMM1) _(XMM2) _(XMM3) _(XMM4) _(XMM5) _(XMM6) _(XMM7)
+#endif
+#define VRIDDEF(_) \
+  _(MRM)
+
+#define RIDENUM(name)	RID_##name,
+
+enum {
+  GPRDEF(RIDENUM)		/* General-purpose registers (GPRs). */
+  FPRDEF(RIDENUM)		/* Floating-point registers (FPRs). */
+  RID_MAX,
+  RID_MRM = RID_MAX,		/* Pseudo-id for ModRM operand. */
+
+  /* Calling conventions. */
+  RID_SP = RID_ESP,
+  RID_RET = RID_EAX,
+#if LJ_64
+  RID_FPRET = RID_XMM0,
+#else
+  RID_RETLO = RID_EAX,
+  RID_RETHI = RID_EDX,
+#endif
+
+  /* These definitions must match with the *.dasc file(s): */
+  RID_BASE = RID_EDX,		/* Interpreter BASE. */
+#if LJ_64 && !LJ_ABI_WIN
+  RID_LPC = RID_EBX,		/* Interpreter PC. */
+  RID_DISPATCH = RID_R14D,	/* Interpreter DISPATCH table. */
+#else
+  RID_LPC = RID_ESI,		/* Interpreter PC. */
+  RID_DISPATCH = RID_EBX,	/* Interpreter DISPATCH table. */
+#endif
+
+  /* Register ranges [min, max) and number of registers. */
+  RID_MIN_GPR = RID_EAX,
+  RID_MIN_FPR = RID_XMM0,
+  RID_MAX_GPR = RID_MIN_FPR,
+  RID_MAX_FPR = RID_MAX,
+  RID_NUM_GPR = RID_MAX_GPR - RID_MIN_GPR,
+  RID_NUM_FPR = RID_MAX_FPR - RID_MIN_FPR,
+};
+
+/* -- Register sets ------------------------------------------------------- */
+
+/* Make use of all registers, except the stack pointer. */
+#define RSET_GPR	(RSET_RANGE(RID_MIN_GPR, RID_MAX_GPR)-RID2RSET(RID_ESP))
+#define RSET_FPR	(RSET_RANGE(RID_MIN_FPR, RID_MAX_FPR))
+#define RSET_ALL	(RSET_GPR|RSET_FPR)
+#define RSET_INIT	RSET_ALL
+
+#if LJ_64
+/* Note: this requires the use of FORCE_REX! */
+#define RSET_GPR8	RSET_GPR
+#else
+#define RSET_GPR8	(RSET_RANGE(RID_EAX, RID_EBX+1))
+#endif
+
+/* ABI-specific register sets. */
+#define RSET_ACD	(RID2RSET(RID_EAX)|RID2RSET(RID_ECX)|RID2RSET(RID_EDX))
+#if LJ_64
+#if LJ_ABI_WIN
+/* Windows x64 ABI. */
+#define RSET_SCRATCH \
+  (RSET_ACD|RSET_RANGE(RID_R8D, RID_R11D+1)|RSET_RANGE(RID_XMM0, RID_XMM5+1))
+#define REGARG_GPRS \
+  (RID_ECX|((RID_EDX|((RID_R8D|(RID_R9D<<5))<<5))<<5))
+#define REGARG_NUMGPR	4
+#define REGARG_NUMFPR	4
+#define REGARG_FIRSTFPR	RID_XMM0
+#define REGARG_LASTFPR	RID_XMM3
+#define STACKARG_OFS	(4*8)
+#else
+/* The rest of the civilized x64 world has a common ABI. */
+#define RSET_SCRATCH \
+  (RSET_ACD|RSET_RANGE(RID_ESI, RID_R11D+1)|RSET_FPR)
+#define REGARG_GPRS \
+  (RID_EDI|((RID_ESI|((RID_EDX|((RID_ECX|((RID_R8D|(RID_R9D \
+   <<5))<<5))<<5))<<5))<<5))
+#define REGARG_NUMGPR	6
+#define REGARG_NUMFPR	8
+#define REGARG_FIRSTFPR	RID_XMM0
+#define REGARG_LASTFPR	RID_XMM7
+#define STACKARG_OFS	0
+#endif
+#else
+/* Common x86 ABI. */
+#define RSET_SCRATCH	(RSET_ACD|RSET_FPR)
+#define REGARG_GPRS	(RID_ECX|(RID_EDX<<5))  /* Fastcall only. */
+#define REGARG_NUMGPR	2  /* Fastcall only. */
+#define REGARG_NUMFPR	0
+#define STACKARG_OFS	0
+#endif
+
+#if LJ_64
+/* Prefer the low 8 regs of each type to reduce REX prefixes. */
+#undef rset_picktop
+#define rset_picktop(rs)	(lj_fls(lj_bswap(rs)) ^ 0x18)
+#endif
+
+/* -- Spill slots --------------------------------------------------------- */
+
+/* Spill slots are 32 bit wide. An even/odd pair is used for FPRs.
+**
+** SPS_FIXED: Available fixed spill slots in interpreter frame.
+** This definition must match with the *.dasc file(s).
+**
+** SPS_FIRST: First spill slot for general use. Reserve min. two 32 bit slots.
+*/
+#if LJ_64
+#if LJ_ABI_WIN
+#define SPS_FIXED	(4*2)
+#define SPS_FIRST	(4*2)	/* Don't use callee register save area. */
+#else
+#if LJ_GC64
+#define SPS_FIXED	2
+#else
+#define SPS_FIXED	4
+#endif
+#define SPS_FIRST	2
+#endif
+#else
+#define SPS_FIXED	6
+#define SPS_FIRST	2
+#endif
+
+#define SPOFS_TMP	0
+
+#define sps_scale(slot)		(4 * (int32_t)(slot))
+#define sps_align(slot)		(((slot) - SPS_FIXED + 3) & ~3)
+
+/* -- Exit state ---------------------------------------------------------- */
+
+/* This definition must match with the *.dasc file(s). */
+typedef struct {
+  lua_Number fpr[RID_NUM_FPR];	/* Floating-point registers. */
+  intptr_t gpr[RID_NUM_GPR];	/* General-purpose registers. */
+  int32_t spill[256];		/* Spill slots. */
+} ExitState;
+
+/* Limited by the range of a short fwd jump (127): (2+2)*(32-1)-2 = 122. */
+#define EXITSTUB_SPACING	(2+2)
+#define EXITSTUBS_PER_GROUP	32
+
+/* -- x86 ModRM operand encoding ------------------------------------------ */
+
+typedef enum {
+  XM_OFS0 = 0x00, XM_OFS8 = 0x40, XM_OFS32 = 0x80, XM_REG = 0xc0,
+  XM_SCALE1 = 0x00, XM_SCALE2 = 0x40, XM_SCALE4 = 0x80, XM_SCALE8 = 0xc0,
+  XM_MASK = 0xc0
+} x86Mode;
+
+/* Structure to hold variable ModRM operand. */
+typedef struct {
+  int32_t ofs;		/* Offset. */
+  uint8_t base;		/* Base register or RID_NONE. */
+  uint8_t idx;		/* Index register or RID_NONE. */
+  uint8_t scale;	/* Index scale (XM_SCALE1 .. XM_SCALE8). */
+} x86ModRM;
+
+/* -- Opcodes ------------------------------------------------------------- */
+
+/* Macros to construct variable-length x86 opcodes. -(len+1) is in LSB. */
+#define XO_(o)		((uint32_t)(0x0000fe + (0x##o<<24)))
+#define XO_FPU(a,b)	((uint32_t)(0x00fd + (0x##a<<16)+(0x##b<<24)))
+#define XO_0f(o)	((uint32_t)(0x0f00fd + (0x##o<<24)))
+#define XO_66(o)	((uint32_t)(0x6600fd + (0x##o<<24)))
+#define XO_660f(o)	((uint32_t)(0x0f66fc + (0x##o<<24)))
+#define XO_f20f(o)	((uint32_t)(0x0ff2fc + (0x##o<<24)))
+#define XO_f30f(o)	((uint32_t)(0x0ff3fc + (0x##o<<24)))
+
+/* This list of x86 opcodes is not intended to be complete. Opcodes are only
+** included when needed. Take a look at DynASM or jit.dis_x86 to see the
+** whole mess.
+*/
+typedef enum {
+  /* Fixed length opcodes. XI_* prefix. */
+  XI_NOP =	0x90,
+  XI_XCHGa =	0x90,
+  XI_CALL =	0xe8,
+  XI_JMP =	0xe9,
+  XI_JMPs =	0xeb,
+  XI_PUSH =	0x50, /* Really 50+r. */
+  XI_JCCs =	0x70, /* Really 7x. */
+  XI_JCCn =	0x80, /* Really 0f8x. */
+  XI_LEA =	0x8d,
+  XI_MOVrib =	0xb0, /* Really b0+r. */
+  XI_MOVri =	0xb8, /* Really b8+r. */
+  XI_ARITHib =	0x80,
+  XI_ARITHi =	0x81,
+  XI_ARITHi8 =	0x83,
+  XI_PUSHi8 =	0x6a,
+  XI_TESTb =	0x84,
+  XI_TEST =	0x85,
+  XI_MOVmi =	0xc7,
+  XI_GROUP5 =	0xff,
+
+  /* Note: little-endian byte-order! */
+  XI_FLDZ =	0xeed9,
+  XI_FLD1 =	0xe8d9,
+  XI_FLDLG2 =	0xecd9,
+  XI_FLDLN2 =	0xedd9,
+  XI_FDUP =	0xc0d9,  /* Really fld st0. */
+  XI_FPOP =	0xd8dd,  /* Really fstp st0. */
+  XI_FPOP1 =	0xd9dd,  /* Really fstp st1. */
+  XI_FRNDINT =	0xfcd9,
+  XI_FSIN =	0xfed9,
+  XI_FCOS =	0xffd9,
+  XI_FPTAN =	0xf2d9,
+  XI_FPATAN =	0xf3d9,
+  XI_FSCALE =	0xfdd9,
+  XI_FYL2X =	0xf1d9,
+
+  /* Variable-length opcodes. XO_* prefix. */
+  XO_MOV =	XO_(8b),
+  XO_MOVto =	XO_(89),
+  XO_MOVtow =	XO_66(89),
+  XO_MOVtob =	XO_(88),
+  XO_MOVmi =	XO_(c7),
+  XO_MOVmib =	XO_(c6),
+  XO_LEA =	XO_(8d),
+  XO_ARITHib =	XO_(80),
+  XO_ARITHi =	XO_(81),
+  XO_ARITHi8 =	XO_(83),
+  XO_ARITHiw8 =	XO_66(83),
+  XO_SHIFTi =	XO_(c1),
+  XO_SHIFT1 =	XO_(d1),
+  XO_SHIFTcl =	XO_(d3),
+  XO_IMUL =	XO_0f(af),
+  XO_IMULi =	XO_(69),
+  XO_IMULi8 =	XO_(6b),
+  XO_CMP =	XO_(3b),
+  XO_TESTb =	XO_(84),
+  XO_TEST =	XO_(85),
+  XO_GROUP3b =	XO_(f6),
+  XO_GROUP3 =	XO_(f7),
+  XO_GROUP5b =	XO_(fe),
+  XO_GROUP5 =	XO_(ff),
+  XO_MOVZXb =	XO_0f(b6),
+  XO_MOVZXw =	XO_0f(b7),
+  XO_MOVSXb =	XO_0f(be),
+  XO_MOVSXw =	XO_0f(bf),
+  XO_MOVSXd =	XO_(63),
+  XO_BSWAP =	XO_0f(c8),
+  XO_CMOV =	XO_0f(40),
+
+  XO_MOVSD =	XO_f20f(10),
+  XO_MOVSDto =	XO_f20f(11),
+  XO_MOVSS =	XO_f30f(10),
+  XO_MOVSSto =	XO_f30f(11),
+  XO_MOVLPD =	XO_660f(12),
+  XO_MOVAPS =	XO_0f(28),
+  XO_XORPS =	XO_0f(57),
+  XO_ANDPS =	XO_0f(54),
+  XO_ADDSD =	XO_f20f(58),
+  XO_SUBSD =	XO_f20f(5c),
+  XO_MULSD =	XO_f20f(59),
+  XO_DIVSD =	XO_f20f(5e),
+  XO_SQRTSD =	XO_f20f(51),
+  XO_MINSD =	XO_f20f(5d),
+  XO_MAXSD =	XO_f20f(5f),
+  XO_ROUNDSD =	0x0b3a0ffc,  /* Really 66 0f 3a 0b. See asm_fpmath. */
+  XO_UCOMISD =	XO_660f(2e),
+  XO_CVTSI2SD =	XO_f20f(2a),
+  XO_CVTTSD2SI=	XO_f20f(2c),
+  XO_CVTSI2SS =	XO_f30f(2a),
+  XO_CVTTSS2SI=	XO_f30f(2c),
+  XO_CVTSS2SD =	XO_f30f(5a),
+  XO_CVTSD2SS =	XO_f20f(5a),
+  XO_ADDSS =	XO_f30f(58),
+  XO_MOVD =	XO_660f(6e),
+  XO_MOVDto =	XO_660f(7e),
+
+  XO_FLDd =	XO_(d9), XOg_FLDd = 0,
+  XO_FLDq =	XO_(dd), XOg_FLDq = 0,
+  XO_FILDd =	XO_(db), XOg_FILDd = 0,
+  XO_FILDq =	XO_(df), XOg_FILDq = 5,
+  XO_FSTPd =	XO_(d9), XOg_FSTPd = 3,
+  XO_FSTPq =	XO_(dd), XOg_FSTPq = 3,
+  XO_FISTPq =	XO_(df), XOg_FISTPq = 7,
+  XO_FISTTPq =	XO_(dd), XOg_FISTTPq = 1,
+  XO_FADDq =	XO_(dc), XOg_FADDq = 0,
+  XO_FLDCW =	XO_(d9), XOg_FLDCW = 5,
+  XO_FNSTCW =	XO_(d9), XOg_FNSTCW = 7
+} x86Op;
+
+/* x86 opcode groups. */
+typedef uint32_t x86Group;
+
+#define XG_(i8, i, g)	((x86Group)(((i8) << 16) + ((i) << 8) + (g)))
+#define XG_ARITHi(g)	XG_(XI_ARITHi8, XI_ARITHi, g)
+#define XG_TOXOi(xg)	((x86Op)(0x000000fe + (((xg)<<16) & 0xff000000)))
+#define XG_TOXOi8(xg)	((x86Op)(0x000000fe + (((xg)<<8) & 0xff000000)))
+
+#define XO_ARITH(a)	((x86Op)(0x030000fe + ((a)<<27)))
+#define XO_ARITHw(a)	((x86Op)(0x036600fd + ((a)<<27)))
+
+typedef enum {
+  XOg_ADD, XOg_OR, XOg_ADC, XOg_SBB, XOg_AND, XOg_SUB, XOg_XOR, XOg_CMP,
+  XOg_X_IMUL
+} x86Arith;
+
+typedef enum {
+  XOg_ROL, XOg_ROR, XOg_RCL, XOg_RCR, XOg_SHL, XOg_SHR, XOg_SAL, XOg_SAR
+} x86Shift;
+
+typedef enum {
+  XOg_TEST, XOg_TEST_, XOg_NOT, XOg_NEG, XOg_MUL, XOg_IMUL, XOg_DIV, XOg_IDIV
+} x86Group3;
+
+typedef enum {
+  XOg_INC, XOg_DEC, XOg_CALL, XOg_CALLfar, XOg_JMP, XOg_JMPfar, XOg_PUSH
+} x86Group5;
+
+/* x86 condition codes. */
+typedef enum {
+  CC_O, CC_NO, CC_B, CC_NB, CC_E, CC_NE, CC_BE, CC_NBE,
+  CC_S, CC_NS, CC_P, CC_NP, CC_L, CC_NL, CC_LE, CC_NLE,
+  CC_C = CC_B, CC_NAE = CC_C, CC_NC = CC_NB, CC_AE = CC_NB,
+  CC_Z = CC_E, CC_NZ = CC_NE, CC_NA = CC_BE, CC_A = CC_NBE,
+  CC_PE = CC_P, CC_PO = CC_NP, CC_NGE = CC_L, CC_GE = CC_NL,
+  CC_NG = CC_LE, CC_G = CC_NLE
+} x86CC;
+
+#endif

data/lua-hooks/ext/luajit/src/lj_trace.c

@@ -0,0 +1,859 @@
+/*
+** Trace management.
+** Copyright (C) 2005-2015 Mike Pall. See Copyright Notice in luajit.h
+*/
+
+#define lj_trace_c
+#define LUA_CORE
+
+#include "lj_obj.h"
+
+#if LJ_HASJIT
+
+#include "lj_gc.h"
+#include "lj_err.h"
+#include "lj_debug.h"
+#include "lj_str.h"
+#include "lj_frame.h"
+#include "lj_state.h"
+#include "lj_bc.h"
+#include "lj_ir.h"
+#include "lj_jit.h"
+#include "lj_iropt.h"
+#include "lj_mcode.h"
+#include "lj_trace.h"
+#include "lj_snap.h"
+#include "lj_gdbjit.h"
+#include "lj_record.h"
+#include "lj_asm.h"
+#include "lj_dispatch.h"
+#include "lj_vm.h"
+#include "lj_vmevent.h"
+#include "lj_target.h"
+
+/* -- Error handling ------------------------------------------------------ */
+
+/* Synchronous abort with error message. */
+void lj_trace_err(jit_State *J, TraceError e)
+{
+  setnilV(&J->errinfo);  /* No error info. */
+  setintV(J->L->top++, (int32_t)e);
+  lj_err_throw(J->L, LUA_ERRRUN);
+}
+
+/* Synchronous abort with error message and error info. */
+void lj_trace_err_info(jit_State *J, TraceError e)
+{
+  setintV(J->L->top++, (int32_t)e);
+  lj_err_throw(J->L, LUA_ERRRUN);
+}
+
+/* -- Trace management ---------------------------------------------------- */
+
+/* The current trace is first assembled in J->cur. The variable length
+** arrays point to shared, growable buffers (J->irbuf etc.). When trace
+** recording ends successfully, the current trace and its data structures
+** are copied to a new (compact) GCtrace object.
+*/
+
+/* Find a free trace number. */
+static TraceNo trace_findfree(jit_State *J)
+{
+  MSize osz, lim;
+  if (J->freetrace == 0)
+    J->freetrace = 1;
+  for (; J->freetrace < J->sizetrace; J->freetrace++)
+    if (traceref(J, J->freetrace) == NULL)
+      return J->freetrace++;
+  /* Need to grow trace array. */
+  lim = (MSize)J->param[JIT_P_maxtrace] + 1;
+  if (lim < 2) lim = 2; else if (lim > 65535) lim = 65535;
+  osz = J->sizetrace;
+  if (osz >= lim)
+    return 0;  /* Too many traces. */
+  lj_mem_growvec(J->L, J->trace, J->sizetrace, lim, GCRef);
+  for (; osz < J->sizetrace; osz++)
+    setgcrefnull(J->trace[osz]);
+  return J->freetrace;
+}
+
+#define TRACE_APPENDVEC(field, szfield, tp) \
+  T->field = (tp *)p; \
+  memcpy(p, J->cur.field, J->cur.szfield*sizeof(tp)); \
+  p += J->cur.szfield*sizeof(tp);
+
+#ifdef LUAJIT_USE_PERFTOOLS
+/*
+** Create symbol table of JIT-compiled code. For use with Linux perf tools.
+** Example usage:
+**   perf record -f -e cycles luajit test.lua
+**   perf report -s symbol
+**   rm perf.data /tmp/perf-*.map
+*/
+#include <stdio.h>
+#include <unistd.h>
+
+static void perftools_addtrace(GCtrace *T)
+{
+  static FILE *fp;
+  GCproto *pt = &gcref(T->startpt)->pt;
+  const BCIns *startpc = mref(T->startpc, const BCIns);
+  const char *name = proto_chunknamestr(pt);
+  BCLine lineno;
+  if (name[0] == '@' || name[0] == '=')
+    name++;
+  else
+    name = "(string)";
+  lua_assert(startpc >= proto_bc(pt) && startpc < proto_bc(pt) + pt->sizebc);
+  lineno = lj_debug_line(pt, proto_bcpos(pt, startpc));
+  if (!fp) {
+    char fname[40];
+    sprintf(fname, "/tmp/perf-%d.map", getpid());
+    if (!(fp = fopen(fname, "w"))) return;
+    setlinebuf(fp);
+  }
+  fprintf(fp, "%lx %x TRACE_%d::%s:%u\n",
+	  (long)T->mcode, T->szmcode, T->traceno, name, lineno);
+}
+#endif
+
+/* Allocate space for copy of trace. */
+static GCtrace *trace_save_alloc(jit_State *J)
+{
+  size_t sztr = ((sizeof(GCtrace)+7)&~7);
+  size_t szins = (J->cur.nins-J->cur.nk)*sizeof(IRIns);
+  size_t sz = sztr + szins +
+	      J->cur.nsnap*sizeof(SnapShot) +
+	      J->cur.nsnapmap*sizeof(SnapEntry);
+  return lj_mem_newt(J->L, (MSize)sz, GCtrace);
+}
+
+/* Save current trace by copying and compacting it. */
+static void trace_save(jit_State *J, GCtrace *T)
+{
+  size_t sztr = ((sizeof(GCtrace)+7)&~7);
+  size_t szins = (J->cur.nins-J->cur.nk)*sizeof(IRIns);
+  char *p = (char *)T + sztr;
+  memcpy(T, &J->cur, sizeof(GCtrace));
+  setgcrefr(T->nextgc, J2G(J)->gc.root);
+  setgcrefp(J2G(J)->gc.root, T);
+  newwhite(J2G(J), T);
+  T->gct = ~LJ_TTRACE;
+  T->ir = (IRIns *)p - J->cur.nk;
+  memcpy(p, J->cur.ir+J->cur.nk, szins);
+  p += szins;
+  TRACE_APPENDVEC(snap, nsnap, SnapShot)
+  TRACE_APPENDVEC(snapmap, nsnapmap, SnapEntry)
+  J->cur.traceno = 0;
+  setgcrefp(J->trace[T->traceno], T);
+  lj_gc_barriertrace(J2G(J), T->traceno);
+  lj_gdbjit_addtrace(J, T);
+#ifdef LUAJIT_USE_PERFTOOLS
+  perftools_addtrace(T);
+#endif
+}
+
+void LJ_FASTCALL lj_trace_free(global_State *g, GCtrace *T)
+{
+  jit_State *J = G2J(g);
+  if (T->traceno) {
+    lj_gdbjit_deltrace(J, T);
+    if (T->traceno < J->freetrace)
+      J->freetrace = T->traceno;
+    setgcrefnull(J->trace[T->traceno]);
+  }
+  lj_mem_free(g, T,
+    ((sizeof(GCtrace)+7)&~7) + (T->nins-T->nk)*sizeof(IRIns) +
+    T->nsnap*sizeof(SnapShot) + T->nsnapmap*sizeof(SnapEntry));
+}
+
+/* Re-enable compiling a prototype by unpatching any modified bytecode. */
+void lj_trace_reenableproto(GCproto *pt)
+{
+  if ((pt->flags & PROTO_ILOOP)) {
+    BCIns *bc = proto_bc(pt);
+    BCPos i, sizebc = pt->sizebc;;
+    pt->flags &= ~PROTO_ILOOP;
+    if (bc_op(bc[0]) == BC_IFUNCF)
+      setbc_op(&bc[0], BC_FUNCF);
+    for (i = 1; i < sizebc; i++) {
+      BCOp op = bc_op(bc[i]);
+      if (op == BC_IFORL || op == BC_IITERL || op == BC_ILOOP)
+	setbc_op(&bc[i], (int)op+(int)BC_LOOP-(int)BC_ILOOP);
+    }
+  }
+}
+
+/* Unpatch the bytecode modified by a root trace. */
+static void trace_unpatch(jit_State *J, GCtrace *T)
+{
+  BCOp op = bc_op(T->startins);
+  BCIns *pc = mref(T->startpc, BCIns);
+  UNUSED(J);
+  if (op == BC_JMP)
+    return;  /* No need to unpatch branches in parent traces (yet). */
+  switch (bc_op(*pc)) {
+  case BC_JFORL:
+    lua_assert(traceref(J, bc_d(*pc)) == T);
+    *pc = T->startins;
+    pc += bc_j(T->startins);
+    lua_assert(bc_op(*pc) == BC_JFORI);
+    setbc_op(pc, BC_FORI);
+    break;
+  case BC_JITERL:
+  case BC_JLOOP:
+    lua_assert(op == BC_ITERL || op == BC_LOOP || bc_isret(op));
+    *pc = T->startins;
+    break;
+  case BC_JMP:
+    lua_assert(op == BC_ITERL);
+    pc += bc_j(*pc)+2;
+    if (bc_op(*pc) == BC_JITERL) {
+      lua_assert(traceref(J, bc_d(*pc)) == T);
+      *pc = T->startins;
+    }
+    break;
+  case BC_JFUNCF:
+    lua_assert(op == BC_FUNCF);
+    *pc = T->startins;
+    break;
+  default:  /* Already unpatched. */
+    break;
+  }
+}
+
+/* Flush a root trace. */
+static void trace_flushroot(jit_State *J, GCtrace *T)
+{
+  GCproto *pt = &gcref(T->startpt)->pt;
+  lua_assert(T->root == 0 && pt != NULL);
+  /* First unpatch any modified bytecode. */
+  trace_unpatch(J, T);
+  /* Unlink root trace from chain anchored in prototype. */
+  if (pt->trace == T->traceno) {  /* Trace is first in chain. Easy. */
+    pt->trace = T->nextroot;
+  } else if (pt->trace) {  /* Otherwise search in chain of root traces. */
+    GCtrace *T2 = traceref(J, pt->trace);
+    if (T2) {
+      for (; T2->nextroot; T2 = traceref(J, T2->nextroot))
+	if (T2->nextroot == T->traceno) {
+	  T2->nextroot = T->nextroot;  /* Unlink from chain. */
+	  break;
+	}
+    }
+  }
+}
+
+/* Flush a trace. Only root traces are considered. */
+void lj_trace_flush(jit_State *J, TraceNo traceno)
+{
+  if (traceno > 0 && traceno < J->sizetrace) {
+    GCtrace *T = traceref(J, traceno);
+    if (T && T->root == 0)
+      trace_flushroot(J, T);
+  }
+}
+
+/* Flush all traces associated with a prototype. */
+void lj_trace_flushproto(global_State *g, GCproto *pt)
+{
+  while (pt->trace != 0)
+    trace_flushroot(G2J(g), traceref(G2J(g), pt->trace));
+}
+
+/* Flush all traces. */
+int lj_trace_flushall(lua_State *L)
+{
+  jit_State *J = L2J(L);
+  ptrdiff_t i;
+  if ((J2G(J)->hookmask & HOOK_GC))
+    return 1;
+  for (i = (ptrdiff_t)J->sizetrace-1; i > 0; i--) {
+    GCtrace *T = traceref(J, i);
+    if (T) {
+      if (T->root == 0)
+	trace_flushroot(J, T);
+      lj_gdbjit_deltrace(J, T);
+      T->traceno = 0;
+      setgcrefnull(J->trace[i]);
+    }
+  }
+  J->cur.traceno = 0;
+  J->freetrace = 0;
+  /* Clear penalty cache. */
+  memset(J->penalty, 0, sizeof(J->penalty));
+  /* Free the whole machine code and invalidate all exit stub groups. */
+  lj_mcode_free(J);
+  memset(J->exitstubgroup, 0, sizeof(J->exitstubgroup));
+  lj_vmevent_send(L, TRACE,
+    setstrV(L, L->top++, lj_str_newlit(L, "flush"));
+  );
+  return 0;
+}
+
+/* Initialize JIT compiler state. */
+void lj_trace_initstate(global_State *g)
+{
+  jit_State *J = G2J(g);
+  TValue *tv;
+  /* Initialize SIMD constants. */
+  tv = LJ_KSIMD(J, LJ_KSIMD_ABS);
+  tv[0].u64 = U64x(7fffffff,ffffffff);
+  tv[1].u64 = U64x(7fffffff,ffffffff);
+  tv = LJ_KSIMD(J, LJ_KSIMD_NEG);
+  tv[0].u64 = U64x(80000000,00000000);
+  tv[1].u64 = U64x(80000000,00000000);
+}
+
+/* Free everything associated with the JIT compiler state. */
+void lj_trace_freestate(global_State *g)
+{
+  jit_State *J = G2J(g);
+#ifdef LUA_USE_ASSERT
+  {  /* This assumes all traces have already been freed. */
+    ptrdiff_t i;
+    for (i = 1; i < (ptrdiff_t)J->sizetrace; i++)
+      lua_assert(i == (ptrdiff_t)J->cur.traceno || traceref(J, i) == NULL);
+  }
+#endif
+  lj_mcode_free(J);
+  lj_ir_k64_freeall(J);
+  lj_mem_freevec(g, J->snapmapbuf, J->sizesnapmap, SnapEntry);
+  lj_mem_freevec(g, J->snapbuf, J->sizesnap, SnapShot);
+  lj_mem_freevec(g, J->irbuf + J->irbotlim, J->irtoplim - J->irbotlim, IRIns);
+  lj_mem_freevec(g, J->trace, J->sizetrace, GCRef);
+}
+
+/* -- Penalties and blacklisting ------------------------------------------ */
+
+/* Blacklist a bytecode instruction. */
+static void blacklist_pc(GCproto *pt, BCIns *pc)
+{
+  setbc_op(pc, (int)bc_op(*pc)+(int)BC_ILOOP-(int)BC_LOOP);
+  pt->flags |= PROTO_ILOOP;
+}
+
+/* Penalize a bytecode instruction. */
+static void penalty_pc(jit_State *J, GCproto *pt, BCIns *pc, TraceError e)
+{
+  uint32_t i, val = PENALTY_MIN;
+  for (i = 0; i < PENALTY_SLOTS; i++)
+    if (mref(J->penalty[i].pc, const BCIns) == pc) {  /* Cache slot found? */
+      /* First try to bump its hotcount several times. */
+      val = ((uint32_t)J->penalty[i].val << 1) +
+	    LJ_PRNG_BITS(J, PENALTY_RNDBITS);
+      if (val > PENALTY_MAX) {
+	blacklist_pc(pt, pc);  /* Blacklist it, if that didn't help. */
+	return;
+      }
+      goto setpenalty;
+    }
+  /* Assign a new penalty cache slot. */
+  i = J->penaltyslot;
+  J->penaltyslot = (J->penaltyslot + 1) & (PENALTY_SLOTS-1);
+  setmref(J->penalty[i].pc, pc);
+setpenalty:
+  J->penalty[i].val = (uint16_t)val;
+  J->penalty[i].reason = e;
+  hotcount_set(J2GG(J), pc+1, val);
+}
+
+/* -- Trace compiler state machine ---------------------------------------- */
+
+/* Start tracing. */
+static void trace_start(jit_State *J)
+{
+  lua_State *L;
+  TraceNo traceno;
+
+  if ((J->pt->flags & PROTO_NOJIT)) {  /* JIT disabled for this proto? */
+    if (J->parent == 0 && J->exitno == 0) {
+      /* Lazy bytecode patching to disable hotcount events. */
+      lua_assert(bc_op(*J->pc) == BC_FORL || bc_op(*J->pc) == BC_ITERL ||
+		 bc_op(*J->pc) == BC_LOOP || bc_op(*J->pc) == BC_FUNCF);
+      setbc_op(J->pc, (int)bc_op(*J->pc)+(int)BC_ILOOP-(int)BC_LOOP);
+      J->pt->flags |= PROTO_ILOOP;
+    }
+    J->state = LJ_TRACE_IDLE;  /* Silently ignored. */
+    return;
+  }
+
+  /* Get a new trace number. */
+  traceno = trace_findfree(J);
+  if (LJ_UNLIKELY(traceno == 0)) {  /* No free trace? */
+    lua_assert((J2G(J)->hookmask & HOOK_GC) == 0);
+    lj_trace_flushall(J->L);
+    J->state = LJ_TRACE_IDLE;  /* Silently ignored. */
+    return;
+  }
+  setgcrefp(J->trace[traceno], &J->cur);
+
+  /* Setup enough of the current trace to be able to send the vmevent. */
+  memset(&J->cur, 0, sizeof(GCtrace));
+  J->cur.traceno = traceno;
+  J->cur.nins = J->cur.nk = REF_BASE;
+  J->cur.ir = J->irbuf;
+  J->cur.snap = J->snapbuf;
+  J->cur.snapmap = J->snapmapbuf;
+  J->mergesnap = 0;
+  J->needsnap = 0;
+  J->bcskip = 0;
+  J->guardemit.irt = 0;
+  J->postproc = LJ_POST_NONE;
+  lj_resetsplit(J);
+  J->retryrec = 0;
+  setgcref(J->cur.startpt, obj2gco(J->pt));
+
+  L = J->L;
+  lj_vmevent_send(L, TRACE,
+    setstrV(L, L->top++, lj_str_newlit(L, "start"));
+    setintV(L->top++, traceno);
+    setfuncV(L, L->top++, J->fn);
+    setintV(L->top++, proto_bcpos(J->pt, J->pc));
+    if (J->parent) {
+      setintV(L->top++, J->parent);
+      setintV(L->top++, J->exitno);
+    }
+  );
+  lj_record_setup(J);
+}
+
+/* Stop tracing. */
+static void trace_stop(jit_State *J)
+{
+  BCIns *pc = mref(J->cur.startpc, BCIns);
+  BCOp op = bc_op(J->cur.startins);
+  GCproto *pt = &gcref(J->cur.startpt)->pt;
+  TraceNo traceno = J->cur.traceno;
+  GCtrace *T = trace_save_alloc(J);  /* Do this first. May throw OOM. */
+  lua_State *L;
+
+  switch (op) {
+  case BC_FORL:
+    setbc_op(pc+bc_j(J->cur.startins), BC_JFORI);  /* Patch FORI, too. */
+    /* fallthrough */
+  case BC_LOOP:
+  case BC_ITERL:
+  case BC_FUNCF:
+    /* Patch bytecode of starting instruction in root trace. */
+    setbc_op(pc, (int)op+(int)BC_JLOOP-(int)BC_LOOP);
+    setbc_d(pc, traceno);
+  addroot:
+    /* Add to root trace chain in prototype. */
+    J->cur.nextroot = pt->trace;
+    pt->trace = (TraceNo1)traceno;
+    break;
+  case BC_RET:
+  case BC_RET0:
+  case BC_RET1:
+    *pc = BCINS_AD(BC_JLOOP, J->cur.snap[0].nslots, traceno);
+    goto addroot;
+  case BC_JMP:
+    /* Patch exit branch in parent to side trace entry. */
+    lua_assert(J->parent != 0 && J->cur.root != 0);
+    lj_asm_patchexit(J, traceref(J, J->parent), J->exitno, J->cur.mcode);
+    /* Avoid compiling a side trace twice (stack resizing uses parent exit). */
+    traceref(J, J->parent)->snap[J->exitno].count = SNAPCOUNT_DONE;
+    /* Add to side trace chain in root trace. */
+    {
+      GCtrace *root = traceref(J, J->cur.root);
+      root->nchild++;
+      J->cur.nextside = root->nextside;
+      root->nextside = (TraceNo1)traceno;
+    }
+    break;
+  case BC_CALLM:
+  case BC_CALL:
+  case BC_ITERC:
+    /* Trace stitching: patch link of previous trace. */
+    traceref(J, J->exitno)->link = traceno;
+    break;
+  default:
+    lua_assert(0);
+    break;
+  }
+
+  /* Commit new mcode only after all patching is done. */
+  lj_mcode_commit(J, J->cur.mcode);
+  J->postproc = LJ_POST_NONE;
+  trace_save(J, T);
+
+  L = J->L;
+  lj_vmevent_send(L, TRACE,
+    setstrV(L, L->top++, lj_str_newlit(L, "stop"));
+    setintV(L->top++, traceno);
+    setfuncV(L, L->top++, J->fn);
+  );
+}
+
+/* Start a new root trace for down-recursion. */
+static int trace_downrec(jit_State *J)
+{
+  /* Restart recording at the return instruction. */
+  lua_assert(J->pt != NULL);
+  lua_assert(bc_isret(bc_op(*J->pc)));
+  if (bc_op(*J->pc) == BC_RETM)
+    return 0;  /* NYI: down-recursion with RETM. */
+  J->parent = 0;
+  J->exitno = 0;
+  J->state = LJ_TRACE_RECORD;
+  trace_start(J);
+  return 1;
+}
+
+/* Abort tracing. */
+static int trace_abort(jit_State *J)
+{
+  lua_State *L = J->L;
+  TraceError e = LJ_TRERR_RECERR;
+  TraceNo traceno;
+
+  J->postproc = LJ_POST_NONE;
+  lj_mcode_abort(J);
+  if (tvisnumber(L->top-1))
+    e = (TraceError)numberVint(L->top-1);
+  if (e == LJ_TRERR_MCODELM) {
+    L->top--;  /* Remove error object */
+    J->state = LJ_TRACE_ASM;
+    return 1;  /* Retry ASM with new MCode area. */
+  }
+  /* Penalize or blacklist starting bytecode instruction. */
+  if (J->parent == 0 && !bc_isret(bc_op(J->cur.startins))) {
+    if (J->exitno == 0) {
+      BCIns *startpc = mref(J->cur.startpc, BCIns);
+      if (e == LJ_TRERR_RETRY)
+	hotcount_set(J2GG(J), startpc+1, 1);  /* Immediate retry. */
+      else
+	penalty_pc(J, &gcref(J->cur.startpt)->pt, startpc, e);
+    } else {
+      traceref(J, J->exitno)->link = J->exitno;  /* Self-link is blacklisted. */
+    }
+  }
+
+  /* Is there anything to abort? */
+  traceno = J->cur.traceno;
+  if (traceno) {
+    ptrdiff_t errobj = savestack(L, L->top-1);  /* Stack may be resized. */
+    J->cur.link = 0;
+    J->cur.linktype = LJ_TRLINK_NONE;
+    lj_vmevent_send(L, TRACE,
+      TValue *frame;
+      const BCIns *pc;
+      GCfunc *fn;
+      setstrV(L, L->top++, lj_str_newlit(L, "abort"));
+      setintV(L->top++, traceno);
+      /* Find original Lua function call to generate a better error message. */
+      frame = J->L->base-1;
+      pc = J->pc;
+      while (!isluafunc(frame_func(frame))) {
+	pc = (frame_iscont(frame) ? frame_contpc(frame) : frame_pc(frame)) - 1;
+	frame = frame_prev(frame);
+      }
+      fn = frame_func(frame);
+      setfuncV(L, L->top++, fn);
+      setintV(L->top++, proto_bcpos(funcproto(fn), pc));
+      copyTV(L, L->top++, restorestack(L, errobj));
+      copyTV(L, L->top++, &J->errinfo);
+    );
+    /* Drop aborted trace after the vmevent (which may still access it). */
+    setgcrefnull(J->trace[traceno]);
+    if (traceno < J->freetrace)
+      J->freetrace = traceno;
+    J->cur.traceno = 0;
+  }
+  L->top--;  /* Remove error object */
+  if (e == LJ_TRERR_DOWNREC)
+    return trace_downrec(J);
+  else if (e == LJ_TRERR_MCODEAL)
+    lj_trace_flushall(L);
+  return 0;
+}
+
+/* Perform pending re-patch of a bytecode instruction. */
+static LJ_AINLINE void trace_pendpatch(jit_State *J, int force)
+{
+  if (LJ_UNLIKELY(J->patchpc)) {
+    if (force || J->bcskip == 0) {
+      *J->patchpc = J->patchins;
+      J->patchpc = NULL;
+    } else {
+      J->bcskip = 0;
+    }
+  }
+}
+
+/* State machine for the trace compiler. Protected callback. */
+static TValue *trace_state(lua_State *L, lua_CFunction dummy, void *ud)
+{
+  jit_State *J = (jit_State *)ud;
+  UNUSED(dummy);
+  do {
+  retry:
+    switch (J->state) {
+    case LJ_TRACE_START:
+      J->state = LJ_TRACE_RECORD;  /* trace_start() may change state. */
+      trace_start(J);
+      lj_dispatch_update(J2G(J));
+      break;
+
+    case LJ_TRACE_RECORD:
+      trace_pendpatch(J, 0);
+      setvmstate(J2G(J), RECORD);
+      lj_vmevent_send_(L, RECORD,
+	/* Save/restore tmptv state for trace recorder. */
+	TValue savetv = J2G(J)->tmptv;
+	TValue savetv2 = J2G(J)->tmptv2;
+	setintV(L->top++, J->cur.traceno);
+	setfuncV(L, L->top++, J->fn);
+	setintV(L->top++, J->pt ? (int32_t)proto_bcpos(J->pt, J->pc) : -1);
+	setintV(L->top++, J->framedepth);
+      ,
+	J2G(J)->tmptv = savetv;
+	J2G(J)->tmptv2 = savetv2;
+      );
+      lj_record_ins(J);
+      break;
+
+    case LJ_TRACE_END:
+      trace_pendpatch(J, 1);
+      J->loopref = 0;
+      if ((J->flags & JIT_F_OPT_LOOP) &&
+	  J->cur.link == J->cur.traceno && J->framedepth + J->retdepth == 0) {
+	setvmstate(J2G(J), OPT);
+	lj_opt_dce(J);
+	if (lj_opt_loop(J)) {  /* Loop optimization failed? */
+	  J->cur.link = 0;
+	  J->cur.linktype = LJ_TRLINK_NONE;
+	  J->loopref = J->cur.nins;
+	  J->state = LJ_TRACE_RECORD;  /* Try to continue recording. */
+	  break;
+	}
+	J->loopref = J->chain[IR_LOOP];  /* Needed by assembler. */
+      }
+      lj_opt_split(J);
+      lj_opt_sink(J);
+      if (!J->loopref) J->cur.snap[J->cur.nsnap-1].count = SNAPCOUNT_DONE;
+      J->state = LJ_TRACE_ASM;
+      break;
+
+    case LJ_TRACE_ASM:
+      setvmstate(J2G(J), ASM);
+      lj_asm_trace(J, &J->cur);
+      trace_stop(J);
+      setvmstate(J2G(J), INTERP);
+      J->state = LJ_TRACE_IDLE;
+      lj_dispatch_update(J2G(J));
+      return NULL;
+
+    default:  /* Trace aborted asynchronously. */
+      setintV(L->top++, (int32_t)LJ_TRERR_RECERR);
+      /* fallthrough */
+    case LJ_TRACE_ERR:
+      trace_pendpatch(J, 1);
+      if (trace_abort(J))
+	goto retry;
+      setvmstate(J2G(J), INTERP);
+      J->state = LJ_TRACE_IDLE;
+      lj_dispatch_update(J2G(J));
+      return NULL;
+    }
+  } while (J->state > LJ_TRACE_RECORD);
+  return NULL;
+}
+
+/* -- Event handling ------------------------------------------------------ */
+
+/* A bytecode instruction is about to be executed. Record it. */
+void lj_trace_ins(jit_State *J, const BCIns *pc)
+{
+  /* Note: J->L must already be set. pc is the true bytecode PC here. */
+  J->pc = pc;
+  J->fn = curr_func(J->L);
+  J->pt = isluafunc(J->fn) ? funcproto(J->fn) : NULL;
+  while (lj_vm_cpcall(J->L, NULL, (void *)J, trace_state) != 0)
+    J->state = LJ_TRACE_ERR;
+}
+
+/* A hotcount triggered. Start recording a root trace. */
+void LJ_FASTCALL lj_trace_hot(jit_State *J, const BCIns *pc)
+{
+  /* Note: pc is the interpreter bytecode PC here. It's offset by 1. */
+  ERRNO_SAVE
+  /* Reset hotcount. */
+  hotcount_set(J2GG(J), pc, J->param[JIT_P_hotloop]*HOTCOUNT_LOOP);
+  /* Only start a new trace if not recording or inside __gc call or vmevent. */
+  if (J->state == LJ_TRACE_IDLE &&
+      !(J2G(J)->hookmask & (HOOK_GC|HOOK_VMEVENT))) {
+    J->parent = 0;  /* Root trace. */
+    J->exitno = 0;
+    J->state = LJ_TRACE_START;
+    lj_trace_ins(J, pc-1);
+  }
+  ERRNO_RESTORE
+}
+
+/* Check for a hot side exit. If yes, start recording a side trace. */
+static void trace_hotside(jit_State *J, const BCIns *pc)
+{
+  SnapShot *snap = &traceref(J, J->parent)->snap[J->exitno];
+  if (!(J2G(J)->hookmask & (HOOK_GC|HOOK_VMEVENT)) &&
+      isluafunc(curr_func(J->L)) &&
+      snap->count != SNAPCOUNT_DONE &&
+      ++snap->count >= J->param[JIT_P_hotexit]) {
+    lua_assert(J->state == LJ_TRACE_IDLE);
+    /* J->parent is non-zero for a side trace. */
+    J->state = LJ_TRACE_START;
+    lj_trace_ins(J, pc);
+  }
+}
+
+/* Stitch a new trace to the previous trace. */
+void LJ_FASTCALL lj_trace_stitch(jit_State *J, const BCIns *pc)
+{
+  /* Only start a new trace if not recording or inside __gc call or vmevent. */
+  if (J->state == LJ_TRACE_IDLE &&
+      !(J2G(J)->hookmask & (HOOK_GC|HOOK_VMEVENT))) {
+    J->parent = 0;  /* Have to treat it like a root trace. */
+    /* J->exitno is set to the invoking trace. */
+    J->state = LJ_TRACE_START;
+    lj_trace_ins(J, pc);
+  }
+}
+
+
+/* Tiny struct to pass data to protected call. */
+typedef struct ExitDataCP {
+  jit_State *J;
+  void *exptr;		/* Pointer to exit state. */
+  const BCIns *pc;	/* Restart interpreter at this PC. */
+} ExitDataCP;
+
+/* Need to protect lj_snap_restore because it may throw. */
+static TValue *trace_exit_cp(lua_State *L, lua_CFunction dummy, void *ud)
+{
+  ExitDataCP *exd = (ExitDataCP *)ud;
+  cframe_errfunc(L->cframe) = -1;  /* Inherit error function. */
+  exd->pc = lj_snap_restore(exd->J, exd->exptr);
+  UNUSED(dummy);
+  return NULL;
+}
+
+#ifndef LUAJIT_DISABLE_VMEVENT
+/* Push all registers from exit state. */
+static void trace_exit_regs(lua_State *L, ExitState *ex)
+{
+  int32_t i;
+  setintV(L->top++, RID_NUM_GPR);
+  setintV(L->top++, RID_NUM_FPR);
+  for (i = 0; i < RID_NUM_GPR; i++) {
+    if (sizeof(ex->gpr[i]) == sizeof(int32_t))
+      setintV(L->top++, (int32_t)ex->gpr[i]);
+    else
+      setnumV(L->top++, (lua_Number)ex->gpr[i]);
+  }
+#if !LJ_SOFTFP
+  for (i = 0; i < RID_NUM_FPR; i++) {
+    setnumV(L->top, ex->fpr[i]);
+    if (LJ_UNLIKELY(tvisnan(L->top)))
+      setnanV(L->top);
+    L->top++;
+  }
+#endif
+}
+#endif
+
+#ifdef EXITSTATE_PCREG
+/* Determine trace number from pc of exit instruction. */
+static TraceNo trace_exit_find(jit_State *J, MCode *pc)
+{
+  TraceNo traceno;
+  for (traceno = 1; traceno < J->sizetrace; traceno++) {
+    GCtrace *T = traceref(J, traceno);
+    if (T && pc >= T->mcode && pc < (MCode *)((char *)T->mcode + T->szmcode))
+      return traceno;
+  }
+  lua_assert(0);
+  return 0;
+}
+#endif
+
+/* A trace exited. Restore interpreter state. */
+int LJ_FASTCALL lj_trace_exit(jit_State *J, void *exptr)
+{
+  ERRNO_SAVE
+  lua_State *L = J->L;
+  ExitState *ex = (ExitState *)exptr;
+  ExitDataCP exd;
+  int errcode;
+  const BCIns *pc;
+  void *cf;
+  GCtrace *T;
+#ifdef EXITSTATE_PCREG
+  J->parent = trace_exit_find(J, (MCode *)(intptr_t)ex->gpr[EXITSTATE_PCREG]);
+#endif
+  T = traceref(J, J->parent); UNUSED(T);
+#ifdef EXITSTATE_CHECKEXIT
+  if (J->exitno == T->nsnap) {  /* Treat stack check like a parent exit. */
+    lua_assert(T->root != 0);
+    J->exitno = T->ir[REF_BASE].op2;
+    J->parent = T->ir[REF_BASE].op1;
+    T = traceref(J, J->parent);
+  }
+#endif
+  lua_assert(T != NULL && J->exitno < T->nsnap);
+  exd.J = J;
+  exd.exptr = exptr;
+  errcode = lj_vm_cpcall(L, NULL, &exd, trace_exit_cp);
+  if (errcode)
+    return -errcode;  /* Return negated error code. */
+
+  if (!(LJ_HASPROFILE && (G(L)->hookmask & HOOK_PROFILE)))
+    lj_vmevent_send(L, TEXIT,
+      lj_state_checkstack(L, 4+RID_NUM_GPR+RID_NUM_FPR+LUA_MINSTACK);
+      setintV(L->top++, J->parent);
+      setintV(L->top++, J->exitno);
+      trace_exit_regs(L, ex);
+    );
+
+  pc = exd.pc;
+  cf = cframe_raw(L->cframe);
+  setcframe_pc(cf, pc);
+  if (LJ_HASPROFILE && (G(L)->hookmask & HOOK_PROFILE)) {
+    /* Just exit to interpreter. */
+  } else if (G(L)->gc.state == GCSatomic || G(L)->gc.state == GCSfinalize) {
+    if (!(G(L)->hookmask & HOOK_GC))
+      lj_gc_step(L);  /* Exited because of GC: drive GC forward. */
+  } else {
+    trace_hotside(J, pc);
+  }
+  if (bc_op(*pc) == BC_JLOOP) {
+    BCIns *retpc = &traceref(J, bc_d(*pc))->startins;
+    if (bc_isret(bc_op(*retpc))) {
+      if (J->state == LJ_TRACE_RECORD) {
+	J->patchins = *pc;
+	J->patchpc = (BCIns *)pc;
+	*J->patchpc = *retpc;
+	J->bcskip = 1;
+      } else {
+	pc = retpc;
+	setcframe_pc(cf, pc);
+      }
+    }
+  }
+  /* Return MULTRES or 0. */
+  ERRNO_RESTORE
+  switch (bc_op(*pc)) {
+  case BC_CALLM: case BC_CALLMT:
+    return (int)((BCReg)(L->top - L->base) - bc_a(*pc) - bc_c(*pc) + LJ_FR2);
+  case BC_RETM:
+    return (int)((BCReg)(L->top - L->base) + 1 - bc_a(*pc) - bc_d(*pc));
+  case BC_TSETM:
+    return (int)((BCReg)(L->top - L->base) + 1 - bc_a(*pc));
+  default:
+    if (bc_op(*pc) >= BC_FUNCF)
+      return (int)((BCReg)(L->top - L->base) + 1);
+    return 0;
+  }
+}
+
+#endif