immunio 0.15.4 → 0.16.0

data/lua-hooks/ext/luajit/src/lj_mcode.c

@@ -0,0 +1,386 @@
+/*
+** Machine code management.
+** Copyright (C) 2005-2015 Mike Pall. See Copyright Notice in luajit.h
+*/
+
+#define lj_mcode_c
+#define LUA_CORE
+
+#include "lj_obj.h"
+#if LJ_HASJIT
+#include "lj_gc.h"
+#include "lj_err.h"
+#include "lj_jit.h"
+#include "lj_mcode.h"
+#include "lj_trace.h"
+#include "lj_dispatch.h"
+#endif
+#if LJ_HASJIT || LJ_HASFFI
+#include "lj_vm.h"
+#endif
+
+/* -- OS-specific functions ----------------------------------------------- */
+
+#if LJ_HASJIT || LJ_HASFFI
+
+/* Define this if you want to run LuaJIT with Valgrind. */
+#ifdef LUAJIT_USE_VALGRIND
+#include <valgrind/valgrind.h>
+#endif
+
+#if LJ_TARGET_IOS
+void sys_icache_invalidate(void *start, size_t len);
+#endif
+
+/* Synchronize data/instruction cache. */
+void lj_mcode_sync(void *start, void *end)
+{
+#ifdef LUAJIT_USE_VALGRIND
+  VALGRIND_DISCARD_TRANSLATIONS(start, (char *)end-(char *)start);
+#endif
+#if LJ_TARGET_X86ORX64
+  UNUSED(start); UNUSED(end);
+#elif LJ_TARGET_IOS
+  sys_icache_invalidate(start, (char *)end-(char *)start);
+#elif LJ_TARGET_PPC
+  lj_vm_cachesync(start, end);
+#elif defined(__GNUC__)
+  __clear_cache(start, end);
+#else
+#error "Missing builtin to flush instruction cache"
+#endif
+}
+
+#endif
+
+#if LJ_HASJIT
+
+#if LJ_TARGET_WINDOWS
+
+#define WIN32_LEAN_AND_MEAN
+#include <windows.h>
+
+#define MCPROT_RW	PAGE_READWRITE
+#define MCPROT_RX	PAGE_EXECUTE_READ
+#define MCPROT_RWX	PAGE_EXECUTE_READWRITE
+
+static void *mcode_alloc_at(jit_State *J, uintptr_t hint, size_t sz, DWORD prot)
+{
+  void *p = VirtualAlloc((void *)hint, sz,
+			 MEM_RESERVE|MEM_COMMIT|MEM_TOP_DOWN, prot);
+  if (!p && !hint)
+    lj_trace_err(J, LJ_TRERR_MCODEAL);
+  return p;
+}
+
+static void mcode_free(jit_State *J, void *p, size_t sz)
+{
+  UNUSED(J); UNUSED(sz);
+  VirtualFree(p, 0, MEM_RELEASE);
+}
+
+static int mcode_setprot(void *p, size_t sz, DWORD prot)
+{
+  DWORD oprot;
+  return !VirtualProtect(p, sz, prot, &oprot);
+}
+
+#elif LJ_TARGET_POSIX
+
+#include <sys/mman.h>
+
+#ifndef MAP_ANONYMOUS
+#define MAP_ANONYMOUS	MAP_ANON
+#endif
+
+#define MCPROT_RW	(PROT_READ|PROT_WRITE)
+#define MCPROT_RX	(PROT_READ|PROT_EXEC)
+#define MCPROT_RWX	(PROT_READ|PROT_WRITE|PROT_EXEC)
+
+static void *mcode_alloc_at(jit_State *J, uintptr_t hint, size_t sz, int prot)
+{
+  void *p = mmap((void *)hint, sz, prot, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
+  if (p == MAP_FAILED) {
+    if (!hint) lj_trace_err(J, LJ_TRERR_MCODEAL);
+    p = NULL;
+  }
+  return p;
+}
+
+static void mcode_free(jit_State *J, void *p, size_t sz)
+{
+  UNUSED(J);
+  munmap(p, sz);
+}
+
+static int mcode_setprot(void *p, size_t sz, int prot)
+{
+  return mprotect(p, sz, prot);
+}
+
+#elif LJ_64
+
+#error "Missing OS support for explicit placement of executable memory"
+
+#else
+
+/* Fallback allocator. This will fail if memory is not executable by default. */
+#define LUAJIT_UNPROTECT_MCODE
+#define MCPROT_RW	0
+#define MCPROT_RX	0
+#define MCPROT_RWX	0
+
+static void *mcode_alloc_at(jit_State *J, uintptr_t hint, size_t sz, int prot)
+{
+  UNUSED(hint); UNUSED(prot);
+  return lj_mem_new(J->L, sz);
+}
+
+static void mcode_free(jit_State *J, void *p, size_t sz)
+{
+  lj_mem_free(J2G(J), p, sz);
+}
+
+#endif
+
+/* -- MCode area protection ----------------------------------------------- */
+
+/* Define this ONLY if page protection twiddling becomes a bottleneck. */
+#ifdef LUAJIT_UNPROTECT_MCODE
+
+/* It's generally considered to be a potential security risk to have
+** pages with simultaneous write *and* execute access in a process.
+**
+** Do not even think about using this mode for server processes or
+** apps handling untrusted external data (such as a browser).
+**
+** The security risk is not in LuaJIT itself -- but if an adversary finds
+** any *other* flaw in your C application logic, then any RWX memory page
+** simplifies writing an exploit considerably.
+*/
+#define MCPROT_GEN	MCPROT_RWX
+#define MCPROT_RUN	MCPROT_RWX
+
+static void mcode_protect(jit_State *J, int prot)
+{
+  UNUSED(J); UNUSED(prot);
+}
+
+#else
+
+/* This is the default behaviour and much safer:
+**
+** Most of the time the memory pages holding machine code are executable,
+** but NONE of them is writable.
+**
+** The current memory area is marked read-write (but NOT executable) only
+** during the short time window while the assembler generates machine code.
+*/
+#define MCPROT_GEN	MCPROT_RW
+#define MCPROT_RUN	MCPROT_RX
+
+/* Protection twiddling failed. Probably due to kernel security. */
+static LJ_NOINLINE void mcode_protfail(jit_State *J)
+{
+  lua_CFunction panic = J2G(J)->panic;
+  if (panic) {
+    lua_State *L = J->L;
+    setstrV(L, L->top++, lj_err_str(L, LJ_ERR_JITPROT));
+    panic(L);
+  }
+}
+
+/* Change protection of MCode area. */
+static void mcode_protect(jit_State *J, int prot)
+{
+  if (J->mcprot != prot) {
+    if (LJ_UNLIKELY(mcode_setprot(J->mcarea, J->szmcarea, prot)))
+      mcode_protfail(J);
+    J->mcprot = prot;
+  }
+}
+
+#endif
+
+/* -- MCode area allocation ----------------------------------------------- */
+
+#if LJ_TARGET_X64
+#define mcode_validptr(p)	((p) && (uintptr_t)(p) < (uintptr_t)1<<47)
+#else
+#define mcode_validptr(p)	((p) && (uintptr_t)(p) < 0xffff0000)
+#endif
+
+#ifdef LJ_TARGET_JUMPRANGE
+
+/* Get memory within relative jump distance of our code in 64 bit mode. */
+static void *mcode_alloc(jit_State *J, size_t sz)
+{
+  /* Target an address in the static assembler code (64K aligned).
+  ** Try addresses within a distance of target-range/2+1MB..target+range/2-1MB.
+  ** Use half the jump range so every address in the range can reach any other.
+  */
+#if LJ_TARGET_MIPS
+  /* Use the middle of the 256MB-aligned region. */
+  uintptr_t target = ((uintptr_t)(void *)lj_vm_exit_handler & 0xf0000000u) +
+		     0x08000000u;
+#else
+  uintptr_t target = (uintptr_t)(void *)lj_vm_exit_handler & ~(uintptr_t)0xffff;
+#endif
+  const uintptr_t range = (1u << (LJ_TARGET_JUMPRANGE-1)) - (1u << 21);
+  /* First try a contiguous area below the last one. */
+  uintptr_t hint = J->mcarea ? (uintptr_t)J->mcarea - sz : 0;
+  int i;
+  for (i = 0; i < 32; i++) {  /* 32 attempts ought to be enough ... */
+    if (mcode_validptr(hint)) {
+      void *p = mcode_alloc_at(J, hint, sz, MCPROT_GEN);
+
+      if (mcode_validptr(p) &&
+	  ((uintptr_t)p + sz - target < range || target - (uintptr_t)p < range))
+	return p;
+      if (p) mcode_free(J, p, sz);  /* Free badly placed area. */
+    }
+    /* Next try probing pseudo-random addresses. */
+    do {
+      hint = (0x78fb ^ LJ_PRNG_BITS(J, 15)) << 16;  /* 64K aligned. */
+    } while (!(hint + sz < range));
+    hint = target + hint - (range>>1);
+  }
+  lj_trace_err(J, LJ_TRERR_MCODEAL);  /* Give up. OS probably ignores hints? */
+  return NULL;
+}
+
+#else
+
+/* All memory addresses are reachable by relative jumps. */
+static void *mcode_alloc(jit_State *J, size_t sz)
+{
+#ifdef __OpenBSD__
+  /* Allow better executable memory allocation for OpenBSD W^X mode. */
+  void *p = mcode_alloc_at(J, 0, sz, MCPROT_RUN);
+  if (p && mcode_setprot(p, sz, MCPROT_GEN)) {
+    mcode_free(J, p, sz);
+    return NULL;
+  }
+  return p;
+#else
+  return mcode_alloc_at(J, 0, sz, MCPROT_GEN);
+#endif
+}
+
+#endif
+
+/* -- MCode area management ----------------------------------------------- */
+
+/* Linked list of MCode areas. */
+typedef struct MCLink {
+  MCode *next;		/* Next area. */
+  size_t size;		/* Size of current area. */
+} MCLink;
+
+/* Allocate a new MCode area. */
+static void mcode_allocarea(jit_State *J)
+{
+  MCode *oldarea = J->mcarea;
+  size_t sz = (size_t)J->param[JIT_P_sizemcode] << 10;
+  sz = (sz + LJ_PAGESIZE-1) & ~(size_t)(LJ_PAGESIZE - 1);
+  J->mcarea = (MCode *)mcode_alloc(J, sz);
+  J->szmcarea = sz;
+  J->mcprot = MCPROT_GEN;
+  J->mctop = (MCode *)((char *)J->mcarea + J->szmcarea);
+  J->mcbot = (MCode *)((char *)J->mcarea + sizeof(MCLink));
+  ((MCLink *)J->mcarea)->next = oldarea;
+  ((MCLink *)J->mcarea)->size = sz;
+  J->szallmcarea += sz;
+}
+
+/* Free all MCode areas. */
+void lj_mcode_free(jit_State *J)
+{
+  MCode *mc = J->mcarea;
+  J->mcarea = NULL;
+  J->szallmcarea = 0;
+  while (mc) {
+    MCode *next = ((MCLink *)mc)->next;
+    mcode_free(J, mc, ((MCLink *)mc)->size);
+    mc = next;
+  }
+}
+
+/* -- MCode transactions -------------------------------------------------- */
+
+/* Reserve the remainder of the current MCode area. */
+MCode *lj_mcode_reserve(jit_State *J, MCode **lim)
+{
+  if (!J->mcarea)
+    mcode_allocarea(J);
+  else
+    mcode_protect(J, MCPROT_GEN);
+  *lim = J->mcbot;
+  return J->mctop;
+}
+
+/* Commit the top part of the current MCode area. */
+void lj_mcode_commit(jit_State *J, MCode *top)
+{
+  J->mctop = top;
+  mcode_protect(J, MCPROT_RUN);
+}
+
+/* Abort the reservation. */
+void lj_mcode_abort(jit_State *J)
+{
+  if (J->mcarea)
+    mcode_protect(J, MCPROT_RUN);
+}
+
+/* Set/reset protection to allow patching of MCode areas. */
+MCode *lj_mcode_patch(jit_State *J, MCode *ptr, int finish)
+{
+#ifdef LUAJIT_UNPROTECT_MCODE
+  UNUSED(J); UNUSED(ptr); UNUSED(finish);
+  return NULL;
+#else
+  if (finish) {
+    if (J->mcarea == ptr)
+      mcode_protect(J, MCPROT_RUN);
+    else if (LJ_UNLIKELY(mcode_setprot(ptr, ((MCLink *)ptr)->size, MCPROT_RUN)))
+      mcode_protfail(J);
+    return NULL;
+  } else {
+    MCode *mc = J->mcarea;
+    /* Try current area first to use the protection cache. */
+    if (ptr >= mc && ptr < (MCode *)((char *)mc + J->szmcarea)) {
+      mcode_protect(J, MCPROT_GEN);
+      return mc;
+    }
+    /* Otherwise search through the list of MCode areas. */
+    for (;;) {
+      mc = ((MCLink *)mc)->next;
+      lua_assert(mc != NULL);
+      if (ptr >= mc && ptr < (MCode *)((char *)mc + ((MCLink *)mc)->size)) {
+	if (LJ_UNLIKELY(mcode_setprot(mc, ((MCLink *)mc)->size, MCPROT_GEN)))
+	  mcode_protfail(J);
+	return mc;
+      }
+    }
+  }
+#endif
+}
+
+/* Limit of MCode reservation reached. */
+void lj_mcode_limiterr(jit_State *J, size_t need)
+{
+  size_t sizemcode, maxmcode;
+  lj_mcode_abort(J);
+  sizemcode = (size_t)J->param[JIT_P_sizemcode] << 10;
+  sizemcode = (sizemcode + LJ_PAGESIZE-1) & ~(size_t)(LJ_PAGESIZE - 1);
+  maxmcode = (size_t)J->param[JIT_P_maxmcode] << 10;
+  if ((size_t)need > sizemcode)
+    lj_trace_err(J, LJ_TRERR_MCODEOV);  /* Too long for any area. */
+  if (J->szallmcarea + sizemcode > maxmcode)
+    lj_trace_err(J, LJ_TRERR_MCODEAL);
+  mcode_allocarea(J);
+  lj_trace_err(J, LJ_TRERR_MCODELM);  /* Retry with new area. */
+}
+
+#endif

data/lua-hooks/ext/luajit/src/lj_mcode.h

@@ -0,0 +1,30 @@
+/*
+** Machine code management.
+** Copyright (C) 2005-2015 Mike Pall. See Copyright Notice in luajit.h
+*/
+
+#ifndef _LJ_MCODE_H
+#define _LJ_MCODE_H
+
+#include "lj_obj.h"
+
+#if LJ_HASJIT || LJ_HASFFI
+LJ_FUNC void lj_mcode_sync(void *start, void *end);
+#endif
+
+#if LJ_HASJIT
+
+#include "lj_jit.h"
+
+LJ_FUNC void lj_mcode_free(jit_State *J);
+LJ_FUNC MCode *lj_mcode_reserve(jit_State *J, MCode **lim);
+LJ_FUNC void lj_mcode_commit(jit_State *J, MCode *m);
+LJ_FUNC void lj_mcode_abort(jit_State *J);
+LJ_FUNC MCode *lj_mcode_patch(jit_State *J, MCode *ptr, int finish);
+LJ_FUNC_NORET void lj_mcode_limiterr(jit_State *J, size_t need);
+
+#define lj_mcode_commitbot(J, m)	(J->mcbot = (m))
+
+#endif
+
+#endif

data/lua-hooks/ext/luajit/src/lj_meta.c

@@ -0,0 +1,477 @@
+/*
+** Metamethod handling.
+** Copyright (C) 2005-2015 Mike Pall. See Copyright Notice in luajit.h
+**
+** Portions taken verbatim or adapted from the Lua interpreter.
+** Copyright (C) 1994-2008 Lua.org, PUC-Rio. See Copyright Notice in lua.h
+*/
+
+#define lj_meta_c
+#define LUA_CORE
+
+#include "lj_obj.h"
+#include "lj_gc.h"
+#include "lj_err.h"
+#include "lj_buf.h"
+#include "lj_str.h"
+#include "lj_tab.h"
+#include "lj_meta.h"
+#include "lj_frame.h"
+#include "lj_bc.h"
+#include "lj_vm.h"
+#include "lj_strscan.h"
+#include "lj_strfmt.h"
+#include "lj_lib.h"
+
+/* -- Metamethod handling ------------------------------------------------- */
+
+/* String interning of metamethod names for fast indexing. */
+void lj_meta_init(lua_State *L)
+{
+#define MMNAME(name)	"__" #name
+  const char *metanames = MMDEF(MMNAME);
+#undef MMNAME
+  global_State *g = G(L);
+  const char *p, *q;
+  uint32_t mm;
+  for (mm = 0, p = metanames; *p; mm++, p = q) {
+    GCstr *s;
+    for (q = p+2; *q && *q != '_'; q++) ;
+    s = lj_str_new(L, p, (size_t)(q-p));
+    /* NOBARRIER: g->gcroot[] is a GC root. */
+    setgcref(g->gcroot[GCROOT_MMNAME+mm], obj2gco(s));
+  }
+}
+
+/* Negative caching of a few fast metamethods. See the lj_meta_fast() macro. */
+cTValue *lj_meta_cache(GCtab *mt, MMS mm, GCstr *name)
+{
+  cTValue *mo = lj_tab_getstr(mt, name);
+  lua_assert(mm <= MM_FAST);
+  if (!mo || tvisnil(mo)) {  /* No metamethod? */
+    mt->nomm |= (uint8_t)(1u<<mm);  /* Set negative cache flag. */
+    return NULL;
+  }
+  return mo;
+}
+
+/* Lookup metamethod for object. */
+cTValue *lj_meta_lookup(lua_State *L, cTValue *o, MMS mm)
+{
+  GCtab *mt;
+  if (tvistab(o))
+    mt = tabref(tabV(o)->metatable);
+  else if (tvisudata(o))
+    mt = tabref(udataV(o)->metatable);
+  else
+    mt = tabref(basemt_obj(G(L), o));
+  if (mt) {
+    cTValue *mo = lj_tab_getstr(mt, mmname_str(G(L), mm));
+    if (mo)
+      return mo;
+  }
+  return niltv(L);
+}
+
+#if LJ_HASFFI
+/* Tailcall from C function. */
+int lj_meta_tailcall(lua_State *L, cTValue *tv)
+{
+  TValue *base = L->base;
+  TValue *top = L->top;
+  const BCIns *pc = frame_pc(base-1);  /* Preserve old PC from frame. */
+  copyTV(L, base-1-LJ_FR2, tv);  /* Replace frame with new object. */
+  if (LJ_FR2)
+    (top++)->u64 = LJ_CONT_TAILCALL;
+  else
+    top->u32.lo = LJ_CONT_TAILCALL;
+  setframe_pc(top++, pc);
+  if (LJ_FR2) top++;
+  setframe_gc(top, obj2gco(L), LJ_TTHREAD);  /* Dummy frame object. */
+  setframe_ftsz(top, ((char *)(top+1) - (char *)base) + FRAME_CONT);
+  L->base = L->top = top+1;
+  /*
+  ** before:   [old_mo|PC]    [... ...]
+  **                         ^base     ^top
+  ** after:    [new_mo|itype] [... ...] [NULL|PC] [dummy|delta]
+  **                                                           ^base/top
+  ** tailcall: [new_mo|PC]    [... ...]
+  **                         ^base     ^top
+  */
+  return 0;
+}
+#endif
+
+/* Setup call to metamethod to be run by Assembler VM. */
+static TValue *mmcall(lua_State *L, ASMFunction cont, cTValue *mo,
+		    cTValue *a, cTValue *b)
+{
+  /*
+  **           |-- framesize -> top       top+1       top+2 top+3
+  ** before:   [func slots ...]
+  ** mm setup: [func slots ...] [cont|?]  [mo|tmtype] [a]   [b]
+  ** in asm:   [func slots ...] [cont|PC] [mo|delta]  [a]   [b]
+  **           ^-- func base                          ^-- mm base
+  ** after mm: [func slots ...]           [result]
+  **                ^-- copy to base[PC_RA] --/     for lj_cont_ra
+  **                          istruecond + branch   for lj_cont_cond*
+  **                                       ignore   for lj_cont_nop
+  ** next PC:  [func slots ...]
+  */
+  TValue *top = L->top;
+  if (curr_funcisL(L)) top = curr_topL(L);
+  setcont(top++, cont);  /* Assembler VM stores PC in upper word or FR2. */
+  if (LJ_FR2) setnilV(top++);
+  copyTV(L, top++, mo);  /* Store metamethod and two arguments. */
+  if (LJ_FR2) setnilV(top++);
+  copyTV(L, top, a);
+  copyTV(L, top+1, b);
+  return top;  /* Return new base. */
+}
+
+/* -- C helpers for some instructions, called from assembler VM ----------- */
+
+/* Helper for TGET*. __index chain and metamethod. */
+cTValue *lj_meta_tget(lua_State *L, cTValue *o, cTValue *k)
+{
+  int loop;
+  for (loop = 0; loop < LJ_MAX_IDXCHAIN; loop++) {
+    cTValue *mo;
+    if (LJ_LIKELY(tvistab(o))) {
+      GCtab *t = tabV(o);
+      cTValue *tv = lj_tab_get(L, t, k);
+      if (!tvisnil(tv) ||
+	  !(mo = lj_meta_fast(L, tabref(t->metatable), MM_index)))
+	return tv;
+    } else if (tvisnil(mo = lj_meta_lookup(L, o, MM_index))) {
+      lj_err_optype(L, o, LJ_ERR_OPINDEX);
+      return NULL;  /* unreachable */
+    }
+    if (tvisfunc(mo)) {
+      L->top = mmcall(L, lj_cont_ra, mo, o, k);
+      return NULL;  /* Trigger metamethod call. */
+    }
+    o = mo;
+  }
+  lj_err_msg(L, LJ_ERR_GETLOOP);
+  return NULL;  /* unreachable */
+}
+
+/* Helper for TSET*. __newindex chain and metamethod. */
+TValue *lj_meta_tset(lua_State *L, cTValue *o, cTValue *k)
+{
+  TValue tmp;
+  int loop;
+  for (loop = 0; loop < LJ_MAX_IDXCHAIN; loop++) {
+    cTValue *mo;
+    if (LJ_LIKELY(tvistab(o))) {
+      GCtab *t = tabV(o);
+      cTValue *tv = lj_tab_get(L, t, k);
+      if (LJ_LIKELY(!tvisnil(tv))) {
+	t->nomm = 0;  /* Invalidate negative metamethod cache. */
+	lj_gc_anybarriert(L, t);
+	return (TValue *)tv;
+      } else if (!(mo = lj_meta_fast(L, tabref(t->metatable), MM_newindex))) {
+	t->nomm = 0;  /* Invalidate negative metamethod cache. */
+	lj_gc_anybarriert(L, t);
+	if (tv != niltv(L))
+	  return (TValue *)tv;
+	if (tvisnil(k)) lj_err_msg(L, LJ_ERR_NILIDX);
+	else if (tvisint(k)) { setnumV(&tmp, (lua_Number)intV(k)); k = &tmp; }
+	else if (tvisnum(k) && tvisnan(k)) lj_err_msg(L, LJ_ERR_NANIDX);
+	return lj_tab_newkey(L, t, k);
+      }
+    } else if (tvisnil(mo = lj_meta_lookup(L, o, MM_newindex))) {
+      lj_err_optype(L, o, LJ_ERR_OPINDEX);
+      return NULL;  /* unreachable */
+    }
+    if (tvisfunc(mo)) {
+      L->top = mmcall(L, lj_cont_nop, mo, o, k);
+      /* L->top+2 = v filled in by caller. */
+      return NULL;  /* Trigger metamethod call. */
+    }
+    copyTV(L, &tmp, mo);
+    o = &tmp;
+  }
+  lj_err_msg(L, LJ_ERR_SETLOOP);
+  return NULL;  /* unreachable */
+}
+
+static cTValue *str2num(cTValue *o, TValue *n)
+{
+  if (tvisnum(o))
+    return o;
+  else if (tvisint(o))
+    return (setnumV(n, (lua_Number)intV(o)), n);
+  else if (tvisstr(o) && lj_strscan_num(strV(o), n))
+    return n;
+  else
+    return NULL;
+}
+
+/* Helper for arithmetic instructions. Coercion, metamethod. */
+TValue *lj_meta_arith(lua_State *L, TValue *ra, cTValue *rb, cTValue *rc,
+		      BCReg op)
+{
+  MMS mm = bcmode_mm(op);
+  TValue tempb, tempc;
+  cTValue *b, *c;
+  if ((b = str2num(rb, &tempb)) != NULL &&
+      (c = str2num(rc, &tempc)) != NULL) {  /* Try coercion first. */
+    setnumV(ra, lj_vm_foldarith(numV(b), numV(c), (int)mm-MM_add));
+    return NULL;
+  } else {
+    cTValue *mo = lj_meta_lookup(L, rb, mm);
+    if (tvisnil(mo)) {
+      mo = lj_meta_lookup(L, rc, mm);
+      if (tvisnil(mo)) {
+	if (str2num(rb, &tempb) == NULL) rc = rb;
+	lj_err_optype(L, rc, LJ_ERR_OPARITH);
+	return NULL;  /* unreachable */
+      }
+    }
+    return mmcall(L, lj_cont_ra, mo, rb, rc);
+  }
+}
+
+/* Helper for CAT. Coercion, iterative concat, __concat metamethod. */
+TValue *lj_meta_cat(lua_State *L, TValue *top, int left)
+{
+  int fromc = 0;
+  if (left < 0) { left = -left; fromc = 1; }
+  do {
+    if (!(tvisstr(top) || tvisnumber(top)) ||
+	!(tvisstr(top-1) || tvisnumber(top-1))) {
+      cTValue *mo = lj_meta_lookup(L, top-1, MM_concat);
+      if (tvisnil(mo)) {
+	mo = lj_meta_lookup(L, top, MM_concat);
+	if (tvisnil(mo)) {
+	  if (tvisstr(top-1) || tvisnumber(top-1)) top++;
+	  lj_err_optype(L, top-1, LJ_ERR_OPCAT);
+	  return NULL;  /* unreachable */
+	}
+      }
+      /* One of the top two elements is not a string, call __cat metamethod:
+      **
+      ** before:    [...][CAT stack .........................]
+      **                                 top-1     top         top+1 top+2
+      ** pick two:  [...][CAT stack ...] [o1]      [o2]
+      ** setup mm:  [...][CAT stack ...] [cont|?]  [mo|tmtype] [o1]  [o2]
+      ** in asm:    [...][CAT stack ...] [cont|PC] [mo|delta]  [o1]  [o2]
+      **            ^-- func base                              ^-- mm base
+      ** after mm:  [...][CAT stack ...] <--push-- [result]
+      ** next step: [...][CAT stack .............]
+      */
+      copyTV(L, top+2*LJ_FR2+2, top);  /* Carefully ordered stack copies! */
+      copyTV(L, top+2*LJ_FR2+1, top-1);
+      copyTV(L, top+LJ_FR2, mo);
+      setcont(top-1, lj_cont_cat);
+      if (LJ_FR2) { setnilV(top); setnilV(top+2); top += 2; }
+      return top+1;  /* Trigger metamethod call. */
+    } else {
+      /* Pick as many strings as possible from the top and concatenate them:
+      **
+      ** before:    [...][CAT stack ...........................]
+      ** pick str:  [...][CAT stack ...] [...... strings ......]
+      ** concat:    [...][CAT stack ...] [result]
+      ** next step: [...][CAT stack ............]
+      */
+      TValue *e, *o = top;
+      uint64_t tlen = tvisstr(o) ? strV(o)->len : STRFMT_MAXBUF_NUM;
+      char *p, *buf;
+      do {
+	o--; tlen += tvisstr(o) ? strV(o)->len : STRFMT_MAXBUF_NUM;
+      } while (--left > 0 && (tvisstr(o-1) || tvisnumber(o-1)));
+      if (tlen >= LJ_MAX_STR) lj_err_msg(L, LJ_ERR_STROV);
+      p = buf = lj_buf_tmp(L, (MSize)tlen);
+      for (e = top, top = o; o <= e; o++) {
+	if (tvisstr(o)) {
+	  GCstr *s = strV(o);
+	  MSize len = s->len;
+	  p = lj_buf_wmem(p, strdata(s), len);
+	} else if (tvisint(o)) {
+	  p = lj_strfmt_wint(p, intV(o));
+	} else {
+	  lua_assert(tvisnum(o));
+	  p = lj_strfmt_wnum(p, o);
+	}
+      }
+      setstrV(L, top, lj_str_new(L, buf, (size_t)(p-buf)));
+    }
+  } while (left >= 1);
+  if (LJ_UNLIKELY(G(L)->gc.total >= G(L)->gc.threshold)) {
+    if (!fromc) L->top = curr_topL(L);
+    lj_gc_step(L);
+  }
+  return NULL;
+}
+
+/* Helper for LEN. __len metamethod. */
+TValue * LJ_FASTCALL lj_meta_len(lua_State *L, cTValue *o)
+{
+  cTValue *mo = lj_meta_lookup(L, o, MM_len);
+  if (tvisnil(mo)) {
+    if (LJ_52 && tvistab(o))
+      tabref(tabV(o)->metatable)->nomm |= (uint8_t)(1u<<MM_len);
+    else
+      lj_err_optype(L, o, LJ_ERR_OPLEN);
+    return NULL;
+  }
+  return mmcall(L, lj_cont_ra, mo, o, LJ_52 ? o : niltv(L));
+}
+
+/* Helper for equality comparisons. __eq metamethod. */
+TValue *lj_meta_equal(lua_State *L, GCobj *o1, GCobj *o2, int ne)
+{
+  /* Field metatable must be at same offset for GCtab and GCudata! */
+  cTValue *mo = lj_meta_fast(L, tabref(o1->gch.metatable), MM_eq);
+  if (mo) {
+    TValue *top;
+    uint32_t it;
+    if (tabref(o1->gch.metatable) != tabref(o2->gch.metatable)) {
+      cTValue *mo2 = lj_meta_fast(L, tabref(o2->gch.metatable), MM_eq);
+      if (mo2 == NULL || !lj_obj_equal(mo, mo2))
+	return (TValue *)(intptr_t)ne;
+    }
+    top = curr_top(L);
+    setcont(top++, ne ? lj_cont_condf : lj_cont_condt);
+    if (LJ_FR2) setnilV(top++);
+    copyTV(L, top++, mo);
+    if (LJ_FR2) setnilV(top++);
+    it = ~(uint32_t)o1->gch.gct;
+    setgcV(L, top, o1, it);
+    setgcV(L, top+1, o2, it);
+    return top;  /* Trigger metamethod call. */
+  }
+  return (TValue *)(intptr_t)ne;
+}
+
+#if LJ_HASFFI
+TValue * LJ_FASTCALL lj_meta_equal_cd(lua_State *L, BCIns ins)
+{
+  ASMFunction cont = (bc_op(ins) & 1) ? lj_cont_condf : lj_cont_condt;
+  int op = (int)bc_op(ins) & ~1;
+  TValue tv;
+  cTValue *mo, *o2, *o1 = &L->base[bc_a(ins)];
+  cTValue *o1mm = o1;
+  if (op == BC_ISEQV) {
+    o2 = &L->base[bc_d(ins)];
+    if (!tviscdata(o1mm)) o1mm = o2;
+  } else if (op == BC_ISEQS) {
+    setstrV(L, &tv, gco2str(proto_kgc(curr_proto(L), ~(ptrdiff_t)bc_d(ins))));
+    o2 = &tv;
+  } else if (op == BC_ISEQN) {
+    o2 = &mref(curr_proto(L)->k, cTValue)[bc_d(ins)];
+  } else {
+    lua_assert(op == BC_ISEQP);
+    setpriV(&tv, ~bc_d(ins));
+    o2 = &tv;
+  }
+  mo = lj_meta_lookup(L, o1mm, MM_eq);
+  if (LJ_LIKELY(!tvisnil(mo)))
+    return mmcall(L, cont, mo, o1, o2);
+  else
+    return (TValue *)(intptr_t)(bc_op(ins) & 1);
+}
+#endif
+
+/* Helper for ordered comparisons. String compare, __lt/__le metamethods. */
+TValue *lj_meta_comp(lua_State *L, cTValue *o1, cTValue *o2, int op)
+{
+  if (LJ_HASFFI && (tviscdata(o1) || tviscdata(o2))) {
+    ASMFunction cont = (op & 1) ? lj_cont_condf : lj_cont_condt;
+    MMS mm = (op & 2) ? MM_le : MM_lt;
+    cTValue *mo = lj_meta_lookup(L, tviscdata(o1) ? o1 : o2, mm);
+    if (LJ_UNLIKELY(tvisnil(mo))) goto err;
+    return mmcall(L, cont, mo, o1, o2);
+  } else if (LJ_52 || itype(o1) == itype(o2)) {
+    /* Never called with two numbers. */
+    if (tvisstr(o1) && tvisstr(o2)) {
+      int32_t res = lj_str_cmp(strV(o1), strV(o2));
+      return (TValue *)(intptr_t)(((op&2) ? res <= 0 : res < 0) ^ (op&1));
+    } else {
+    trymt:
+      while (1) {
+	ASMFunction cont = (op & 1) ? lj_cont_condf : lj_cont_condt;
+	MMS mm = (op & 2) ? MM_le : MM_lt;
+	cTValue *mo = lj_meta_lookup(L, o1, mm);
+#if LJ_52
+	if (tvisnil(mo) && tvisnil((mo = lj_meta_lookup(L, o2, mm))))
+#else
+	cTValue *mo2 = lj_meta_lookup(L, o2, mm);
+	if (tvisnil(mo) || !lj_obj_equal(mo, mo2))
+#endif
+	{
+	  if (op & 2) {  /* MM_le not found: retry with MM_lt. */
+	    cTValue *ot = o1; o1 = o2; o2 = ot;  /* Swap operands. */
+	    op ^= 3;  /* Use LT and flip condition. */
+	    continue;
+	  }
+	  goto err;
+	}
+	return mmcall(L, cont, mo, o1, o2);
+      }
+    }
+  } else if (tvisbool(o1) && tvisbool(o2)) {
+    goto trymt;
+  } else {
+  err:
+    lj_err_comp(L, o1, o2);
+    return NULL;
+  }
+}
+
+/* Helper for ISTYPE and ISNUM. Implicit coercion or error. */
+void lj_meta_istype(lua_State *L, BCReg ra, BCReg tp)
+{
+  L->top = curr_topL(L);
+  ra++; tp--;
+  lua_assert(LJ_DUALNUM || tp != ~LJ_TNUMX);  /* ISTYPE -> ISNUM broken. */
+  if (LJ_DUALNUM && tp == ~LJ_TNUMX) lj_lib_checkint(L, ra);
+  else if (tp == ~LJ_TNUMX+1) lj_lib_checknum(L, ra);
+  else if (tp == ~LJ_TSTR) lj_lib_checkstr(L, ra);
+  else lj_err_argtype(L, ra, lj_obj_itypename[tp]);
+}
+
+/* Helper for calls. __call metamethod. */
+void lj_meta_call(lua_State *L, TValue *func, TValue *top)
+{
+  cTValue *mo = lj_meta_lookup(L, func, MM_call);
+  TValue *p;
+  if (!tvisfunc(mo))
+    lj_err_optype_call(L, func);
+  for (p = top; p > func+2*LJ_FR2; p--) copyTV(L, p, p-1);
+  if (LJ_FR2) copyTV(L, func+2, func);
+  copyTV(L, func, mo);
+}
+
+/* Helper for FORI. Coercion. */
+void LJ_FASTCALL lj_meta_for(lua_State *L, TValue *o)
+{
+  if (!lj_strscan_numberobj(o)) lj_err_msg(L, LJ_ERR_FORINIT);
+  if (!lj_strscan_numberobj(o+1)) lj_err_msg(L, LJ_ERR_FORLIM);
+  if (!lj_strscan_numberobj(o+2)) lj_err_msg(L, LJ_ERR_FORSTEP);
+  if (LJ_DUALNUM) {
+    /* Ensure all slots are integers or all slots are numbers. */
+    int32_t k[3];
+    int nint = 0;
+    ptrdiff_t i;
+    for (i = 0; i <= 2; i++) {
+      if (tvisint(o+i)) {
+	k[i] = intV(o+i); nint++;
+      } else {
+	k[i] = lj_num2int(numV(o+i)); nint += ((lua_Number)k[i] == numV(o+i));
+      }
+    }
+    if (nint == 3) {  /* Narrow to integers. */
+      setintV(o, k[0]);
+      setintV(o+1, k[1]);
+      setintV(o+2, k[2]);
+    } else if (nint != 0) {  /* Widen to numbers. */
+      if (tvisint(o)) setnumV(o, (lua_Number)intV(o));
+      if (tvisint(o+1)) setnumV(o+1, (lua_Number)intV(o+1));
+      if (tvisint(o+2)) setnumV(o+2, (lua_Number)intV(o+2));
+    }
+  }
+}
+