immunio 0.15.4 → 0.16.0

data/lua-hooks/ext/luajit/src/lj_jit.h

@@ -0,0 +1,440 @@
+/*
+** Common definitions for the JIT compiler.
+** Copyright (C) 2005-2015 Mike Pall. See Copyright Notice in luajit.h
+*/
+
+#ifndef _LJ_JIT_H
+#define _LJ_JIT_H
+
+#include "lj_obj.h"
+#include "lj_ir.h"
+
+/* JIT engine flags. */
+#define JIT_F_ON		0x00000001
+
+/* CPU-specific JIT engine flags. */
+#if LJ_TARGET_X86ORX64
+#define JIT_F_SSE2		0x00000010
+#define JIT_F_SSE3		0x00000020
+#define JIT_F_SSE4_1		0x00000040
+#define JIT_F_PREFER_IMUL	0x00000080
+#define JIT_F_LEA_AGU		0x00000100
+
+/* Names for the CPU-specific flags. Must match the order above. */
+#define JIT_F_CPU_FIRST		JIT_F_SSE2
+#define JIT_F_CPUSTRING		"\4SSE2\4SSE3\6SSE4.1\3AMD\4ATOM"
+#elif LJ_TARGET_ARM
+#define JIT_F_ARMV6_		0x00000010
+#define JIT_F_ARMV6T2_		0x00000020
+#define JIT_F_ARMV7		0x00000040
+#define JIT_F_VFPV2		0x00000080
+#define JIT_F_VFPV3		0x00000100
+
+#define JIT_F_ARMV6		(JIT_F_ARMV6_|JIT_F_ARMV6T2_|JIT_F_ARMV7)
+#define JIT_F_ARMV6T2		(JIT_F_ARMV6T2_|JIT_F_ARMV7)
+#define JIT_F_VFP		(JIT_F_VFPV2|JIT_F_VFPV3)
+
+/* Names for the CPU-specific flags. Must match the order above. */
+#define JIT_F_CPU_FIRST		JIT_F_ARMV6_
+#define JIT_F_CPUSTRING		"\5ARMv6\7ARMv6T2\5ARMv7\5VFPv2\5VFPv3"
+#elif LJ_TARGET_PPC
+#define JIT_F_SQRT		0x00000010
+#define JIT_F_ROUND		0x00000020
+
+/* Names for the CPU-specific flags. Must match the order above. */
+#define JIT_F_CPU_FIRST		JIT_F_SQRT
+#define JIT_F_CPUSTRING		"\4SQRT\5ROUND"
+#elif LJ_TARGET_MIPS
+#define JIT_F_MIPS32R2		0x00000010
+
+/* Names for the CPU-specific flags. Must match the order above. */
+#define JIT_F_CPU_FIRST		JIT_F_MIPS32R2
+#define JIT_F_CPUSTRING		"\010MIPS32R2"
+#else
+#define JIT_F_CPU_FIRST		0
+#define JIT_F_CPUSTRING		""
+#endif
+
+/* Optimization flags. */
+#define JIT_F_OPT_MASK		0x0fff0000
+
+#define JIT_F_OPT_FOLD		0x00010000
+#define JIT_F_OPT_CSE		0x00020000
+#define JIT_F_OPT_DCE		0x00040000
+#define JIT_F_OPT_FWD		0x00080000
+#define JIT_F_OPT_DSE		0x00100000
+#define JIT_F_OPT_NARROW	0x00200000
+#define JIT_F_OPT_LOOP		0x00400000
+#define JIT_F_OPT_ABC		0x00800000
+#define JIT_F_OPT_SINK		0x01000000
+#define JIT_F_OPT_FUSE		0x02000000
+
+/* Optimizations names for -O. Must match the order above. */
+#define JIT_F_OPT_FIRST		JIT_F_OPT_FOLD
+#define JIT_F_OPTSTRING	\
+  "\4fold\3cse\3dce\3fwd\3dse\6narrow\4loop\3abc\4sink\4fuse"
+
+/* Optimization levels set a fixed combination of flags. */
+#define JIT_F_OPT_0	0
+#define JIT_F_OPT_1	(JIT_F_OPT_FOLD|JIT_F_OPT_CSE|JIT_F_OPT_DCE)
+#define JIT_F_OPT_2	(JIT_F_OPT_1|JIT_F_OPT_NARROW|JIT_F_OPT_LOOP)
+#define JIT_F_OPT_3	(JIT_F_OPT_2|\
+  JIT_F_OPT_FWD|JIT_F_OPT_DSE|JIT_F_OPT_ABC|JIT_F_OPT_SINK|JIT_F_OPT_FUSE)
+#define JIT_F_OPT_DEFAULT	JIT_F_OPT_3
+
+#if LJ_TARGET_WINDOWS || LJ_64
+/* See: http://blogs.msdn.com/oldnewthing/archive/2003/10/08/55239.aspx */
+#define JIT_P_sizemcode_DEFAULT		64
+#else
+/* Could go as low as 4K, but the mmap() overhead would be rather high. */
+#define JIT_P_sizemcode_DEFAULT		32
+#endif
+
+/* Optimization parameters and their defaults. Length is a char in octal! */
+#define JIT_PARAMDEF(_) \
+  _(\010, maxtrace,	1000)	/* Max. # of traces in cache. */ \
+  _(\011, maxrecord,	4000)	/* Max. # of recorded IR instructions. */ \
+  _(\012, maxirconst,	500)	/* Max. # of IR constants of a trace. */ \
+  _(\007, maxside,	100)	/* Max. # of side traces of a root trace. */ \
+  _(\007, maxsnap,	500)	/* Max. # of snapshots for a trace. */ \
+  _(\011, minstitch,	0)	/* Min. # of IR ins for a stitched trace. */ \
+  \
+  _(\007, hotloop,	56)	/* # of iter. to detect a hot loop/call. */ \
+  _(\007, hotexit,	10)	/* # of taken exits to start a side trace. */ \
+  _(\007, tryside,	4)	/* # of attempts to compile a side trace. */ \
+  \
+  _(\012, instunroll,	4)	/* Max. unroll for instable loops. */ \
+  _(\012, loopunroll,	15)	/* Max. unroll for loop ops in side traces. */ \
+  _(\012, callunroll,	3)	/* Max. unroll for recursive calls. */ \
+  _(\011, recunroll,	2)	/* Min. unroll for true recursion. */ \
+  \
+  /* Size of each machine code area (in KBytes). */ \
+  _(\011, sizemcode,	JIT_P_sizemcode_DEFAULT) \
+  /* Max. total size of all machine code areas (in KBytes). */ \
+  _(\010, maxmcode,	512) \
+  /* End of list. */
+
+enum {
+#define JIT_PARAMENUM(len, name, value)	JIT_P_##name,
+JIT_PARAMDEF(JIT_PARAMENUM)
+#undef JIT_PARAMENUM
+  JIT_P__MAX
+};
+
+#define JIT_PARAMSTR(len, name, value)	#len #name
+#define JIT_P_STRING	JIT_PARAMDEF(JIT_PARAMSTR)
+
+/* Trace compiler state. */
+typedef enum {
+  LJ_TRACE_IDLE,	/* Trace compiler idle. */
+  LJ_TRACE_ACTIVE = 0x10,
+  LJ_TRACE_RECORD,	/* Bytecode recording active. */
+  LJ_TRACE_START,	/* New trace started. */
+  LJ_TRACE_END,		/* End of trace. */
+  LJ_TRACE_ASM,		/* Assemble trace. */
+  LJ_TRACE_ERR		/* Trace aborted with error. */
+} TraceState;
+
+/* Post-processing action. */
+typedef enum {
+  LJ_POST_NONE,		/* No action. */
+  LJ_POST_FIXCOMP,	/* Fixup comparison and emit pending guard. */
+  LJ_POST_FIXGUARD,	/* Fixup and emit pending guard. */
+  LJ_POST_FIXGUARDSNAP,	/* Fixup and emit pending guard and snapshot. */
+  LJ_POST_FIXBOOL,	/* Fixup boolean result. */
+  LJ_POST_FIXCONST,	/* Fixup constant results. */
+  LJ_POST_FFRETRY	/* Suppress recording of retried fast functions. */
+} PostProc;
+
+/* Machine code type. */
+#if LJ_TARGET_X86ORX64
+typedef uint8_t MCode;
+#else
+typedef uint32_t MCode;
+#endif
+
+/* Stack snapshot header. */
+typedef struct SnapShot {
+  uint16_t mapofs;	/* Offset into snapshot map. */
+  IRRef1 ref;		/* First IR ref for this snapshot. */
+  uint8_t nslots;	/* Number of valid slots. */
+  uint8_t topslot;	/* Maximum frame extent. */
+  uint8_t nent;		/* Number of compressed entries. */
+  uint8_t count;	/* Count of taken exits for this snapshot. */
+} SnapShot;
+
+#define SNAPCOUNT_DONE	255	/* Already compiled and linked a side trace. */
+
+/* Compressed snapshot entry. */
+typedef uint32_t SnapEntry;
+
+#define SNAP_FRAME		0x010000	/* Frame slot. */
+#define SNAP_CONT		0x020000	/* Continuation slot. */
+#define SNAP_NORESTORE		0x040000	/* No need to restore slot. */
+#define SNAP_SOFTFPNUM		0x080000	/* Soft-float number. */
+LJ_STATIC_ASSERT(SNAP_FRAME == TREF_FRAME);
+LJ_STATIC_ASSERT(SNAP_CONT == TREF_CONT);
+
+#define SNAP(slot, flags, ref)	(((SnapEntry)(slot) << 24) + (flags) + (ref))
+#define SNAP_TR(slot, tr) \
+  (((SnapEntry)(slot) << 24) + ((tr) & (TREF_CONT|TREF_FRAME|TREF_REFMASK)))
+#define SNAP_MKPC(pc)		((SnapEntry)u32ptr(pc))
+#define SNAP_MKFTSZ(ftsz)	((SnapEntry)(ftsz))
+#define snap_ref(sn)		((sn) & 0xffff)
+#define snap_slot(sn)		((BCReg)((sn) >> 24))
+#define snap_isframe(sn)	((sn) & SNAP_FRAME)
+#define snap_pc(sn)		((const BCIns *)(uintptr_t)(sn))
+#define snap_setref(sn, ref)	(((sn) & (0xffff0000&~SNAP_NORESTORE)) | (ref))
+
+/* Snapshot and exit numbers. */
+typedef uint32_t SnapNo;
+typedef uint32_t ExitNo;
+
+/* Trace number. */
+typedef uint32_t TraceNo;	/* Used to pass around trace numbers. */
+typedef uint16_t TraceNo1;	/* Stored trace number. */
+
+/* Type of link. ORDER LJ_TRLINK */
+typedef enum {
+  LJ_TRLINK_NONE,		/* Incomplete trace. No link, yet. */
+  LJ_TRLINK_ROOT,		/* Link to other root trace. */
+  LJ_TRLINK_LOOP,		/* Loop to same trace. */
+  LJ_TRLINK_TAILREC,		/* Tail-recursion. */
+  LJ_TRLINK_UPREC,		/* Up-recursion. */
+  LJ_TRLINK_DOWNREC,		/* Down-recursion. */
+  LJ_TRLINK_INTERP,		/* Fallback to interpreter. */
+  LJ_TRLINK_RETURN,		/* Return to interpreter. */
+  LJ_TRLINK_STITCH		/* Trace stitching. */
+} TraceLink;
+
+/* Trace object. */
+typedef struct GCtrace {
+  GCHeader;
+  uint8_t topslot;	/* Top stack slot already checked to be allocated. */
+  uint8_t linktype;	/* Type of link. */
+  IRRef nins;		/* Next IR instruction. Biased with REF_BIAS. */
+#if LJ_GC64
+  uint32_t unused_gc64;
+#endif
+  GCRef gclist;
+  IRIns *ir;		/* IR instructions/constants. Biased with REF_BIAS. */
+  IRRef nk;		/* Lowest IR constant. Biased with REF_BIAS. */
+  uint16_t nsnap;	/* Number of snapshots. */
+  uint16_t nsnapmap;	/* Number of snapshot map elements. */
+  SnapShot *snap;	/* Snapshot array. */
+  SnapEntry *snapmap;	/* Snapshot map. */
+  GCRef startpt;	/* Starting prototype. */
+  MRef startpc;		/* Bytecode PC of starting instruction. */
+  BCIns startins;	/* Original bytecode of starting instruction. */
+  MSize szmcode;	/* Size of machine code. */
+  MCode *mcode;		/* Start of machine code. */
+  MSize mcloop;		/* Offset of loop start in machine code. */
+  uint16_t nchild;	/* Number of child traces (root trace only). */
+  uint16_t spadjust;	/* Stack pointer adjustment (offset in bytes). */
+  TraceNo1 traceno;	/* Trace number. */
+  TraceNo1 link;	/* Linked trace (or self for loops). */
+  TraceNo1 root;	/* Root trace of side trace (or 0 for root traces). */
+  TraceNo1 nextroot;	/* Next root trace for same prototype. */
+  TraceNo1 nextside;	/* Next side trace of same root trace. */
+  uint8_t sinktags;	/* Trace has SINK tags. */
+  uint8_t unused1;
+#ifdef LUAJIT_USE_GDBJIT
+  void *gdbjit_entry;	/* GDB JIT entry. */
+#endif
+} GCtrace;
+
+#define gco2trace(o)	check_exp((o)->gch.gct == ~LJ_TTRACE, (GCtrace *)(o))
+#define traceref(J, n) \
+  check_exp((n)>0 && (MSize)(n)<J->sizetrace, (GCtrace *)gcref(J->trace[(n)]))
+
+LJ_STATIC_ASSERT(offsetof(GChead, gclist) == offsetof(GCtrace, gclist));
+
+static LJ_AINLINE MSize snap_nextofs(GCtrace *T, SnapShot *snap)
+{
+  if (snap+1 == &T->snap[T->nsnap])
+    return T->nsnapmap;
+  else
+    return (snap+1)->mapofs;
+}
+
+/* Round-robin penalty cache for bytecodes leading to aborted traces. */
+typedef struct HotPenalty {
+  MRef pc;		/* Starting bytecode PC. */
+  uint16_t val;		/* Penalty value, i.e. hotcount start. */
+  uint16_t reason;	/* Abort reason (really TraceErr). */
+} HotPenalty;
+
+#define PENALTY_SLOTS	64	/* Penalty cache slot. Must be a power of 2. */
+#define PENALTY_MIN	(36*2)	/* Minimum penalty value. */
+#define PENALTY_MAX	60000	/* Maximum penalty value. */
+#define PENALTY_RNDBITS	4	/* # of random bits to add to penalty value. */
+
+/* Round-robin backpropagation cache for narrowing conversions. */
+typedef struct BPropEntry {
+  IRRef1 key;		/* Key: original reference. */
+  IRRef1 val;		/* Value: reference after conversion. */
+  IRRef mode;		/* Mode for this entry (currently IRCONV_*). */
+} BPropEntry;
+
+/* Number of slots for the backpropagation cache. Must be a power of 2. */
+#define BPROP_SLOTS	16
+
+/* Scalar evolution analysis cache. */
+typedef struct ScEvEntry {
+  MRef pc;		/* Bytecode PC of FORI. */
+  IRRef1 idx;		/* Index reference. */
+  IRRef1 start;		/* Constant start reference. */
+  IRRef1 stop;		/* Constant stop reference. */
+  IRRef1 step;		/* Constant step reference. */
+  IRType1 t;		/* Scalar type. */
+  uint8_t dir;		/* Direction. 1: +, 0: -. */
+} ScEvEntry;
+
+/* Reverse bytecode map (IRRef -> PC). Only for selected instructions. */
+typedef struct RBCHashEntry {
+  MRef pc;		/* Bytecode PC. */
+  GCRef pt;		/* Prototype. */
+  IRRef ref;		/* IR reference. */
+} RBCHashEntry;
+
+/* Number of slots in the reverse bytecode hash table. Must be a power of 2. */
+#define RBCHASH_SLOTS	8
+
+/* 128 bit SIMD constants. */
+enum {
+  LJ_KSIMD_ABS,
+  LJ_KSIMD_NEG,
+  LJ_KSIMD__MAX
+};
+
+/* Get 16 byte aligned pointer to SIMD constant. */
+#define LJ_KSIMD(J, n) \
+  ((TValue *)(((intptr_t)&J->ksimd[2*(n)] + 15) & ~(intptr_t)15))
+
+/* Set/reset flag to activate the SPLIT pass for the current trace. */
+#if LJ_SOFTFP || (LJ_32 && LJ_HASFFI)
+#define lj_needsplit(J)		(J->needsplit = 1)
+#define lj_resetsplit(J)	(J->needsplit = 0)
+#else
+#define lj_needsplit(J)		UNUSED(J)
+#define lj_resetsplit(J)	UNUSED(J)
+#endif
+
+/* Fold state is used to fold instructions on-the-fly. */
+typedef struct FoldState {
+  IRIns ins;		/* Currently emitted instruction. */
+  IRIns left;		/* Instruction referenced by left operand. */
+  IRIns right;		/* Instruction referenced by right operand. */
+} FoldState;
+
+/* JIT compiler state. */
+typedef struct jit_State {
+  GCtrace cur;		/* Current trace. */
+
+  lua_State *L;		/* Current Lua state. */
+  const BCIns *pc;	/* Current PC. */
+  GCfunc *fn;		/* Current function. */
+  GCproto *pt;		/* Current prototype. */
+  TRef *base;		/* Current frame base, points into J->slots. */
+
+  uint32_t flags;	/* JIT engine flags. */
+  BCReg maxslot;	/* Relative to baseslot. */
+  BCReg baseslot;	/* Current frame base, offset into J->slots. */
+
+  uint8_t mergesnap;	/* Allowed to merge with next snapshot. */
+  uint8_t needsnap;	/* Need snapshot before recording next bytecode. */
+  IRType1 guardemit;	/* Accumulated IRT_GUARD for emitted instructions. */
+  uint8_t bcskip;	/* Number of bytecode instructions to skip. */
+
+  FoldState fold;	/* Fold state. */
+
+  const BCIns *bc_min;	/* Start of allowed bytecode range for root trace. */
+  MSize bc_extent;	/* Extent of the range. */
+
+  TraceState state;	/* Trace compiler state. */
+
+  int32_t instunroll;	/* Unroll counter for instable loops. */
+  int32_t loopunroll;	/* Unroll counter for loop ops in side traces. */
+  int32_t tailcalled;	/* Number of successive tailcalls. */
+  int32_t framedepth;	/* Current frame depth. */
+  int32_t retdepth;	/* Return frame depth (count of RETF). */
+
+  MRef k64;		/* Pointer to chained array of 64 bit constants. */
+  TValue ksimd[LJ_KSIMD__MAX*2+1];  /* 16 byte aligned SIMD constants. */
+
+  IRIns *irbuf;		/* Temp. IR instruction buffer. Biased with REF_BIAS. */
+  IRRef irtoplim;	/* Upper limit of instuction buffer (biased). */
+  IRRef irbotlim;	/* Lower limit of instuction buffer (biased). */
+  IRRef loopref;	/* Last loop reference or ref of final LOOP (or 0). */
+
+  MSize sizesnap;	/* Size of temp. snapshot buffer. */
+  SnapShot *snapbuf;	/* Temp. snapshot buffer. */
+  SnapEntry *snapmapbuf;  /* Temp. snapshot map buffer. */
+  MSize sizesnapmap;	/* Size of temp. snapshot map buffer. */
+
+  PostProc postproc;	/* Required post-processing after execution. */
+#if LJ_SOFTFP || (LJ_32 && LJ_HASFFI)
+  uint8_t needsplit;	/* Need SPLIT pass. */
+#endif
+  uint8_t retryrec;	/* Retry recording. */
+
+  GCRef *trace;		/* Array of traces. */
+  TraceNo freetrace;	/* Start of scan for next free trace. */
+  MSize sizetrace;	/* Size of trace array. */
+
+  IRRef1 chain[IR__MAX];  /* IR instruction skip-list chain anchors. */
+  TRef slot[LJ_MAX_JSLOTS+LJ_STACK_EXTRA];  /* Stack slot map. */
+
+  int32_t param[JIT_P__MAX];  /* JIT engine parameters. */
+
+  MCode *exitstubgroup[LJ_MAX_EXITSTUBGR];  /* Exit stub group addresses. */
+
+  HotPenalty penalty[PENALTY_SLOTS];  /* Penalty slots. */
+  uint32_t penaltyslot;	/* Round-robin index into penalty slots. */
+  uint32_t prngstate;	/* PRNG state. */
+
+#ifdef LUAJIT_ENABLE_TABLE_BUMP
+  RBCHashEntry rbchash[RBCHASH_SLOTS];  /* Reverse bytecode map. */
+#endif
+
+  BPropEntry bpropcache[BPROP_SLOTS];  /* Backpropagation cache slots. */
+  uint32_t bpropslot;	/* Round-robin index into bpropcache slots. */
+
+  ScEvEntry scev;	/* Scalar evolution analysis cache slots. */
+
+  const BCIns *startpc;	/* Bytecode PC of starting instruction. */
+  TraceNo parent;	/* Parent of current side trace (0 for root traces). */
+  ExitNo exitno;	/* Exit number in parent of current side trace. */
+
+  BCIns *patchpc;	/* PC for pending re-patch. */
+  BCIns patchins;	/* Instruction for pending re-patch. */
+
+  int mcprot;		/* Protection of current mcode area. */
+  MCode *mcarea;	/* Base of current mcode area. */
+  MCode *mctop;		/* Top of current mcode area. */
+  MCode *mcbot;		/* Bottom of current mcode area. */
+  size_t szmcarea;	/* Size of current mcode area. */
+  size_t szallmcarea;	/* Total size of all allocated mcode areas. */
+
+  TValue errinfo;	/* Additional info element for trace errors. */
+
+#if LJ_HASPROFILE
+  GCproto *prev_pt;	/* Previous prototype. */
+  BCLine prev_line;	/* Previous line. */
+  int prof_mode;	/* Profiling mode: 0, 'f', 'l'. */
+#endif
+}
+#if LJ_TARGET_ARM
+LJ_ALIGN(16)		/* For DISPATCH-relative addresses in assembler part. */
+#endif
+jit_State;
+
+/* Trivial PRNG e.g. used for penalty randomization. */
+static LJ_AINLINE uint32_t LJ_PRNG_BITS(jit_State *J, int bits)
+{
+  /* Yes, this LCG is very weak, but that doesn't matter for our use case. */
+  J->prngstate = J->prngstate * 1103515245 + 12345;
+  return J->prngstate >> (32-bits);
+}
+
+#endif

data/lua-hooks/ext/luajit/src/lj_lex.c

@@ -0,0 +1,482 @@
+/*
+** Lexical analyzer.
+** Copyright (C) 2005-2015 Mike Pall. See Copyright Notice in luajit.h
+**
+** Major portions taken verbatim or adapted from the Lua interpreter.
+** Copyright (C) 1994-2008 Lua.org, PUC-Rio. See Copyright Notice in lua.h
+*/
+
+#define lj_lex_c
+#define LUA_CORE
+
+#include "lj_obj.h"
+#include "lj_gc.h"
+#include "lj_err.h"
+#include "lj_buf.h"
+#include "lj_str.h"
+#if LJ_HASFFI
+#include "lj_tab.h"
+#include "lj_ctype.h"
+#include "lj_cdata.h"
+#include "lualib.h"
+#endif
+#include "lj_state.h"
+#include "lj_lex.h"
+#include "lj_parse.h"
+#include "lj_char.h"
+#include "lj_strscan.h"
+#include "lj_strfmt.h"
+
+/* Lua lexer token names. */
+static const char *const tokennames[] = {
+#define TKSTR1(name)		#name,
+#define TKSTR2(name, sym)	#sym,
+TKDEF(TKSTR1, TKSTR2)
+#undef TKSTR1
+#undef TKSTR2
+  NULL
+};
+
+/* -- Buffer handling ----------------------------------------------------- */
+
+#define LEX_EOF			(-1)
+#define lex_iseol(ls)		(ls->c == '\n' || ls->c == '\r')
+
+/* Get more input from reader. */
+static LJ_NOINLINE LexChar lex_more(LexState *ls)
+{
+  size_t sz;
+  const char *p = ls->rfunc(ls->L, ls->rdata, &sz);
+  if (p == NULL || sz == 0) return LEX_EOF;
+  ls->pe = p + sz;
+  ls->p = p + 1;
+  return (LexChar)(uint8_t)p[0];
+}
+
+/* Get next character. */
+static LJ_AINLINE LexChar lex_next(LexState *ls)
+{
+  return (ls->c = ls->p < ls->pe ? (LexChar)(uint8_t)*ls->p++ : lex_more(ls));
+}
+
+/* Save character. */
+static LJ_AINLINE void lex_save(LexState *ls, LexChar c)
+{
+  lj_buf_putb(&ls->sb, c);
+}
+
+/* Save previous character and get next character. */
+static LJ_AINLINE LexChar lex_savenext(LexState *ls)
+{
+  lex_save(ls, ls->c);
+  return lex_next(ls);
+}
+
+/* Skip line break. Handles "\n", "\r", "\r\n" or "\n\r". */
+static void lex_newline(LexState *ls)
+{
+  LexChar old = ls->c;
+  lua_assert(lex_iseol(ls));
+  lex_next(ls);  /* Skip "\n" or "\r". */
+  if (lex_iseol(ls) && ls->c != old) lex_next(ls);  /* Skip "\n\r" or "\r\n". */
+  if (++ls->linenumber >= LJ_MAX_LINE)
+    lj_lex_error(ls, ls->tok, LJ_ERR_XLINES);
+}
+
+/* -- Scanner for terminals ----------------------------------------------- */
+
+/* Parse a number literal. */
+static void lex_number(LexState *ls, TValue *tv)
+{
+  StrScanFmt fmt;
+  LexChar c, xp = 'e';
+  lua_assert(lj_char_isdigit(ls->c));
+  if ((c = ls->c) == '0' && (lex_savenext(ls) | 0x20) == 'x')
+    xp = 'p';
+  while (lj_char_isident(ls->c) || ls->c == '.' ||
+	 ((ls->c == '-' || ls->c == '+') && (c | 0x20) == xp)) {
+    c = ls->c;
+    lex_savenext(ls);
+  }
+  lex_save(ls, '\0');
+  fmt = lj_strscan_scan((const uint8_t *)sbufB(&ls->sb), tv,
+	  (LJ_DUALNUM ? STRSCAN_OPT_TOINT : STRSCAN_OPT_TONUM) |
+	  (LJ_HASFFI ? (STRSCAN_OPT_LL|STRSCAN_OPT_IMAG) : 0));
+  if (LJ_DUALNUM && fmt == STRSCAN_INT) {
+    setitype(tv, LJ_TISNUM);
+  } else if (fmt == STRSCAN_NUM) {
+    /* Already in correct format. */
+#if LJ_HASFFI
+  } else if (fmt != STRSCAN_ERROR) {
+    lua_State *L = ls->L;
+    GCcdata *cd;
+    lua_assert(fmt == STRSCAN_I64 || fmt == STRSCAN_U64 || fmt == STRSCAN_IMAG);
+    if (!ctype_ctsG(G(L))) {
+      ptrdiff_t oldtop = savestack(L, L->top);
+      luaopen_ffi(L);  /* Load FFI library on-demand. */
+      L->top = restorestack(L, oldtop);
+    }
+    if (fmt == STRSCAN_IMAG) {
+      cd = lj_cdata_new_(L, CTID_COMPLEX_DOUBLE, 2*sizeof(double));
+      ((double *)cdataptr(cd))[0] = 0;
+      ((double *)cdataptr(cd))[1] = numV(tv);
+    } else {
+      cd = lj_cdata_new_(L, fmt==STRSCAN_I64 ? CTID_INT64 : CTID_UINT64, 8);
+      *(uint64_t *)cdataptr(cd) = tv->u64;
+    }
+    lj_parse_keepcdata(ls, tv, cd);
+#endif
+  } else {
+    lua_assert(fmt == STRSCAN_ERROR);
+    lj_lex_error(ls, TK_number, LJ_ERR_XNUMBER);
+  }
+}
+
+/* Skip equal signs for "[=...=[" and "]=...=]" and return their count. */
+static int lex_skipeq(LexState *ls)
+{
+  int count = 0;
+  LexChar s = ls->c;
+  lua_assert(s == '[' || s == ']');
+  while (lex_savenext(ls) == '=')
+    count++;
+  return (ls->c == s) ? count : (-count) - 1;
+}
+
+/* Parse a long string or long comment (tv set to NULL). */
+static void lex_longstring(LexState *ls, TValue *tv, int sep)
+{
+  lex_savenext(ls);  /* Skip second '['. */
+  if (lex_iseol(ls))  /* Skip initial newline. */
+    lex_newline(ls);
+  for (;;) {
+    switch (ls->c) {
+    case LEX_EOF:
+      lj_lex_error(ls, TK_eof, tv ? LJ_ERR_XLSTR : LJ_ERR_XLCOM);
+      break;
+    case ']':
+      if (lex_skipeq(ls) == sep) {
+	lex_savenext(ls);  /* Skip second ']'. */
+	goto endloop;
+      }
+      break;
+    case '\n':
+    case '\r':
+      lex_save(ls, '\n');
+      lex_newline(ls);
+      if (!tv) lj_buf_reset(&ls->sb);  /* Don't waste space for comments. */
+      break;
+    default:
+      lex_savenext(ls);
+      break;
+    }
+  } endloop:
+  if (tv) {
+    GCstr *str = lj_parse_keepstr(ls, sbufB(&ls->sb) + (2 + (MSize)sep),
+				      sbuflen(&ls->sb) - 2*(2 + (MSize)sep));
+    setstrV(ls->L, tv, str);
+  }
+}
+
+/* Parse a string. */
+static void lex_string(LexState *ls, TValue *tv)
+{
+  LexChar delim = ls->c;  /* Delimiter is '\'' or '"'. */
+  lex_savenext(ls);
+  while (ls->c != delim) {
+    switch (ls->c) {
+    case LEX_EOF:
+      lj_lex_error(ls, TK_eof, LJ_ERR_XSTR);
+      continue;
+    case '\n':
+    case '\r':
+      lj_lex_error(ls, TK_string, LJ_ERR_XSTR);
+      continue;
+    case '\\': {
+      LexChar c = lex_next(ls);  /* Skip the '\\'. */
+      switch (c) {
+      case 'a': c = '\a'; break;
+      case 'b': c = '\b'; break;
+      case 'f': c = '\f'; break;
+      case 'n': c = '\n'; break;
+      case 'r': c = '\r'; break;
+      case 't': c = '\t'; break;
+      case 'v': c = '\v'; break;
+      case 'x':  /* Hexadecimal escape '\xXX'. */
+	c = (lex_next(ls) & 15u) << 4;
+	if (!lj_char_isdigit(ls->c)) {
+	  if (!lj_char_isxdigit(ls->c)) goto err_xesc;
+	  c += 9 << 4;
+	}
+	c += (lex_next(ls) & 15u);
+	if (!lj_char_isdigit(ls->c)) {
+	  if (!lj_char_isxdigit(ls->c)) goto err_xesc;
+	  c += 9;
+	}
+	break;
+      case 'z':  /* Skip whitespace. */
+	lex_next(ls);
+	while (lj_char_isspace(ls->c))
+	  if (lex_iseol(ls)) lex_newline(ls); else lex_next(ls);
+	continue;
+      case '\n': case '\r': lex_save(ls, '\n'); lex_newline(ls); continue;
+      case '\\': case '\"': case '\'': break;
+      case LEX_EOF: continue;
+      default:
+	if (!lj_char_isdigit(c))
+	  goto err_xesc;
+	c -= '0';  /* Decimal escape '\ddd'. */
+	if (lj_char_isdigit(lex_next(ls))) {
+	  c = c*10 + (ls->c - '0');
+	  if (lj_char_isdigit(lex_next(ls))) {
+	    c = c*10 + (ls->c - '0');
+	    if (c > 255) {
+	    err_xesc:
+	      lj_lex_error(ls, TK_string, LJ_ERR_XESC);
+	    }
+	    lex_next(ls);
+	  }
+	}
+	lex_save(ls, c);
+	continue;
+      }
+      lex_save(ls, c);
+      lex_next(ls);
+      continue;
+      }
+    default:
+      lex_savenext(ls);
+      break;
+    }
+  }
+  lex_savenext(ls);  /* Skip trailing delimiter. */
+  setstrV(ls->L, tv,
+	  lj_parse_keepstr(ls, sbufB(&ls->sb)+1, sbuflen(&ls->sb)-2));
+}
+
+/* -- Main lexical scanner ------------------------------------------------ */
+
+/* Get next lexical token. */
+static LexToken lex_scan(LexState *ls, TValue *tv)
+{
+  lj_buf_reset(&ls->sb);
+  for (;;) {
+    if (lj_char_isident(ls->c)) {
+      GCstr *s;
+      if (lj_char_isdigit(ls->c)) {  /* Numeric literal. */
+	lex_number(ls, tv);
+	return TK_number;
+      }
+      /* Identifier or reserved word. */
+      do {
+	lex_savenext(ls);
+      } while (lj_char_isident(ls->c));
+      s = lj_parse_keepstr(ls, sbufB(&ls->sb), sbuflen(&ls->sb));
+      setstrV(ls->L, tv, s);
+      if (s->reserved > 0)  /* Reserved word? */
+	return TK_OFS + s->reserved;
+      return TK_name;
+    }
+    switch (ls->c) {
+    case '\n':
+    case '\r':
+      lex_newline(ls);
+      continue;
+    case ' ':
+    case '\t':
+    case '\v':
+    case '\f':
+      lex_next(ls);
+      continue;
+    case '-':
+      lex_next(ls);
+      if (ls->c != '-') return '-';
+      lex_next(ls);
+      if (ls->c == '[') {  /* Long comment "--[=*[...]=*]". */
+	int sep = lex_skipeq(ls);
+	lj_buf_reset(&ls->sb);  /* `lex_skipeq' may dirty the buffer */
+	if (sep >= 0) {
+	  lex_longstring(ls, NULL, sep);
+	  lj_buf_reset(&ls->sb);
+	  continue;
+	}
+      }
+      /* Short comment "--.*\n". */
+      while (!lex_iseol(ls) && ls->c != LEX_EOF)
+	lex_next(ls);
+      continue;
+    case '[': {
+      int sep = lex_skipeq(ls);
+      if (sep >= 0) {
+	lex_longstring(ls, tv, sep);
+	return TK_string;
+      } else if (sep == -1) {
+	return '[';
+      } else {
+	lj_lex_error(ls, TK_string, LJ_ERR_XLDELIM);
+	continue;
+      }
+      }
+    case '=':
+      lex_next(ls);
+      if (ls->c != '=') return '='; else { lex_next(ls); return TK_eq; }
+    case '<':
+      lex_next(ls);
+      if (ls->c != '=') return '<'; else { lex_next(ls); return TK_le; }
+    case '>':
+      lex_next(ls);
+      if (ls->c != '=') return '>'; else { lex_next(ls); return TK_ge; }
+    case '~':
+      lex_next(ls);
+      if (ls->c != '=') return '~'; else { lex_next(ls); return TK_ne; }
+    case ':':
+      lex_next(ls);
+      if (ls->c != ':') return ':'; else { lex_next(ls); return TK_label; }
+    case '"':
+    case '\'':
+      lex_string(ls, tv);
+      return TK_string;
+    case '.':
+      if (lex_savenext(ls) == '.') {
+	lex_next(ls);
+	if (ls->c == '.') {
+	  lex_next(ls);
+	  return TK_dots;   /* ... */
+	}
+	return TK_concat;   /* .. */
+      } else if (!lj_char_isdigit(ls->c)) {
+	return '.';
+      } else {
+	lex_number(ls, tv);
+	return TK_number;
+      }
+    case LEX_EOF:
+      return TK_eof;
+    default: {
+      LexChar c = ls->c;
+      lex_next(ls);
+      return c;  /* Single-char tokens (+ - / ...). */
+    }
+    }
+  }
+}
+
+/* -- Lexer API ----------------------------------------------------------- */
+
+/* Setup lexer state. */
+int lj_lex_setup(lua_State *L, LexState *ls)
+{
+  int header = 0;
+  ls->L = L;
+  ls->fs = NULL;
+  ls->pe = ls->p = NULL;
+  ls->vstack = NULL;
+  ls->sizevstack = 0;
+  ls->vtop = 0;
+  ls->bcstack = NULL;
+  ls->sizebcstack = 0;
+  ls->tok = 0;
+  ls->lookahead = TK_eof;  /* No look-ahead token. */
+  ls->linenumber = 1;
+  ls->lastline = 1;
+  lex_next(ls);  /* Read-ahead first char. */
+  if (ls->c == 0xef && ls->p + 2 <= ls->pe && (uint8_t)ls->p[0] == 0xbb &&
+      (uint8_t)ls->p[1] == 0xbf) {  /* Skip UTF-8 BOM (if buffered). */
+    ls->p += 2;
+    lex_next(ls);
+    header = 1;
+  }
+  if (ls->c == '#') {  /* Skip POSIX #! header line. */
+    do {
+      lex_next(ls);
+      if (ls->c == LEX_EOF) return 0;
+    } while (!lex_iseol(ls));
+    lex_newline(ls);
+    header = 1;
+  }
+  if (ls->c == LUA_SIGNATURE[0]) {  /* Bytecode dump. */
+    if (header) {
+      /*
+      ** Loading bytecode with an extra header is disabled for security
+      ** reasons. This may circumvent the usual check for bytecode vs.
+      ** Lua code by looking at the first char. Since this is a potential
+      ** security violation no attempt is made to echo the chunkname either.
+      */
+      setstrV(L, L->top++, lj_err_str(L, LJ_ERR_BCBAD));
+      lj_err_throw(L, LUA_ERRSYNTAX);
+    }
+    return 1;
+  }
+  return 0;
+}
+
+/* Cleanup lexer state. */
+void lj_lex_cleanup(lua_State *L, LexState *ls)
+{
+  global_State *g = G(L);
+  lj_mem_freevec(g, ls->bcstack, ls->sizebcstack, BCInsLine);
+  lj_mem_freevec(g, ls->vstack, ls->sizevstack, VarInfo);
+  lj_buf_free(g, &ls->sb);
+}
+
+/* Return next lexical token. */
+void lj_lex_next(LexState *ls)
+{
+  ls->lastline = ls->linenumber;
+  if (LJ_LIKELY(ls->lookahead == TK_eof)) {  /* No lookahead token? */
+    ls->tok = lex_scan(ls, &ls->tokval);  /* Get next token. */
+  } else {  /* Otherwise return lookahead token. */
+    ls->tok = ls->lookahead;
+    ls->lookahead = TK_eof;
+    ls->tokval = ls->lookaheadval;
+  }
+}
+
+/* Look ahead for the next token. */
+LexToken lj_lex_lookahead(LexState *ls)
+{
+  lua_assert(ls->lookahead == TK_eof);
+  ls->lookahead = lex_scan(ls, &ls->lookaheadval);
+  return ls->lookahead;
+}
+
+/* Convert token to string. */
+const char *lj_lex_token2str(LexState *ls, LexToken tok)
+{
+  if (tok > TK_OFS)
+    return tokennames[tok-TK_OFS-1];
+  else if (!lj_char_iscntrl(tok))
+    return lj_strfmt_pushf(ls->L, "%c", tok);
+  else
+    return lj_strfmt_pushf(ls->L, "char(%d)", tok);
+}
+
+/* Lexer error. */
+void lj_lex_error(LexState *ls, LexToken tok, ErrMsg em, ...)
+{
+  const char *tokstr;
+  va_list argp;
+  if (tok == 0) {
+    tokstr = NULL;
+  } else if (tok == TK_name || tok == TK_string || tok == TK_number) {
+    lex_save(ls, '\0');
+    tokstr = sbufB(&ls->sb);
+  } else {
+    tokstr = lj_lex_token2str(ls, tok);
+  }
+  va_start(argp, em);
+  lj_err_lex(ls->L, ls->chunkname, tokstr, ls->linenumber, em, argp);
+  va_end(argp);
+}
+
+/* Initialize strings for reserved words. */
+void lj_lex_init(lua_State *L)
+{
+  uint32_t i;
+  for (i = 0; i < TK_RESERVED; i++) {
+    GCstr *s = lj_str_newz(L, tokennames[i]);
+    fixstring(s);  /* Reserved words are never collected. */
+    s->reserved = (uint8_t)(i+1);
+  }
+}
+