immunio 1.0.4 → 1.0.5

data/lib/immunio/request.rb

@@ -20,7 +20,7 @@ module Immunio
       @paused_times = {}
       @start_time = Time.now
 
-      Immunio.logger.debug "Created new Request #{id} with data #{@data}"
+      Immunio.logger.debug { "Created new Request #{id} with data #{@data}" }
     end
 
     def id
@@ -78,7 +78,7 @@ module Immunio
       result = Immunio.run_hook "request", "should_report"
       # If the hook doesn't exist, turn a nil report value into a false value
       report = !!result["report"]
-      Immunio.logger.debug "Report request: #{report}"
+      Immunio.logger.debug { "Report request: #{report}" }
       report
     end
 
@@ -98,7 +98,7 @@ module Immunio
 
     def self.current=(request)
       Thread.current["immunio.request"] = request
-      Immunio.logger.debug "Setting current request for thread #{Thread.current.object_id} to #{request && request.id || nil}"
+      Immunio.logger.debug { "Setting current request for thread #{Thread.current.object_id} to #{request && request.id || nil}" }
     end
 
     # This shim exists to preserve the stack depth into hooks when self.time and self.pause are

data/lib/immunio/version.rb

@@ -1,5 +1,5 @@
 module Immunio
   AGENT_TYPE = "agent-ruby"
-  VERSION = "1.0.4"
+  VERSION = "1.0.5"
   VM_VERSION = "2.2.0"
 end

data/lib/immunio/vm.rb

@@ -54,7 +54,7 @@ module Immunio
         @functions = functions
         @utils = utils
       }
-      Immunio.logger.info "Lua code updated to version #{version}."
+      Immunio.logger.info { "Lua code updated to version #{version}." }
     end
 
     def update_data(version, data)
@@ -65,7 +65,7 @@ module Immunio
         @data_version = version
         @data = new_data
       }
-      Immunio.logger.debug "Lua data updated to version #{version}."
+      Immunio.logger.debug { "Lua data updated to version #{version}." }
     end
 
     def current_state
@@ -76,7 +76,7 @@ module Immunio
     end
 
     def new_vm
-      Immunio.logger.debug "Creating new Lua VM"
+      Immunio.logger.debug { "Creating new Lua VM" }
       dev_mode_hook_update if @dev_mode
 
       @update_lock.synchronize {
@@ -87,7 +87,7 @@ module Immunio
     def dev_mode_hook_update
       # Load default hook handlers if present (they do not ship with the gem)
       path = File.expand_path("#{Immunio::DIR}/../lua-hooks/hooks")
-      Immunio.logger.info "Checking hook handlers from #{path}"
+      Immunio.logger.info { "Checking hook handlers from #{path}" }
 
       unique_files = Dir[path + "/*.lua"].collect do |file|
         name = File.basename(file, ".*")
@@ -100,7 +100,7 @@ module Immunio
       end
       checksum = unique_files.join
       if checksum == @dev_checksum
-        Immunio.logger.info "Reusing dev vm, files unchanged"
+        Immunio.logger.info { "Reusing dev vm, files unchanged" }
         return
       end
 
@@ -118,7 +118,7 @@ module Immunio
         handlers[name] = data
         handlers
       end
-      Immunio.logger.info "Updating dev vm code, files changed"
+      Immunio.logger.info { "Updating dev vm code, files changed" }
       update_code("DEV-#{Time.now.strftime("%H:%M:%S")}", functions)
       @dev_checksum = checksum
     end

data/lua-hooks/Makefile

@@ -14,16 +14,61 @@ SRC = \
 	ext/lpeg/lpprint.c \
 	ext/lpeg/lpvm.c
 
+LUA_SRC = \
+	lib/cookie.lua \
+	lib/DataDumper.lua \
+	lib/date.lua \
+	lib/defence.lua \
+	lib/escape.lua \
+	lib/extensions.lua \
+	lib/hooks.lua \
+	lib/idn.lua \
+	lib/lexgraph.lua \
+	lib/neturl.lua \
+	lib/paths.lua \
+	lib/permit.lua \
+	lib/sanitize_sql.lua \
+	lib/semver.lua \
+	lib/sha1.lua \
+	lib/snap.lua \
+	lib/utils.lua \
+	lib/lexers/bash_dqstr.lua \
+	lib/lexers/bash.lua \
+	lib/lexers/css_attr.lua \
+	lib/lexers/css.lua \
+	lib/lexers/html.lua \
+	lib/lexers/javascript.lua \
+	lib/lexer.lua \
+	lib/hooks/authenticate.lua \
+	lib/hooks/bad_cookie.lua \
+	lib/hooks/custom_threat.lua \
+	lib/hooks/eval.lua \
+	lib/hooks/exception.lua \
+	lib/hooks/file_io.lua \
+	lib/hooks/framework_csrf_check.lua \
+	lib/hooks/framework_login.lua \
+	lib/hooks/framework_password_reset.lua \
+	lib/hooks/framework_redirect.lua \
+	lib/hooks/framework_session.lua \
+	lib/hooks/framework_user.lua \
+	lib/hooks/http_request_finish.lua \
+	lib/hooks/http_request_start.lua \
+	lib/hooks/http_response_start.lua \
+	lib/hooks/should_report.lua \
+	lib/hooks/sql_execute.lua \
+	lib/hooks/template_render_done.lua
+
 OBJ = ${SRC:.c=.o}
 
 SHA1OBJ = ext/sha1/sha1.o
+OBJ = ${SRC:.c=.o} ${SHA1OBJ}
 
 # Library archive. Used for compiling along agent bindings.
 SO_OUT = libimmunio.so
 A_OUT = libimmunio.a
 
 # CLI for running tests
-CLI = lua
+CLI = ./lua
 CLI_SRC = ext/luajit/src/luajit.c ${SRC}
 
 XCFLAGS =
@@ -59,7 +104,7 @@ all: ${CLI} ${INIT_HOOK} ${HOOKS_TARBALL} ${HOOKS_SRCS_TARBALL}
 ${SHA1OBJ}:
 	${CC} -O -c ${INCS} -o ${SHA1OBJ} ${SHA1OBJ:.o=.c}
 
-${SO_OUT}: ${OBJ} ${LUAJIT_OBJ} ${SHA1OBJ}
+${SO_OUT}: ${OBJ} ${LUAJIT_OBJ}
 	${CC} -shared ${CFLAGS} ${LIBS} -o $@ -lc $^
 
 ${A_OUT}: ${OBJ}
@@ -77,11 +122,9 @@ ${CLI}: ${CLI_SRC} ${LUAJIT_OBJ} ${SHA1OBJ}
 	${CC} ${CFLAGS} -DLUA_UNSAFE_MODE ${INCS} -o $@ $^ ${LIBS}
 
 # Concatenate init hooks into one __init__.lua hook with two newlines in between
-${INIT_HOOK}: hooks/init/*.lua hooks/init/__header__
+${INIT_HOOK}: ${LUA_SRC} ${CLI}
 	rm -f hooks/__init__.lua
-	cat hooks/init/__header__ >> hooks/__init__.lua
-	for file in hooks/init/*.lua; do cat "$$file" >> hooks/__init__.lua; printf "\n\n" >> hooks/__init__.lua; done
-	echo "return utils" >> hooks/__init__.lua
+	${CLI} ./luald.lua ${LUA_SRC} > hooks/__init__.lua
 
 build/%.lua: hooks/%.lua ${CLI}
 	@mkdir -p build

data/lua-hooks/lib/boot.lua

@@ -1,290 +1,62 @@
--- This file is executed when the Lua VM boots.
-require 'encode'
-
--- This is required to make lexers load from test harness.
--- In VM the path is handled for us by vm.rb --ol
-lexer_path='lib/lexers/?.lua'
-package.path = package.path..';'..lexer_path
-
--- Define the environment available to code executing in the VM.
--- All available functions must be declared here.
--- Make sure the function is safe before adding it here.
--- See http://lua-users.org/wiki/SandBoxes
-SANDBOX_ENV = {
-  -- Lua libs
-  ipairs = ipairs,
-  next = next,
-  pairs = pairs,
-  pcall = pcall,
-  tonumber = tonumber,
-  tostring = tostring,
-  type = type,
-  unpack = unpack,
-  assert = assert,
-  error = error,
-  getmetatable = getmetatable,
-  setmetatable = setmetatable,
-  rawget = rawget,
-  rawset = rawset,
-  collectgarbage = collectgarbage,
-  math = math,
-  string = string,
-  bit = {
-    band = bit.band,
-    extract = bit.extract,
-    bor = bit.bor,
-    bnot = bit.bnot,
-    arshift = bit.arshift,
-    rshift = bit.rshift,
-    rrotate = bit.rrotate,
-    replace = bit.replace,
-    lshift = bit.lshift,
-    lrotate = bit.lrotate,
-    btest = bit.btest,
-    bxor = bit.bxor
-  },
-  coroutine = {
-    create = coroutine.create,
-    resume = coroutine.resume,
-    running = coroutine.running,
-    status = coroutine.status,
-    wrap = coroutine.wrap,
-    yield = coroutine.yield,
-  },
-  debug = {
-    -- Block most debug in sandbox, but allow tracebacks
-    traceback = debug.traceback
-  },
-  select = select,
-  sha1 = sha1,
-  utf8 = {
-    byte = utf8.byte,
-    char = utf8.char,
-    find = utf8.find,
-    format = utf8.format,
-    gmatch = utf8.gmatch,
-    gsub = utf8.gsub,
-    len = utf8.len,
-    lower = utf8.lower,
-    match = utf8.match,
-    rep = utf8.rep,
-    reverse = utf8.reverse,
-    sub = utf8.sub,
-    upper = utf8.upper,
-    split = utf8.split,
-    escape = utf8.escape,
-    charpos = utf8.charpos,
-    insert = utf8.insert,
-    remove = utf8.remove,
-    next = utf8.next,
-    ncasecmp = utf8.ncasecmp,
-  },
-  table = {
-    insert = table.insert,
-    maxn = table.maxn,
-    remove = table.remove,
-    sort = table.sort,
-    map = table.map,
-    reduce = table.reduce,
-    length = table.length,
-    concat = table.concat,
-  },
-  libinjection = {
-    sqli = libinjection.sqli,
-    fingerprint = libinjection.fingerprint,
-    xss = libinjection.xss,
-    sqli_tokenize = libinjection.sqli_tokenize
-  },
-  -- LPeg Library
-  lpeg = {
-    ptree = lpeg.ptree,
-    pcode = lpeg.pcode,
-    match = lpeg.match,
-    B = lpeg.B,
-    V = lpeg.V,
-    C = lpeg.C,
-    Cc = lpeg.Cc,
-    Cmt = lpeg.Cmt,
-    Cb = lpeg.Cb,
-    Carg = lpeg.Carg,
-    Cp = lpeg.Cp,
-    Cs = lpeg.Cs,
-    Ct = lpeg.Ct,
-    Cf = lpeg.Cf,
-    Cg = lpeg.Cg,
-    P = lpeg.P,
-    S = lpeg.S,
-    R = lpeg.R,
-    locale = lpeg.locale,
-    version = lpeg.version,
-    setmaxstack = lpeg.setmaxstack,
-    type = lpeg.type,
-  },
-  -- pre built lexer library
-  -- the call to load here will both load the code
-  -- and compile the LPeg grammar
-  lexers = {
-    lexer = require('lexers/lexer'),
-    bash = require('lexers/lexer').load('bash'), -- bash
-    bash_dqstr = require('lexers/lexer').load('bash_dqstr'),  -- bash strings
-    html = require('lexers/lexer').load('html'),
-    javascript = require('lexers/lexer').load('javascript'),
-    css = require('lexers/lexer').load('css'),
-    css_attr = require('lexers/lexer').load('css_attr'),
-  },
-  -- Immunio vars
-  serverdata = {}, -- Default empty serverdata
-  agentdata = {},
-  utils = {}, -- Used to store utility functions declared in the sandbox.
-  -- pass mode flags into the VM
-  DEV_MODE = DEV_MODE,
-  DEBUG_MODE = DEBUG_MODE,
-  LUA_PLATFORM = LUA_PLATFORM or 'unix',
-  IMMUNIO_KEY = IMMUNIO_KEY,
-  IMMUNIO_SECRET = IMMUNIO_SECRET
-}
-
--- Enable a few more things in dev mode. For debugging.
-if DEBUG_MODE or DEV_MODE then
-  SANDBOX_ENV.print = print
-  SANDBOX_ENV.snapshot = snapshot
-else
-  SANDBOX_ENV.print = function(...) end
+VM_VERSION = 1
+-- Global reference to hold the VM itself.
+VM = nil
+
+-- Create global agentdata and serverdata
+agentdata = agentdata or {}
+serverdata = serverdata or {}
+
+-- XXX Java agent has built in assumption that this function exists before VM
+-- initialisation.
+-- Encode a Lua object to be sent to the server.
+function encode(object)
+  return cmsgpack.pack(object)
 end
 
-
--- Perform a VM call a method of a lua pseudo-object
-function sandboxed_method_call(method, object, vars)
-  if DEBUG_MODE then
-    SANDBOX_ENV.utils.debug_prefix = "UNKNOWN"
-      -- Change the values here to toggle debugging per module.
-    SANDBOX_ENV.utils.debug_module_prefixes = {
-      UNKNOWN = true,
-      IO = true,
-      SQLi = true,
-      ExceptionHandler = true,
-      Redirect = true,
-      XSS = true,
-      Eval = true,
-    }
-  end
-  -- Merges the vars and the default sandbox env.
-  -- The vars can override the sandbox environment.
-  -- The table is copied to keep data from leaking
-  --   out of the functions.
-  local merged_vars = {}
-  merged_vars._G = merged_vars
-  for k, v in pairs(SANDBOX_ENV) do
-    merged_vars[k] = v
-  end
-
+-- Function called by the Agent to call and sandbox a function.
+function sandboxed_call(method, vars)
+  local rval = nil
+  -- Merge caller supplied vars
   if vars then
+    -- utils.debug_dump("Merge vars from agent: ", vars)
     for k, v in pairs(vars) do
-      merged_vars[k] = v
-    end
-  end
-
-  -- XXX Open sandbox in DEBUG_MODE
-  if DEBUG_MODE then
-    merged_vars['__REAL_G'] = _G
-  end
-  -- Sets the environment of the function.
-  setfenv(method, merged_vars)
-  -- Call it!
-  local rval = nil
-  if object then
-    rval = method(object)
-  else
-    rval = method()
-  end
-  -- Hint the lua VM GC that the references held to values in merged_vars don't
-  -- count anymore. If we omit this line the function environment is held onto
-  -- by the GC and we leak the universe... --ol
-  setmetatable( merged_vars, {__mode = "v"} )
-  -- Remove merged_vars from function environment so it can be collected sooner
-  setfenv(method, _G)
-  return rval
-end
-
--- Function called by the VM to call and sandbox a function.
-function sandboxed_call(func, vars)
-  return sandboxed_method_call(func, nil, vars)
-end
-
-if DEBUG_MODE then
--- Memory Snapshot Debugger
-  local saved_snapshot = {}
-  local saved_usage = 0
-  function dump_snapshot( label )
-    collectgarbage()
-    collectgarbage()
-    saved_snapshot = snapshot.snapshot()
-    saved_usage = collectgarbage('count')
-    print("------------------------\nSNAPSHOT\n")
-    print("USAGE: " .. saved_usage .. "\n")
-    if label then print(label) end
-    for k,v in pairs(saved_snapshot) do
-        print( "ALLOCATION:" .. tostring(k):gsub("userdata:", "") .. " " .. v)
-    end
-  end
-
-  function update_snapshot()
-    collectgarbage()
-    collectgarbage()
-    saved_snapshot = snapshot.snapshot()
-    saved_usage = collectgarbage('count')
-  end
-
-  function diff_snapshot( update )
-    collectgarbage()
-    collectgarbage()
-    local S = snapshot.snapshot()
-    local U = collectgarbage('count')
-    local output = ("------------------------\nDIFF SNAPSHOT\n")
-    output = output .. "USAGE DELTA: " .. U - saved_usage .. "\n"
-    for k,v in pairs(S) do
-      if saved_snapshot[k] == nil then
-        output = output .. "ALLOCATION:" .. tostring(k):gsub("userdata:", "") .. " " .. v .. "\n"
-
+      if k ~= "utils" then -- XXX agent API compatability. Ignore utils.
+        _G[k] = v
       end
     end
-    if update then saved_snapshot = S end
-    return output
   end
-
-  function diff_count_snapshot( update )
-    collectgarbage()
-    collectgarbage()
-    local S = snapshot.snapshot()
-    local U = collectgarbage('count')
-    local total = 0
-    local count = 0
-    local output = ("------------------------\nCOUNT SNAPSHOT\n")
-    for k,v in pairs(S) do
-      total = total + 1
-      if saved_snapshot[k] == nil then
-        count = count + 1
+  if type(method) == "function" then
+    -- legacy hooks. Convert to new hook call
+    rval = method(vars)
+    if rval and type(rval) == 'table' then
+      -- Special handling for the __init__ hook
+      -- If rval contains a _vm_update key then (re)install the package loader
+      -- and (re)load packages and hooks
+      if rval._vm_update then
+        VM = rval._vm_update(DEV_MODE, VM_VERSION)
+        rval = {}
       end
-    end
-    output = output .. "DELTA USAGE: " .. U - saved_usage .. "\n"
-    output = output .. "\n*** NEW ALLOCATIONS ***\nTOTAL: " .. total .. "\nNEW: " .. count .. "\nUSAGE: " .. U .. "\n"
-    total = 0
-    count = 0
-    for k,v in pairs(saved_snapshot) do
-      total = total + 1
-      if S[k] == nil then
-        count = count + 1
+      -- Special handling for the other hooks
+      if rval._vm_hook_call then
+        -- swap _vm_hook_call string into method which will be passed to hooks.run below
+        method = rval._vm_hook_call
+      else
+        -- Just punt the rval back to the caller.
+        return rval
       end
     end
-    output = output .. "\n*** OLD ALLOCATIONS ***\nTOTAL: " .. total .. "\nFREED: " .. count .. "\nUSAGE: " .. saved_usage .. "\n"
-    if update then saved_snapshot = S end
-    return output
   end
-
-  -- Uncomment for snapshot tracing
-  --snapshot.tron()
-  -- Uncomment to generate a snapshot at boot.
-  --dump_snapshot('BOOT')
-  --snapshot.troff()
+  if type(method) == "string" then
+    -- new style hook call
+    if VM then
+      -- Call it!
+      rval = VM.run(method, vars)
+    else
+      error("Attempt to call hook '" .. method .. "' before VM code has loaded.")
+    end
+  elseif type(method) ~= 'function' then
+    error("sandboxed_call called with a method of type " .. type(method) .. ".")
+  end
+  return rval
 end