sqreen 0.1.0.pre → 0.7.01461158029

data/lib/sqreen/parsers/sql_tokenizer.rb

@@ -0,0 +1,266 @@
+# Copyright (c) 2015 Sqreen. All Rights Reserved.
+# Please refer to our terms for more information: https://www.sqreen.io/terms.html
+
+# Inspired by
+# https://practicingruby.com/articles/parsing-json-the-hard-way
+
+require 'strscan'
+require 'sqreen/exception'
+
+module Sqreen
+  module Parsers
+    class ParsingException < ::Sqreen::Exception
+    end
+
+    # Forward declaration
+    class SQLTokenizer; end
+
+    # https://www.sqlite.org/lang_select.html
+    # https://www.sqlite.org/lang_expr.html
+    class SQLiteTokenizer < SQLTokenizer
+      def initialize(_db_infos,
+                     _backslash_escape = false,
+                     _has_hex          = false)
+        @name = 'SQLite'
+      end
+    end
+
+    # How string litterals work in MySQL
+    # https://dev.mysql.com/doc/refman/5.0/en/string-literals.html
+    # https://dev.mysql.com/doc/refman/5.0/en/number-literals.html
+    #
+    # Configuration modifiers:
+    # sqlmode: The backslash charcter does not perform escapes
+    # https://dev.mysql.com/doc/refman/5.0/en/sql-mode.html#sqlmode_ansi_quotes
+    # sqlmode: No backslash escapes
+    # https://dev.mysql.com/doc/refman/5.0/en/sql-mode.html#sqlmode_no_backslash_escapes
+
+    # The double quote cannot be a string delimiter
+    #
+    # Double quote strangeness:
+    #
+    # mysql> SELECT 'hello', '"hello"', '""hello""', 'hel''lo', '\'hello';
+    # +-------+---------+-----------+--------+--------+
+    # | hello | "hello" | ""hello"" | hel'lo | 'hello |
+    # +-------+---------+-----------+--------+--------+
+    #
+    # mysql> SELECT "hello", "'hello'", "''hello''", "hel""lo", "\"hello";
+    # +-------+---------+-----------+--------+--------+
+    # | hello | 'hello' | ''hello'' | hel"lo | "hello |
+    # +-------+---------+-----------+--------+--------+
+    class MySQLTokenizer < SQLTokenizer
+      def initialize(db_infos,
+                     backslash_escape = true,
+                     has_hex          = true)
+        super
+        @name = 'MySQL'
+      end
+    end
+
+    class SQLTokenizer
+      NUMBER = /[-+]*(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?/
+      COMMAND = /[a-zA-Z_][a-zA-Z_0-9]*/
+      HEX = /0x[0-9a-fA-F]+/
+
+      binary_operators_list = %w(\+ - ~ \! <=> <> != <= = >= < > := % NOT OR \|\| AND IS IN BETWEEN LIKE REGEXP)
+      binary_operators_list << 'SOUNDS LIKE' << 'IS NULL' << 'IS NOT NULL'
+      BINARY_OPERATORS_LIST = binary_operators_list.freeze
+
+      BITS_OPERATORS_LIST = %w(\| & << >> \+ - \* / DIV MOD % ^).freeze
+
+      BINARY_OPERATORS = Regexp.new(BINARY_OPERATORS_LIST.join('|'))
+      BITS_OPERATORS = Regexp.new(BITS_OPERATORS_LIST.join('|'))
+
+      SQ = "'".freeze
+      DQ = '"'.freeze
+      BQ = '`'.freeze
+
+      attr_reader :ss
+
+      def initialize(db_infos,
+                     backslash_escape = true,
+                     has_hex          = true)
+
+        @db_infos = db_infos
+        @ss = nil
+        @backslash_escape = backslash_escape
+        @has_hex = has_hex
+      end
+
+      def tokenize(str)
+        @ss = StringScanner.new(str)
+      end
+
+      def each_token
+        while token = next_token
+          type = get_type(token[0])
+          puts token.inspect       if $DEBUG
+          puts 'After: ' + ss.rest if $DEBUG
+          yield(token, type)
+        end
+      end
+
+      # Can raise a ParsingException.
+      def next_token
+        @ss.skip(/^[ \t\n]+/)
+
+        return if @ss.eos?
+
+        # FIXME: multiple commands (; separated)
+        # FIXME endline comments (-- )
+
+        res = if text = @ss.scan(COMMAND)
+                [:COMMAND, text]
+              elsif @has_hex && text = @ss.scan(HEX)
+                [:HEX_STRING, text]
+              elsif text = @ss.scan(NUMBER)
+                [:NUMBER, text]
+              elsif text = extract_string(SQ)
+                [:SINGLE_QUOTED_STRING, SQ + text + SQ]
+              elsif text = extract_string(DQ)
+                [:DOUBLE_QUOTED_STRING, DQ + text + DQ]
+              elsif text = extract_string(BQ)
+                [:BACK_QUOTED_STRING, BQ + text + BQ]
+              elsif @ss.peek(1) == '*' && @ss.pos += 1
+                [:ASTERISK, '*']
+              elsif @ss.peek(3) == '/*!' && @ss.pos += 3 and text = @ss.search_full(/\*\//, true, true)
+                [:INLINE_COMMENT, '/*!' + text]
+              elsif @ss.peek(2) == '/*' && @ss.pos += 2 and text = @ss.search_full(/\*\//, true, true)
+                [:INLINE_COMMENT, '/*' + text]
+              elsif text = @ss.scan(/\)/)
+                [:PAR_CLOSE, text]
+              elsif text = @ss.scan(/\(/)
+                [:PAR_OPEN, text]
+              elsif text = @ss.scan(/,/)
+                [:COMA, text]
+              elsif @ss.peek(1) == '?' && @ss.pos += 1
+                [:QUESTION_MARK, '?']
+              elsif text = @ss.scan(/;/)
+                [:QUERY_END, text]
+              elsif @ss.peek(1) == ':' && @ss.pos += 1
+                [:LABEL_MARK, ':']
+              elsif @ss.peek(1) == '.' && @ss.pos += 1
+                [:DOT, '.']
+              elsif text = @ss.scan(BINARY_OPERATORS)
+                [:BINARY_OPERATOR, text]
+              # elsif text = @ss.scan(BITS_OPERATORS) then
+              #     [:BITS_OPERATOR, text]
+              else
+                pos = @ss.pos
+                @ss.reset
+                request = @ss.rest
+                @ss.pos = pos
+                raise ParsingException, format('could not find anything %s', request.inspect)
+              end
+
+        res
+      end
+
+      TYPES_MAPPING = {
+        :literal_string => [
+          :SINGLE_QUOTED_STRING,
+          :DOUBLE_QUOTED_STRING,
+          :HEX_STRING,
+        ],
+        :literal_number => [
+          :NUMBER,
+        ],
+        :object => [
+          :BACK_QUOTED_STRING,
+        ],
+        :command => [
+          :COMMAND,
+        ],
+      }.freeze
+
+      INVERT_TYPE_MAPPING = TYPES_MAPPING.inject({}) do |new, types_h|
+        meta_type, types = types_h
+        for type in types
+          new[type] = meta_type
+        end
+        new
+      end
+
+      def get_type(token_type)
+        INVERT_TYPE_MAPPING[token_type]
+      end
+
+      def extract_string(delim)
+        return unless @ss.peek(1) == delim
+        @ss.pos += 1
+        find_string(delim)
+      end
+
+      def find_dq_str
+        find_string('"')
+      end
+
+      def find_sq_str(_string_scanner)
+        find_string("'")
+      end
+
+      def find_string(delim)
+        remain = @ss.rest
+        end_idx = find_impair_delim_pos(remain, delim)
+        raise ParsingException, format('no end for string %s', remain.inspect) if end_idx.nil?
+
+        # So we point on the last internal char of string
+        res = remain[0...end_idx]
+
+        # Ruby 1.9.3: String.bytes = Enumerator has no size method
+        bytes_size = 0
+        res.bytes.each { bytes_size += 1 }
+
+        # Remain is a unicode string. @ss the StringScanner works with
+        # bytes, so we need to move it forward by numbre of bytes in remain.
+        @ss.pos += bytes_size + 1
+        res
+      end
+
+      # MySQL string exctraction is a pain since:
+      #  - a string may hold antislashes, which escapes the next char
+      #  - a string may hold their delimiter (' or ") twice: '' or "", which
+      #    translates \' or \"
+      # Hence we browse the string. We skip \*, and if we find delimiters, we
+      # count how many there are.
+      #  - If its an odd number, (1, 3...) this is the end of the string.
+      #  - Else, they have all been escaped, we can ignore them.
+      def find_impair_delim_pos(str, delim)
+        skip = false
+        first = nil
+        str.chars.each_with_index do |char, i|
+          if skip
+            skip = false
+            next
+          end
+          if @backslash_escape && char == '\\'
+            # if we have an escape, let's skip it
+            skip = true
+            next
+          end
+          if char == delim
+            first = i unless first
+          else
+            if first
+              # first time we reach the end of a `delim` sequence
+              nb = i - first
+              if nb.odd?
+                # we had an odd number of delim
+                return i - 1
+              else
+                # we had an even number of delim, keep searching
+                first = nil
+              end
+            end
+          end
+        end
+        i = str.size - 1
+        if first
+          nb = i - first + 1
+          return i if nb.odd?
+        end
+        nil
+      end
+    end
+  end
+end

data/lib/sqreen/parsers/unix.rb

@@ -0,0 +1,110 @@
+# Copyright (c) 2015 Sqreen. All Rights Reserved.
+# Please refer to our terms for more information: https://www.sqreen.io/terms.html
+
+require 'strscan'
+require 'sqreen/exception'
+
+module Sqreen
+  module Parsers
+    # A command fragment
+    class UnixAtom
+      attr_accessor :kind, :val, :start, :end
+
+      def initialize(kind, value, start)
+        self.kind = kind
+        self.val = value
+        self.start = start
+        self.end = start + value.size
+      end
+
+      def executable?
+        kind == :exec
+      end
+    end
+
+    # A basic Shell command parser
+    class Unix
+      attr_reader :atoms
+
+      def initialize
+        @ifs = ENV.fetch('IFS', " \t\n")
+        @white = Regexp.new("[#{@ifs}]+")
+        @nonwhite = Regexp.new("[^#{@ifs}]+")
+        @control = ';&|'
+        @nonwhite_and_control = Regexp.new("[^#{@ifs}#{@control}]+")
+      end
+
+      # Parse a shell command.
+      #
+      # fills atoms with parsing results
+      # @param cmd [String] Command to be parsed
+      # @return [Boolean] true if successful
+      def parse(cmd)
+        @atoms = []
+        each_token(cmd) do |atom|
+          @atoms << atom
+        end
+
+        true
+      end
+
+      protected
+
+      def each_token(cmd)
+        scan = StringScanner.new(cmd)
+
+        @first = true
+        loop do
+          token = next_token(scan)
+          break if token.nil?
+          yield token
+        end
+      end
+
+      def next_token(scan)
+        scan.skip(@white)
+
+        return nil if scan.eos?
+
+        pos = scan.pos
+        text = scan.scan(Regexp.new("#{@nonwhite}="))
+        return UnixAtom.new(:param, text + find_word(scan), pos) if text
+        next_ch = scan.peek(1)
+        if @control.include?(next_ch)
+          @first = true
+          return UnixAtom.new(:control, scan.scan(Regexp.new("[#{@control}]+")), pos)
+        end
+        word = find_word(scan)
+        return nil unless word
+        if @first
+          @first = false
+          return UnixAtom.new(:exec, word, pos)
+        else
+          return UnixAtom.new(:field, word, pos)
+        end
+      end
+
+      def find_word(scan)
+        first = scan.peek(1)
+        return '' if first.match(@white)
+
+        if first == "'"
+          scan.getch
+          "'" + scan.scan_until(/'|\z/)
+        elsif first == '"'
+          txt = ['"']
+          scan.getch
+          until scan.eos?
+            txt << scan.scan_until(/"|\z/)
+            escaped = txt[-1].match(/\\+"\z/)
+            break unless escaped
+            break if escaped.size.even?
+          end
+          txt.join
+        else
+          scan.scan(@nonwhite_and_control)
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/payload_creator.rb

@@ -0,0 +1,132 @@
+# Copyright (c) 2015 Sqreen. All Rights Reserved.
+# Please refer to our terms for more information: https://www.sqreen.io/terms.html
+
+require 'sqreen/context'
+require 'sqreen/runtime_infos'
+require 'sqreen/events/remote_exception'
+
+module Sqreen
+  # Create a payload from a given query
+  #
+  # Template elements are made of sections and subsections.
+  # This class is able to send the full content of section or
+  # only the required subsections as needed.
+  #
+  # The payload will always be outputed as a
+  # Hash of section => subsection.
+  class PayloadCreator
+    def initialize(query)
+      self.query = query
+    end
+
+    def query=(keys)
+      @sections = {}
+      keys.each do |key|
+        section, subsection = key.split('.', 2)
+        @sections[section] = true if subsection.nil?
+        next if @sections[section] == true
+        @sections[section] ||= []
+        @sections[section].push(subsection)
+      end
+    end
+
+    def payload(framework, rule = {})
+      ret = {}
+      METHODS.each_key do |section|
+        ret = fill(section, ret, framework, rule)
+      end
+      ret
+    end
+
+    protected
+
+    def fill(key, base, framework, rule)
+      subsection = @sections[key]
+      return base if subsection.nil?
+      if subsection == true
+        return base.merge!(key => full_section(key, framework, rule))
+      end
+      return base if subsection.size == 0
+      base[key] = fields(key, framework, rule)
+      base
+    end
+
+    FULL_SECTIONS = {
+      'request' => 'request_infos',
+      'params' => 'filtered_request_params',
+      'local' => 'local_infos',
+    }.freeze
+
+    METHODS = {
+      'request' => {
+        'addr' => 'client_ip',
+        'rid' => 'request_id',
+      },
+      'local' => {
+        'name' => 'hostname',
+      },
+      'params' => {
+        'form' => 'form_params',
+        'query' => 'query_params',
+        'cookies' => 'cookies_params',
+        'rails' => 'rails_params',
+      },
+      'rule' => {},
+      'context' => {
+        'backtrace' => 'get_current_backtrace',
+      },
+    }.freeze
+
+    def section_object(section, framework, rule)
+      return RuntimeInfos if section == 'local'
+      return rule if section == 'rule'
+      return Context.new if section == 'context'
+      framework
+    end
+
+    def full_section(section, framework, rule)
+      return section_rule(framework, rule) if section == 'rule'
+      return section_context(framework, rule) if section == 'context'
+      so = section_object(section, framework, rule)
+      so.send(FULL_SECTIONS[section])
+    end
+
+    def fields(section, framework, rule)
+      out = {}
+      object = section_object(section, framework, rule)
+      remove = []
+      @sections[section].each do |key|
+        meth = METHODS[section][key]
+        invoke(out, key, object, meth || key, remove)
+      end
+      remove.each { |k| @sections[section].delete(k) }
+      Hash[out]
+    end
+
+    def invoke(out, key, object, method, remove)
+      out[key] = if object.respond_to?(:[])
+                   object[method]
+                 else
+                   object.send(method)
+                 end
+    rescue NoMethodError => e
+      remove.push(key)
+      Sqreen::RemoteException.record(e)
+    end
+
+    def section_context(framework, rule)
+      obj = section_object('context', framework, rule)
+      {
+        'backtrace' => obj.get_current_backtrace,
+      }
+    end
+
+    def section_rule(_framework, rule)
+      {
+        'name' => rule['name'],
+        'rulespack_id' => rule['rulespack_id'],
+        'test' => rule['test'],
+      }
+    end
+  end
+end