sqreen 0.8.11465220943 → 1.0.0.pre1480953244

data/lib/sqreen/runtime_infos.rb

@@ -45,7 +45,8 @@ module Sqreen
     end
 
     def time
-      { :time => Time.now.to_s }
+      # FIXME: That should maybe be called local-time
+      { :time => Time.now }
     end
 
     def ssl
@@ -120,7 +121,7 @@ module Sqreen
       ret['remotes'] = opts['remotes'] if opts['remotes']
       ret['uri'] = opts['uri'] if opts['uri']
       # FIXME: scrub any auth data in uris
-      ret['path'] = opts['path'] if opts['path']
+      ret['path'] = opts['path'].to_s if opts['path']
       ret
     end
   end

data/lib/sqreen/serializer.rb

@@ -0,0 +1,46 @@
+# Copyright (c) 2015 Sqreen. All Rights Reserved.
+# Please refer to our terms for more information: https://www.sqreen.io/terms.html
+
+module Sqreen
+  # Serialization functions: convert Hash -> simple ruby types
+  module Serializer
+    # Serialize a deep hash/array to more simple types
+    def self.serialize(obj, max_depth = 10)
+      if obj.is_a?(Array)
+        new_obj = []
+        i = -1
+        to_do = obj.map { |v| [new_obj, i += 1, v, 0] }
+      else
+        new_obj = {}
+        to_do = obj.map { |k, v| [new_obj, k, v, 0] }
+      end
+      until to_do.empty?
+        where, key, value, deepness = to_do.pop
+        safe_key = key.kind_of?(Integer) ? key : key.to_s
+        if value.is_a?(Hash) && deepness < max_depth
+          where[safe_key] = {}
+          to_do += value.map { |k, v| [where[safe_key], k, v, deepness + 1] }
+        elsif value.is_a?(Array) && deepness < max_depth
+          where[safe_key] = []
+          i = -1
+          to_do += value.map { |v| [where[safe_key], i += 1, v, deepness + 1] }
+        else
+          case value
+          when Symbol
+            where[safe_key] = value.to_s
+          when Rational
+            where[safe_key] = value.to_f
+          when Time
+            where[safe_key] = value.iso8601
+          when Numeric, String, TrueClass, FalseClass, NilClass
+            where[safe_key] = value
+          else
+            where[safe_key] = "#{value.class.name}:#{value.inspect}"
+          end
+        end
+      end
+
+      new_obj
+    end
+  end
+end

data/lib/sqreen/session.rb

@@ -2,8 +2,10 @@
 # Please refer to our terms for more information: https://www.sqreen.io/terms.html
 
 require 'sqreen/log'
+require 'sqreen/serializer'
 require 'sqreen/runtime_infos'
 require 'sqreen/events/remote_exception'
+require 'sqreen/exception'
 
 require 'net/https'
 require 'json'
@@ -46,6 +48,7 @@ module Sqreen
       @server_url = server_url
       @request_compression = false
       @connected = nil
+      @con = nil
 
       uri = parse_uri(server_url)
       use_ssl = (uri.scheme == 'https')
@@ -71,6 +74,7 @@ module Sqreen
     end
 
     def prefix_path(path)
+      return '/sqreen/v1/' + path if path == 'app-login' || path == 'app-beat'
       @@path_prefix + path
     end
 
@@ -180,7 +184,11 @@ module Sqreen
                  when :GET
                    @con.get(path, headers)
                  when :POST
-                   json_data = compress(self.class.encode_payload(data))
+                   json_data = nil
+                   unless data.nil?
+                     serialized = Serializer.serialize(data)
+                     json_data = compress(self.class.encode_payload(serialized))
+                   end
                    @con.post(path, json_data, headers)
                  else
                    Sqreen.log.debug format('unknown method %s', method)
@@ -203,7 +211,7 @@ module Sqreen
 
     def self.encode_payload(data)
       JSON.generate(data)
-    rescue JSON::GeneratorError
+    rescue JSON::GeneratorError, Encoding::UndefinedConversionError
       Sqreen.log.debug('Payload could not be encoded enforcing recode')
       JSON.generate(rencode_payload(data))
     end
@@ -242,12 +250,14 @@ module Sqreen
 
     def self.enforce_encoding(str)
       return str unless str.is_a?(String)
-      return str if str.valid_encoding?
+      return str if str.ascii_only?
+      encoded8bit = str.encoding.name == 'ASCII-8BIT'
+      return str if !encoded8bit && str.valid_encoding?
       str.chars.map do |v|
-        if v.valid_encoding?
-          v
-        else
+        if !v.valid_encoding? || (encoded8bit && !v.ascii_only?)
           v.bytes.map { |c| "\\x#{c.to_s(16).upcase}" }.join
+        else
+          v
         end
       end.join
     end
@@ -267,19 +277,19 @@ module Sqreen
       @session_id = res['session_id']
       Sqreen.log.debug "received session_id #{@session_id}"
       Sqreen.logged_in = true
-      res.fetch('features', {})
+      res
     end
 
     def rules
       resilient_get('rulespack')
     end
 
-    def heartbeat
-      get('app-beat', {}, 5)
-    end
+    def heartbeat(cmd_res = {}, metrics = [])
+      payload = {}
+      payload['metrics'] = metrics unless metrics.nil? || metrics.empty?
+      payload['command_results'] = cmd_res unless cmd_res.nil? || cmd_res.empty?
 
-    def post_commands_result(res)
-      resilient_post('commands', res)
+      post('app-beat', payload.empty? ? nil : payload, {}, 5)
     end
 
     def post_metrics(metrics)

data/lib/sqreen/version.rb

@@ -2,5 +2,5 @@
 # Please refer to our terms for more information: https://www.sqreen.io/terms.html
 # Warning This file is auto generated! DO NOT edit.
 module Sqreen
-  VERSION = "0.8.11465220943".freeze
+  VERSION = "1.0.0.pre1480953244".freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sqreen
 version: !ruby/object:Gem::Version
-  version: 0.8.11465220943
+  version: 1.0.0.pre1480953244
 platform: ruby
 authors:
 - Sqreen
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-06-06 00:00:00.000000000 Z
+date: 2016-12-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: execjs
@@ -60,9 +60,6 @@ files:
 - lib/sqreen/context.rb
 - lib/sqreen/deliveries/batch.rb
 - lib/sqreen/deliveries/simple.rb
-- lib/sqreen/detect.rb
-- lib/sqreen/detect/shell_injection.rb
-- lib/sqreen/detect/sql_injection.rb
 - lib/sqreen/event.rb
 - lib/sqreen/events/attack.rb
 - lib/sqreen/events/remote_exception.rb
@@ -81,9 +78,6 @@ files:
 - lib/sqreen/metrics/collect.rb
 - lib/sqreen/metrics/sum.rb
 - lib/sqreen/metrics_store.rb
-- lib/sqreen/parsers/sql.rb
-- lib/sqreen/parsers/sql_tokenizer.rb
-- lib/sqreen/parsers/unix.rb
 - lib/sqreen/payload_creator.rb
 - lib/sqreen/performance_notifications.rb
 - lib/sqreen/performance_notifications/log.rb
@@ -106,15 +100,13 @@ files:
 - lib/sqreen/rules_callbacks/record_request_context.rb
 - lib/sqreen/rules_callbacks/reflected_xss.rb
 - lib/sqreen/rules_callbacks/regexp_rule.rb
-- lib/sqreen/rules_callbacks/shell.rb
 - lib/sqreen/rules_callbacks/shell_env.rb
-- lib/sqreen/rules_callbacks/sql.rb
-- lib/sqreen/rules_callbacks/system_shell.rb
 - lib/sqreen/rules_callbacks/url_matches.rb
 - lib/sqreen/rules_callbacks/user_agent_matches.rb
 - lib/sqreen/rules_signature.rb
 - lib/sqreen/runner.rb
 - lib/sqreen/runtime_infos.rb
+- lib/sqreen/serializer.rb
 - lib/sqreen/session.rb
 - lib/sqreen/stats.rb
 - lib/sqreen/version.rb
@@ -132,12 +124,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.4
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: Sqreen Ruby agent

data/lib/sqreen/detect.rb

@@ -1,14 +0,0 @@
-# Copyright (c) 2015 Sqreen. All Rights Reserved.
-# Please refer to our terms for more information: https://www.sqreen.io/terms.html
-
-require 'sqreen/detect/sql_injection'
-require 'sqreen/detect/shell_injection'
-
-module Sqreen
-  module Detect
-    def sql_injection?(request, params, db_type, db_infos = {})
-      inj = SQLInjection.new(db_type, db_infos)
-      inj.user_escape?(request, params)
-    end
-  end
-end

data/lib/sqreen/detect/shell_injection.rb

@@ -1,61 +0,0 @@
-# Copyright (c) 2015 Sqreen. All Rights Reserved.
-# Please refer to our terms for more information: https://www.sqreen.io/terms.html
-
-require 'sqreen/parsers/unix'
-
-module Sqreen
-  module Detect
-    # Detector class for shell injections
-    # Find instance of user parameters injections into executable commands
-    # It work by:
-    #  1 - Highlighting the cmd for executable sections
-    #  2 - Highlighting the cmd for traces of user parameters
-    #  3 - Comparing if there is any intersection
-    class ShellInjection
-      def initialize
-        @parser = Sqreen::Parsers::Unix.new
-      end
-
-      # Is there a user injection in cmd
-      # @param cmd [String] command to analyze
-      # @param params [Hash] Hash of user parameters
-      def user_escape?(cmd, params)
-        Sqreen.log.info format('escape? %s', [cmd, params].inspect)
-
-        # We found the user query inside the cmd. A risk exists.
-        @parser.parse(cmd)
-        execs = @parser.atoms.select(&:executable?)
-
-        each_param_scalar(params) do |v|
-          next unless v
-          value = v.to_s
-          next unless value.size > 0
-          offset = 0
-          loop do
-            match_start = cmd.index(value, offset)
-            break if match_start.nil?
-            match_end = match_start + value.size
-            offset = match_end
-            covered = execs.any? do |exec|
-              match_end >= exec.start && match_start < exec.end
-            end
-            next unless covered
-            Sqreen.log.info format('injection for parameter %s', value.inspect)
-            return true
-          end
-        end
-        false
-      end
-
-      # FIXME: deduplicate code
-      def each_param_scalar(params, &block)
-        case params
-        when Hash  then params.each { |_k, v| each_param_scalar(v, &block) }
-        when Array then params.each { |v| each_param_scalar(v, &block) }
-        else
-          yield params
-        end
-      end
-    end
-  end
-end

data/lib/sqreen/detect/sql_injection.rb

@@ -1,115 +0,0 @@
-# Copyright (c) 2015 Sqreen. All Rights Reserved.
-# Please refer to our terms for more information: https://www.sqreen.io/terms.html
-
-require 'sqreen/parsers/sql'
-require 'strscan'
-
-module Sqreen
-  module Detect
-    class SQLInjection
-      PARAM_SIZE_LIMIT = 0
-
-      def self.parser(db_type, db_infos)
-        @parsers ||= {}
-        res = nil
-        results = @parsers[db_type]
-        res = results.find { |infos, _| infos == db_infos } unless results.nil?
-        return res.last unless res.nil?
-        @parsers[db_type] ||= []
-        parser = Sqreen::Parsers::SQL.new(db_type, db_infos)
-        @parsers[db_type] << [db_infos, parser]
-        parser
-      end
-
-      attr_accessor :db_type, :db_infos
-      def initialize(db_type, db_infos)
-        @db_type = db_type
-        @db_infos = db_infos
-        @parser = SQLInjection.parser(db_type, db_infos)
-      end
-
-      # FIXME: we are likely to find false postive
-      # As is, high risk of false positive with extremely short parameters.
-      # E.g. if parameter value is 'e', it will be found in request, (e in
-      # select) but not in literals.
-      #
-      # We may want to skip:
-      #  - too short parameters (e.g. < 10 letters?)
-      #  - non suspicious parameters (e.g. without blanks or comments?)
-      def user_escape?(request, params)
-        included = count_user_params_in_request(request, params)
-        return false if included == {}
-
-        escape_found = false
-
-        # We found the user query inside the request. A risk exists.
-        @parser.parse(request)
-        literals = @parser.atoms.select(&:is_literal?)
-        included.each do |param, expected_count|
-          param_count = 0
-          literals.each do |literal|
-            # Count number of literals that fully include the user query
-            param_count += count_substring_nb(literal.val, param)
-          end
-
-          # puts "%s in raw request: %d, in atoms: %d" % [param, expected_count, param_count]
-          next unless param_count != expected_count
-          Sqreen.log.info format('injection for parameter %s', param.inspect)
-          # require request aborption
-          # log attack
-          escape_found = true
-        end
-        escape_found
-      end
-
-      # What if a string can be prefixed itself?
-      # E.g. substr = 'a b c a b c'
-      # If str = 'a b c a b c a b c' we will return 2:
-      #           \----1----/
-      #                 \----2----/
-      def count_substring_nb(str, substr)
-        s = StringScanner.new(str)
-        nb = 0
-        quote = Regexp.quote(substr)
-        re = Regexp.new(quote)
-        nb += 1 while s.search_full(re, true, false)
-        nb
-      end
-
-      def each_param_scalar(params, &block)
-        case params
-        when Hash  then params.each { |_k, v| each_param_scalar(v, &block) }
-        when Array then params.each { |v| each_param_scalar(v, &block) }
-        else
-          yield params
-        end
-      end
-
-      # FIXME: this work on params values. We might wnat to work on parameters
-      # names? High risk of false positive since a parameter name is often the
-      # database column name.
-      def count_user_params_in_request(request, params_hash)
-        res = {}
-        params_hash.each do |_type, params|
-          next if params.nil?
-          each_param_scalar(params) do |value|
-            next unless value
-            v = value.to_s
-            next if v.size <= PARAM_SIZE_LIMIT
-            next if v =~ /\A\.+\z/
-            next if v =~ /\A\s+\z/
-            next if v =~ /\A(\w+|\w[\w\.]+\w)\z/i
-
-            # We need to overwrite the count of equal parameters that
-            # came from different ways (e.g. Cookie and query).
-            next if res.key? v
-            nb = count_substring_nb(request, v)
-
-            res[v] = nb if nb > 0
-          end
-        end
-        res
-      end
-    end
-  end
-end

data/lib/sqreen/parsers/sql.rb

@@ -1,98 +0,0 @@
-# Copyright (c) 2015 Sqreen. All Rights Reserved.
-# Please refer to our terms for more information: https://www.sqreen.io/terms.html
-
-require 'sqreen/parsers/sql_tokenizer'
-require 'sqreen/exception'
-
-module Sqreen
-  module Parsers
-    # SELECT * FROM contacts WHERE name = #{name}
-    # name = 'a' 'b'
-
-    class SQLAtom
-      attr_reader :val, :type, :delim
-      def initialize(val, typ = :unkown, delim = nil, token_type = nil)
-        @val = val
-        @type = typ
-        @token_type = token_type
-        @delim = delim
-      end
-
-      def ==(other)
-        return false if other.nil?
-        @val == other.val &&
-          @type == other.type &&
-          @delim == other.delim
-      end
-
-      def is_literal?
-        [:literal_string, :literal_number].include? @type
-      end
-
-      def to_s
-        res = ":#{@type || @token_type}: #{@val}"
-        res += " (#{@delim})" if delim
-        res
-      end
-    end
-
-    class SQL
-      attr_reader :atoms, :tokenizer
-
-      def initialize(db_type, db_infos)
-        klass = case db_type
-                when :mysql
-                  MySQLTokenizer
-                when :sqlite
-                  SQLiteTokenizer
-                else
-                  Sqreen.log.warn format('warning: db_type %s not found, falling back to default SQL type', db_type)
-                  SQLTokenizer
-                end
-        @tokenizer = klass.new(db_type, db_infos)
-      end
-
-      def to_s
-        res = "Parsed: #{@req}\n"
-        nb = 0
-        @atoms.each do |atom|
-          res << "\t" + nb.to_s + ': ' + atom.to_s + "\n"
-          nb += 1
-        end
-        res
-      end
-
-      def parse(req)
-        @req = req
-        @atoms = []
-        @tokenizer.tokenize(req)
-        @tokenizer.each_token do |token, meta_type|
-          value = token[1]
-          type = token[0]
-          delim = case type
-                  when :SINGLE_QUOTED_STRING
-                    "'"
-                  when :DOUBLE_QUOTED_STRING
-                    '"'
-                  end
-          atom = SQLAtom.new(value, meta_type, delim, type)
-          @atoms << atom
-        end
-      end
-
-      def to_atoms(ary, type = :unknown)
-        ary.map { |frag| to_atom(frag, nil, type) }
-      end
-
-      def to_atom(_frag, _type = :unknown, _delim = nil)
-      end
-    end
-  end
-end
-
-if $0 == __FILE__
-  s = Sqreen::Parsers::SQL.new :mysql, {}
-  s.parse(ARGV[0])
-  puts s
-
-end