tcell_agent 0.2.12 → 0.2.13

data/lib/tcell_agent/rails/middleware/body_filter_middleware.rb

@@ -5,7 +5,6 @@ require 'uri'
 require 'tcell_agent/logger'
 require 'tcell_agent/agent'
 require 'tcell_agent/sensor_events/sensor'
-require 'tcell_agent/sensor_events/app_sensor'
 require 'tcell_agent/sensor_events/server_agent'
 require 'tcell_agent/sensor_events/util/sanitizer_utilities'
 require 'tcell_agent/sensor_events/util/redirect_utils'

data/lib/tcell_agent/rails/middleware/context_middleware.rb

@@ -5,7 +5,6 @@ require 'uri'
 require 'tcell_agent/logger'
 require 'tcell_agent/agent'
 require 'tcell_agent/sensor_events/sensor'
-require 'tcell_agent/sensor_events/app_sensor'
 require 'tcell_agent/sensor_events/server_agent'
 require 'tcell_agent/sensor_events/util/sanitizer_utilities'
 require 'tcell_agent/sensor_events/util/redirect_utils'

data/lib/tcell_agent/rails/middleware/global_middleware.rb

@@ -5,7 +5,6 @@ require 'uri'
 require 'tcell_agent/logger'
 require 'tcell_agent/agent'
 require 'tcell_agent/sensor_events/sensor'
-require 'tcell_agent/sensor_events/app_sensor'
 require 'tcell_agent/sensor_events/server_agent'
 require 'tcell_agent/sensor_events/util/sanitizer_utilities'
 require 'tcell_agent/sensor_events/util/redirect_utils'

data/lib/tcell_agent/rails/middleware/headers_middleware.rb

@@ -5,7 +5,7 @@ require 'uri'
 require 'tcell_agent/logger'
 require 'tcell_agent/agent'
 require 'tcell_agent/sensor_events/sensor'
-require 'tcell_agent/sensor_events/app_sensor'
+require 'tcell_agent/sensor_events/appsensor_meta_event'
 require 'tcell_agent/sensor_events/server_agent'
 require 'tcell_agent/sensor_events/util/sanitizer_utilities'
 require 'tcell_agent/sensor_events/util/redirect_utils'
@@ -137,45 +137,17 @@ module TCellAgent
           def _handle_appsensor(request, response)
             TCellAgent::Instrumentation.safe_block("Handling AppSensor") {
               status_code, response_headers, response_body = response
-              appsensor_policy = TCellAgent.policy(TCellAgent::PolicyTypes::AppSensor)
 
               if (!(response_body.is_a?(Array)) && !(response_body.is_a?(Rack::BodyProxy)))
                 return response
               end
+
               rack_response = Rack::Response.new(response)
 
-              if appsensor_policy && appsensor_policy.enabled
-                event = TCellAgent::SensorEvents::TCellAppSensorEventProcessor.new
-                event.request_headers = request.env
-                event.request_content_length = (request.content_length || "0").to_i
-                event.response_content_length = (rack_response.length || "0").to_i
-                event.request_content_type = request.content_type
-                event.remote_addr = request.ip
-                event.uri = "#{request.base_url}#{request.fullpath}"
-                event.get_params = request.GET
-                event.post_params = request.POST
-                event.request_body = request.body.gets
-                event.cookies = request.cookies
-                event.request_method = request.request_method
-
-                event.request_size = event.request_content_length
-                #status, headers, active_response = response
-                #if active_response.class != ActionDispatch::Response
-                #  return response
-                #end
-                event.status_code = status_code
-                event.response_headers = response_headers
-
-                event.route_id = request.env["tcell.request_data"].route_id
-                event.transaction_id = request.env["tcell.request_data"].transaction_id
-                event.session_id = request.env["tcell.request_data"].hmac_session_id
-                event.user_id = request.env["tcell.request_data"].user_id
-
-                #puts event.response_headers
-                #puts event.cookies, event.post_params, event.get_params, event.uri, event.remote_addr
-                #puts event.request_headers
-                TCellAgent.send_event(event)
-              end
+              event = TCellAgent::SensorEvents::AppSensorMetaEvent.build(
+                request, rack_response, status_code, response_headers
+              )
+              TCellAgent.send_event(event)
             }
             response
           end

data/lib/tcell_agent/sensor_events/appsensor_event.rb

@@ -0,0 +1,59 @@
+require 'tcell_agent/sensor_events/sensor'
+
+
+module TCellAgent
+  module SensorEvents
+
+    class TCellAppSensorEvent < TCellSensorEvent
+
+      def initialize(location,
+                     detection_point,
+                     method,
+                     remote_addr,
+                     param,
+                     route_id,
+                     data=nil,
+                     transaction_id=nil,
+                     session_id=nil,
+                     user_id=nil,
+                     payload=nil,
+                     pattern=nil)
+        super("as")
+        self["dp"] = detection_point
+        self["param"] = param
+        self["remote_addr"] = remote_addr
+        if (route_id)
+          self["rou"] = route_id
+        end
+        self["m"] = method
+        @raw_location = location
+        @user_id = user_id
+        @transaction_id = transaction_id
+        @raw_session_id = session_id
+        @payload = payload
+        if pattern
+          self["pattern"] = pattern
+        end
+      end
+
+      def post_process
+        self["loc"] = Util.strip_uri_values(@raw_location)
+        if @user_id
+          self["uid"] = @user_id.to_s
+        end
+        if @transaction_id
+          self["tid"] = @transaction_id
+        end
+        if @raw_session_id
+          hmac_key = Util.getHmacKey()
+          self["sid"] = Util.hmac(@raw_session_id, hmac_key)
+        end
+        if @payload
+          self["payload"] = @payload[0..150]
+        end
+      end
+
+    end
+
+  end
+end

data/lib/tcell_agent/sensor_events/appsensor_meta_event.rb

@@ -0,0 +1,95 @@
+# encoding: utf-8
+# See the file "LICENSE" for the full license governing this code.
+
+require 'tcell_agent/sensor_events/sensor'
+
+require 'tcell_agent/agent'
+require 'tcell_agent/agent/policy_types'
+require 'tcell_agent/policies/appsensor_policy'
+
+# Some Rules Originate from ModSecurity
+#   ModSecurity for Apache 2.x, http://www.modsecurity.org/
+#   Copyright (c) 2004-2013 Trustwave Holdings, Inc. (http://www.trustwave.com/)
+
+module TCellAgent
+  module SensorEvents
+
+    class AppSensorMetaEvent < TCellSensorEvent
+
+      class << self
+        def build(request, rack_response, response_code, response_headers)
+          meta_event = AppSensorMetaEvent.new
+
+          meta_event.remote_address = request.ip
+          meta_event.method = request.request_method
+          meta_event.location = "#{request.base_url}#{request.fullpath}"
+          meta_event.request_headers = request.env
+          meta_event.request_content_len = (request.content_length || "0").to_i
+          meta_event.response_content_len = (rack_response.length || "0").to_i
+          meta_event.get_dict = request.GET
+          meta_event.post_dict = request.POST
+          meta_event.cookie_dict = request.cookies
+
+          meta_event.response_code = response_code
+          meta_event.response_headers = response_headers
+
+          meta_event.route_id = request.env["tcell.request_data"].route_id
+          meta_event.transaction_id = request.env["tcell.request_data"].transaction_id
+          meta_event.session_id = request.env["tcell.request_data"].hmac_session_id
+          meta_event.user_id = request.env["tcell.request_data"].user_id
+
+          meta_event.set_body_dict(
+            meta_event.request_content_len,
+            request.content_type,
+            request.body.gets
+          )
+
+          meta_event
+        end
+      end
+
+
+      attr_accessor :remote_address, :method, :location, :route_id, :session_id, :user_id, :transaction_id,
+        :request_content_len, :get_dict, :post_dict, :body_dict, :cookie_dict, :response_content_len, :response_code
+
+      attr_accessor :request_headers, :response_headers
+
+      def initialize
+        @request_content_len = 0
+        @response_content_len = 0
+        @send = false
+        @body_dict = {}
+        @get_dict = {}
+        @post_dict = {}
+        @cookie_dict = {}
+      end
+
+      def set_body_dict(request_content_len, request_content_type, request_body)
+        if request_content_len > 2000000
+          @body_dict = {}
+
+        else
+          if request_content_type =~ %r{application/json}i && request_body
+            begin
+              @body_dict = JSON.parse(request_body)
+            rescue
+              TCellAgent.logger.debug("JSON body parameter parsing failed")
+              @body_dict = {}
+            end
+          else
+            @body_dict = {}
+          end
+        end
+      end
+
+      def post_process
+        appsensor_policy = TCellAgent.policy(TCellAgent::PolicyTypes::AppSensor)
+        return unless appsensor_policy
+
+        appsensor_policy.process_meta_event(self)
+      end
+
+    end
+
+  end
+end

data/lib/tcell_agent/servers/rails_server.rb

@@ -2,19 +2,28 @@
 # `bundle exec rails s` command. The appropriate server will be
 # launched through Rack::Server interface
 
-require("tcell_agent/servers/unicorn") if defined?(Unicorn::HttpServer)
-require("tcell_agent/servers/webrick") if defined?(Rack::Handler::WEBrick)
-require("tcell_agent/servers/thin") if defined?(Thin::Server)
+Rails::Server.class_eval do
 
-if defined?(Puma::Server)
-  Puma::Server.class_eval do
+  alias_method :tcell_start, :start
+  def start(&blk)
+    require("tcell_agent/servers/unicorn") if defined?(Unicorn::HttpServer)
+    require("tcell_agent/servers/webrick") if defined?(Rack::Handler::WEBrick)
+    require("tcell_agent/servers/thin") if defined?(Thin::Server)
 
-    alias_method :original_run, :run
-    def run(background=true)
-      TCellAgent.run_instrumentation("Puma Single Mode")
+    if defined?(Puma::Server)
+      Puma::Server.class_eval do
 
-      original_run(background)
+        alias_method :original_run, :run
+        def run(background=true)
+          TCellAgent.run_instrumentation("Puma Single Mode")
+
+          original_run(background)
+        end
+
+      end
     end
 
+    tcell_start(&blk)
   end
+
 end

data/lib/tcell_agent/utils/params.rb

@@ -0,0 +1,40 @@
+module TCellAgent
+  module Utils
+    module Params
+      GET_PARAM = "get"
+      POST_PARAM = "post"
+      JSON_PARAM = "json"
+      COOKIE_PARAM = "cookies"
+
+      def param_deep_loop(param_name, param_value, &block)
+        return nil unless param_name and param_value
+
+        if param_value.is_a?(Hash)
+          param_value.each do |k, v|
+            result = param_deep_loop(k, v, &block)
+            if result
+              return result
+            end
+          end
+
+        elsif param_value.is_a?(Array)
+          param_value.each do |v|
+            result = param_deep_loop(param_name, v, &block)
+            return result if result
+          end
+
+        elsif param_value.is_a?(String)
+          #if isinstance(param_value, bytes):
+            #param_value = param_value.decode('utf-8')
+          #param_type = str(type(param_value))
+
+          match = block.call(param_name, param_value)
+          return match if match
+        end
+
+        return nil
+      end
+
+    end
+  end
+end

data/lib/tcell_agent/version.rb

@@ -1,5 +1,5 @@
 # See the file "LICENSE" for the full license governing this code.
 
 module TCellAgent
-  VERSION = "0.2.12"
+  VERSION = "0.2.13"
 end

data/spec/lib/tcell_agent/appsensor/rules/appsensor_rule_manager_spec.rb

@@ -0,0 +1,39 @@
+require 'spec_helper'
+
+module TCellAgent
+
+  describe AppSensorRuleManager do
+    describe "#initialize" do
+      context "loading default baserules" do
+        it "should initialize all the sensors" do
+          rule_manager = AppSensorRuleManager.new(get_test_resource_path("baserules.json"))
+
+          expect(rule_manager.rule_info.empty?).to eq(false)
+        end
+      end
+    end
+
+    describe "#load_rules_file" do
+      context "with nonexistent file" do
+        it "should do nothing" do
+          rule_manager = AppSensorRuleManager.new()
+          rule_manager.load_rules_file("non-existent-file.json")
+
+          expect(rule_manager.rule_info.empty?).to eq(true)
+        end
+      end
+    end
+
+    describe "#load_default_rules_file" do
+      it "should attempt to load default rules file" do
+        expect_any_instance_of(AppSensorRuleManager).to receive(:load_rules_file).with(
+          /tcell_agent\/appsensor\/rules\/baserules.json/
+        )
+
+        rule_manager = AppSensorRuleManager.new()
+        rule_manager.load_default_rules_file()
+      end
+    end
+  end
+
+end

data/spec/lib/tcell_agent/appsensor/rules/appsensor_rule_set_spec.rb

@@ -0,0 +1,152 @@
+require 'spec_helper'
+
+module TCellAgent
+
+  describe AppSensorRuleSet do
+
+    describe "#add_pattern_from_dict" do
+      context "adding from nil dict" do
+        it "should do nothing" do
+          rule_set = AppSensorRuleSet.new
+          rule_set.add_pattern_from_dict(nil)
+          expect(rule_set.safe_pattern).to eq(nil)
+          expect(rule_set.patterns).to eq([])
+        end
+      end
+
+      context "adding from empty dict" do
+        it "should do nothing" do
+          rule_set = AppSensorRuleSet.new
+          rule_set.add_pattern_from_dict({})
+          expect(rule_set.safe_pattern).to eq(nil)
+          expect(rule_set.patterns).to eq([])
+        end
+      end
+
+      context "adding a ruby pattern" do
+        it "should add the pattern" do
+          rule_set = AppSensorRuleSet.new
+          rule_set.add_pattern_from_dict({
+            "id" => 1,
+            "common" => "<(iframe)",
+            "ruby" => "<(script)"
+          })
+
+          expect(rule_set.safe_pattern).to eq(nil)
+          expect(rule_set.patterns.size).to eq(1)
+
+          arp = rule_set.patterns[0]
+          expect(arp.enabled).to eq(true)
+          expect(arp.pattern_id).to eq(1)
+          expect(arp.pattern_regex).to_not eq(nil)
+          expect("<script".match(arp.pattern_regex).captures).to eq(["script"])
+        end
+      end
+
+      context "adding a common pattern" do
+        it "should add the pattern" do
+          rule_set = AppSensorRuleSet.new
+          rule_set.add_pattern_from_dict({
+            "id" => 1,
+            "common" => "<(script|iframe|embed|frame|frameset|object|img|applet|body|html|style|layer|link|ilayer|meta|bgsound)"
+          })
+
+          expect(rule_set.safe_pattern).to eq(nil)
+          expect(rule_set.patterns.size).to eq(1)
+
+          arp = rule_set.patterns[0]
+          expect(arp.enabled).to eq(true)
+          expect(arp.pattern_id).to eq(1)
+          expect(arp.pattern_regex).to_not eq(nil)
+          expect("<script".match(arp.pattern_regex).captures).to eq(["script"])
+        end
+      end
+    end
+
+    describe "#set_safe_pattern_from_string" do
+      it "should set the safe pattern" do
+        rule_set = AppSensorRuleSet.new
+        rule_set.set_safe_pattern_from_string("^[a-zA-Z0-9_\s\r\n\t]*$")
+
+        expect(rule_set.safe_pattern).to eq(Regexp.new("^[a-zA-Z0-9_\s\r\n\t]*$"))
+        expect(rule_set.patterns.size).to eq(0)
+      end
+    end
+
+    describe "#check_violation" do
+      before(:each) do
+        @rule_set = AppSensorRuleSet.new
+        @rule_set.set_safe_pattern_from_string("super_safe")
+        @rule_set.add_pattern_from_dict({
+          "id" => 1,
+          "common" => "<(script)"
+        })
+        @rule_set.add_pattern_from_dict({
+          "id" => 2,
+          "common" => "<(iframe)"
+        })
+      end
+
+      context "param value is nil" do
+        it "should return nil" do
+          expect(@rule_set.check_violation(nil, nil, {}, true)).to eq(nil)
+        end
+      end
+
+      context "param value is empty" do
+        it "should return nil" do
+          expect(@rule_set.check_violation(nil, nil, {}, true)).to eq(nil)
+        end
+      end
+
+      context "param value is present" do
+        context "param value matches safe pattern" do
+          it "should return nil" do
+            expect(@rule_set.check_violation("param_name", "super_safe", {}, true)).to eq(nil)
+          end
+        end
+
+        context "param value does not match anything" do
+          it "should return nil" do
+            expect(@rule_set.check_violation("param_name", "weeee", {}, true)).to eq(nil)
+          end
+        end
+
+        context "param value matches a pattern" do
+          it "should return the match" do
+            match_data = @rule_set.check_violation("param_name", "evil <script>", {}, true)
+            expect(match_data).to eq({"param"=>"param_name", "value"=>"evil <script>", "pattern"=>1})
+          end
+        end
+
+        context "v1_compatability is off" do
+          context "all patterns are disabled" do
+            context "param value contains evil pattern" do
+              it "should return nil" do
+                match_data = @rule_set.check_violation("param_name", "evil <script>", {}, false)
+                expect(match_data).to eq(nil)
+              end
+            end
+          end
+
+          context "one pattern is disabled" do
+            context "evil param_value matches disabled pattern" do
+              it "should return nil" do
+                match_data = @rule_set.check_violation("param_name", "evil <script>", {2 => true}, false)
+                expect(match_data).to eq(nil)
+              end
+            end
+
+            context "evil param_value matches enabled pattern" do
+              it "should return the match" do
+                match_data = @rule_set.check_violation("param_name", "evil <iframe>", {2 => true}, false)
+                expect(match_data).to eq({"param"=>"param_name", "value"=>"evil <iframe>", "pattern"=>2})
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+
+end