tcell_agent 0.2.2 → 0.2.4

data/lib/tcell_agent/sensor_events/discovery.rb

@@ -0,0 +1,30 @@
+require 'tcell_agent/sensor_events/sensor'        
+
+module TCellAgent
+    module SensorEvents
+        class DiscoveryEvent < TCellSensorEvent
+            DATABASE_TYPE = "db"
+            def initialize(route_id=nil)
+                super("discovery")
+                if route_id
+                    self["rid"] = route_id
+                end
+            end
+            def for_database(database, schema, table, field)
+                self["type"] = "db"
+                self["db"] = database
+                self["schema"] = schema
+                self["table"] = table
+                self["field"] = field
+            end
+            def for_database_fields(database, schema, table, fields)
+                self["type"] = "db"
+                self["db"] = database
+                self["schema"] = schema
+                self["table"] = table
+                self["fields"] = fields
+                return self
+            end
+        end #/DiscoveryEvent
+    end #/SensorEvents
+end #/TCellAgent

data/lib/tcell_agent/sensor_events/dlp.rb

@@ -16,9 +16,11 @@ module TCellAgent
             REQUEST_CONTEXT_COOKIE = "cookie"
             REQUEST_CONTEXT_HEADER = "header"
 
-            def initialize(route_id, raw_uri, found_in, hmac_session_id=nil, user_id=nil)
+            def initialize(route_id, raw_uri, found_in, id=nil, hmac_session_id=nil, user_id=nil)
                 super("dlp")
-                self["rid"] = route_id
+                if route_id
+                    self["rid"] = route_id
+                end
                 self["found_in"] = found_in
                 @raw_uri = raw_uri
                 if hmac_session_id
@@ -27,6 +29,9 @@ module TCellAgent
                 if user_id
                     self["uid"] = user_id
                 end
+                if id
+                    self["id"] = id
+                end
             end
             def for_database(database, schema, table, field)
                 self["type"] = "db"
@@ -37,13 +42,13 @@ module TCellAgent
                 return self
             end
             def for_framework(variable)
-                self["type"] = "framework"
+                self["type"] = "fw"
                 self["context"] = "framework"
                 self["variable"] = variable
                 return self
             end
             def for_request(variable_context, variable)
-                self["type"] = "request"
+                self["type"] = "req"
                 self["context"] = variable_context
                 self["variable"] = variable
                 return self

data/lib/tcell_agent/sensor_events/sensor.rb

@@ -24,6 +24,9 @@ module TCellAgent
                 # This is called in the background thread, so any
                 #   santization, analysis, etc doesn't get in the way
             end
+            def bucket_key
+                return nil
+            end
         end
         class TCellHttpTxSensorEvent < TCellSensorEvent
             def initialize(request, response)

data/lib/tcell_agent/sensor_events/server_agent.rb

@@ -1,5 +1,6 @@
 # See the file "LICENSE" for the full license governing this code.
 
+require 'tcell_agent/logger'
 require 'tcell_agent/sensor_events/util/sanitizer_utilities'
 require 'tcell_agent/sensor_events/sensor'
 require 'tcell_agent/sensor_events/util/utils'
@@ -12,10 +13,25 @@ module TCellAgent
                 super("server_agent_details")
                 @flush = true
                 @ensure = true
-                login = Etc.getlogin
-                self["user"] = Etc.getlogin
-                info = Etc.getpwnam(login)
-                self["group"] = info.gid.to_s
+                begin
+                    login = Etc.getlogin
+                    self["user"] = login
+                    TCellAgent.logger.debug("User #{login}")
+                    begin
+                        info = Etc.getpwnam(login)
+                        self["group"] = info.gid.to_s
+                        TCellAgent.logger.debug("Group #{info.gid.to_s}")
+                    rescue Exception => te
+                        TCellAgent.logger.warn("Exception finding group id: #{te.message}")
+                        TCellAgent.logger.debug(te.backtrace)
+                        self["group"] = "unknown"
+                    end
+                rescue Exception => to
+                    self["user"] = "unknown"
+                    self["group"] = "unknown"
+                    TCellAgent.logger.warn("Exception finding user & group: #{to.message}")
+                    TCellAgent.logger.debug(te.backtrace)
+                end
             end
         end
         class ServerAgentAppFrameworkEvent < TCellSensorEvent
@@ -34,8 +50,16 @@ module TCellAgent
                 @ensure = true
                 packages = []
                 Gem.loaded_specs.values.map { |x| 
-                    package = {"n"=>x.name, "v"=>x.version.version}
-                    packages.push(package)
+                    begin
+                        if x.name
+                            package = {"n"=>x.name, "v"=>x.version.version}
+                            packages.push(package)
+                            TCellAgent.logger.debug("Adding packages #{x.name}")
+                        end
+                    rescue Exception => te
+                        TCellAgent.logger.error("Exception adding package: #{te.message}")
+                        TCellAgent.logger.debug(te.backtrace)
+                    end
                 }
                 self["packages"] = packages
             end

data/lib/tcell_agent/sensor_events/util/utils.rb

@@ -14,7 +14,11 @@ module TCellAgent
         end 
       end
       def self.calculateRouteId(method, path, params=nil)
-        jhash("#{method}|#{path}")
+        route_id = jhash("#{method}|#{path}")
+        if (route_id)
+          route_id = route_id.to_s
+        end
+        route_id
       end
     end
   end

data/lib/tcell_agent/start_background_thread.rb

@@ -3,9 +3,9 @@
 require 'tcell_agent/logger'
 require 'tcell_agent/agent'
 require 'tcell_agent/configuration'
+require 'thread'
 
-agent = ::TCellAgent::Agent.new(Process.pid)
-TCellAgent.thread_agent = agent
+TCellAgent.thread_agent.start
 
 # creates a fork and pipes a string from parent to child
 if File.basename($0) != 'rake'
@@ -14,8 +14,8 @@ if File.basename($0) != 'rake'
       (defined?(WEBrick) && (Rack::Handler.default == Rack::Handler::WEBrick))))
     TCellAgent.logger.debug("Initializing background thread: Thin")
     begin
-        TCellAgent.thread_agent = agent
-        agent.start
+        TCellAgent.thread_agent.start
+        #TCellAgent.ensure_event_processor_running
     rescue Exception => e
       TCellAgent.logger.error("Could not start worker.", e.message)
     end
@@ -25,38 +25,43 @@ if File.basename($0) != 'rake'
       TCellAgent.logger.debug("Initializing background thread: Puma Preload")
       puma_worker_start = Proc.new do
         begin
-            agent = TCellAgent::Agent.new(Process.pid)
-            TCellAgent.thread_agent = agent
-            agent.start
+            TCellAgent.thread_agent.start
+            #TCellAgent.ensure_event_processor_running
         rescue Exception => e
-          TCellAgent.logger.error("Could not start worker.", e.message)
+          TCellAgent.logger.error("Could not start worker." + e.message)
         end
       end
       Puma.cli_config.options[:before_worker_boot].push(puma_worker_start)
-  elsif defined?(Unicorn) && Unicorn::HttpServer::LISTENERS.length == 0
-      agent.start_event_processor(false)
+  elsif defined?(Unicorn)
       TCellAgent.logger.debug("Initializing background thread: Unicorn Preload")
       class Unicorn::HttpServer
+        alias _tcell_start spawn_missing_workers
+        def spawn_missing_workers
+          TCellAgent.logger.debug("Stopping (pre-spawn) agent " + Process.pid.to_s)
+          return _tcell_start
+        end
         alias old_init_worker_process init_worker_process
         def init_worker_process(work)
+            start_process = old_init_worker_process(work)
             begin
-                agent = TCellAgent::Agent.new(Process.pid)
-                TCellAgent.thread_agent = agent
-                agent.start
+                TCellAgent.logger.debug("Starting (post-spawn) agent " + Process.pid.to_s)
+                TCellAgent.thread_agent.policy_polling_worker_mutex = Mutex.new
+                TCellAgent.thread_agent.policy_polling_thread = nil
+                TCellAgent.thread_agent.start
+                #TCellAgent.ensure_event_processor_running
             rescue Exception => e
-              TCellAgent.logger.error("Could not start worker.", e.message)
+                TCellAgent.logger.error("Could not start worker.")
             end
-            return old_init_worker_process(work)
+            start_process
         end
       end
   else
-    TCellAgent.logger.debug("Initializing background thread (no-preload).")
-    begin
-        TCellAgent.thread_agent = agent
-        agent.start
-    rescue Exception => e
-      TCellAgent.logger.error("Could not start worker.", e.message)
-    end
+      TCellAgent.logger.debug("Initializing background thread (no-preload).")
+      begin
+          TCellAgent.thread_agent.start
+      rescue Exception => e
+        TCellAgent.logger.error("Could not start worker.")
+      end
   end
 end
 

data/lib/tcell_agent/utils/queue_with_timeout.rb

@@ -5,6 +5,69 @@ require 'thread'
 require 'logger'
 
 module TCellAgent
+  class BoundedQueue
+    def initialize(max_size = :infinite)
+      @lock  = Mutex.new
+      @items = []
+      @item_available = ConditionVariable.new
+      @max_size = max_size
+      @space_available = ConditionVariable.new
+    end
+
+    def push(obj, timeout=:never, &timeout_policy)
+      timeout_policy ||= -> do
+        raise "Push timed out"
+      end
+      wait_for_condition(
+        @space_available,
+        ->{!full?},
+        timeout,
+        timeout_policy) do
+
+        @items.push(obj)
+        @item_available.signal
+      end
+    end
+
+    def pop(timeout = :never, &timeout_policy)
+      timeout_policy ||= ->{nil}
+      wait_for_condition(
+        @item_available,
+        ->{@items.any?},
+        timeout,
+        timeout_policy) do 
+        @items.shift 
+      end
+    end
+
+    def full?
+      return false if @max_size == :infinite
+      @max_size <= @items.size
+    end
+
+    private
+
+    def wait_for_condition(
+        cv, condition_predicate, timeout=:never, timeout_policy=->{nil})
+      deadline = timeout == :never ? :never : Time.now + timeout
+      @lock.synchronize do
+        loop do
+          cv_timeout = timeout == :never ? nil : deadline - Time.now
+          if !condition_predicate.call && cv_timeout.to_f >= 0
+            cv.wait(@lock, cv_timeout) 
+          end
+          if condition_predicate.call
+            return yield
+          elsif deadline == :never || deadline > Time.now
+            next
+          else
+            return timeout_policy.call
+          end
+        end
+      end
+    end
+  end
+
   class QueueWithTimeout
     def initialize
       @mutex = Mutex.new
@@ -51,7 +114,8 @@ module TCellAgent
         if @queue.empty?
           @recieved.wait(@mutex, timeout) if timeout != 0
           #if we're still empty after the timeout, raise exception
-          raise ThreadError, "queue empty" if @queue.empty?
+          return nil if @queue.empty?
+          #raise ThreadError, "queue empty" if @queue.empty?
         end
         @queue.shift
       end

data/lib/tcell_agent/version.rb

@@ -1,5 +1,5 @@
 # See the file "LICENSE" for the full license governing this code.
 
 module TCellAgent
-  VERSION = "0.2.2"
+  VERSION = "0.2.4"
 end

data/spec/lib/tcell_agent/instrumentation_spec.rb

@@ -0,0 +1,198 @@
+require 'spec_helper'
+
+module TCellAgent
+  class MockAgent < Agent
+  end
+end
+
+
+module TCellAgent
+  module Instrumentation
+    describe Instrumentation do
+        context "Body - SessionId Filters" do
+          it "Tests Redaction and Events in Body" do
+            action = TCellAgent::Policies::DataLossPolicy::FilterActions.new
+            action.body_redact = true
+            action.action_id = 5
+            policy_json_two = {
+                "policy_id"=>"x1a1",
+                "data"=>{
+                    "session_id_protection"=>{"body"=>["redact"], "log"=>["event"]}
+                }
+            }
+            session_id_policy = TCellAgent::Policies::DataLossPolicy.fromJson(policy_json_two)
+            mock_agent = MockAgent.new(-1)
+            mock_agent.policies[TCellAgent::PolicyTypes::DataLoss] = session_id_policy
+            TCellAgent.set_thread_agent(mock_agent)
+
+            context = TCellData.new
+            context.session_id = "tim123123my"
+
+            body = "this is about tim123123my 3123123."
+            TCellAgent.empty_event_queue
+            context.filter_body(body)
+            expect(body).to eq("this is about [redacted] 3123123.")
+            expect(TCellAgent.event_queue.length).to eq(1)
+            TCellAgent.set_thread_agent(nil)
+          end
+          it "Tests Events in Body" do
+            action = TCellAgent::Policies::DataLossPolicy::FilterActions.new
+            action.body_redact = true
+            action.action_id = 5
+            policy_json_two = {
+                "policy_id"=>"x1a1",
+                "data"=>{
+                    "session_id_protection"=>{"body"=>["event"], "log"=>["redact"]}
+                }
+            }
+            session_id_policy = TCellAgent::Policies::DataLossPolicy.fromJson(policy_json_two)
+            mock_agent = MockAgent.new(-1)
+            mock_agent.policies[TCellAgent::PolicyTypes::DataLoss] = session_id_policy
+            TCellAgent.set_thread_agent(mock_agent)
+
+            context = TCellData.new
+            context.session_id = "tim123123my"
+
+            body = "this is about tim123123my 3123123."
+            TCellAgent.empty_event_queue
+            context.filter_body(body)
+            expect(body).to eq("this is about tim123123my 3123123.")
+            expect(TCellAgent.event_queue.length).to eq(1)
+            TCellAgent.set_thread_agent(nil)
+          end
+        end
+        context "Log - SessionId Filters" do
+          it "Tests Redaction and Events in Body" do
+            action = TCellAgent::Policies::DataLossPolicy::FilterActions.new
+            action.body_redact = true
+            action.action_id = 5
+            policy_json_two = {
+                "policy_id"=>"x1a1",
+                "data"=>{
+                    "session_id_protection"=>{"body"=>["redact"], "log"=>["redact"]}
+                }
+            }
+            session_id_policy = TCellAgent::Policies::DataLossPolicy.fromJson(policy_json_two)
+            mock_agent = MockAgent.new(-1)
+            mock_agent.policies[TCellAgent::PolicyTypes::DataLoss] = session_id_policy
+            TCellAgent.set_thread_agent(mock_agent)
+
+            context = TCellData.new
+            context.session_id = "tim123123my"
+
+            body = "this is about tim123123my 3123123."
+            TCellAgent.empty_event_queue
+            context.filter_log(body)
+            expect(body).to eq("this is about [redacted] 3123123.")
+            expect(TCellAgent.event_queue.length).to eq(1)
+            TCellAgent.set_thread_agent(nil)
+          end
+          it "Tests Events Only" do
+            action = TCellAgent::Policies::DataLossPolicy::FilterActions.new
+            action.body_redact = true
+            action.action_id = 5
+            policy_json_two = {
+                "policy_id"=>"x1a1",
+                "data"=>{
+                    "session_id_protection"=>{"body"=>["redact"], "log"=>["event"]}
+                }
+            }
+            session_id_policy = TCellAgent::Policies::DataLossPolicy.fromJson(policy_json_two)
+            mock_agent = MockAgent.new(-1)
+            mock_agent.policies[TCellAgent::PolicyTypes::DataLoss] = session_id_policy
+            TCellAgent.set_thread_agent(mock_agent)
+
+            context = TCellData.new
+            context.session_id = "tim123123my"
+
+            body = "this is about tim123123my 3123123."
+            TCellAgent.empty_event_queue
+            context.filter_log(body)
+            expect(body).to eq("this is about tim123123my 3123123.")
+            expect(TCellAgent.event_queue.length).to eq(1)
+            TCellAgent.set_thread_agent(nil)
+          end
+        end
+        context "Body - Database Filters" do
+          it "Tests Redaction and Events in Body" do
+            action = TCellAgent::Policies::DataLossPolicy::FilterActions.new
+            action.body_redact = true
+            action.action_id = 5
+            context = TCellData.new
+            context.add_response_db_filter("timmy", action, "don", "sam", "tim", "fred")
+            context.add_response_db_filter("timmy23", action, "don", "sam", "tim", "fred")
+            context.add_response_db_filter("3123123", action, "don", "sam", "tim", "fred")
+            context.add_response_db_filter("tim123my", action, "don", "sam", "tim", "fred")
+            context.add_response_db_filter("timmy1", action, "don", "sam", "tim", "fred")
+            context.add_response_db_filter("tim123123my", action, "don", "sam", "tim", "fred")
+            context.add_response_db_filter("ti21312mmy", action, "don", "sam", "tim", "fred")
+            context.add_response_db_filter("ti123123mmy", action, "don", "sam", "tim", "fred")
+            body = "this is about timmy1 3123123."
+            TCellAgent.empty_event_queue
+            context.filter_body(body)
+            expect(body).to eq("this is about [redacted] [redacted].")
+            expect(TCellAgent.event_queue.length).to eq(2)
+          end
+          it "Tests Event Only Match in Body" do
+            action = TCellAgent::Policies::DataLossPolicy::FilterActions.new
+            action.body_event = true
+            action.action_id = 5
+            context = TCellData.new
+            context.add_response_db_filter("timmy", action, "don", "sam", "tim", "fred")
+            context.add_response_db_filter("timmy23", action, "don", "sam", "tim", "fred")
+            context.add_response_db_filter("3123123", action, "don", "sam", "tim", "fred")
+            context.add_response_db_filter("tim123my", action, "don", "sam", "tim", "fred")
+            context.add_response_db_filter("timmy1", action, "don", "sam", "tim", "fred")
+            context.add_response_db_filter("tim123123my", action, "don", "sam", "tim", "fred")
+            context.add_response_db_filter("ti21312mmy", action, "don", "sam", "tim", "fred")
+            context.add_response_db_filter("ti123123mmy", action, "don", "sam", "tim", "fred")
+            body = "this is about timmy1 3123123."
+            TCellAgent.empty_event_queue
+            context.filter_body(body)
+            expect(body).to eq("this is about timmy1 3123123.")
+            expect(TCellAgent.event_queue.length).to eq(3) # timmy, timmy1, 3123123
+          end
+        end
+        context "Log - Database Filters" do
+          it "Tests Redaction and Events" do
+            action = TCellAgent::Policies::DataLossPolicy::FilterActions.new
+            action.log_redact = true
+            action.action_id = 5
+            context = TCellData.new
+            context.add_response_db_filter("timmy", action, "don", "sam", "tim", "fred")
+            context.add_response_db_filter("timmy23", action, "don", "sam", "tim", "fred")
+            context.add_response_db_filter("3123123", action, "don", "sam", "tim", "fred")
+            context.add_response_db_filter("tim123my", action, "don", "sam", "tim", "fred")
+            context.add_response_db_filter("timmy1", action, "don", "sam", "tim", "fred")
+            context.add_response_db_filter("tim123123my", action, "don", "sam", "tim", "fred")
+            context.add_response_db_filter("ti21312mmy", action, "don", "sam", "tim", "fred")
+            context.add_response_db_filter("ti123123mmy", action, "don", "sam", "tim", "fred")
+            body = "this is about timmy1 3123123."
+            TCellAgent.empty_event_queue
+            context.filter_log(body)
+            expect(body).to eq("this is about [redacted] [redacted].")
+            expect(TCellAgent.event_queue.length).to eq(2)
+          end
+          it "Tests Report-Only and Events" do
+            action = TCellAgent::Policies::DataLossPolicy::FilterActions.new
+            action.log_event = true
+            action.action_id = 5
+            context = TCellData.new
+            context.add_response_db_filter("timmy", action, "don", "sam", "tim", "fred")
+            context.add_response_db_filter("timmy23", action, "don", "sam", "tim", "fred")
+            context.add_response_db_filter("3123123", action, "don", "sam", "tim", "fred")
+            context.add_response_db_filter("tim123my", action, "don", "sam", "tim", "fred")
+            context.add_response_db_filter("timmy1", action, "don", "sam", "tim", "fred")
+            context.add_response_db_filter("tim123123my", action, "don", "sam", "tim", "fred")
+            context.add_response_db_filter("ti21312mmy", action, "don", "sam", "tim", "fred")
+            context.add_response_db_filter("ti123123mmy", action, "don", "sam", "tim", "fred")
+            body = "this is about timmy1 3123123."
+            TCellAgent.empty_event_queue
+            context.filter_log(body)
+            expect(body).to eq("this is about timmy1 3123123.")
+            expect(TCellAgent.event_queue.length).to eq(3) # timmy, timmy1, 3123123
+          end
+        end
+      end
+  end
+end