tcell_agent 0.2.8 → 0.2.9

data/lib/tcell_agent/sensor_events/sensor.rb

@@ -44,13 +44,16 @@ module TCellAgent
             end
         end
         class TCellRedirectSensorEvent < TCellSensorEvent
-            def initialize(redirect_domain, original_domain, original_url, method, status_code, remote_addr, user_id=nil, session_id=nil)
+            def initialize(redirect_domain, original_domain, original_url, method, route_id, status_code, remote_addr, session_id=nil, user_id=nil)
                 super("redirect")
                 @raw_original_url = original_url
                 self["method"] = method
                 self["from_domain"] = original_domain
                 self["status_code"] = status_code
                 self["remote_addr"] = remote_addr
+                if route_id
+                    self["rid"] = route_id
+                end
                 @raw_redirect_domain = redirect_domain
                 @user_id = user_id
                 @raw_session_id = session_id

data/lib/tcell_agent/servers/thin.rb

@@ -1,3 +1,4 @@
+require("tcell_agent/servers/unicorn") if defined?(Unicorn::HttpServer)
 
 Thin::Server.class_eval do
 

data/lib/tcell_agent/servers/unicorn.rb

@@ -18,6 +18,80 @@ Unicorn::HttpServer.class_eval do
     original_join
   end
 
+  # This gets called when unicorn receives the HUP signal to reload its config.
+  # Tcell also needs to ensure its config is reloaded and services are started
+  # or stopped accordingly
+  alias_method :original_load_config!, :load_config!
+  def load_config!
+    original_load_config!
+
+    TCellAgent::Instrumentation.safe_block("Reloading Tcell Config") do
+      new_config = TCellAgent::Configuration.new
+      TCellAgent.logger.debug("Reloading config")
+      TCellAgent.logger.debug(
+        "ENABLED:#{new_config.enabled}" +
+        "|ENABLE_EVENT_MANAGER:#{new_config.enable_event_manager}" +
+        "|ENABLE_EVENT_CONSUMER:#{new_config.enable_event_consumer}" +
+        "|ENABLE_POLICY_POLLING:#{new_config.enable_policy_polling}" +
+        "|ENABLE_INSTRUMENTATION:#{new_config.enable_instrumentation}" +
+        "|ENABLE_INTERCEPT_REQUESTS:#{new_config.enable_intercept_requests}"
+      )
+      old_config = TCellAgent.configuration
+
+      TCellAgent.configuration = new_config
+
+      if new_config.enabled ^ old_config.enabled
+        if new_config.enabled
+          TCellAgent.run_instrumentation("Unicorn")
+
+        else
+          TCellAgent.thread_agent.stop_event_processor
+          TCellAgent.thread_agent.stop_metrics_event_thread
+          TCellAgent.thread_agent.stop_policy_polling
+        end
+      end
+
+      if new_config.enable_event_manager ^ old_config.enable_event_manager
+        if new_config.enable_event_manager
+          TCellAgent.run_instrumentation("Unicorn Restart")
+        else
+          TCellAgent.thread_agent.stop_event_processor
+        end
+      else
+        # Just in case
+        if new_config.enable_event_manager
+          TCellAgent.thread_agent.ensure_event_processor_running
+        end
+      end
+
+      if new_config.enable_event_consumer ^ old_config.enable_event_consumer
+        if new_config.enable_event_consumer
+          TCellAgent.thread_agent.ensure_metrics_event_thread_running
+        else
+          TCellAgent.thread_agent.stop_metrics_event_thread
+        end
+      else
+        # Just in case
+        if new_config.enable_event_consumer
+          TCellAgent.thread_agent.ensure_metrics_event_thread_running
+        end
+      end
+
+      if new_config.enable_policy_polling ^ old_config.enable_policy_polling
+        if new_config.enable_policy_polling
+          TCellAgent.thread_agent.ensure_policy_polling_running
+        else
+          TCellAgent.thread_agent.stop_policy_polling
+        end
+      else
+        # Just in case
+        if new_config.enable_policy_polling
+          TCellAgent.thread_agent.ensure_policy_polling_running
+        end
+      end
+    end
+  end
+
   # this only runs when preload_app=true because when preload_app=false
   # the gems aren't loaded early enough for tcell to override
   # the class definitions
@@ -25,13 +99,15 @@ Unicorn::HttpServer.class_eval do
   def init_worker_process(work)
     start_process = original_init_worker_process(work)
 
-    begin
-      TCellAgent.thread_agent.policy_polling_worker_mutex = Mutex.new
-      TCellAgent.thread_agent.policy_polling_thread = nil
-      TCellAgent.thread_agent.start
+    if TCellAgent.configuration.enabled && TCellAgent.configuration.should_instrument?
+      begin
+        TCellAgent.thread_agent.policy_polling_worker_mutex = Mutex.new
+        TCellAgent.thread_agent.policy_polling_thread = nil
+        TCellAgent.thread_agent.start
 
-    rescue Exception => e
-      TCellAgent.logger.error("Could not start thread agent. #{e.message}")
+      rescue Exception => e
+        TCellAgent.logger.error("Could not start thread agent. #{e.message}")
+      end
     end
 
     start_process

data/lib/tcell_agent/start_background_thread.rb

@@ -5,22 +5,22 @@ require 'tcell_agent/agent'
 require 'tcell_agent/configuration'
 require 'thread'
 
-if TCellAgent.configuration.enabled
-  module TCellAgent
-    #require 'tcell_agent/sinatra' if defined?(Sinatra)
-    require 'tcell_agent/rails' if defined?(Rails)
+module TCellAgent
+  #require 'tcell_agent/sinatra' if defined?(Sinatra)
+  require 'tcell_agent/rails' if defined?(Rails)
 
-    def self.run_instrumentation(server_name)
+  def self.run_instrumentation(server_name)
 
-      require 'tcell_agent/rails/on_start' if defined?(Rails)
+    require 'tcell_agent/rails/on_start' if defined?(Rails)
 
-      begin
-        TCellAgent.logger.debug("Instrumenting: #{server_name}")
-        TCellAgent.thread_agent.start
-      rescue Exception => e
-        TCellAgent.logger.error("Could not start thread agent. #{e.message}")
-      end
+    begin
+      TCellAgent.logger.debug("Instrumenting: #{server_name}")
+      TCellAgent.thread_agent.start
+    rescue Exception => e
+      TCellAgent.logger.error("Could not start thread agent. #{e.message}")
+    end
 
+    if TCellAgent.configuration.should_instrument?
       Thread.abort_on_exception = TCellAgent.configuration.raise_exceptions
       Thread.new do
 
@@ -34,36 +34,34 @@ if TCellAgent.configuration.enabled
           TCellAgent.send_event(event)
         end
 
-        if (TCellAgent.configuration.enabled && TCellAgent.configuration.should_instrument? && defined?(Rails))
+        if defined?(Rails)
           TCellAgent::Instrumentation.safe_block("Instrumenting routes") do
             TCellAgent::Instrumentation::Rails.instrument_routes
           end
         end
-
       end
 
     end
 
   end
 
-  tcell_server = ENV["TCELL_AGENT_SERVER"]
+end
+
+tcell_server = ENV["TCELL_AGENT_SERVER"]
 
+if TCellAgent.configuration.should_instrument?
   if (!(tcell_server && tcell_server == "mock"))
 
     if (tcell_server && tcell_server == "webrick") || defined?(Rails::Server)
-
       require("tcell_agent/servers/rails_server")
 
     elsif (tcell_server && tcell_server == "thin") || defined?(Thin)
-
       require("tcell_agent/servers/thin")
 
     elsif (tcell_server && tcell_server == "puma") || defined?(Puma)
-
       require("tcell_agent/servers/puma")
 
     elsif (tcell_server && tcell_server == "unicorn") || defined?(Unicorn)
-
       require("tcell_agent/servers/unicorn")
 
     else
@@ -74,4 +72,11 @@ if TCellAgent.configuration.enabled
       puts "[tCell.io] **********************************************************************"
     end
   end
+
+else
+
+  # unicorn is always instrumented to support rolling restarts
+  if (tcell_server && tcell_server == "unicorn") || defined?(Unicorn)
+    require("tcell_agent/servers/unicorn")
+  end
 end

data/lib/tcell_agent/version.rb

@@ -1,5 +1,5 @@
 # See the file "LICENSE" for the full license governing this code.
 
 module TCellAgent
-  VERSION = "0.2.8"
+  VERSION = "0.2.9"
 end

data/spec/lib/tcell_agent/agent/policy_manager_spec.rb

@@ -0,0 +1,218 @@
+require 'spec_helper'
+
+module TCellAgent
+  module SensorEvents
+
+    describe Agent do
+
+      describe "#cache" do
+        context "with an existing cached file" do
+
+          context "with two processes" do
+            context "while one process is writing to the cached file" do
+              before(:each) do
+                TCellAgent.thread_agent.cache(
+                  "http-redirect",
+                  {
+                    "app_id"=>"raftest-EyJZR",
+                    "policy_id"=>"363b8b60-a9a8-11e5-bb10-a9372a56b8a7",
+                    "data"=> {
+                      "enabled"=>true,
+                      "block"=>false,
+                      "whitelist"=>[]
+                    }
+                  }
+                )
+              end
+
+              it "should raise a timeout exception when the other process tries to write to it" do
+                file_lock_holder = ForkBreak::Process.new do |breakpoints|
+
+                  original_dump = JSON.method(:dump)
+                  JSON.stub(:dump) do |*args|
+                    breakpoints << :before_dump
+                    original_dump.call(*args)
+                  end
+
+                  TCellAgent.thread_agent.cache(
+                    "http-redirect",
+                    {
+                      "app_id"=>"raftest-EyJZR",
+                      "policy_id"=>"363b8b60-a9a8-11e5-bb10-a9372a56b8a7",
+                      "data"=> {
+                        "enabled"=>true,
+                        "block"=>false,
+                        "whitelist"=>[]
+                      }
+                    }
+                  )
+                end
+
+                file_lock_holder.run_until(:before_dump).wait
+
+                logger = double("logger")
+                expect(TCellAgent).to receive(:logger).and_return(logger)
+                expect(logger).to receive(:error).with("execution expired")
+
+                TCellAgent.thread_agent.cache(
+                  "http-redirect",
+                  {
+                    "app_id"=>"raftest-EyJZR",
+                    "policy_id"=>"363b8b60-a9a8-11e5-bb10-a9372a56b8a7",
+                    "data"=> {
+                      "enabled"=>true,
+                      "block"=>false,
+                      "whitelist"=>[]
+                    }
+                  }
+                )
+
+                file_lock_holder.finish.wait
+              end
+
+
+              after(:each) do
+                File.delete(TCellAgent.configuration.cache_filename_with_app_id)
+              end
+            end
+          end
+        end
+
+        context "with 10 processes updating the cached file" do
+          it "should update the cached file with all updates" do
+            processes = 5.times.map do |process_number|
+              ForkBreak::Process.new do |breakpoints|
+                original_dump = JSON.method(:dump)
+                JSON.stub(:dump) do |*args|
+                  breakpoints << :before_dump
+                  original_dump.call(*args)
+                end
+
+                TCellAgent.thread_agent.cache(
+                  "process_#{process_number}",
+                  {
+                    "app_id"=>"raftest-EyJZR",
+                    "policy_id"=>"policy_id",
+                    "data"=> { "enabled"=>true }
+                  }
+                )
+
+              end
+            end
+
+            processes.map(&:finish).map(&:wait)
+
+            policies = JSON.parse(open(TCellAgent.configuration.cache_filename_with_app_id).read)
+
+            File.delete(TCellAgent.configuration.cache_filename_with_app_id)
+
+            expect(policies).to eq({
+              "process_0"=>{"app_id"=>"raftest-EyJZR", "policy_id"=>"policy_id", "data"=>{"enabled"=>true}},
+              "process_1"=>{"app_id"=>"raftest-EyJZR", "policy_id"=>"policy_id", "data"=>{"enabled"=>true}},
+              "process_2"=>{"app_id"=>"raftest-EyJZR", "policy_id"=>"policy_id", "data"=>{"enabled"=>true}},
+              "process_3"=>{"app_id"=>"raftest-EyJZR", "policy_id"=>"policy_id", "data"=>{"enabled"=>true}},
+              "process_4"=>{"app_id"=>"raftest-EyJZR", "policy_id"=>"policy_id", "data"=>{"enabled"=>true}}
+            })
+          end
+        end
+      end
+
+      describe "#policies_from_cachefile" do
+        context "with no cache file" do
+          it "should return nil" do
+
+            expect(File).to receive(:exist?).with(/tcell\/cache\/tcell_agent.cache/).and_return(false)
+            expect_any_instance_of(TCellAgent::Agent).to_not receive(:processPolicyJson)
+            agent = TCellAgent::Agent.new(Process.pid)
+
+            expect(agent.policies).to eq({})
+          end
+        end
+
+        context "with a cache file present" do
+          context "that is empty" do
+            it "should raise a json error" do
+              cache_file = double("cache_file")
+              expect(File).to receive(:exist?).with(/tcell\/cache\/tcell_agent.cache/).and_return(true)
+              expect(File).to receive(:open).and_return(cache_file)
+              expect(cache_file).to receive(:flock).and_return(true)
+              expect(cache_file).to receive(:read).and_return("")
+              expect(cache_file).to receive(:close)
+
+              logger = double("logger")
+              expect(TCellAgent).to receive(:logger).and_return(logger)
+              expect(logger).to receive(:error).with("A JSON text must at least contain two octets!")
+
+              expect_any_instance_of(TCellAgent::Agent).to_not receive(:processPolicyJson)
+
+              agent = TCellAgent::Agent.new(Process.pid)
+
+              expect(agent.policies).to eq({})
+            end
+          end
+
+          context "that has content" do
+            context "that is malformed" do
+              it "should raise a json error" do
+                cache_file = double("cache_file")
+                expect(File).to receive(:exist?).with(/tcell\/cache\/tcell_agent.cache/).and_return(true)
+                expect(File).to receive(:open).and_return(cache_file)
+                expect(cache_file).to receive(:flock).and_return(true)
+                expect(cache_file).to receive(:read).and_return("bad_json")
+                expect(cache_file).to receive(:close)
+
+                logger = double("logger")
+                expect(TCellAgent).to receive(:logger).and_return(logger)
+                expect(logger).to receive(:error).with("757: unexpected token at 'bad_json'")
+                expect_any_instance_of(TCellAgent::Agent).to_not receive(:processPolicyJson)
+
+                agent = TCellAgent::Agent.new(Process.pid)
+
+                expect(agent.policies).to eq({})
+              end
+            end
+
+            context "that is an empty json object" do
+              it "should raise a json error" do
+                cache_file = double("cache_file")
+                expect(File).to receive(:exist?).with(/tcell\/cache\/tcell_agent.cache/).and_return(true)
+                expect(File).to receive(:open).and_return(cache_file)
+                expect(cache_file).to receive(:flock).and_return(true)
+                expect(cache_file).to receive(:read).and_return("{}")
+                expect(cache_file).to receive(:close)
+
+                expect(TCellAgent).to_not receive(:logger)
+
+                expect_any_instance_of(TCellAgent::Agent).to receive(:processPolicyJson).with({}, false)
+
+                agent = TCellAgent::Agent.new(Process.pid)
+              end
+            end
+
+            context "that is a valid policy" do
+              it "should set the policies" do
+                cache_file = double("cache_file")
+                expect(File).to receive(:exist?).with(/tcell\/cache\/tcell_agent.cache/).and_return(true)
+                expect(File).to receive(:open).and_return(cache_file)
+                expect(cache_file).to receive(:flock).and_return(true)
+                expect(cache_file).to receive(:read).and_return(
+                  {
+                    process_0: {app_id: "raftest-EyJZR", policy_id: "policy_id", data: {enabled: true}}
+                  }.to_json
+                )
+                expect(cache_file).to receive(:close)
+                expect_any_instance_of(TCellAgent::Agent).to receive(:processPolicyJson).with(
+                  {"process_0"=>{"app_id"=>"raftest-EyJZR", "policy_id"=>"policy_id", "data"=>{"enabled"=>true}}},
+                  false
+                )
+
+                TCellAgent::Agent.new(Process.pid)
+              end
+            end
+          end
+        end
+      end
+    end
+
+  end
+end

data/spec/lib/tcell_agent/policies/http_redirect_policy_spec.rb

@@ -46,14 +46,14 @@ module TCellAgent
           http_redirect_from_json.enabled = false
           http_redirect_from_json.block = true
           http_redirect_from_json.whitelist = ["*.google.com"]
-          result = http_redirect_from_json.enforce("https://test.google.com", "www.test.com", "/path/a", "GET", "1.1.1.1", 400)
+          result = http_redirect_from_json.enforce("https://test.google.com", "www.test.com", "/path/a", "GET", "routex", "1.1.1.1", 400)
           expect(result).to eq(nil)
         end
         it "domain enforce enabled true, block true" do
           http_redirect_from_json.enabled = true
           http_redirect_from_json.block = true
           http_redirect_from_json.whitelist = ["good.com"]
-          result = http_redirect_from_json.enforce("https://www.google.com/abc/def", "localhost", "/path/a", "GET", "1.1.1.1", 400)
+          result = http_redirect_from_json.enforce("https://www.google.com/abc/def", "localhost", "/path/a", "GET", "routey", "1.1.1.1", 400)
           expect(result).to eq("/")
         end
       end