tcell_agent 1.0.0 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e3bfdacf0f493764e58e984f770dc53d04787a98
-  data.tar.gz: 27c3464c78adda7a3224e5ea080e2615ff857975
+  metadata.gz: 2b7c816872f23014224453ddbea3a8b582ab52a2
+  data.tar.gz: 2dcf9eff93ea31bbc2ebb51d759311769eaf3f91
 SHA512:
-  metadata.gz: ef098d1aeb0feb1398c0ce002d706d0cef7b1a82ce1f9dd600eda39230e2bd718e60800ee3c9d7bb23ed345e8728f8fefb4b6bd5bda59a379868aea5f63ae0c7
-  data.tar.gz: d320becd35d19b4e124dfa04d703264b538191ccaf27c699fbdfde939769c4b8e97ff9ca6ac0cca16a47fe4d71ccdba83feb28c29eae7ff4a218aadfd29a7994
+  metadata.gz: 5b835851060db61f226730552a94b08ea3ccd520ac9c7f67edb2cdfe60cc05f475ebbc1f8fe1ae7426f3d5c563e5ca09eb9f521c622e31a94bad58571441d036
+  data.tar.gz: 0a9e816f3a2c6abf942c9536a8cfb0ce9c8af853f8a6f554d3c25cda43153d15322416260598833da575a150d071ac89b444b3e488821d61a90a3aeaf7c988d5

data/Readme.txt

@@ -0,0 +1,7 @@
+Config goes in config/tcell_agent.config
+Fill in API key, Company Name, App Name
+
+You can add 
+        "tcell_api_url":"http://10.0.2.2:8000/api/v1",
+        "tcell_input_url":"http://10.0.2.2:3000"
+to specify other servers to use

data/bin/tcell_agent

@@ -254,8 +254,12 @@ elsif command == 'test'
 
   printf '%-50s', 'Make test API call for policies... '
   api = TCellAgent::TCellApi.new
-  api.poll_api
-  puts 'passed'
+  if api.poll_api
+    puts 'passed'
+  else
+    puts 'failed'
+    Kernel.exit(1)
+  end
 
   printf '%-50s', 'Sending a Test event... '
   send_succeeded = api.send_event_set([])

data/lib/tcell_agent.rb

@@ -8,11 +8,8 @@ require 'tcell_agent/configuration'
 
 require 'tcell_agent/agent'
 
-require 'tcell_agent/policies/content_security_policy'
 require 'tcell_agent/policies/http_tx_policy'
 require 'tcell_agent/policies/http_redirect_policy'
-require 'tcell_agent/policies/secure_headers_policy'
-require 'tcell_agent/policies/clickjacking_policy'
 require 'tcell_agent/policies/login_fraud_policy'
 require 'tcell_agent/policies/dataloss_policy'
 

data/lib/tcell_agent/agent/event_processor.rb

@@ -7,11 +7,8 @@ require "tcell_agent/version"
 require "tcell_agent/api"
 require "tcell_agent/configuration"
 
-require "tcell_agent/policies/content_security_policy"
-require "tcell_agent/policies/clickjacking_policy"
 require "tcell_agent/policies/http_tx_policy"
 require "tcell_agent/policies/http_redirect_policy"
-require "tcell_agent/policies/secure_headers_policy"
 
 require "tcell_agent/sensor_events/server_agent"
 require "tcell_agent/sensor_events/metrics"
@@ -34,7 +31,7 @@ module TCellAgent
     # Startup scripts are likely to run shell commands. It's not a good idea to startup the event
     # processor before worker processses are forked, so the safest thing to do is let a different
     # event start the event processor to avoid deadlocking worker processes.
-    def is_it_safe_to_send_cmdi_events?()
+    def safe_to_send_cmdi_events?()
       event_processor_running?
     end
 

data/lib/tcell_agent/agent/policy_manager.rb

@@ -9,11 +9,8 @@ require "tcell_agent/configuration"
 
 require "tcell_agent/agent/policy_types"
 
-require "tcell_agent/policies/content_security_policy"
-require "tcell_agent/policies/clickjacking_policy"
 require "tcell_agent/policies/http_tx_policy"
 require "tcell_agent/policies/http_redirect_policy"
-require "tcell_agent/policies/secure_headers_policy"
 
 require "tcell_agent/sensor_events/server_agent"
 
@@ -100,10 +97,6 @@ module TCellAgent
 
           return [failure_sleep_time, last_poll_time]
 
-        elsif policy_jsons.key?("last_timestamp")
-          if policy_jsons["last_timestamp"] != 0
-            last_poll_time = policy_jsons["last_timestamp"]
-          end
         elsif policy_jsons.key?("last_id")
           if policy_jsons["last_id"] != 0
             last_poll_time = policy_jsons["last_id"]
@@ -148,7 +141,11 @@ module TCellAgent
 
       if cache_the_policy
         (TCellAgent::PolicyTypes::ClassMap.keys +
-        [TCellAgent::PolicyTypes::AppSensor,
+        [TCellAgent::PolicyTypes::CSP,
+         TCellAgent::PolicyTypes::Clickjacking,
+         TCellAgent::PolicyTypes::SecureHeaders,
+         TCellAgent::PolicyTypes::JSAgentInjection,
+         TCellAgent::PolicyTypes::AppSensor,
          TCellAgent::PolicyTypes::Patches,
          TCellAgent::PolicyTypes::CommandInjection,
          TCellAgent::PolicyTypes::Regex]).each do |policy_type|

data/lib/tcell_agent/agent/policy_types.rb

@@ -2,12 +2,8 @@
 
 # See the file "LICENSE" for the full license governing this code.
 
-require "tcell_agent/policies/content_security_policy"
-require "tcell_agent/policies/clickjacking_policy"
-
 require "tcell_agent/policies/http_tx_policy"
 require "tcell_agent/policies/http_redirect_policy"
-require "tcell_agent/policies/secure_headers_policy"
 require "tcell_agent/policies/login_fraud_policy"
 require "tcell_agent/policies/dataloss_policy"
 require "tcell_agent/policies/rust_policies"
@@ -26,11 +22,9 @@ module TCellAgent
     CommandInjection = "cmdi"
     Regex = "regex"
     Rust = "rust"
+    JSAgentInjection = "jsagentinjection"
 
     ClassMap = {
-      CSP=>TCellAgent::Policies::ContentSecurityPolicy,
-      Clickjacking=>TCellAgent::Policies::ClickjackingPolicy,
-      SecureHeaders=>TCellAgent::Policies::SecureHeadersPolicy,
       HttpTx=>TCellAgent::Policies::HttpTxPolicy,
       HttpRedirect=>TCellAgent::Policies::HttpRedirectPolicy,
       LoginFraud=>TCellAgent::Policies::LoginFraudPolicy,

data/lib/tcell_agent/agent/static_agent.rb

@@ -61,7 +61,7 @@ module TCellAgent
     self.thread_agent.ensure_event_processor_running
   end
 
-  def self.is_it_safe_to_send_cmdi_events?
-    self.thread_agent.is_it_safe_to_send_cmdi_events?
+  def self.safe_to_send_cmdi_events?
+    self.thread_agent.safe_to_send_cmdi_events?
   end
 end

data/lib/tcell_agent/api.rb

@@ -14,15 +14,16 @@ module TCellAgent
     def initialize
     end
 
-    def poll_api(last_timestamp=nil)
+    def poll_api(last_id=nil)
       if !TCellAgent.configuration || !TCellAgent.configuration.tcell_api_url || !TCellAgent.configuration.app_id
         raise TCellAgent::ConfigurationException.new("Config Information Not Found, can't poll for policy updates")
       end
 
-      full_url = TCellAgent.configuration.tcell_api_url + "/app/" + TCellAgent.configuration.app_id + "/update"
-      if (last_timestamp && last_timestamp != "")
-        full_url = full_url + "?last_timestamp=" + last_timestamp.to_s
-      end
+      full_url = TCellAgent.configuration.tcell_api_url.sub(
+        '{app_id}',
+        TCellAgent.configuration.app_id
+      )
+      full_url += "&last_id=#{last_id.to_s}" if last_id
 
       TCellAgent.logger.debug "tCell.io API Request: " + full_url
 
@@ -39,10 +40,7 @@ module TCellAgent
 
       if res.is_a?(Net::HTTPSuccess)
         TCellAgent.logger.debug("tCell.io API Response: #{res.body}".force_encoding("UTF-8"))
-        response_json = JSON.parse(res.body)
-        if (response_json && response_json.has_key?("result"))
-          return response_json["result"]
-        end
+        return JSON.parse(res.body)
       else
         TCellAgent.logger.error("Received error response while contacting api: #{res.inspect}")
       end

data/lib/tcell_agent/configuration.rb

@@ -21,8 +21,12 @@ module TCellAgent
   end
 
   class Configuration
-    attr_accessor :version, :app_id, :api_key, :hmac_key,
-      :tcell_api_url, :tcell_input_url,
+    attr_accessor :version,
+      :app_id,
+      :api_key,
+      :hmac_key,
+      :tcell_api_url,
+      :tcell_input_url,
       :logging_options,
       :logger,
       :appfirewall_payloads_logger, # appfirewall_payloads_logger can be specified from initializers
@@ -156,8 +160,8 @@ module TCellAgent
       read_config_using_env
 
       if @demomode
-        @event_batch_size_limit = 2
-        @event_time_limit_seconds = 5
+        @event_batch_size_limit = 1
+        @event_time_limit_seconds = 2
       end
 
       if ENV["TCELL_AGENT_ALLOW_UNENCRYPTED_APPSENSOR_PAYLOADS"]
@@ -178,9 +182,9 @@ module TCellAgent
         @allow_payloads = [true, "true", "yes", "1"].include?(ENV["TCELL_AGENT_ALLOW_PAYLOADS"])
       end
 
-      @tcell_api_url ||= "https://api.tcell.io/api/v1"
+      @tcell_api_url = compose_api_url
       @tcell_input_url ||= "https://input.tcell.io/api/v1"
-      @js_agent_api_base_url ||= nil
+      @js_agent_api_base_url ||= @tcell_api_url
       @js_agent_url ||= "https://jsagent.tcell.io/tcellagent.min.js"
 
       if (@host_identifier == nil)
@@ -194,6 +198,38 @@ module TCellAgent
       @uuid = SecureRandom.uuid
     end
 
+    def compose_api_url
+      @tcell_api_url ||= "https://api.tcell.io"
+      parsed_uri = URI.parse(@tcell_api_url)
+
+      api_url = [
+        parsed_uri.scheme,
+        "://",
+        parsed_uri.host
+      ]
+
+      api_url.push(":#{parsed_uri.port}") unless [80, 443].include?(parsed_uri.port)
+
+      [
+        api_url.join(''),
+        "/agents/api/v1/apps/",
+        "{app_id}",
+        "/policies/latest",
+        "?",
+        "type=jsagentinjection:v1",
+        "&type=http-redirect:v1",
+        "&type=clickjacking:v1",
+        "&type=secure-headers:v1",
+        "&type=cmdi:v1",
+        "&type=csp-headers:v1",
+        "&type=dlp:v1",
+        "&type=login:v1",
+        "&type=regex:v1",
+        "&type=appsensor:v2",
+        "&type=patches:v1"
+      ].join('')
+    end
+
     def cache_filename_with_app_id
       @cache_filename ||= File.join(@agent_home_dir, "cache", "tcell_agent.cache")
 

data/lib/tcell_agent/policies/rust_policies.rb

@@ -18,6 +18,8 @@ module TCellAgent
         @appfirewall_enabled = false
         @patches_enabled = false
         @cmdi_enabled = false
+        @headers_enabled = false
+        @jsagent_enabled = false
         @agent_ptr = nil
 
         whisper = TCellAgent::Rust::Whisperer.create_agent()
@@ -29,18 +31,20 @@ module TCellAgent
       end
 
       def update_policies(policies_json)
-        return unless @agent_ptr && policies_json
+        return if @agent_ptr.nil? || policies_json.nil? || policies_json.empty?
 
-        whisper = TCellAgent::Rust::Whisperer.update_policies(@agent_ptr, {"result" => policies_json})
+        whisper = TCellAgent::Rust::Whisperer.update_policies(@agent_ptr, policies_json)
         if whisper["errors"]
           whisper["errors"].each do |error|
             TCellAgent.logger.error("Error updating policies: #{error}")
           end
         else
           enablements = whisper["enablements"]
-          @appfirewall_enabled = enablements["appfirewall"]
-          @patches_enabled = enablements["patches"]
-          @cmdi_enabled = enablements["cmdi"]
+          @appfirewall_enabled = !!enablements["appfirewall"]
+          @patches_enabled = !!enablements["patches"]
+          @cmdi_enabled = !!enablements["cmdi"]
+          @headers_enabled = !!enablements["headers"]
+          @jsagent_enabled = !!enablements["jsagentinjection"]
         end
       end
 
@@ -72,9 +76,12 @@ module TCellAgent
       end
 
       def block_command?(command, tcell_context)
-        return false unless @agent_ptr && @cmdi_enabled && TCellAgent.is_it_safe_to_send_cmdi_events?
-
-        whisper = TCellAgent::Rust::Whisperer.apply_cmdi(@agent_ptr, command)
+        return false unless @agent_ptr &&
+                            @cmdi_enabled &&
+                            TCellAgent.safe_to_send_cmdi_events?
+        whisper = TCellAgent::Rust::Whisperer.apply_cmdi(
+          @agent_ptr, command, tcell_context
+        )
         apply_response = whisper.fetch("apply_response", {})
         cmdi_event =
           TCellAgent::SensorEvents::CommandInjectionEvent.build_from_native_lib_response_and_tcell_context(apply_response,
@@ -85,6 +92,24 @@ module TCellAgent
 
         apply_response.fetch("blocked", false)
       end
+
+      def get_headers(tcell_context)
+        return [] unless @agent_ptr &&
+                         @headers_enabled
+        whisper = TCellAgent::Rust::Whisperer.get_headers(
+          @agent_ptr, tcell_context
+        )
+        return whisper['headers'] || []
+      end
+
+      def get_js_agent_script_tag(tcell_context)
+        return nil unless @agent_ptr &&
+                          @jsagent_enabled
+        whisper = TCellAgent::Rust::Whisperer.get_js_agent_script_tag(
+          @agent_ptr, tcell_context
+        )
+        return whisper['script_tag']
+      end
     end
   end
 end

data/lib/tcell_agent/rails/js_agent_insert.rb

@@ -13,11 +13,10 @@ module TCellAgent
               new_content_length = 0
               newbody = []
               rack_body.each { |str|
-                # this modifies str itself
-                js_agent_handler.call(script_insert, str)
+                modified_str = js_agent_handler.call(script_insert, str)
 
-                newbody << str
-                new_content_length += str.bytesize
+                newbody << modified_str
+                new_content_length += modified_str.bytesize
               }
               rack_body.close if rack_body.respond_to?(:close)
 
@@ -30,34 +29,34 @@ module TCellAgent
         end
 
         def self.handle_js_agent_insert(script_insert, response)
+          new_response = response
           TCellAgent::Instrumentation.safe_block("Handling JSAgent insert") do
-            return !!response.sub!(
+            new_response = response.sub(
               TCellAgent::Instrumentation::Rails::JSAgent::HEAD_SEARCH_REGEX,
               "<head>#{script_insert}"
             )
           end
 
-          false
+          new_response
         end
 
-        def self.get_handler_and_script_insert(response_headers)
+        def self.get_handler_and_script_insert(request, response_headers)
           js_agent_handler = nil
           script_insert = nil
 
           TCellAgent::Instrumentation.safe_block("JSAgent get handler and script insert") do
             if (response_headers.fetch("Content-Type","").start_with?'text/html')
-              script_tag_policy = TCellAgent.policy(TCellAgent::PolicyTypes::CSP)
-              if (script_tag_policy && script_tag_policy.js_agent_api_key)
-                base_url_vars = ""
-                if (script_tag_policy.js_agent_api_base_url)
-                  base_url_vars = " tcellbaseurl=\"#{script_tag_policy.js_agent_api_base_url}\""
+              rust_policy = TCellAgent.policy(TCellAgent::PolicyTypes::Rust)
+              if rust_policy
+                js_script_tag = rust_policy.get_js_agent_script_tag(
+                  request.env[TCellAgent::Instrumentation::TCELL_ID]
+                )
+                if js_script_tag
+                  script_insert = js_script_tag
+                  js_agent_handler = proc { |si, resp|
+                    self.handle_js_agent_insert(si, resp)
+                  }
                 end
-                script_insert = "<script src=\"#{script_tag_policy.js_agent_url}\" "
-                script_insert += "tcellapikey=\"#{script_tag_policy.js_agent_api_key}\" "
-                script_insert += "tcellappid=\"#{script_tag_policy.js_agent_app_id}\"#{base_url_vars}></script>\n"
-                js_agent_handler = proc { |si, resp|
-                  self.handle_js_agent_insert(si, resp)
-                }
               end
             end
           end

data/lib/tcell_agent/rails/middleware/headers_middleware.rb

@@ -39,9 +39,7 @@ module TCellAgent
                   tcell_response = self._handle_appsensor_js_agent_and_dlp(request, tcell_response)
                 end
                 tcell_response = self._handle_redirect(request, tcell_response)
-                tcell_response = self._set_csp_header(request, tcell_response)
-                tcell_response = self._set_clickjacking_header(request, tcell_response)
-                tcell_response = self._set_secure_headers(request, tcell_response)
+                tcell_response = self._set_headers(request, tcell_response)
                 response = tcell_response
               }
             end
@@ -49,66 +47,27 @@ module TCellAgent
             response
           end
 
-          def _set_csp_header(request, response)
-            TCellAgent::Instrumentation.safe_block("Setting CSP Headers") {
-              status, headers, active_response = response
-
-              content_security_policy = TCellAgent.policy(TCellAgent::PolicyTypes::CSP)
-
-              if content_security_policy
-                tcell_context = request.env[TCellAgent::Instrumentation::TCELL_ID]
-                content_security_policy.each_header_pair(
-                  tcell_context.transaction_id,
-                  tcell_context.route_id,
-                  tcell_context.hmac_session_id,
-                  tcell_context.user_id,
-                  tcell_context.path
-                ) do |header_name, header_value|
+          def _set_headers(request, response)
+            status, headers, active_response = response
+
+            rust_policies = TCellAgent.policy(TCellAgent::PolicyTypes::Rust)
+            if rust_policies
+              policy_headers = rust_policies.get_headers(
+                request.env[TCellAgent::Instrumentation::TCELL_ID]
+              )
+              policy_headers.each do |header_info|
+                header_name = header_info['name']
+                header_value = header_info['value']
+                existing_header_value = headers[header_name]
+                if existing_header_value
+                  headers[header_name] = "#{existing_header_value}, #{header_value}"
+                else
                   headers[header_name] = header_value
                 end
               end
               response = [status, headers, active_response]
-            }
-            response
-          end
-
-          def _set_clickjacking_header(request, response)
-            TCellAgent::Instrumentation.safe_block("Setting Clickjacking Headers") {
-              status, headers, active_response = response
-              clickjacking_policy = TCellAgent.policy(TCellAgent::PolicyTypes::Clickjacking)
-
-              if clickjacking_policy
-                  clickjacking_policy.each(
-                    request.env[TCellAgent::Instrumentation::TCELL_ID].transaction_id,
-                    request.env[TCellAgent::Instrumentation::TCELL_ID].hmac_session_id,
-                    request.env[TCellAgent::Instrumentation::TCELL_ID].user_id) do | header_pair |
-                  header_name = header_pair["name"]
-                  header_value = header_pair["value"]
-                  if (headers.has_key?header_name)
-                    headers[header_name] = headers[header_name] + "," + header_value
-                  else
-                    headers[header_name] = header_value
-                  end
-                end
-              end #if
-
-              response = [status, headers, active_response]
-            }
-            response
-          end
-
-          def _set_secure_headers(request, response)
-            TCellAgent::Instrumentation.safe_block("Setting Secure Headers") {
-              status, headers, active_response = response
+            end
 
-              secure_headers_policy = TCellAgent.policy(TCellAgent::PolicyTypes::SecureHeaders)
-              if secure_headers_policy
-                secure_headers_policy.headers.each do | secure_header |
-                  headers[secure_header.name] = secure_header.value
-                end
-              end
-              response = [status, headers, active_response]
-            }
             response
           end
 
@@ -144,7 +103,7 @@ module TCellAgent
               status_code, response_headers, response_body = response
 
               js_agent_handler, script_insert =
-                TCellAgent::Instrumentation::Rails::JSAgent.get_handler_and_script_insert(response_headers)
+                TCellAgent::Instrumentation::Rails::JSAgent.get_handler_and_script_insert(request, response_headers)
               dlp_handler, tcell_context =
                 TCellAgent::Instrumentation::Rails::DLPHandler.get_handler_and_context(request, response_headers)