tcell_agent 1.0.0 → 1.1.0

data/lib/tcell_agent/rails/tcell_body_proxy.rb

@@ -44,7 +44,7 @@ module TCellAgent
           return to_enum(:each) unless block_given?
 
           @body.each { |body_chunk|
-            process_body!(body_chunk)
+            body_chunk = process_body(body_chunk)
 
             yield body_chunk
           }
@@ -58,7 +58,7 @@ module TCellAgent
           @body.__send__(method_name, *args, &block)
         end
 
-        def process_body!(body)
+        def process_body(body)
           TCellAgent::Instrumentation.safe_block("Processing tcell body proxy body") do
             chunked_response_match = nil
             if body.class.name == "String"
@@ -68,23 +68,27 @@ module TCellAgent
               end
             end
 
+            new_body = body
             if body.class.name == "ActionView::OutputBuffer" ||
                 (body.class.name == "String" && !chunked_response_match)
-              @content_length += body.bytesize
-
               if @process_js_and_dlp
                 if @js_agent_insertion_proc
-                  if @js_agent_insertion_proc.call(@script_insert, body)
+                  new_body = @js_agent_insertion_proc.call(@script_insert, body)
+                  if new_body != body
                     # js agent was successfully inserted so no need to keep
                     # calling this proc
                     @js_agent_insertion_proc = nil
                   end
                 end
                 if @dlp_cleaner_proc
-                  @dlp_cleaner_proc.call(@tcell_context, body)
+                  @dlp_cleaner_proc.call(@tcell_context, new_body)
                 end
               end
+
+              @content_length += new_body.bytesize
             end
+
+            new_body
           end
         end
       end

data/lib/tcell_agent/rust/whisperer.rb

@@ -9,7 +9,7 @@ module TCellAgent
     module Wrapper
       extend FFI::Library
 
-      VERSION = "0.11.1"
+      VERSION = "0.19.5"
       prefix = "lib"
       extension = ".so"
       if /cygwin|mswin|mingw|bccwin|wince|emx/ =~ RUBY_PLATFORM
@@ -22,35 +22,21 @@ module TCellAgent
       begin
         ffi_lib File.join(File.dirname(__FILE__), "#{prefix}tcellagent-#{VERSION}#{extension}")
 
+        # All the rust library calls have the following response api:
+        #
         # result [int]: 0+ length of buffer_out answer
         #               -1 general error
         #               -2 buffer_out is not big enough for response
         #               -3 buffer_out is null
-        attach_function :create_agent, [:pointer, :size_t, :pointer, :size_t], :int
 
-        # result [int]: 0+ length of buffer_out answer
-        #               -1 general error
-        #               -2 buffer_out is not big enough for response
-        #               -3 buffer_out is null
+        attach_function :create_agent, [:pointer, :size_t, :pointer, :size_t], :int
+        attach_function :free_agent, [:pointer], :int
         attach_function :update_policies, [:pointer, :pointer, :size_t, :pointer, :size_t], :int
-
-        # result [int]: 0+ length of buffer_out answer
-        #               -1 general error
-        #               -2 buffer_out is not big enough for response
-        #               -3 buffer_out is null
         attach_function :appfirewall_apply, [:pointer, :pointer, :size_t, :pointer, :size_t], :int
-
-        # result [int]: 0+ length of buffer_out answer
-        #               -1 general error
-        #               -2 buffer_out is not big enough for response
-        #               -3 buffer_out is null
         attach_function :patches_apply, [:pointer, :pointer, :size_t, :pointer, :size_t], :int
-
-        # result [int]: 0+ length of buffer_out answer
-        #               -1 general error
-        #               -2 buffer_out is not big enough for response
-        #               -3 buffer_out is null
         attach_function :cmdi_apply, [:pointer, :pointer, :size_t, :pointer, :size_t], :int
+        attach_function :get_headers, [:pointer, :pointer, :size_t, :pointer, :size_t], :int
+        attach_function :get_js_agent_script_tag, [:pointer, :pointer, :size_t, :pointer, :size_t], :int
 
         def self.common_lib_available?
           true
@@ -74,7 +60,13 @@ module TCellAgent
           )
         else
           begin
-            return JSON.parse(result.get_string(0, result_size))
+            response = JSON.parse(result.get_string(0, result_size))
+            if response['error']
+              Logger.error(function_called + ' returned an error: ' + response['error'])
+              response = {}
+            end
+
+            return response
           rescue JSON::ParserError
             # don't log the actual error since it might contain payload information
             TCellAgent.logger.error(
@@ -88,15 +80,42 @@ module TCellAgent
 
       def self.create_agent()
         if TCellAgent::Rust::Wrapper.common_lib_available?
+          allow_payloads = !!TCellAgent.configuration.allow_payloads
           agent_config = {
+            "application" => {
+              "app_id" => TCellAgent.configuration.app_id,
+              "api_key" =>  TCellAgent.configuration.api_key,
+              "tcell_api_url" => "",
+              "tcell_input_url" => "",
+              "allow_payloads" => allow_payloads,
+              "js_agent_api_base_url" => TCellAgent.configuration.js_agent_api_base_url,
+              "js_agent_url" => TCellAgent.configuration.js_agent_url
+            },
             "appfirewall" => {
               "enable_body_xxe_inspection" => false,
               "enable_body_json_inspection" => false,
-              "allow_send_payloads" => TCellAgent.configuration.allow_payloads,
+              "allow_send_payloads" => allow_payloads,
               "allow_log_payloads" => true
-            }
+            },
+            "policy_versions" => {
+              "patches" => 1,
+              "login" => 1,
+              "appsensor" => 2,
+              "regex" => 1,
+              "csp-headers" => 1,
+              "http-redirect" => 1,
+              "clickjacking" => 1,
+              "secure-headers" => 1,
+              "canaries" => 1,
+              "dlp" => 1,
+              "cmdi" => 1,
+              "jsagentinjection" => 1
+            },
+            "max_header_size" => TCellAgent.configuration.max_csp_header_bytes || (1024 * 1024)
           }
-          config_pointer = FFI::MemoryPointer.from_string(JSON.dump(agent_config))
+          config_pointer = FFI::MemoryPointer.from_string(
+            JSON.dump(agent_config)
+          )
 
           buf = FFI::MemoryPointer.new(:uint8, 1024 * 8)
           # config_pointer.size - 1: strips null terminator
@@ -109,16 +128,31 @@ module TCellAgent
         return {}
       end
 
+      def self.free_agent(agent_ptr)
+        if TCellAgent::Rust::Wrapper.common_lib_available? &&
+            agent_ptr
+          TCellAgent::Rust::Wrapper.free_agent(
+            FFI::Pointer.new(agent_ptr)
+          )
+        end
+      end
+
       def self.update_policies(agent_ptr, policies)
         if TCellAgent::Rust::Wrapper.common_lib_available? &&
             agent_ptr &&
             TCellAgent::Utils::Strings.present?(policies)
-          policies_pointer = FFI::MemoryPointer.from_string(JSON.dump(policies))
+          policies_pointer = FFI::MemoryPointer.from_string(
+            JSON.dump(policies)
+          )
 
           buf = FFI::MemoryPointer.new(:uint8, 1024 * 8)
           # policies_pointer.size - 1: strips null terminator
           result_size = TCellAgent::Rust::Wrapper.update_policies(
-            FFI::Pointer.new(agent_ptr), policies_pointer, policies_pointer.size - 1, buf, buf.size
+            FFI::Pointer.new(agent_ptr),
+            policies_pointer,
+            policies_pointer.size - 1,
+            buf,
+            buf.size
           )
           return self.convert_result("update_policies", result_size, buf)
         end
@@ -130,13 +164,21 @@ module TCellAgent
          if TCellAgent::Rust::Wrapper.common_lib_available? &&
              agent_ptr &&
              appsensor_meta
-           request_response_json = TCellAgent::Rust::Models.create_request_response(appsensor_meta)
-           request_response_pointer = FFI::MemoryPointer.from_string(JSON.dump(request_response_json))
+           request_response_json = TCellAgent::Rust::Models.create_request_response(
+             appsensor_meta
+           )
+           request_response_pointer = FFI::MemoryPointer.from_string(
+             JSON.dump(request_response_json)
+           )
 
-           buf = FFI::MemoryPointer.new(:uint8, 1024 * 8)
+           buf = FFI::MemoryPointer.new(:uint8, 1024 * 32)
            # request_response_pointer.size - 1: strips null terminator
            result_size = TCellAgent::Rust::Wrapper.appfirewall_apply(
-             FFI::Pointer.new(agent_ptr), request_response_pointer, request_response_pointer.size - 1, buf, buf.size
+             FFI::Pointer.new(agent_ptr),
+             request_response_pointer,
+             request_response_pointer.size - 1,
+             buf,
+             buf.size
            )
            return self.convert_result("apply_appfirewall", result_size, buf)
          end
@@ -148,13 +190,21 @@ module TCellAgent
         if TCellAgent::Rust::Wrapper.common_lib_available? &&
             agent_ptr &&
             appsensor_meta
-          patches_request_json = TCellAgent::Rust::Models.create_patches_request(appsensor_meta)
-          patches_request_pointer = FFI::MemoryPointer.from_string(JSON.dump(patches_request_json))
+          patches_request_json = TCellAgent::Rust::Models.create_patches_request(
+            appsensor_meta
+          )
+          patches_request_pointer = FFI::MemoryPointer.from_string(
+            JSON.dump(patches_request_json)
+          )
 
-          buf = FFI::MemoryPointer.new(:uint8, 1024 * 8)
+          buf = FFI::MemoryPointer.new(:uint8, 1024 * 32)
           # patches_request_pointer.size - 1: strips null terminator
           result_size = TCellAgent::Rust::Wrapper.patches_apply(
-            FFI::Pointer.new(agent_ptr), patches_request_pointer, patches_request_pointer.size - 1, buf, buf.size
+            FFI::Pointer.new(agent_ptr),
+            patches_request_pointer,
+            patches_request_pointer.size - 1,
+            buf,
+            buf.size
           )
           return self.convert_result("apply_patches", result_size, buf)
         end
@@ -162,22 +212,98 @@ module TCellAgent
         return {}
       end
 
-      def self.apply_cmdi(agent_ptr, command)
+      def self.apply_cmdi(agent_ptr, command, tcell_context)
+
         if TCellAgent::Rust::Wrapper.common_lib_available? &&
             agent_ptr &&
             TCellAgent::Utils::Strings.present?(command)
-          command_pointer = FFI::MemoryPointer.from_string(command)
+          method = tcell_context && tcell_context.request_method
+          path = tcell_context && tcell_context.path
+          command_info = {
+            "command" => command,
+            "method" => method,
+            "path" => path
+          }
+          command_pointer = FFI::MemoryPointer.from_string(
+            JSON.dump(command_info)
+          )
 
-          buf = FFI::MemoryPointer.new(:uint8, 1024 * 8)
+          buf = FFI::MemoryPointer.new(:uint8, 1024 * 32)
           # patches_request_pointer.size - 1: strips null terminator
           result_size = TCellAgent::Rust::Wrapper.cmdi_apply(
-            FFI::Pointer.new(agent_ptr), command_pointer, command_pointer.size - 1, buf, buf.size
+            FFI::Pointer.new(agent_ptr),
+            command_pointer,
+            command_pointer.size - 1,
+            buf,
+            buf.size
           )
           return self.convert_result("apply_cmdi", result_size, buf)
         end
 
         return {}
       end
+
+      def self.get_headers(agent_ptr, tcell_context)
+        if TCellAgent::Rust::Wrapper.common_lib_available? &&
+            agent_ptr &&
+            tcell_context
+          method = tcell_context.request_method
+          path = tcell_context.path
+          route_id = tcell_context.route_id
+          session_id = tcell_context.session_id
+          headers_request = {
+            method: method,
+            path: path,
+            route_id: route_id && route_id.to_s,
+            session_id: session_id && session_id.to_s
+          }
+          headers_request_pointer = FFI::MemoryPointer.from_string(
+            JSON.dump(headers_request)
+          )
+
+          buf = FFI::MemoryPointer.new(:uint8, 1024 * 16)
+          # patches_request_pointer.size - 1: strips null terminator
+          result_size = TCellAgent::Rust::Wrapper.get_headers(
+            FFI::Pointer.new(agent_ptr),
+            headers_request_pointer,
+            headers_request_pointer.size - 1,
+            buf,
+            buf.size
+          )
+          return self.convert_result("get_headers", result_size, buf)
+        end
+
+        return {}
+      end
+
+      def self.get_js_agent_script_tag(agent_ptr, tcell_context)
+        if TCellAgent::Rust::Wrapper.common_lib_available? &&
+            agent_ptr &&
+            tcell_context
+          method = tcell_context.request_method
+          path = tcell_context.path
+          jsagent_request = {
+            method: method,
+            path: path
+          }
+          jsagent_request_pointer = FFI::MemoryPointer.from_string(
+            JSON.dump(jsagent_request)
+          )
+
+          buf = FFI::MemoryPointer.new(:uint8, 1024 * 8)
+          # patches_request_pointer.size - 1: strips null terminator
+          result_size = TCellAgent::Rust::Wrapper.get_js_agent_script_tag(
+            FFI::Pointer.new(agent_ptr),
+            jsagent_request_pointer,
+            jsagent_request_pointer.size - 1,
+            buf,
+            buf.size
+          )
+          return self.convert_result("get_js_agent_script_tag", result_size, buf)
+        end
+
+        return {}
+      end
     end
   end
 end

data/lib/tcell_agent/sensor_events/patches.rb

@@ -14,6 +14,8 @@ module TCellAgent
         self["m"] = appsensor_meta.method if appsensor_meta.method
         self["remote_addr"] = appsensor_meta.remote_address if appsensor_meta.remote_address
         self["uri"] = TCellAgent::SensorEvents::Util.strip_uri_values(appsensor_meta.location) if appsensor_meta.location
+        self["regex_pid"] = rust_response["regex_pid"] if rust_response["regex_pid"]
+        self["payload"] = rust_response["payload"] if rust_response["payload"]
       end
     end
 

data/lib/tcell_agent/sinatra.rb

@@ -12,23 +12,26 @@ module TCellAgent
       alias_method :original_finish, :finish
       def finish
         status, headers, response = original_finish
-        TCellAgent::Instrumentation.safe_block("Setting CSP Headers") {
-          content_security_policy = TCellAgent.policy(TCellAgent::PolicyTypes::CSP)
-          if content_security_policy
-            content_security_policy.each_header_pair do |header_name, header_value|
-              headers[header_name] = header_value
-            end
-          end
-        }
 
-        TCellAgent::Instrumentation.safe_block("Setting Secure Headers") {
-          secure_headers_policy = TCellAgent.policy(TCellAgent::PolicyTypes::SecureHeaders)
-          if secure_headers_policy
-            secure_headers_policy.headers.each do | secure_header |
-              headers[secure_header.name] = secure_header.value
+        TCellAgent::Instrumentation.safe_block("Setting Headers") do
+          rust_policies = TCellAgent.policy(TCellAgent::PolicyTypes::Rust)
+          if rust_policies
+            policy_headers = rust_policies.get_headers(
+              request.env[TCellAgent::Instrumentation::TCELL_ID]
+            )
+            policy_headers.each do |header_info|
+              header_name = header_info['name']
+              header_value = header_info['value']
+              existing_header_value = headers[header_name]
+              if existing_header_value
+                headers[header_name] = "#{existing_header_value}, #{header_value}"
+              else
+                headers[header_name] = header_value
+              end
             end
+            response = [status, headers, active_response]
           end
-        }
+        end
 
         [status, headers, response]
       end

data/lib/tcell_agent/version.rb

@@ -1,5 +1,5 @@
 # See the file "LICENSE" for the full license governing this code.
 
 module TCellAgent
-  VERSION = "1.0.0"
+  VERSION = "1.1.0"
 end

data/spec/lib/tcell_agent/agent/policy_manager_spec.rb

@@ -203,6 +203,23 @@ module TCellAgent
           context 'with two processes' do
             context 'while one process is writing to the cached file' do
               before(:each) do
+                configuration = double(
+                  'configuration',
+                  {
+                    'app_id' => 'app_id',
+                    'api_key' => 'api_key',
+                    'allow_payloads' => true,
+                    'js_agent_api_base_url' => 'http://api.tcell.com/',
+                    'js_agent_url' => 'https://jsagent.tcell.io/tcellagent.min.js',
+                    'max_csp_header_bytes' => nil,
+                    'event_time_limit_seconds' => 15,
+                    'event_batch_size_limit' => 50,
+                    'preload_policy_filename' => nil,
+                    'cache_filename_with_app_id' => '/tcellagent_src/tcell/cache/tcell_agent.cache',
+                    'agent_home_owner' => nil
+                  }
+                )
+                expect(TCellAgent).to receive(:configuration).and_return(configuration).at_least(:once)
                 TCellAgent.thread_agent.cache(
                   'http-redirect',
                   {

data/spec/lib/tcell_agent/api/api_spec.rb

@@ -17,19 +17,22 @@ module TCellAgent
           ' \'none\'" ,"report-uri":"http://localhost:3000/csp/cab5e750e66d614bd46fd07a7078db1e74b4f427b2a135b2c96eca684a642707"}]}}}'
         end
         uri_template =
-          Addressable::Template.new 'https://api.tcell.io/api/v1/app/{app}/update'
-        stub_request(:any, uri_template)
-          .to_return(lambda { |request|
+          Addressable::Template.new 'https://api.tcell.io/agents/api/v1/apps/test-appid/policies/latest?type=patches:v1'
+
+        stub_request(:any, uri_template).to_return(
+          lambda { |request|
             {
-              :body => checkreq(request),  :status => 200,
-              :headers => { 'Content-Tyoe' => 'application/json' }
+              :body => checkreq(request),
+              :status => 200,
+              :headers => { 'Content-Type' => 'application/json' }
             }
-          })
+          }
+        )
 
         result = tapi.poll_api
         TCellAgent.configuration.app_id = nil
         TCellAgent.configuration.api_key = nil
-        expect(result['csp-headers']['app_id']).to eq('testapp-Becwu')
+        expect(result['result']['csp-headers']['app_id']).to eq('testapp-Becwu')
       end
     end
   end