sqreen 1.20.0 → 1.20.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6f18004c730291041540b696217f1c70b449e179328f959705a07a148364bfe9
-  data.tar.gz: 417f8bb15cbee7faf4f1aa127552e38f0c929255493341d0f683daae37ab9894
+  metadata.gz: 7be645cedf514d1af5754fc7244738e17f817aad09761d2da7d7fa032ac55801
+  data.tar.gz: f81295c9ac98714284d6368bc5e01c1a857317c8dad98d53366b24e53cc6cc47
 SHA512:
-  metadata.gz: f891f7785362829c23028206d6ba656ae3860e8c13cb265a839608c2cf02ba5be6576337c8d2630d1b0c8eb5c69e9dc732d7c68581f9bb472157e900f7a49a15
-  data.tar.gz: b1e2e6708cfc177b66e5fe25dc85a9839e784dd4afffd49585f06050311d821c3eee9fe06ddff63a4eddabfc94b9307ca54e4a7301ebbcd5fc4d82096ec8efa7
+  metadata.gz: 986748fc2c11ee0a6ea7a997d661246d2840e170fdcbe5d7c1221fb189e3b6a9281e4feb1025f58f1ac6712ff10f598caa7d81867a7a62898138e743eadc9262
+  data.tar.gz: 39b246ce351e780f539a0d2fe3df9791996d8203056545ab5472f0deec2e1526351bfc9cbb4f8c4cd5c145969db0b177b1b559a7491e4a8a15de4e9dd3721665

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 1.20.1
+
+* Add fallback mechanisms when connecting to new Sqreen backend API domains
+
 ## 1.20.0
 
 * Enable new instrumentation engine by default

data/lib/sqreen/agent_message.rb

@@ -0,0 +1,20 @@
+require 'digest'
+
+module Sqreen
+  class AgentMessage
+    def initialize(kind, message, id = nil)
+      id ||= message + "\x00" + kind
+      @hash_hex = Digest::SHA1.hexdigest(id)
+      @kind = kind
+      @message = message
+    end
+
+    def to_h
+      {
+        id: @hash_hex,
+        kind: @kind,
+        message: @message,
+      }
+    end
+  end
+end

data/lib/sqreen/ca.crt

@@ -70,3 +70,27 @@ WE9gyn6CagsCqiUXObXbf+eEZSqVir2G3l6BFoMtEMze/aiCKm0oHw0LxOXnGiYZ
 4fQRbxC1lfznQgUy286dUV4otp6F01vvpX1FQHKOtw5rDgb7MzVIcbidJ4vEZV8N
 hnacRHr2lVz2XTIIM6RUthg/aFzyQkqFOFSDX9HoLPKsEdao7WNq
 -----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIID7zCCAtegAwIBAgIBADANBgkqhkiG9w0BAQsFADCBmDELMAkGA1UEBhMCVVMx
+EDAOBgNVBAgTB0FyaXpvbmExEzARBgNVBAcTClNjb3R0c2RhbGUxJTAjBgNVBAoT
+HFN0YXJmaWVsZCBUZWNobm9sb2dpZXMsIEluYy4xOzA5BgNVBAMTMlN0YXJmaWVs
+ZCBTZXJ2aWNlcyBSb290IENlcnRpZmljYXRlIEF1dGhvcml0eSAtIEcyMB4XDTA5
+MDkwMTAwMDAwMFoXDTM3MTIzMTIzNTk1OVowgZgxCzAJBgNVBAYTAlVTMRAwDgYD
+VQQIEwdBcml6b25hMRMwEQYDVQQHEwpTY290dHNkYWxlMSUwIwYDVQQKExxTdGFy
+ZmllbGQgVGVjaG5vbG9naWVzLCBJbmMuMTswOQYDVQQDEzJTdGFyZmllbGQgU2Vy
+dmljZXMgUm9vdCBDZXJ0aWZpY2F0ZSBBdXRob3JpdHkgLSBHMjCCASIwDQYJKoZI
+hvcNAQEBBQADggEPADCCAQoCggEBANUMOsQq+U7i9b4Zl1+OiFOxHz/Lz58gE20p
+OsgPfTz3a3Y4Y9k2YKibXlwAgLIvWX/2h/klQ4bnaRtSmpDhcePYLQ1Ob/bISdm2
+8xpWriu2dBTrz/sm4xq6HZYuajtYlIlHVv8loJNwU4PahHQUw2eeBGg6345AWh1K
+Ts9DkTvnVtYAcMtS7nt9rjrnvDH5RfbCYM8TWQIrgMw0R9+53pBlbQLPLJGmpufe
+hRhJfGZOozptqbXuNC66DQO4M99H67FrjSXZm86B0UVGMpZwh94CDklDhbZsc7tk
+6mFBrMnUVN+HL8cisibMn1lUaJ/8viovxFUcdUBgF4UCVTmLfwUCAwEAAaNCMEAw
+DwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYwHQYDVR0OBBYEFJxfAN+q
+AdcwKziIorhtSpzyEZGDMA0GCSqGSIb3DQEBCwUAA4IBAQBLNqaEd2ndOxmfZyMI
+bw5hyf2E3F/YNoHN2BtBLZ9g3ccaaNnRbobhiCPPE95Dz+I0swSdHynVv/heyNXB
+ve6SbzJ08pGCL72CQnqtKrcgfU28elUSwhXqvfdqlS5sdJ/PHLTyxQGjhdByPq1z
+qwubdQxtRbeOlKyWN7Wg0I8VRw7j6IPdj/3vQQF3zCepYoUz8jcI73HPdwbeyBkd
+iEDPfUYd/x7H4c7/I9vG+o1VTqkC50cRRj70/b17KSa7qWFiNyi2LSr2EIZkyXCn
+0q23KXB56jzaYyWf/Wi3MOxw+3WKt21gZ7IeyLnp2KhvAotnDU0mV3HaIPzBSlCN
+sSi6
+-----END CERTIFICATE-----

data/lib/sqreen/configuration.rb

@@ -43,9 +43,9 @@ module Sqreen
     { :env => :SQREEN_WEAVE_STRATEGY, :name => :weave_strategy,
       :default => :prepend, :convert => :to_sym },
     { :env => :SQREEN_URL, :name => :url,
-      :default => 'https://back.sqreen.io' },
+      :default => nil },
     { :env => :SQREEN_INGESTION_URL, :name => :ingestion_url,
-      :default => 'https://ingestion.sqreen.com/' },
+      :default => nil },
     { :env => :SQREEN_PROXY_URL, :name => :proxy_url,
       :default => nil },
     { :env => :SQREEN_TOKEN,     :name => :token,
@@ -78,6 +78,8 @@ module Sqreen
       :default => nil },
     { :env => :SQREEN_STRIP_SENSITIVE_REGEX, :name => :strip_sensitive_regex,
       :default => nil },
+    { :env => :SQREEN_NO_SNIFF_DOMAINS, :name => :no_sniff_domains,
+      :default => false },
 
   ].freeze
 

data/lib/sqreen/endpoint_testing.rb

@@ -0,0 +1,184 @@
+require 'net/https'
+require 'sqreen/agent_message'
+require 'sqreen/log/loggable'
+
+module Sqreen
+  class EndpointTesting
+    Endpoint = Struct.new(:url, :ca_store)
+    class ChosenEndpoints
+      def initialize
+        @messages = []
+      end
+
+      # @return [Sqreen::EndpointTesting::Endpoint]
+      attr_accessor :control
+
+      # @return [Sqreen::EndpointTesting::Endpoint]
+      attr_accessor :ingestion
+
+      # @return [Array<Sqreen::AgentMessage>]
+      attr_reader :messages
+
+      # @param [Sqreen::AgentMessage] message
+      def add_message(message)
+        @messages << message
+      end
+    end
+
+    MAIN_CONTROL_HOST = 'back.sqreen.com'.freeze
+    MAIN_INJECTION_HOST = 'ingestion.sqreen.com'.freeze
+    FALLBACK_ENDPOINT_URL = 'https://back.sqreen.io/'.freeze
+    GLOBAL_TIMEOUT = 30
+
+    CONTROL_ERROR_KIND = 'back_sqreen_com_unavailable'.freeze
+    INGESTION_ERROR_KIND = 'ingestion_sqreen_com_unavailable'.freeze
+
+    class << self
+      include Log::Loggable
+
+      # reproduces behaviour before endpoint testing was introduced
+      def no_test_endpoints(config_url, config_ingestion_url)
+        endpoints = ChosenEndpoints.new
+
+        endpoints.control = Endpoint.new(
+          config_url || "https://#{MAIN_CONTROL_HOST}/", cert_store
+        )
+        endpoints.ingestion = Endpoint.new(
+          config_ingestion_url || "https://#{MAIN_INJECTION_HOST}/", nil
+        )
+
+        endpoints
+      end
+
+      def test_endpoints(proxy_url, config_url, config_ingestion_url)
+        proxy_params = create_proxy_params(proxy_url)
+
+        # execute the tests in separate threads and wait for them
+        thread_control = Thread.new do
+          thread_main(config_url, proxy_params, MAIN_CONTROL_HOST)
+        end
+        thread_injection = Thread.new do
+          thread_main(config_ingestion_url, proxy_params, MAIN_INJECTION_HOST)
+        end
+
+        wait_for_threads(thread_control, thread_injection)
+
+        # build and return result
+        fallback = Endpoint.new(FALLBACK_ENDPOINT_URL, cert_store)
+        endpoints = ChosenEndpoints.new
+        endpoints.control = thread_control[:endpoint] || fallback
+        endpoints.ingestion = thread_injection[:endpoint] || fallback
+
+        if thread_control[:endpoint_error]
+          msg = AgentMessage.new(CONTROL_ERROR_KIND, thread_control[:endpoint_error])
+          endpoints.add_message msg
+        end
+        if thread_injection[:endpoint_error]
+          msg = AgentMessage.new(INGESTION_ERROR_KIND, thread_injection[:endpoint_error])
+          endpoints.add_message msg
+        end
+
+        endpoints
+      end
+
+      private
+
+      def thread_main(configured_url, proxy_params, host)
+        res = if configured_url
+                Endpoint.new(configured_url, nil)
+              else
+                EndpointTesting.send(:test_with_store_variants, proxy_params, host)
+              end
+
+        Thread.current[:endpoint] = res
+      rescue StandardError => e
+        Thread.current[:endpoint_error] = e.message
+      end
+
+      def create_proxy_params(proxy_url)
+        return [] unless proxy_url
+
+        proxy = URI.parse(proxy_url)
+
+        return [] unless proxy.scheme == 'http'
+
+        [proxy.host, proxy.port, proxy.user, proxy.password]
+      end
+
+      def test_with_store_variants(proxy_params, server_name)
+        # first without custom store
+        do_test(proxy_params, server_name, false)
+      rescue StandardError => _e
+        do_test(proxy_params, server_name, true)
+      end
+
+      # @param [Array] proxy_params
+      # @param [String] server_name
+      # @param [Boolean] custom_store
+      def do_test(proxy_params, server_name, custom_store)
+        http = Net::HTTP.new(server_name, 443, *proxy_params)
+        http.use_ssl = true
+        http.verify_mode = OpenSSL::SSL::VERIFY_NONE if ENV['SQREEN_SSL_NO_VERIFY']
+        http.verify_callback = lambda do |preverify_ok, ctx|
+          unless preverify_ok
+            logger.warn do
+              "Certificate validation failure for certificate issued to " \
+              "#{ctx.chain[0].subject}: #{ctx.error_string}"
+            end
+          end
+
+          preverify_ok
+        end
+
+        http.open_timeout = 13
+        http.ssl_timeout = 7
+        http.read_timeout = 7
+        http.close_on_empty_response = true
+
+        http.cert_store = cert_store if custom_store
+
+        resp = http.get('/ping')
+
+        logger.info do
+          "Got response from #{server_name}'s ping endpoint. " \
+          "Status code is #{resp.code} (custom CA store: #{custom_store})"
+        end
+
+        unless resp.code == '200'
+          raise "Response code for /ping is #{resp.code}, not 200"
+        end
+
+        Endpoint.new("https://#{server_name}/", http.cert_store)
+      rescue StandardError => e
+        logger.info do
+          "Error in request to #{server_name} " \
+          "(custom store: #{custom_store}): #{e.message}"
+        end
+
+        raise "Error in request to #{server_name}: #{e.message}"
+      end
+
+      def wait_for_threads(thread_control, thread_injection)
+        deadline = Time.now + GLOBAL_TIMEOUT
+        [thread_control, thread_injection].each do |thread|
+          rem = deadline - Time.now
+          rem = 0.1 if rem < 0.1
+          next if thread.join(rem)
+          logger.debug { "Timeout for thread #{thread}" }
+          thread.kill
+          thread[:endpoint_error] = "Timeout doing endpoint testing"
+        end
+      end
+
+      def cert_store
+        @cert_store ||= begin
+          cert_file = File.join(File.dirname(__FILE__), 'ca.crt')
+          cert_store = OpenSSL::X509::Store.new
+          cert_store.add_file cert_file
+
+          cert_store
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/log/loggable.rb

@@ -23,6 +23,6 @@ module Sqreen::Log::Loggable
   end
 
   def logger
-    @logger || self.class.logger
+    @logger || singleton_class.logger
   end
 end

data/lib/sqreen/runner.rb

@@ -11,6 +11,7 @@ require 'sqreen/events/attack'
 
 require 'sqreen/log'
 
+require 'sqreen/agent_message'
 require 'sqreen/rules'
 require 'sqreen/session'
 require 'sqreen/remote_command'
@@ -18,6 +19,7 @@ require 'sqreen/capped_queue'
 require 'sqreen/metrics_store'
 require 'sqreen/deliveries/simple'
 require 'sqreen/deliveries/batch'
+require 'sqreen/endpoint_testing'
 require 'sqreen/performance_notifications/metrics'
 require 'sqreen/performance_notifications/binned_metrics'
 require 'sqreen/legacy/instrumentation'
@@ -113,19 +115,23 @@ module Sqreen
       @next_metrics = []
       @running = true
 
+      @proxy_url = @configuration.get(:proxy_url)
+      chosen_endpoints = determine_endpoints
+
       @token = @configuration.get(:token)
       @app_name = @configuration.get(:app_name)
-      @url = @configuration.get(:url)
-      @proxy_url = @configuration.get(:proxy_url)
+      @url = chosen_endpoints.control.url
+      @cert_store = chosen_endpoints.control.ca_store
+
       Sqreen.update_whitelisted_paths([])
       Sqreen.update_whitelisted_ips({})
       Sqreen.update_performance_budget(nil)
-      raise(Sqreen::Exception, 'no url found') unless @url
       raise(Sqreen::TokenNotFoundException, 'no token found') unless @token
 
       Sqreen::Kit::Configuration.logger = Sqreen.log
-      Sqreen::Kit::Configuration.ingestion_url = @configuration.get(:ingestion_url)
-      Sqreen::Kit::Configuration.proxy_url = @configuration.get(:proxy_url)
+      Sqreen::Kit::Configuration.ingestion_url = chosen_endpoints.ingestion.url
+      Sqreen::Kit::Configuration.certificate_store = chosen_endpoints.ingestion.ca_store
+      Sqreen::Kit::Configuration.proxy_url = @proxy_url
 
       register_exit_cb if set_at_exit
 
@@ -143,6 +149,7 @@ module Sqreen
 
       Sqreen.log.debug "Using token #{@token}"
       response = create_session(session_class)
+      post_endpoint_testing_msgs(chosen_endpoints)
       wanted_features = response.fetch('features', {})
       conf_initial_features = configuration.get(:initial_features)
       unless conf_initial_features.nil?
@@ -155,7 +162,7 @@ module Sqreen
           wanted_features = wanted_features.merge(conf_features)
         rescue
           Sqreen.log.warn do
-            "NOT using invalid inital features #{conf_initial_features}"
+            "NOT using invalid initial features #{conf_initial_features}"
           end
         end
       end
@@ -171,7 +178,7 @@ module Sqreen
     end
 
     def create_session(session_class)
-      @session = session_class.new(@url, @token, @app_name, @proxy_url)
+      @session = session_class.new(@url, @cert_store, @token, @app_name, @proxy_url)
       session.login(@framework)
     end
 
@@ -508,6 +515,28 @@ module Sqreen
 
     private
 
+    def post_endpoint_testing_msgs(chosen_endpoints)
+      chosen_endpoints.messages.each do |msg|
+        session.post_agent_message(@framework, msg)
+      end
+    rescue => e
+      Sqreen.log.warn "Error submitting agent message: #{e}"
+      RemoteException.record(e)
+    end
+
+    def determine_endpoints
+      # there's no sniffing going on; just a misnamed config setting
+      if @configuration.get(:no_sniff_domains)
+        # reproduces behaviour before endpoint testing was introduced
+        EndpointTesting.no_test_endpoints(@configuration.get(:url),
+                                          @configuration.get(:ingestion_url))
+      else
+        EndpointTesting.test_endpoints(@proxy_url,
+                                       @configuration.get(:url),
+                                       @configuration.get(:ingestion_url))
+      end
+    end
+
     def load_actions(hashes)
       unsupported = Set.new
 

data/lib/sqreen/session.rb

@@ -50,7 +50,7 @@ module Sqreen
 
     attr_accessor :request_compression
 
-    def initialize(server_url, token, app_name = nil, proxy_url = nil)
+    def initialize(server_url, cert_store, token, app_name = nil, proxy_url = nil)
       @token = token
       @app_name = app_name
       @session_id = nil
@@ -73,12 +73,7 @@ module Sqreen
       @http = Net::HTTP.new(uri.host, uri.port, *proxy_params)
       @http.use_ssl = use_ssl
       @http.verify_mode = OpenSSL::SSL::VERIFY_NONE if ENV['SQREEN_SSL_NO_VERIFY'] # for testing
-      if use_ssl
-        cert_file = File.join(File.dirname(__FILE__), 'ca.crt')
-        cert_store = OpenSSL::X509::Store.new
-        cert_store.add_file cert_file
-        @http.cert_store = cert_store
-      end
+      @http.cert_store = cert_store if use_ssl
       self.use_signals = false
     end
 
@@ -240,10 +235,7 @@ module Sqreen
     end
 
     def login(framework)
-      headers = {
-        'x-api-key'  => @token,
-        'x-app-name' => @app_name || framework.application_name,
-      }.reject { |k, v| v ==  nil }
+      headers = prelogin_auth_headers(framework)
 
       Sqreen.log.warn "Using app name: #{headers['x-app-name']}"
 
@@ -312,6 +304,11 @@ module Sqreen
       @evt_sub_strategy.post_batch(events)
     end
 
+    def post_agent_message(framework, agent_message)
+      headers = prelogin_auth_headers(framework)
+      post('app_agent_message', agent_message.to_h, headers, 0)
+    end
+
     # Perform agent logout
     # @param retrying [Boolean] whether to try again on error
     def logout(retrying = true)
@@ -325,5 +322,14 @@ module Sqreen
       Sqreen.logged_in = false
       disconnect
     end
+
+    private
+
+    def prelogin_auth_headers(framework)
+      {
+        'x-api-key' => @token,
+        'x-app-name' => @app_name || framework.application_name,
+      }.reject { |_k, v| v == nil }
+    end
   end
 end

data/lib/sqreen/version.rb

@@ -4,5 +4,5 @@
 # Please refer to our terms for more information: https://www.sqreen.com/terms.html
 
 module Sqreen
-  VERSION = '1.20.0'.freeze
+  VERSION = '1.20.1'.freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sqreen
 version: !ruby/object:Gem::Version
-  version: 1.20.0
+  version: 1.20.1
 platform: ruby
 authors:
 - Sqreen
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-06-18 00:00:00.000000000 Z
+date: 2020-06-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sqreen-backport
@@ -30,14 +30,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.2.0
+        version: 0.2.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.2.0
+        version: 0.2.1
 - !ruby/object:Gem::Dependency
   name: sq_mini_racer
   requirement: !ruby/object:Gem::Requirement
@@ -93,6 +93,7 @@ files:
 - lib/sqreen/actions/user_action_class.rb
 - lib/sqreen/actions/users_index.rb
 - lib/sqreen/agent.rb
+- lib/sqreen/agent_message.rb
 - lib/sqreen/aggregated_metric.rb
 - lib/sqreen/attack_blocked.rb
 - lib/sqreen/attack_detected.html
@@ -122,6 +123,7 @@ files:
 - lib/sqreen/dependency/sentry.rb
 - lib/sqreen/dependency/sinatra.rb
 - lib/sqreen/encoding_sanitizer.rb
+- lib/sqreen/endpoint_testing.rb
 - lib/sqreen/error_handling_middleware.rb
 - lib/sqreen/event.rb
 - lib/sqreen/events/attack.rb