sqreen 0.1.0.pre → 0.7.01461158029

data/lib/sqreen/runner.rb

@@ -0,0 +1,312 @@
+# Copyright (c) 2015 Sqreen. All Rights Reserved.
+# Please refer to our terms for more information: https://www.sqreen.io/terms.html
+
+require 'timeout'
+
+require 'sqreen/events/attack'
+
+require 'sqreen/log'
+
+require 'sqreen/rules'
+require 'sqreen/remote_command'
+require 'sqreen/capped_queue'
+require 'sqreen/metrics_store'
+require 'sqreen/deliveries/simple'
+require 'sqreen/deliveries/batch'
+require 'sqreen/performance_notifications/metrics'
+
+module Sqreen
+  @features = {}
+  @queue = nil
+
+  # Event Queue that enable communication between threads and the reporter
+  MAX_QUEUE_LENGTH = 100
+  MAX_OBS_QUEUE_LENGTH = 1000
+
+  METRICS_EVENT = 'metrics'.freeze
+
+  class << self
+    attr_reader :features
+    def update_features(features)
+      @features = features
+    end
+
+    def queue
+      @queue ||= CappedQueue.new(MAX_QUEUE_LENGTH)
+    end
+
+    def observations_queue
+      @observations_queue ||= CappedQueue.new(MAX_OBS_QUEUE_LENGTH)
+    end
+
+    attr_accessor :instrumentation_ready
+    alias instrumentation_ready? instrumentation_ready
+
+    attr_accessor :logged_in
+    alias logged_in? logged_in
+  end
+
+  # Main running job class for the agent
+  class Runner
+    # At start, heartbeat is every 15 seconds
+    HEARTBEAT_INITIAL_DELAY = 15
+    # During one hour
+    HEARTBEAT_WARMUP = 60 * 60
+    # Then delay raises to 5 minutes
+    HEARTBEAT_MAX_DELAY = 5 * 60
+
+    attr_reader :heartbeat_delay
+    attr_accessor :metrics_engine
+    attr_reader :publish_metrics_delay
+    attr_reader :deliverer
+    attr_reader :session
+    attr_reader :instrumenter
+
+    # we may want to do that in a thread in order to prevent delaying app
+    # startup
+    # set_at_exit do not place a global at_exit (used for testing)
+    def initialize(configuration, framework, set_at_exit = true, session_class = Sqreen::Session)
+      @logged_out_tried = false
+      @configuration = configuration
+      @framework = framework
+      @heartbeat_delay = HEARTBEAT_INITIAL_DELAY
+      @publish_metrics_delay = HEARTBEAT_MAX_DELAY
+      @sleep_delay = HEARTBEAT_INITIAL_DELAY
+      @last_heartbeat_request = Time.at(0)
+      @last_post_metrics_request = Time.now
+      @started = Time.now
+
+      @token = @configuration.get(:token)
+      @url = @configuration.get(:url)
+      raise(Sqreen::Exception, 'no url found') unless @url
+      raise(Sqreen::TokenNotFoundException, 'no token found') unless @token
+
+      register_exit_cb if set_at_exit
+
+      Sqreen.log.warn "using token #{@token}"
+      self.features = create_session(session_class)
+      # Ensure a deliverer is there unless features have set it first
+      self.deliverer ||= Deliveries::Simple.new(session)
+
+      self.metrics_engine = MetricsStore.new
+      @instrumenter = Instrumentation.new(metrics_engine)
+    end
+
+    def create_session(session_class)
+      @session = session_class.new(@url, @token)
+      session.login(@framework)
+    end
+
+    def deliverer=(new_deliverer)
+      deliverer.drain if deliverer
+      @deliverer = new_deliverer
+    end
+
+    def batch_events(batch_size, max_staleness = nil)
+      size = batch_size.to_i
+      self.deliverer = if size < 1
+                         Deliveries::Simple.new(session)
+                       else
+                         staleness = max_staleness.to_i
+                         Deliveries::Batch.new(session, size, staleness)
+                       end
+    end
+
+    def load_rules
+      rules_pack = session.rules
+      rulespack_id = rules_pack['pack_id']
+      rules = rules_pack['rules'].each { |r| r['rulespack_id'] = rulespack_id }
+      Sqreen.log.info format('retrieved rulespack id: %s', rulespack_id)
+      Sqreen.log.debug format('retrieved %d rules', rules.size)
+      local_rules = Sqreen::Rules.local(@configuration) || []
+      rules += local_rules.
+               select { |rule| rule['enabled'] }.
+               each { |r| r['rulespack_id'] = 'local' }
+      Sqreen.log.debug format('rules: %s', rules.
+                              sort_by { |r| r['name'] }.
+                              map { |r| format('(%s, %s)', r['name'], r.to_json.size) }.
+                              join(', '))
+      [rulespack_id, rules]
+    end
+
+    def performance_metrics_period=(value)
+      value = value.to_i
+      if value > 0
+        PerformanceNotifications::Metrics.enable(metrics_engine, value)
+      else
+        PerformanceNotifications::Metrics.disable
+      end
+    end
+
+    def setup_instrumentation
+      Sqreen.log.info 'setup instrumentation'
+      rulespack_id, rules = load_rules
+      @framework.instrument_when_ready!(instrumenter, rules)
+      rulespack_id.to_s
+    end
+
+    def remove_instrumentation
+      Sqreen.log.debug 'removing instrumentation'
+      instrumenter.remove_all_callbacks
+      true
+    end
+
+    def reload_rules
+      Sqreen.log.debug 'Reloading rules'
+      rulespack_id, rules = load_rules
+      instrumenter.remove_all_callbacks
+
+      @framework.instrument_when_ready!(instrumenter, rules)
+      Sqreen.log.debug 'Rules reloaded'
+      rulespack_id.to_s
+    end
+
+    def process_commands(commands)
+      while commands && !commands.empty?
+        res = RemoteCommand.process_list(self, commands)
+        res = session.post_commands_result(res)
+        commands = res['commands']
+      end
+    end
+
+    def do_heartbeat
+      @last_heartbeat_request = Time.now
+      res = session.heartbeat
+      update_heartbeat_delay
+      process_commands(res['commands'])
+    end
+
+    def features
+      Sqreen.features
+    end
+
+    def features=(features)
+      Sqreen.update_features(features)
+      session.request_compression = features['request_compression'] if session
+      self.performance_metrics_period = features['performance_metrics_period']
+      md = features['publish_metrics_delay'].to_i
+      self.publish_metrics_delay = md if md > 0
+      return if features['batch_size'].nil?
+      batch_events(features['batch_size'], features['max_staleness'])
+    end
+
+    def change_features(new_features)
+      old = features
+      self.features = new_features
+      {
+        'was' => old,
+        'now' => new_features,
+      }
+    end
+
+    def aggregate_observations
+      q = Sqreen.observations_queue
+      q.size.times do
+        cat, key, obs, t = q.pop
+        metrics_engine.update(cat, t, key, obs)
+      end
+    end
+
+    def run_watcher_once
+      event = Timeout.timeout(@sleep_delay) do
+        Sqreen.queue.pop
+      end
+    rescue Timeout::Error
+      periodic_cleanup
+    else
+      handle_event(event)
+      if (@last_heartbeat_request + HEARTBEAT_MAX_DELAY) < Time.now
+        Sqreen.log.debug 'Forced an heartbeat'
+        do_heartbeat
+        # Also aggregate/post metrics when cleanup has
+        # not been done for a long time
+        periodic_cleanup
+      end
+    end
+
+    def periodic_cleanup
+      # Nothing occured:
+      # tick delivery, aggregates_metrics
+      # issue a simple heartbeat if it's time (which may return commands)
+      @deliverer.tick
+      aggregate_observations
+      t = Time.now
+      do_heartbeat if (@last_heartbeat_request + heartbeat_delay) < t
+      post_metrics if (@last_post_metrics_request + publish_metrics_delay) < t
+    end
+
+    def handle_event(event)
+      if event == METRICS_EVENT
+        aggregate_observations
+      else
+        @deliverer.post_event(event)
+      end
+    end
+
+    def run_watcher
+      loop do
+        run_watcher_once
+      end
+    end
+
+    def update_heartbeat_delay
+      return unless Time.now - @started > HEARTBEAT_WARMUP
+      return if heartbeat_delay == HEARTBEAT_MAX_DELAY
+      self.heartbeat_delay = HEARTBEAT_MAX_DELAY
+    end
+
+    def update_sleep_delay
+      @sleep_delay = [heartbeat_delay, publish_metrics_delay].min
+      Sqreen.log.debug { format('sleep delay %f', @sleep_delay) }
+    end
+
+    def heartbeat_delay=(x)
+      @heartbeat_delay = x
+      update_sleep_delay
+      x
+    end
+
+    def publish_metrics_delay=(x)
+      @publish_metrics_delay = x
+      update_sleep_delay
+      x
+    end
+
+    def post_metrics
+      return unless metrics_engine
+      @last_post_metrics_request = Time.now
+      session.post_metrics(metrics_engine.publish)
+    end
+
+    # Sinatra is using at_exit to run the application, see:
+    # https://github.com/sinatra/sinatra/blob/cd503e6c590cd48c2c9bb7869522494bfc62cb14/lib/sinatra/main.rb#L25
+    def exit_from_sinatra_startup?
+      defined?(Sinatra::Application) &&
+        Sinatra::Application.respond_to?(:run?) &&
+        !Sinatra::Application.run?
+    end
+
+    def logout(retrying = true)
+      return unless session
+      if @logged_out_tried
+        Sqreen.log.debug('Not running logout twice')
+        return
+      end
+      @logged_out_tried = true
+      @deliverer.drain if @deliverer
+      aggregate_observations
+      post_metrics
+      session.logout(retrying)
+    end
+
+    def register_exit_cb(try_again = true)
+      at_exit do
+        if exit_from_sinatra_startup? && try_again
+          register_exit_cb(false)
+        else
+          logout
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/runtime_infos.rb

@@ -0,0 +1,127 @@
+# Copyright (c) 2015 Sqreen. All Rights Reserved.
+# Please refer to our terms for more information: https://www.sqreen.io/terms.html
+
+require 'sqreen/version'
+require 'sqreen/frameworks'
+
+require 'socket'
+
+module Sqreen
+  module RuntimeInfos
+    module_function
+
+    def all(framework)
+      res = { :various_infos => {} }
+      res.merge! agent
+      res.merge! os
+      res.merge! runtime
+      res.merge! framework.framework_infos
+      res[:various_infos].merge! time
+      res[:various_infos].merge! dependencies
+      res[:various_infos].merge! process
+      res
+    end
+
+    def local_infos
+      {
+        'time' => Time.now.utc,
+        'name' => hostname,
+      }
+    end
+
+    def dependencies
+      gem_info = Gem.loaded_specs
+      gem_info = gem_info.map do |name, spec|
+        {
+          :name => name,
+          :version => spec.version.to_s,
+          :homepage => spec.homepage,
+          :source => (extract_source(spec.source) if spec.respond_to?(:source)),
+        }
+      end
+      {
+        :dependencies => gem_info,
+      }
+    end
+
+    def time
+      { :time => Time.now.to_s }
+    end
+
+    def ssl
+      type = nil
+      version = nil
+      if defined? OpenSSL
+        type = 'OpenSSL'
+        version = OpenSSL::OPENSSL_VERSION if defined? OpenSSL::OPENSSL_VERSION
+      end
+      { :ssl =>
+        {
+          :type => type,
+          :version => version,
+        },
+      }
+    end
+
+    def agent
+      {
+        :agent_type => :ruby,
+        :agent_version => ::Sqreen::VERSION,
+      }
+    end
+
+    def os
+      plat = if defined? ::RUBY_PLATFORM
+               ::RUBY_PLATFORM
+             elsif defined? ::PLATFORM
+               ::PLATFORM
+             else
+               ''
+             end
+      {
+        :os_type => plat,
+        :hostname => hostname,
+      }
+    end
+
+    def hostname
+      Socket.gethostname
+    end
+
+    def process
+      {
+        :pid => Process.pid,
+        :ppid => Process.ppid,
+        :euid => Process.euid,
+        :egid => Process.egid,
+        :uid  => Process.uid,
+        :gid  => Process.gid,
+        :name => $0,
+      }
+    end
+
+    def runtime
+      engine = if defined? ::RUBY_ENGINE
+                 ::RUBY_ENGINE
+               else
+                 'ruby'
+               end
+      {
+        :runtime_type    => engine,
+        :runtime_version => ::RUBY_DESCRIPTION,
+      }
+    end
+
+    def extract_source(source)
+      return nil unless source
+      ret = { 'name' => source.class.name.split(':')[-1] }
+      opts = {}
+      opts = source.options if source.respond_to?(:options)
+      ret['remotes'] = opts['remotes'] if opts['remotes']
+      ret['uri'] = opts['uri'] if opts['uri']
+      # FIXME: scrub any auth data in uris
+      ret['path'] = opts['path'] if opts['path']
+      ret
+    end
+  end
+end

data/lib/sqreen/session.rb

@@ -0,0 +1,340 @@
+# Copyright (c) 2015 Sqreen. All Rights Reserved.
+# Please refer to our terms for more information: https://www.sqreen.io/terms.html
+
+require 'sqreen/log'
+require 'sqreen/runtime_infos'
+require 'sqreen/events/remote_exception'
+
+require 'net/https'
+require 'json'
+require 'uri'
+require 'openssl'
+require 'zlib'
+
+# $ curl -H"x-api-key: ${KEY}"  http://127.0.0.1:5000/sqreen/v0/app-login
+# {
+#       "session_id": "c9171007c27d4da8906312ff343ed41307f65b2f6fdf4a05a445bb7016186657",
+#       "status": true
+# }
+#
+# $ curl -H"x-session-key: ${SESS}" http://127.0.0.1:5000/sqreen/v0/get-rulespack
+
+#
+# FIXME: we should be proxy capable
+# FIXME: we should be multithread aware (when callbacks perform server requests?)
+#
+
+module Sqreen
+  class Session
+    RETRY_CONNECT_SECONDS = 10
+    RETRY_REQUEST_SECONDS = 10
+
+    MAX_DELAY = 60 * 30
+
+    RETRY_LONG = 128
+
+    MUTEX = Mutex.new
+    METRICS_KEY = 'metrics'.freeze
+
+    @@path_prefix = '/sqreen/v0/'
+
+    attr_accessor :request_compression
+
+    def initialize(server_url, token)
+      @token = token
+      @session_id = nil
+      @server_url = server_url
+      @request_compression = false
+      @connected = nil
+
+      uri = parse_uri(server_url)
+      use_ssl = (uri.scheme == 'https')
+
+      @req_nb = 0
+
+      @http = Net::HTTP.new(uri.host, uri.port)
+      @http.use_ssl = use_ssl
+      if use_ssl
+        cert_file = File.join(File.dirname(__FILE__), 'ca.crt')
+        cert_store = OpenSSL::X509::Store.new
+        cert_store.add_file cert_file
+        @http.cert_store = cert_store
+      end
+    end
+
+    def parse_uri(uri)
+      # This regexp is the Ruby constant URI::PATTERN::HOSTNAME augmented
+      # with the _ character that is frequent in Docker linked containers.
+      re = '(?:(?:[a-zA-Z\\d](?:[-_a-zA-Z\\d]*[a-zA-Z\\d])?)\\.)*(?:[a-zA-Z](?:[-_a-zA-Z\\d]*[a-zA-Z\\d])?)\\.?'
+      parser = URI::Parser.new :HOSTNAME => re
+      parser.parse(uri)
+    end
+
+    def prefix_path(path)
+      @@path_prefix + path
+    end
+
+    def connected?
+      @con && @con.started?
+    end
+
+    def disconnect
+      @http.finish if connected?
+    end
+
+    NET_ERRORS = [Timeout::Error,
+                  Errno::EINVAL,
+                  Errno::ECONNRESET,
+                  Errno::ECONNREFUSED,
+                  EOFError,
+                  Net::HTTPBadResponse,
+                  Net::HTTPHeaderSyntaxError,
+                  SocketError,
+                  Net::ProtocolError].freeze
+
+    def connect
+      return if connected?
+      Sqreen.log.warn "connection to #{@server_url}..."
+      @session_id = nil
+      @conn_retry = 0
+      begin
+        @con = @http.start
+      rescue *NET_ERRORS
+        Sqreen.log.debug "Cannot connect, retry in #{RETRY_CONNECT_SECONDS} seconds"
+        sleep RETRY_CONNECT_SECONDS
+        @conn_retry += 1
+        retry
+      else
+        Sqreen.log.warn 'connection success.'
+      end
+    end
+
+    def resilient_post(path, data, headers = {})
+      post(path, data, headers, RETRY_LONG)
+    end
+
+    def resilient_get(path, headers = {})
+      get(path, headers, RETRY_LONG)
+    end
+
+    def post(path, data, headers = {}, max_retry = 2)
+      do_http_request(:POST, path, data, headers, max_retry)
+    end
+
+    def get(path, headers = {}, max_retry = 2)
+      do_http_request(:GET, path, nil, headers, max_retry)
+    end
+
+    def resiliently(retry_request_seconds, max_retry, current_retry = 0)
+      return yield
+    rescue => e
+
+      Sqreen.log.error(e.inspect)
+
+      current_retry += 1
+
+      raise e if current_retry >= max_retry || e.is_a?(Sqreen::NotImplementedYet)
+
+      sleep_delay = [MAX_DELAY, retry_request_seconds * current_retry].min
+      Sqreen.log.debug format('Sleeping %ds', sleep_delay)
+      sleep(sleep_delay)
+
+      retry
+    end
+
+    def thread_id
+      th = Thread.current
+      return '' unless th
+      re = th.to_s.scan(/:(0x.*)>/)
+      return '' unless re && re.size > 0
+      res = re[0]
+      return '' unless res && res.size > 0
+      res[0]
+    end
+
+    def do_http_request(method, path, data, headers = {}, max_retry = 2)
+      connect unless connected?
+      headers['X-Session-Key'] = @session_id if @session_id
+      headers['X-Sqreen-Time'] = Time.now.utc.to_f.to_s
+      headers['X-Sqreen-Agent'] = "Ruby/#{Sqreen::VERSION}"
+      headers['X-Sqreen-Beta'] = format('pid=%d;tid=%s;nb=%d;t=%f',
+                                        Process.pid,
+                                        thread_id,
+                                        @req_nb,
+                                        Time.now.utc.to_f)
+      headers['Content-Type'] = 'application/json'
+      if request_compression && !method.casecmp(:GET).zero?
+        headers['Content-Encoding'] = 'gzip'
+      end
+
+      @req_nb += 1
+
+      path = prefix_path(path)
+      Sqreen.log.debug format('%s %s (%s)', method, path, @token)
+
+      res = {}
+      resiliently(RETRY_REQUEST_SECONDS, max_retry) do
+        json = nil
+        MUTEX.synchronize do
+          json = case method.upcase
+                 when :GET
+                   @con.get(path, headers)
+                 when :POST
+                   json_data = compress(self.class.encode_payload(data))
+                   @con.post(path, json_data, headers)
+                 else
+                   Sqreen.log.debug format('unknown method %s', method)
+                   raise Sqreen::NotImplementedYet
+                 end
+        end
+
+        if json && json.body
+          res = JSON.parse(json.body)
+          unless res['status']
+            Sqreen.log.debug(format('Cannot %s %s.', method, path))
+          end
+        else
+          Sqreen.log.debug 'warning: empty return value'
+        end
+      end
+      Sqreen.log.debug format('%s %s (DONE)', method, path)
+      res
+    end
+
+    def self.encode_payload(data)
+      JSON.generate(data)
+    rescue JSON::GeneratorError
+      Sqreen.log.debug('Payload could not be encoded enforcing recode')
+      JSON.generate(rencode_payload(data))
+    end
+
+    def compress(data)
+      return data unless request_compression
+      out = StringIO.new
+      w = Zlib::GzipWriter.new(out)
+      w.write(data)
+      w.close
+      out.string
+    end
+
+    def self.rencode_payload(obj, max_depth = 20)
+      max_depth -= 1
+      return obj if max_depth < 0
+      return rencode_array(obj, max_depth) if obj.is_a?(Array)
+      return enforce_encoding(obj) unless obj.is_a?(Hash)
+      obj.each do |k, v|
+        case v
+        when Array
+          obj[k] = rencode_array(v, max_depth)
+        when Hash
+          obj[k] = rencode_payload(v, max_depth)
+        when String
+          obj[k] = enforce_encoding(v)
+        end
+      end
+      obj
+    end
+
+    def self.rencode_array(array, max_depth)
+      array.map! { |e| rencode_payload(e, max_depth - 1) }
+      array
+    end
+
+    def self.enforce_encoding(str)
+      return str unless str.is_a?(String)
+      return str if str.valid_encoding?
+      str.chars.map do |v|
+        if v.valid_encoding?
+          v
+        else
+          v.bytes.map { |c| "\\x#{c.to_s(16).upcase}" }.join
+        end
+      end.join
+    end
+
+    def login(framework)
+      headers = { 'x-api-key' => @token }
+
+      res = resilient_post('app-login', RuntimeInfos.all(framework), headers)
+
+      if !res || !res['status']
+        public_error = format('Cannot login. Token may be invalid: %s', @token)
+        Sqreen.log.error public_error
+        raise(Sqreen::TokenInvalidException,
+              format('invalid response: %s', res.inspect))
+      end
+      Sqreen.log.info 'Login success.'
+      @session_id = res['session_id']
+      Sqreen.log.debug "received session_id #{@session_id}"
+      Sqreen.logged_in = true
+      res.fetch('features', {})
+    end
+
+    def rules
+      resilient_get('rulespack')
+    end
+
+    def heartbeat
+      get('app-beat', {}, 5)
+    end
+
+    def post_commands_result(res)
+      resilient_post('commands', res)
+    end
+
+    def post_metrics(metrics)
+      return if metrics.nil? || metrics.empty?
+      payload = { METRICS_KEY => metrics }
+      resilient_post(METRICS_KEY, payload)
+    end
+
+    def post_attack(attack)
+      resilient_post('attack', attack.to_hash)
+    end
+
+    # Post an exception to Sqreen for analysis
+    # @param exception [RemoteException] Exception and context to be sent over
+    def post_sqreen_exception(exception)
+      post('sqreen_exception', exception.to_hash, {}, 5)
+    rescue *NET_ERRORS => e
+      Sqreen.log.error(format('Could not post exception (network down? %s) %s',
+                              e.inspect,
+                              exception.to_hash.inspect))
+      nil
+    end
+
+    BATCH_KEY = 'batch'.freeze
+    EVENT_TYPE_KEY = 'event_type'.freeze
+    def post_batch(events)
+      batch = events.map do |event|
+        h = event.to_hash
+        h[EVENT_TYPE_KEY] = event_kind(event)
+        h
+      end
+      resilient_post(BATCH_KEY, BATCH_KEY => batch)
+    end
+
+    # Perform agent logout
+    # @param retrying [Boolean] whether to try again on error
+    def logout(retrying = true)
+      # Do not try to connect if we are not connected
+      unless connected?
+        Sqreen.log.debug('Not connected: not trying to logout')
+        return
+      end
+      # Perform not very resilient logout not to slow down client app shutdown
+      get('app-logout', {}, retrying ? 2 : 1)
+      Sqreen.logged_in = false
+      disconnect
+    end
+
+    protected
+
+    def event_kind(event)
+      case event
+      when Sqreen::RemoteException then 'sqreen_exception'
+      when Sqreen::Attack then 'attack'
+      end
+    end
+  end
+end