apisonator 2.100.0

data/lib/3scale/backend/logging/logger.rb

@@ -0,0 +1,18 @@
+# Add an application-wide logger UNRELATED to the request's logs.
+#
+# This allows for additional instrumentation and information gathering
+# in production environment.
+#
+require 'logger'
+
+module ThreeScale
+  module Backend
+    module Logging
+      class Logger
+        def self.new(*args)
+          ::Logger.new(*args)
+        end
+      end
+    end
+  end
+end

data/lib/3scale/backend/logging/middleware.rb

@@ -0,0 +1,62 @@
+require '3scale/backend/logging/middleware/writer'
+require '3scale/backend/logging/middleware/text_writer'
+require '3scale/backend/logging/middleware/json_writer'
+
+module ThreeScale
+  module Backend
+    module Logging
+      class Middleware
+        WRITERS = { text: TextWriter, json: JsonWriter }.freeze
+        private_constant :WRITERS
+
+        DEFAULT_WRITERS = [WRITERS[:text].new].freeze
+        private_constant :DEFAULT_WRITERS
+
+        class UnsupportedLoggerType < StandardError
+          def initialize(logger)
+            super "#{logger} is not a supported logger type."
+          end
+        end
+
+        # writers is an array of symbols. WRITERS contains the accepted values
+        def initialize(app, writers: DEFAULT_WRITERS)
+          @app = app
+          @writers = writers
+        end
+
+        def call(env)
+          began_at = Time.now
+          begin
+            status, header, body = @app.call(env)
+          rescue Exception => e
+            @writers.each do |writer|
+              writer.log_error(env, 500, e.message, began_at)
+            end
+            raise e
+          end
+
+          header = ::Rack::Utils::HeaderHash.new(header)
+          body = ::Rack::BodyProxy.new(body) do
+            @writers.each do |writer|
+              writer.log(env, status, header, began_at)
+            end
+          end
+
+          [status, header, body]
+        end
+
+        # Returns the Writer instances that correspond to the loggers given.
+        # If no loggers are given, returns the default writers.
+        def self.writers(loggers)
+          writers = Array(loggers).map do |logger|
+            writer_class =  WRITERS[logger]
+            raise UnsupportedLoggerType.new(logger) unless writer_class
+            writer_class.new
+          end
+
+          writers.empty? ? DEFAULT_WRITERS : writers
+        end
+      end
+    end
+  end
+end

data/lib/3scale/backend/logging/middleware/json_writer.rb

@@ -0,0 +1,21 @@
+require '3scale/backend/logging/middleware/writer'
+
+module ThreeScale
+  module Backend
+    module Logging
+      class Middleware
+        class JsonWriter
+          include Middleware::Writer
+
+          private
+
+          def formatted_log(data)
+            data.to_json + "\n".freeze
+          end
+
+          alias_method :formatted_error, :formatted_log
+        end
+      end
+    end
+  end
+end

data/lib/3scale/backend/logging/middleware/text_writer.rb

@@ -0,0 +1,60 @@
+require '3scale/backend/logging/middleware/writer'
+
+module ThreeScale
+  module Backend
+    module Logging
+      class Middleware
+        class TextWriter
+          include Middleware::Writer
+
+          FORMAT = "%s - %s [%s] \"%s %s%s %s\" %d %s %s 0 0 0 %s %s %s %s %s\n".freeze
+          private_constant :FORMAT
+
+          ERROR_FORMAT = "%s - %s [%s] \"%s %s%s %s\" %d \"%s\" %s %s %s\n".freeze
+          private_constant :ERROR_FORMAT
+
+          SORTED_LOG_FIELDS = [:forwarded_for,
+                               :remote_user,
+                               :time,
+                               :method,
+                               :path_info,
+                               :query_string,
+                               :http_version,
+                               :status,
+                               :length,
+                               :response_time,
+                               :memoizer_size,
+                               :memoizer_count,
+                               :memoizer_hits,
+                               :request_id,
+                               :extensions].freeze
+          private_constant :SORTED_LOG_FIELDS
+
+          SORTED_ERROR_LOG_FIELDS = [:forwarded_for,
+                                     :remote_user,
+                                     :time,
+                                     :method,
+                                     :path_info,
+                                     :query_string,
+                                     :http_version,
+                                     :status,
+                                     :error,
+                                     :response_time,
+                                     :request_id,
+                                     :extensions].freeze
+          private_constant :SORTED_ERROR_LOG_FIELDS
+
+          private
+
+          def formatted_log(data)
+            FORMAT % SORTED_LOG_FIELDS.map { |field| data[field] }
+          end
+
+          def formatted_error(data)
+            ERROR_FORMAT % SORTED_ERROR_LOG_FIELDS.map { |field| data[field] }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/3scale/backend/logging/middleware/writer.rb

@@ -0,0 +1,143 @@
+module ThreeScale
+  module Backend
+    module Logging
+      class Middleware
+        module Writer
+          Z3_RANGE = 0..3.freeze
+          private_constant :Z3_RANGE
+
+          private_constant(*[
+              :HTTP_X_FORWARDED_FOR,
+              :REMOTE_ADDR,
+              :REMOTE_USER,
+              :REQUEST_METHOD,
+              :PATH_INFO,
+              :HTTP_VERSION,
+              :HTTP_X_REQUEST_ID,
+              :QUERY_STRING,
+              :HTTP_3SCALE_OPTIONS
+          ].each do |k|
+            const_set(k, k.to_s.freeze)
+          end)
+
+          private_constant(*{
+              DATE_FORMAT: '%d/%b/%Y %H:%M:%S %Z',
+              STR_PROVIDER_KEY: 'provider_key',
+              STR_POST: 'POST',
+              STR_EQUAL: '=',
+              STR_DASH: '-',
+              STR_AMPERSAND: '&',
+              STR_ZERO: '0',
+              STR_RACK_ERRORS: 'rack.errors',
+              STR_EMPTY: '',
+              STR_QUESTION_MARK: '?',
+              STR_NEWLINE: "\n",
+              STR_DQUOTE: '"',
+              STR_ESCAPED_DQUOTE: '\"',
+              STR_CONTENT_LENGTH: 'Content-Length'
+          }.map do |k, v|
+            const_set(k, v.freeze)
+            k
+          end)
+
+          def initialize(logger=STDOUT)
+            @logger = logger
+          end
+
+          def log(env, status, header, began_at)
+            data = log_data(env, status, header, began_at)
+            log = formatted_log(data)
+            logger(env).write(log)
+          end
+
+          def log_error(env, status, error, began_at)
+            data = log_error_data(env, status, error, began_at)
+            error = formatted_error(data)
+            logger(env).write(error)
+          end
+
+          private
+
+          def logger(env)
+            @logger || env[STR_RACK_ERRORS] || STDERR
+          end
+
+          def log_data(env, status, header, began_at)
+            common_request_data(env, status, began_at)
+                .merge(success_specific_data(header))
+          end
+
+          def log_error_data(env, status, error, began_at)
+            common_request_data(env, status, began_at)
+                .merge(error_specific_data(error))
+          end
+
+          def common_request_data(env, status, began_at)
+            now = Time.now.getutc
+
+            { forwarded_for: env[HTTP_X_FORWARDED_FOR] || env[REMOTE_ADDR] || STR_DASH,
+              remote_user: env[REMOTE_USER] || STR_DASH,
+              time: now.strftime(DATE_FORMAT),
+              method: env[REQUEST_METHOD],
+              path_info: env[PATH_INFO],
+              query_string: extract_query_string(env),
+              http_version: env[HTTP_VERSION],
+              status: status.to_s[Z3_RANGE],
+              response_time: now - began_at,
+              request_id: env[HTTP_X_REQUEST_ID] || STR_DASH,
+              extensions: extensions(env) }
+          end
+
+          def success_specific_data(header)
+            { length: extract_content_length(header) }.merge(memoizer_data)
+          end
+
+          def memoizer_data
+            memoizer = memoizer_stats
+
+            { memoizer_size: memoizer[:size] || STR_DASH,
+              memoizer_count: memoizer[:count] || STR_DASH,
+              memoizer_hits: memoizer[:hits] || STR_DASH }
+          end
+
+          def error_specific_data(error)
+            { error: error }
+          end
+
+          def extract_query_string(env)
+            qs = env[QUERY_STRING]
+            if env[REQUEST_METHOD].to_s.upcase == STR_POST
+              provider_key = begin
+                ::Rack::Request.new(env).params[STR_PROVIDER_KEY]
+              rescue IOError
+                # happens when body does not parse
+                nil
+              end
+              unless provider_key.nil?
+                qs = qs.dup
+                qs << STR_AMPERSAND unless qs.empty?
+                qs << STR_PROVIDER_KEY + STR_EQUAL + provider_key.to_s
+              end
+            end
+
+            qs.empty? ? STR_EMPTY : STR_QUESTION_MARK + qs.tr(STR_NEWLINE, STR_EMPTY)
+          end
+
+          def extract_content_length(headers)
+            value = headers[STR_CONTENT_LENGTH] or return STR_DASH
+            value.to_s == STR_ZERO ? STR_DASH : value
+          end
+
+          def extensions(env)
+            ext = env[HTTP_3SCALE_OPTIONS]
+            ext ? "\"#{ext.gsub(STR_DQUOTE, STR_ESCAPED_DQUOTE)}\"" : STR_DASH
+          end
+
+          def memoizer_stats
+            ThreeScale::Backend::Memoizer.stats
+          end
+        end
+      end
+    end
+  end
+end

data/lib/3scale/backend/logging/worker.rb

@@ -0,0 +1,107 @@
+module ThreeScale
+  module Backend
+    module Logging
+      class Worker
+        include Configurable
+
+        module PlainText
+          private
+
+          def logger_formatter
+            proc do |severity, datetime, _progname, msg|
+              "#{severity} #{Process.pid} #{formatted_datetime(datetime)} #{msg}\n"
+            end
+          end
+
+          def formatted_datetime(datetime)
+            datetime.getutc.strftime("[%d/%b/%Y %H:%M:%S %Z]".freeze)
+          end
+        end
+        private_constant :PlainText
+
+        module Json
+          STRING_COMMON_FIELDS = [:job_class].freeze
+          private_constant :STRING_COMMON_FIELDS
+
+          FLOAT_COMMON_FIELDS = [:runtime, :run_plus_queued_time].freeze
+          private_constant :FLOAT_COMMON_FIELDS
+
+          INT_COMMON_FIELDS = [:memoizer_size, :memoizer_count, :memoizer_hits].freeze
+          private_constant :INT_COMMON_FIELDS
+
+          COMMON_LOG_FIELDS = (STRING_COMMON_FIELDS +
+                               FLOAT_COMMON_FIELDS +
+                               INT_COMMON_FIELDS).freeze
+          private_constant :COMMON_LOG_FIELDS
+
+          private
+
+          def logger_formatter
+            proc do |severity, datetime, _progname, msg|
+              common_fields = { severity: severity,
+                                pid: Process.pid,
+                                time: datetime.getutc.to_s }
+
+              # When there is an error, the msg does not contain run times and
+              # memoizer stats.
+              msg_fields = if severity == 'INFO'.freeze
+                             formatted_msg(msg)
+                           else
+                             { msg: msg }
+                           end
+
+              common_fields.merge(msg_fields).to_json + "\n".freeze
+            end
+          end
+
+          def formatted_msg(msg)
+            # The format of the message depends on the kind of background job
+            # that sends it:
+            # job_class -variable_part- run_time run_time+queued_time
+            # memoizer_size memoizer_count memoizer_hits.
+            # -variable_part- depends on the kind of job. It might contain app
+            # ids, a message (with spaces), etc. The rest of the message is
+            # common for all the jobs. For now, we discard the variable part.
+            fields = msg.split(' '.freeze)
+            common_field_values = [fields.first] + fields.last(5)
+            res = Hash[COMMON_LOG_FIELDS.zip(common_field_values)]
+
+            FLOAT_COMMON_FIELDS.each do |field|
+              res[field] = res[field].to_f
+            end
+
+            INT_COMMON_FIELDS.each do |field|
+              res[field] = res[field].to_i
+            end
+
+            res
+          end
+        end
+        private_constant :Json
+
+        extend(configuration.workers_logger_formatter == :json ? Json : PlainText)
+
+        class << self
+          def configure_logging(worker_class, workers_log_file)
+            log_file = workers_log_file || configuration.workers_log_file || '/dev/null'
+
+            Logging.enable! on: worker_class.singleton_class, with: log_file do |logger|
+              logger.formatter = logger_formatter
+
+              # At this point, we've already configured the logger for Backend
+              # (used in Listeners and rake tasks). We can reuse the notify proc
+              # defined there.
+              logger.define_singleton_method(:notify, backend_logger_notify_proc)
+            end
+          end
+
+          private
+
+          def backend_logger_notify_proc
+            Backend.logger.method(:notify).to_proc
+          end
+        end
+      end
+    end
+  end
+end

data/lib/3scale/backend/manifest.rb

@@ -0,0 +1,80 @@
+require '3scale/backend/version'
+require '3scale/backend/util'
+require '3scale/backend/server'
+
+module ThreeScale
+  module Backend
+    module Manifest
+      class << self
+        LISTENER_WORKERS = 'LISTENER_WORKERS'.freeze
+        private_constant :LISTENER_WORKERS
+        PUMA_WORKERS = 'PUMA_WORKERS'.freeze
+        private_constant :PUMA_WORKERS
+        PUMA_WORKERS_CPUMULT = 8
+        private_constant :PUMA_WORKERS_CPUMULT
+
+        # Thread safety of our application. Turn this on if we ever are MT safe.
+        def thread_safe?
+          false
+        end
+
+        # Compute workers based on LISTENER_WORKERS and PUMA_WORKERS env
+        # variables. The former takes precedence.
+        # If those envs do not exist or are empty, use number of cpus
+        def compute_workers(ncpus)
+          return 0 unless Process.respond_to?(:fork)
+
+          compute_workers_from_env(LISTENER_WORKERS) ||
+            compute_workers_from_env(PUMA_WORKERS) ||
+            ncpus * PUMA_WORKERS_CPUMULT
+        end
+
+        def server_model
+          # Serving model settings here
+          #
+          # compute default workers and threads values
+          # We want to adapt workers and threads to our characteristics.
+          # Note that these values will likely need to be tweaked depending on
+          # the Ruby implementation and how our app behaves!
+          ncpus = ThreeScale::Backend::Util.number_of_cpus
+          workers = compute_workers ncpus
+          # if no workers but mt-safe, we spawn more threads.
+          min_threads, max_threads = if thread_safe?
+                                       shift = workers.zero? ? 2 : 0
+                                       [ncpus << shift, ncpus << 1 + shift]
+                                     else
+                                       [1, 1]
+                                     end
+          {
+            ncpus: ncpus,
+            workers: workers,
+            min_threads: min_threads,
+            max_threads: max_threads
+          }
+        end
+
+        def report
+          {
+            version: ThreeScale::Backend::VERSION,
+            root_dir: ThreeScale::Backend::Util.root_dir,
+            servers: ThreeScale::Backend::Server.list,
+            thread_safe: thread_safe?,
+            server_model: server_model,
+          }
+        end
+
+        private
+
+        def compute_workers_from_env(env_name)
+          if ENV[env_name] && !ENV[env_name].empty?
+            begin
+              Integer(ENV[env_name])
+            rescue => e
+              raise e, "#{env_name} environment var cannot be parsed: #{e.message}"
+            end
+          end
+        end
+      end
+    end
+  end
+end