apisonator 2.100.0

data/lib/3scale/backend/experiment.rb

@@ -0,0 +1,84 @@
+require 'scientist'
+require 'statsd'
+
+module ThreeScale
+  module Backend
+    # This class allows us to perform experiments using the Scientist gem from
+    # GitHub. It is useful for comparing two different behaviors when doing a
+    # refactoring. The idea is that we can use the old code in the normal flow
+    # of the program and at the same time, we can compare its results and
+    # performance against new code. This class notifies the logger if the result
+    # of the two fragments of code does not match. Also, it sends their
+    # execution time to StatsD.
+    #
+    # To use this class, you need to declare the old behavior in a 'use' block,
+    # and the new one in a 'try' block. Also, when instantiating the class, we
+    # need to give the experiment a name, and define the % of times that it
+    # will run.
+    # exp = ThreeScale::Backend::Experiment.new('my-experiment', 50)
+    # exp.use { old_method }
+    # exp.try { new_method }
+    # exp.run
+    #
+    # 'run' returns the result of the block sent to the 'use' method. Its
+    # return value is the one we should use in our program. On the other hand,
+    # 'try' will swallow any exception raised inside the block. The operations
+    # inside the 'try' block should be idempotent. Avoid operations like
+    # writing to the DB.
+
+    class Experiment
+      include Scientist::Experiment
+      include Backend::Logging
+
+      ResultsMismatch = Class.new(StandardError)
+
+      RANDOM = Random.new
+      private_constant :RANDOM
+
+      def initialize(name, perc_exec)
+        @name = name
+        @perc_exec = perc_exec
+        @base_metric = "backend.#{environment}.experiments.#{name}"
+      end
+
+      def publish(result)
+        # This only works for the first candidate (try block)
+        send_to_statsd(result.control.duration, result.candidates.first.duration)
+        check_mismatch(result)
+      end
+
+      def enabled?
+        RANDOM.rand(100) < perc_exec
+      end
+
+      private
+
+      attr_reader :name, :perc_exec, :base_metric
+
+      def send_to_statsd(control_duration, candidate_duration)
+        statsd.timing("#{base_metric}.control", control_duration)
+        statsd.timing("#{base_metric}.candidate", candidate_duration)
+      end
+
+      def check_mismatch(result)
+        if result.mismatched?
+          msg = mismatch_msg(result.control.value, result.candidates.first.value)
+          logger.notify(ResultsMismatch.new(msg))
+        end
+      end
+
+      def mismatch_msg(control_value, candidate_value)
+        "There was a mismatch when running the experiment #{name}. "\
+        "control = #{control_value}, candidate = #{candidate_value}''"
+      end
+
+      def statsd
+        Statsd.instance
+      end
+
+      def environment
+        ThreeScale::Backend.environment
+      end
+    end
+  end
+end

data/lib/3scale/backend/extensions.rb

@@ -0,0 +1,5 @@
+require '3scale/backend/extensions/hash'
+require '3scale/backend/extensions/array'
+require '3scale/backend/extensions/nil_class'
+require '3scale/backend/extensions/string'
+require '3scale/backend/extensions/time'

data/lib/3scale/backend/extensions/array.rb

@@ -0,0 +1,19 @@
+module ThreeScale
+  module Backend
+    module Extensions
+      module Array
+        def valid_encoding?
+          self.each do |v|
+            if v.is_a?(String) || v.is_a?(Array) || v.is_a?(Hash)
+              return false unless v.valid_encoding?
+            end
+          end
+          return true
+        end
+      end
+    end
+  end
+end
+
+Array.send(:include, ThreeScale::Backend::Extensions::Array)
+

data/lib/3scale/backend/extensions/hash.rb

@@ -0,0 +1,26 @@
+module ThreeScale
+  module Backend
+    module Extensions
+      module Hash
+        def symbolize_names
+          inject({}) do |memo, (key, value)|
+            memo[key.to_sym] = value
+            memo
+          end
+        end
+
+        def valid_encoding?
+          self.each do |k, v|
+            return false if k.is_a?(String) && !k.valid_encoding?
+            if v.is_a?(String) || v.is_a?(Array) || v.is_a?(Hash)
+              return false unless v.valid_encoding?
+            end
+          end
+          return true
+        end
+      end
+    end
+  end
+end
+
+Hash.send(:include, ThreeScale::Backend::Extensions::Hash)

data/lib/3scale/backend/extensions/nil_class.rb

@@ -0,0 +1,13 @@
+module ThreeScale
+  module Backend
+    module Extensions
+      module NilClass
+        def blank?
+          true
+        end
+      end
+    end
+  end
+end
+
+NilClass.send(:include, ThreeScale::Backend::Extensions::NilClass)

data/lib/3scale/backend/extensions/redis.rb

@@ -0,0 +1,44 @@
+# Monkey-patches a method in Redis::Client::Connector::Sentinel to fix a bug
+# with sentinel passwords. It applies the fix in
+# https://github.com/redis/redis-rb/pull/856.
+#
+# The fix was included in 4.1.2, but we cannot upgrade because that version
+# drops support for ruby < 2.3.0 which we still need to support.
+#
+# This should only be temporary. It should be deleted when updating the gem.
+class Redis
+  class Client
+    class Connector
+      class Sentinel
+        def sentinel_detect
+          @sentinels.each do |sentinel|
+            client = Redis::Client.new(@options.merge({:host => sentinel[:host],
+                                                       :port => sentinel[:port],
+                                                       password: sentinel[:password],
+                                                       :reconnect_attempts => 0,
+                                                      }))
+
+            begin
+              if result = yield(client)
+                # This sentinel responded. Make sure we ask it first next time.
+                @sentinels.delete(sentinel)
+                @sentinels.unshift(sentinel)
+
+                return result
+              end
+            rescue BaseConnectionError
+            rescue RuntimeError => exception
+              # Needed because when the sentinel address cannot be resolved it
+              # raises this instead of "BaseConnectionError"
+              raise unless exception.message =~ /Name or service not known/
+            ensure
+              client.disconnect
+            end
+          end
+
+          raise CannotConnectError, "No sentinels available."
+        end
+      end
+    end
+  end
+end

data/lib/3scale/backend/extensions/string.rb

@@ -0,0 +1,13 @@
+module ThreeScale
+  module Backend
+    module Extensions
+      module String
+        def blank?
+          self !~ /\S/
+        end
+      end
+    end
+  end
+end
+
+String.send(:include, ThreeScale::Backend::Extensions::String)

data/lib/3scale/backend/extensions/time.rb

@@ -0,0 +1,110 @@
+module ThreeScale
+  module Backend
+    module TimeHacks
+      ONE_MINUTE = 60
+      ONE_HOUR   = 60 * ONE_MINUTE
+      ONE_DAY    = 24 * ONE_HOUR
+
+      def beginning_of_bucket(seconds_in_bucket)
+        if seconds_in_bucket > 30 || seconds_in_bucket < 1 || !seconds_in_bucket.is_a?(Integer)
+          raise Exception, "seconds_in_bucket cannot be larger than 30 seconds or smaller than 1"
+        end
+        norm_sec = (sec/seconds_in_bucket)*seconds_in_bucket
+        self.class.utc(year, month, day, hour, min, norm_sec)
+      end
+
+      # Formats the time using as little characters as possible, but still keeping
+      # readability.
+      #
+      # == Examples
+      #
+      # Time.utc(2010, 5, 6, 17, 24, 22).to_compact_s # "20100506172422"
+      # Time.utc(2010, 5, 6, 17, 24, 00).to_compact_s # "201005061724"
+      # Time.utc(2010, 5, 6, 17, 00, 00).to_compact_s # "2010050617"
+      # Time.utc(2010, 5, 6, 00, 00, 00).to_compact_s # "20100506"
+      #
+      # Careful with cases where hours, minutes or seconds have 2 digits and
+      # the second one is a 0. You might find them a bit counter-intuitive
+      # (notice the missing 0 at the end of the resulting string):
+      # Time.utc(2016, 1, 2, 10, 11, 10).to_compact_s # "2016010210111"
+      # Time.utc(2016, 1, 2, 18, 10, 0).to_compact_s # "20160102181"
+      # Time.utc(2016, 1, 2, 10, 0, 0).to_compact_s # "201601021"
+      #
+      # That behavior does not happen with days ending with a 0:
+      # Time.utc(2016, 1, 20, 0, 0, 0).to_compact_s # "20160120"
+
+      # Leap seconds would map to 60, so include it.
+      MODS = 61.times.map do |i|
+        i % 10
+      end.freeze
+      private_constant :MODS
+
+      DIVS = 61.times.map do |i|
+        i / 10
+      end.freeze
+      private_constant :DIVS
+
+      # This function is equivalent to:
+      # strftime('%Y%m%d%H%M%S').sub(/0{0,6}$/, '').
+      #
+      # When profiling, we found that this method was one of the ones which
+      # consumed more CPU time so we decided to optimize it.
+      def to_compact_s
+        s = year * 10000 +  month * 100 + day
+        if sec != 0
+          s = s * 100000 + hour * 1000 + min * 10
+          MODS[sec] == 0 ? s + DIVS[sec] : s * 10 + sec
+        elsif min != 0
+          s = s * 1000 + hour * 10
+          MODS[min] == 0 ? s + DIVS[min] : s * 10 + min
+        elsif hour != 0
+          s = s * 10
+          MODS[hour] == 0 ? s + DIVS[hour] : s * 10 + hour
+        else
+          s
+        end.to_s
+      end
+
+      def to_not_compact_s
+        (year * 10000000000 + month * 100000000 + day * 1000000 +
+         hour * 10000 + min * 100 + sec).to_s
+      end
+
+      module ClassMethods
+        def parse_to_utc(input)
+          input = input.to_s
+
+          # Test firts for a UNIX timestamp, since it is the most useful way to specify UTC time
+          parse_unix_timestamp(input) || parse_non_unix_timestamp(input)
+        end
+
+        private
+
+        def parse_non_unix_timestamp(ts)
+          parts = Date._parse ts
+
+          if parts.has_key?(:year) && parts.has_key?(:mon) && parts.has_key?(:mday)
+            utc_time = Time.utc(parts[:year],
+                                parts[:mon],
+                                parts[:mday],
+                                parts[:hour],
+                                parts[:min],
+                                parts[:sec],
+                                parts[:sec_fraction])
+
+            parts[:offset] ? utc_time - parts[:offset] : utc_time
+          end
+        rescue
+          # if nothing can be parsed, just return nil
+        end
+
+        def parse_unix_timestamp(ts)
+          Time.at(Integer ts) rescue nil
+        end
+      end
+    end
+  end
+end
+
+Time.send(:include, ThreeScale::Backend::TimeHacks)
+Time.extend(ThreeScale::Backend::TimeHacks::ClassMethods)

data/lib/3scale/backend/failed_jobs_scheduler.rb

@@ -0,0 +1,141 @@
+require '3scale/backend/logging'
+
+module ThreeScale
+  module Backend
+    class FailedJobsScheduler
+      TTL_RESCHEDULE_S = 30
+      private_constant :TTL_RESCHEDULE_S
+
+      # We are going to reschedule a job only if the remaining time for the TTL
+      # is at least SAFE_MARGIN_S. This is to minimize the chance of having 2
+      # jobs running at the same time.
+      SAFE_MARGIN_S = 0.25*TTL_RESCHEDULE_S
+      private_constant :SAFE_MARGIN_S
+
+      # We need to limit the amount of failed jobs that we reschedule each
+      # time. Even a small Redis downtime can cause lots of failed jobs and we
+      # want to avoid spending more time than the TTL defined above. Otherwise,
+      # several reschedule jobs might try to run at the same time.
+      MAX_JOBS_TO_RESCHEDULE = 20_000
+      private_constant :MAX_JOBS_TO_RESCHEDULE
+
+      PATTERN_INTEGER_JOB_ERROR = /undefined method `\[\]=' for .*:Integer*/.freeze
+      private_constant :PATTERN_INTEGER_JOB_ERROR
+
+      NO_JOBS_IN_QUEUE_ERROR = "undefined method `[]=' for nil:NilClass".freeze
+      private_constant :NO_JOBS_IN_QUEUE_ERROR
+
+      class << self
+        include Backend::Logging
+
+        def reschedule_failed_jobs
+          # There might be several callers trying to requeue failed jobs at the
+          # same time. We need to use a lock to avoid rescheduling the same
+          # failed job more than once.
+          key = dist_lock.lock
+
+          ttl_expiration_time = Time.now + TTL_RESCHEDULE_S
+          rescheduled = failed_while_rescheduling = 0
+
+          if key
+            number_of_jobs_to_reschedule.times do
+              break unless time_for_another_reschedule?(ttl_expiration_time)
+
+              requeue_result = requeue_oldest_failed_job
+
+              if requeue_result[:rescheduled?]
+                rescheduled += 1
+              else
+                failed_while_rescheduling += 1
+              end
+
+              # :ok_to_remove? is false only when the requeue() call fails
+              # because there are no more jobs in the queue.
+              requeue_result[:ok_to_remove?] ? remove_oldest_failed_job : break
+            end
+
+            dist_lock.unlock if key == dist_lock.current_lock_key
+          end
+
+          { rescheduled: rescheduled,
+            failed_while_rescheduling: failed_while_rescheduling,
+            failed_current: failed_queue.count }
+        end
+
+        private
+
+        def dist_lock
+          @dist_lock ||= DistributedLock.new(
+              self.name, TTL_RESCHEDULE_S, Resque.redis)
+        end
+
+        def time_for_another_reschedule?(ttl_expiration_time)
+          remaining = ttl_expiration_time - Time.now
+          remaining >= SAFE_MARGIN_S
+        end
+
+        def failed_queue
+          @failed_jobs_queue ||= Resque::Failure
+        end
+
+        def number_of_jobs_to_reschedule
+          [failed_queue.count, MAX_JOBS_TO_RESCHEDULE].min
+        end
+
+        # Returns a hash with two symbol keys. ':rescheduled?' is a boolean
+        # that indicates whether the job has been rescheduled successfully.
+        # ':ok_to_remove?' is a boolean that indicates whether we should remove
+        # the job from the queue. That is true when the job has been
+        # rescheduled successfully and when we want to discard the job because
+        # the error it raised.
+        def requeue_oldest_failed_job
+          failed_queue.requeue(0)
+          { rescheduled?: true, ok_to_remove?: true }
+        rescue Resque::Helpers::DecodeException => e
+          # This means we tried to dequeue a job with invalid encoding.
+          # We just want to delete it from the queue.
+          logger.notify(e)
+          { rescheduled?: false, ok_to_remove?: true }
+        rescue Exception => e
+          logger.notify(e)
+          { rescheduled?: false, ok_to_remove?: ok_to_remove?(e.message)}
+        end
+
+        def ok_to_remove?(error_msg)
+          # The dist lock we use does not guarantee mutual exclusion in all
+          # cases. This can result in a 'NoMethodError' if requeue is
+          # called with an index that is no longer valid.
+          # The error msg raised in this case is the one defined in
+          # NO_JOBS_IN_QUEUE_ERROR. We do not want to remove the job in this
+          # case because the one we wanted to remove no longer exists.
+          #
+          # There are other cases that can result in a 'NoMethodError'.
+          # The format that Resque expects for a job is a hash with fields
+          # like payload, args, failed_at, timestamp, etc. However,
+          # we have seen Fixnums enqueued. The root cause of that is not
+          # clear, but it is a problem. A Fixnum does not raise a
+          # DecodeException, but when Resque receives that 'job', it
+          # raises a 'NoMethodError' because it tries to call [] (remember
+          # that it expects a hash) on that Fixnum.
+          # The error msg raised in this case matches the pattern
+          # PATTERN_INTEGER_JOB_ERROR.
+          # We need to make sure that we remove 'jobs' like this from the
+          # queue, otherwise, they'll be retried forever.
+
+          if PATTERN_INTEGER_JOB_ERROR =~ error_msg
+            true
+          elsif NO_JOBS_IN_QUEUE_ERROR == error_msg
+            false
+          else # Unknown error. Remove the job to avoid retrying it forever.
+            true
+          end
+        end
+
+        def remove_oldest_failed_job
+          failed_queue.remove(0)
+        end
+
+      end
+    end
+  end
+end