semian_extension 0.11.4.1

data/lib/semian/circuit_breaker.rb

@@ -0,0 +1,167 @@
+module Semian
+  class CircuitBreaker #:nodoc:
+    extend Forwardable
+
+    def_delegators :@state, :closed?, :open?, :half_open?
+
+    attr_reader :name, :half_open_resource_timeout, :error_timeout, :state, :last_error
+
+    def initialize(name, exceptions:, success_threshold:, error_threshold:,
+                         error_timeout:, implementation:, half_open_resource_timeout: nil)
+      @name = name.to_sym
+      @success_count_threshold = success_threshold
+      @error_count_threshold = error_threshold
+      @error_timeout = error_timeout
+      @exceptions = exceptions
+      @half_open_resource_timeout = half_open_resource_timeout
+
+      @errors = implementation::SlidingWindow.new(max_size: @error_count_threshold)
+      @successes = implementation::Integer.new
+      @state = implementation::State.new
+
+      reset
+    end
+
+    def acquire(resource = nil, &block)
+      return yield if disabled?
+      transition_to_half_open if transition_to_half_open?
+
+      raise OpenCircuitError unless request_allowed?
+
+      result = nil
+      begin
+        result = maybe_with_half_open_resource_timeout(resource, &block)
+      rescue *@exceptions => error
+        if !error.respond_to?(:marks_semian_circuits?) || error.marks_semian_circuits?
+          mark_failed(error)
+        end
+        raise error
+      else
+        mark_success
+      end
+      result
+    end
+
+    def transition_to_half_open?
+      open? && error_timeout_expired? && !half_open?
+    end
+
+    def request_allowed?
+      closed? || half_open? || transition_to_half_open?
+    end
+
+    def mark_failed(error)
+      push_error(error)
+      push_time(@errors)
+      if closed?
+        transition_to_open if error_threshold_reached?
+      elsif half_open?
+        transition_to_open
+      end
+    end
+
+    def mark_success
+      return unless half_open?
+      @successes.increment
+      transition_to_close if success_threshold_reached?
+    end
+
+    def reset
+      @errors.clear
+      @successes.reset
+      transition_to_close
+    end
+
+    def destroy
+      @errors.destroy
+      @successes.destroy
+      @state.destroy
+    end
+
+    def in_use?
+      return false if error_timeout_expired?
+      @errors.size > 0
+    end
+
+    private
+
+    def transition_to_close
+      notify_state_transition(:closed)
+      log_state_transition(:closed)
+      @state.close!
+      @errors.clear
+    end
+
+    def transition_to_open
+      notify_state_transition(:open)
+      log_state_transition(:open)
+      @state.open!
+    end
+
+    def transition_to_half_open
+      notify_state_transition(:half_open)
+      log_state_transition(:half_open)
+      @state.half_open!
+      @successes.reset
+    end
+
+    def success_threshold_reached?
+      @successes.value >= @success_count_threshold
+    end
+
+    def error_threshold_reached?
+      @errors.size == @error_count_threshold
+    end
+
+    def error_timeout_expired?
+      last_error_time = @errors.last
+      return false unless last_error_time
+      Time.at(last_error_time) + @error_timeout < Time.now
+    end
+
+    def push_error(error)
+      @last_error = error
+    end
+
+    def push_time(window, time: Time.now)
+      window.reject! { |err_time| err_time + @error_timeout < time.to_i }
+      window << time.to_i
+    end
+
+    def log_state_transition(new_state)
+      return if @state.nil? || new_state == @state.value
+
+      str = "[#{self.class.name}] State transition from #{@state.value} to #{new_state}."
+      str << " success_count=#{@successes.value} error_count=#{@errors.size}"
+      str << " success_count_threshold=#{@success_count_threshold} error_count_threshold=#{@error_count_threshold}"
+      str << " error_timeout=#{@error_timeout} error_last_at=\"#{@errors.last}\""
+      str << " name=\"#{@name}\""
+      if new_state == :open && @last_error
+        str << " last_error_message=#{@last_error.message.inspect}"
+      end
+
+      Semian.logger.info(str)
+    end
+
+    def notify_state_transition(new_state)
+      Semian.notify(:state_change, self, nil, nil, state: new_state)
+    end
+
+    def disabled?
+      ENV['SEMIAN_CIRCUIT_BREAKER_DISABLED'] || ENV['SEMIAN_DISABLED']
+    end
+
+    def maybe_with_half_open_resource_timeout(resource, &block)
+      result =
+        if half_open? && @half_open_resource_timeout && resource.respond_to?(:with_resource_timeout)
+          resource.with_resource_timeout(@half_open_resource_timeout) do
+            block.call
+          end
+        else
+          block.call
+        end
+
+      result
+    end
+  end
+end

data/lib/semian/grpc.rb

@@ -0,0 +1,104 @@
+require 'semian/adapter'
+require 'grpc'
+
+module GRPC
+  GRPC::Unavailable.include(::Semian::AdapterError)
+  GRPC::Unknown.include(::Semian::AdapterError)
+  GRPC::ResourceExhausted.include(::Semian::AdapterError)
+
+  class SemianError < GRPC::Unavailable
+    attr_reader :details
+
+    def initialize(semian_identifier, *args)
+      super(*args)
+      @details = message
+      @semian_identifier = semian_identifier
+    end
+  end
+
+  ResourceBusyError = Class.new(SemianError)
+  CircuitOpenError = Class.new(SemianError)
+end
+
+module Semian
+  module GRPC
+    attr_reader :raw_semian_options
+    include Semian::Adapter
+
+    ResourceBusyError = ::GRPC::ResourceBusyError
+    CircuitOpenError = ::GRPC::CircuitOpenError
+
+    class SemianConfigurationChangedError < RuntimeError
+      def initialize(msg = "Cannot re-initialize semian_configuration")
+        super
+      end
+    end
+
+    class << self
+      attr_accessor :exceptions
+      attr_reader :semian_configuration
+
+      def semian_configuration=(configuration)
+        raise Semian::GRPC::SemianConfigurationChangedError unless @semian_configuration.nil?
+        @semian_configuration = configuration
+      end
+
+      def retrieve_semian_configuration(host)
+        @semian_configuration.call(host) if @semian_configuration.respond_to?(:call)
+      end
+    end
+
+    def raw_semian_options
+      @raw_semian_options ||= begin
+        # If the host is empty, it's possible that the adapter was initialized
+        # with the channel. Therefore, we look into the channel to find the host
+        if @host.empty?
+          host = @ch.target
+        else
+          host = @host
+        end
+        @raw_semian_options = Semian::GRPC.retrieve_semian_configuration(host)
+        @raw_semian_options = @raw_semian_options.dup unless @raw_semian_options.nil?
+      end
+    end
+
+    def semian_identifier
+      @semian_identifier ||= raw_semian_options[:name]
+    end
+
+    def resource_exceptions
+      [
+        ::GRPC::DeadlineExceeded,
+        ::GRPC::ResourceExhausted,
+        ::GRPC::Unavailable,
+        ::GRPC::Unknown,
+      ]
+    end
+
+    def disabled?
+      raw_semian_options.nil?
+    end
+
+    def request_response(*, **)
+      return super if disabled?
+      acquire_semian_resource(adapter: :grpc, scope: :request_response) { super }
+    end
+
+    def client_streamer(*, **)
+      return super if disabled?
+      acquire_semian_resource(adapter: :grpc, scope: :client_streamer) { super }
+    end
+
+    def server_streamer(*, **)
+      return super if disabled?
+      acquire_semian_resource(adapter: :grpc, scope: :server_streamer) { super }
+    end
+
+    def bidi_streamer(*, **)
+      return super if disabled?
+      acquire_semian_resource(adapter: :grpc, scope: :bidi_streamer) { super }
+    end
+  end
+end
+
+::GRPC::ClientStub.prepend(Semian::GRPC)

data/lib/semian/instrumentable.rb

@@ -0,0 +1,28 @@
+module Semian
+  module Instrumentable
+    def subscribe(name = rand, &block)
+      subscribers[name] = block
+      name
+    end
+
+    def unsubscribe(name)
+      subscribers.delete(name)
+    end
+
+    # Args:
+    #   event (string)
+    #   resource (Object)
+    #   scope (string)
+    #   adapter (string)
+    #   payload (optional)
+    def notify(*args)
+      subscribers.values.each { |subscriber| subscriber.call(*args) }
+    end
+
+    private
+
+    def subscribers
+      @subscribers ||= {}
+    end
+  end
+end

data/lib/semian/lru_hash.rb

@@ -0,0 +1,174 @@
+class LRUHash
+  # This LRU (Least Recently Used) hash will allow
+  # the cleaning of resources as time goes on.
+  # The goal is to remove the least recently used resources
+  # everytime we set a new resource. A default window of
+  # 5 minutes will allow empty item to stay in the hash
+  # for a maximum of 5 minutes
+  class NoopMutex
+    def synchronize(*)
+      yield
+    end
+
+    def try_lock
+      true
+    end
+
+    def unlock
+      true
+    end
+
+    def locked?
+      true
+    end
+
+    def owned?
+      true
+    end
+  end
+
+  def keys
+    @lock.synchronize { @table.keys }
+  end
+
+  def clear
+    @lock.synchronize { @table.clear }
+  end
+
+  # Create an LRU hash
+  #
+  # Arguments:
+  #   +max_size+ The maximum size of the table
+  #   +min_time+ The minimum time a resource can live in the cache
+  #
+  # Note:
+  #   The +min_time+ is a stronger guarantee than +max_size+. That is, if there are
+  #   more than +max_size+ entries in the cache, but they've all been updated more
+  #   recently than +min_time+, the garbage collection will not remove them and the
+  #   cache can grow without bound. This usually means that you have many active
+  #   circuits to disparate endpoints (or your circuit names are bad).
+  #   If the max_size is 0, the garbage collection will be very aggressive and
+  #   potentially computationally expensive.
+  def initialize(max_size: Semian.maximum_lru_size, min_time: Semian.minimum_lru_time)
+    @max_size = max_size
+    @min_time = min_time
+    @table = {}
+    @lock =
+      if Semian.thread_safe?
+        Mutex.new
+      else
+        NoopMutex.new
+      end
+  end
+
+  def size
+    @lock.synchronize { @table.size }
+  end
+
+  def count(&block)
+    @lock.synchronize { @table.count(&block) }
+  end
+
+  def empty?
+    @lock.synchronize { @table.empty? }
+  end
+
+  def values
+    @lock.synchronize { @table.values }
+  end
+
+  def set(key, resource)
+    @lock.synchronize do
+      @table.delete(key)
+      @table[key] = resource
+      resource.updated_at = Time.now
+    end
+    clear_unused_resources if @table.length > @max_size
+  end
+
+  # This method uses the property that "Hashes enumerate their values in the
+  # order that the corresponding keys were inserted." Deleting a key and
+  # re-inserting it effectively moves it to the front of the cache.
+  # Update the `updated_at` field so we can use it later do decide if the
+  # resource is "in use".
+  def get(key)
+    @lock.synchronize do
+      found = @table.delete(key)
+      if found
+        @table[key] = found
+        found.updated_at = Time.now
+      end
+      found
+    end
+  end
+
+  def delete(key)
+    @lock.synchronize do
+      @table.delete(key)
+    end
+  end
+
+  def []=(key, resource)
+    set(key, resource)
+  end
+
+  def [](key)
+    get(key)
+  end
+
+  private
+
+  def clear_unused_resources
+    payload = {
+        size: @table.size,
+        examined: 0,
+        cleared: 0,
+        elapsed: nil,
+    }
+    timer_start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+    ran = try_synchronize do
+      # Clears resources that have not been used in the last 5 minutes.
+
+      stop_time = Time.now - @min_time # Don't process resources updated after this time
+      @table.each do |_, resource|
+        payload[:examined] += 1
+
+        # The update times of the resources in the LRU are monotonically increasing,
+        # time, so we can stop looking once we find the first resource with an
+        # update time after the stop_time.
+        break if resource.updated_at > stop_time
+
+        next if resource.in_use?
+
+        resource = @table.delete(resource.name)
+        if resource
+          payload[:cleared] += 1
+          resource.destroy
+        end
+      end
+    end
+
+    if ran
+      payload[:elapsed] = Process.clock_gettime(Process::CLOCK_MONOTONIC) - timer_start
+      Semian.notify(:lru_hash_gc, self, nil, nil, payload)
+    end
+  end
+
+  EXCEPTION_NEVER = {Exception => :never}.freeze
+  EXCEPTION_IMMEDIATE = {Exception => :immediate}.freeze
+  private_constant :EXCEPTION_NEVER
+  private_constant :EXCEPTION_IMMEDIATE
+
+  def try_synchronize
+    Thread.handle_interrupt(EXCEPTION_NEVER) do
+      begin
+        return false unless @lock.try_lock
+        Thread.handle_interrupt(EXCEPTION_IMMEDIATE) { yield }
+        true
+      ensure
+        @lock.unlock if @lock.owned?
+      end
+    end
+  end
+end

data/lib/semian/mysql2.rb

@@ -0,0 +1,135 @@
+require 'semian/adapter'
+require 'mysql2'
+
+module Mysql2
+  Mysql2::Error.include(::Semian::AdapterError)
+
+  class SemianError < Mysql2::Error
+    def initialize(semian_identifier, *args)
+      super(*args)
+      @semian_identifier = semian_identifier
+    end
+  end
+
+  ResourceBusyError = Class.new(SemianError)
+  CircuitOpenError = Class.new(SemianError)
+end
+
+module Semian
+  module Mysql2
+    include Semian::Adapter
+
+    CONNECTION_ERROR = Regexp.union(
+      /Can't connect to MySQL server on/i,
+      /Lost connection to MySQL server/i,
+      /MySQL server has gone away/i,
+      /Too many connections/i,
+      /closed MySQL connection/i,
+      /Timeout waiting for a response/i,
+    )
+
+    ResourceBusyError = ::Mysql2::ResourceBusyError
+    CircuitOpenError = ::Mysql2::CircuitOpenError
+    PingFailure = Class.new(::Mysql2::Error)
+
+    DEFAULT_HOST = 'localhost'
+    DEFAULT_PORT = 3306
+
+    QUERY_WHITELIST = Regexp.union(
+      /\A(?:\/\*.*?\*\/)?\s*ROLLBACK/i,
+      /\A(?:\/\*.*?\*\/)?\s*COMMIT/i,
+      /\A(?:\/\*.*?\*\/)?\s*RELEASE\s+SAVEPOINT/i,
+    )
+
+    # The naked methods are exposed as `raw_query` and `raw_connect` for instrumentation purpose
+    def self.included(base)
+      base.send(:alias_method, :raw_query, :query)
+      base.send(:remove_method, :query)
+
+      base.send(:alias_method, :raw_connect, :connect)
+      base.send(:remove_method, :connect)
+
+      base.send(:alias_method, :raw_ping, :ping)
+      base.send(:remove_method, :ping)
+    end
+
+    def semian_identifier
+      @semian_identifier ||= begin
+        unless name = semian_options && semian_options[:name]
+          host = query_options[:host] || DEFAULT_HOST
+          port = query_options[:port] || DEFAULT_PORT
+          name = "#{host}:#{port}"
+        end
+        :"mysql_#{name}"
+      end
+    end
+
+    def ping
+      result = nil
+      acquire_semian_resource(adapter: :mysql, scope: :ping) do
+        result = raw_ping
+        raise PingFailure.new(result.to_s) unless result
+      end
+      result
+    rescue ResourceBusyError, CircuitOpenError, PingFailure
+      false
+    end
+
+    def query(*args)
+      if query_whitelisted?(*args)
+        raw_query(*args)
+      else
+        acquire_semian_resource(adapter: :mysql, scope: :query) { raw_query(*args) }
+      end
+    end
+
+    # TODO: write_timeout and connect_timeout can't be configured currently
+    # dynamically, await https://github.com/brianmario/mysql2/pull/955
+    def with_resource_timeout(temp_timeout)
+      prev_read_timeout = @read_timeout
+
+      begin
+        # C-ext reads this directly, writer method will configure
+        # properly on the client but based on my read--this is good enough
+        # until we get https://github.com/brianmario/mysql2/pull/955 in
+        @read_timeout = temp_timeout
+        yield
+      ensure
+        @read_timeout = prev_read_timeout
+      end
+    end
+
+    private
+
+    def query_whitelisted?(sql, *)
+      QUERY_WHITELIST =~ sql
+    rescue ArgumentError
+      # The above regexp match can fail if the input SQL string contains binary
+      # data that is not recognized as a valid encoding, in which case we just
+      # return false.
+      return false unless sql.valid_encoding?
+      raise
+    end
+
+    def connect(*args)
+      acquire_semian_resource(adapter: :mysql, scope: :connection) { raw_connect(*args) }
+    end
+
+    def acquire_semian_resource(**)
+      super
+    rescue ::Mysql2::Error => error
+      if error.is_a?(PingFailure) || (!error.is_a?(::Mysql2::SemianError) && error.message.match?(CONNECTION_ERROR))
+        semian_resource.mark_failed(error)
+        error.semian_identifier = semian_identifier
+      end
+      raise
+    end
+
+    def raw_semian_options
+      return query_options[:semian] if query_options.key?(:semian)
+      return query_options['semian'.freeze] if query_options.key?('semian'.freeze)
+    end
+  end
+end
+
+::Mysql2::Client.include(Semian::Mysql2)