workarea-circuit_breaker 1.0.3

data/app/controllers/workarea/admin/circuits_controller.rb

@@ -0,0 +1,26 @@
+module Workarea
+  module Admin
+    class CircuitsController < Admin::ApplicationController
+      def index
+        @circuits = Admin::CircuitsViewModel.new
+      end
+
+      def disable
+        circuit = CircuitViewModel.new(CircuitBreaker[params[:id]])
+        circuit.add_failure(message: t('workarea.admin.circuit_breaker.manually_broke_circuit', user: current_user.name))
+        circuit.break!
+
+        flash[:success] = t('workarea.admin.circuit_breaker.flash_messages.turned_off', circuit: circuit.name, break_for: circuit.break_for.inspect)
+        redirect_to circuits_path
+      end
+
+      def enable
+        circuit = CircuitViewModel.new(CircuitBreaker[params[:id]])
+        circuit.set_healthy!
+
+        flash[:success] = t('workarea.admin.circuit_breaker.flash_messages.turned_on', circuit: circuit.name)
+        redirect_to circuits_path
+      end
+    end
+  end
+end

data/app/view_models/workarea/admin/circuit_view_model.rb

@@ -0,0 +1,33 @@
+module Workarea
+  module Admin
+    class CircuitViewModel < ApplicationViewModel
+      def name
+        model.name.to_s.humanize
+      end
+
+      def slug
+        model.name
+      end
+
+      def current_fails
+        timeline.length
+      end
+
+      def broken_until
+        return unless options[:broken_until].present?
+
+        Time.at(options[:broken_until].to_i)
+      end
+
+      def healthy?
+        options[:broken_until].blank?
+      end
+
+      def timeline
+        @timeline ||= options.fetch(:timeline, []).map do |(failure_string, timestamp)|
+          CircuitBreaker::FailureMessage.from_string(failure_string, Time.at(timestamp))
+        end
+      end
+    end
+  end
+end

data/app/view_models/workarea/admin/circuits_view_model.rb

@@ -0,0 +1,41 @@
+module Workarea
+  module Admin
+    class CircuitsViewModel < ApplicationViewModel
+      delegate :each, to: :all
+
+      private
+
+        def all
+          @all ||= circuits.zip(circuits_broken_until, circuits_timeline).map do |circuit, broken_until, timeline|
+            CircuitViewModel.new(
+              circuit,
+              broken_until: broken_until,
+              timeline: timeline
+            )
+          end
+        end
+
+        def circuits
+          @circuits ||= CircuitBreaker.config.circuits.map do |name, options|
+            CircuitBreaker::Circuit.new(name, options)
+          end
+        end
+
+        def circuits_timeline
+          @circuits_timeline ||= CircuitBreaker.redis.with do |redis|
+            redis.pipelined do
+              circuits.each do |circuit|
+                redis.zrange(circuit.redis_failure_set_key, 0, -1, with_scores: true)
+              end
+            end
+          end
+        end
+
+        def circuits_broken_until
+          @circuits_broken_until ||= CircuitBreaker.redis.with do |redis|
+            redis.mget(circuits.map(&:redis_broken_key))
+          end
+        end
+    end
+  end
+end

data/app/views/workarea/admin/circuits/index.html.haml

@@ -0,0 +1,69 @@
+.view
+  .view__header
+    .view__heading
+      = link_to "↑ #{t('workarea.admin.dashboards.main_dashboard')}", root_path
+      %h1= t('workarea.admin.circuit_breaker.title')
+
+  .view__container
+    %p= t('workarea.admin.circuit_breaker.help_text')
+
+    - if @circuits.each.present?
+      %table
+        %thead
+          %tr
+            %th= t('workarea.admin.circuit_breaker.circuit')
+            %th
+              = t('workarea.admin.circuit_breaker.max_fails')
+              = link_to '#circuit-max-fails', data: { tooltip: '' } do
+                = inline_svg('workarea/admin/icons/help.svg', class: 'svg-icon svg-icon--small svg-icon--blue', title: t('workarea.admin.circuit_breaker.max_fails'))
+
+              #circuit-max-fails.tooltip-content
+                %p= t('workarea.admin.circuit_breaker.help.max_fails')
+            %th
+              = t('workarea.admin.circuit_breaker.window')
+              = link_to '#circuit-window', data: { tooltip: '' } do
+                = inline_svg('workarea/admin/icons/help.svg', class: 'svg-icon svg-icon--small svg-icon--blue', title: t('workarea.admin.circuit_breaker.window'))
+              #circuit-window.tooltip-content
+                %p= t('workarea.admin.circuit_breaker.help.window')
+            %th
+              = t('workarea.admin.circuit_breaker.break_for')
+              = link_to '#circuit-break-for', data: { tooltip: '' } do
+                = inline_svg('workarea/admin/icons/help.svg', class: 'svg-icon svg-icon--small svg-icon--blue', title: t('workarea.admin.circuit_breaker.break_for'))
+              #circuit-break-for.tooltip-content
+                %p= t('workarea.admin.circuit_breaker.help.break_for')
+            %th= t('workarea.admin.circuit_breaker.current_fails')
+            %th= t('workarea.admin.circuit_breaker.timeline')
+            %th= t('workarea.admin.circuit_breaker.broken_until')
+            %th
+        %tbody
+          - @circuits.each do |circuit|
+            %tr
+              %td= circuit.name
+              %td= circuit.max_fails
+              %td= circuit.window.inspect
+              %td= circuit.break_for.inspect
+              %td= circuit.current_fails
+              %td
+                - if circuit.timeline.present?
+                  %ul.list-reset
+                    - circuit.timeline.each do |failure_message|
+                      %li
+                        = failure_message.timestamp.to_s(:time_only)
+                        - if failure_message.event_id.present?
+                          %strong= t('workarea.admin.circuit_breaker.sentry')
+                          = failure_message.event_id
+                        %strong= t('workarea.admin.circuit_breaker.message')
+                        = failure_message.message
+              %td
+                - if circuit.broken_until.present?
+                  = circuit.broken_until.to_s(:time_only)
+              - if circuit.healthy?
+                %td
+                  = inline_svg('workarea/admin/valid.svg', class: 'svg-icon svg-icon--green')
+                  = link_to t('workarea.admin.circuit_breaker.break'), disable_circuit_path(circuit.slug), data: { method: 'post' }
+              - else
+                %td
+                  = inline_svg('workarea/admin/invalid.svg', class: 'svg-icon svg-icon--red')
+                  = link_to t('workarea.admin.circuit_breaker.enable'), enable_circuit_path(circuit.slug), data: { method: 'post' }
+    - else
+      %p= t('workarea.admin.circuit_breaker.no_circuits')

data/app/views/workarea/admin/shared/_circuit_breaker_link.html.haml

@@ -0,0 +1 @@
+%li{ class: "primary-nav__item" }= link_to t('workarea.admin.shared.primary_nav.circuit_breaker_title'), circuits_path, class: navigation_link_classes(circuits_path)

data/bin/rails

@@ -0,0 +1,25 @@
+#!/usr/bin/env ruby
+# This command will automatically be run when you run "rails" with Rails gems
+# installed from the root of your application.
+
+ENGINE_ROOT = File.expand_path('..', __dir__)
+ENGINE_PATH = File.expand_path('../lib/workarea/circuit_breaker/engine', __dir__)
+APP_PATH = File.expand_path('../test/dummy/config/application', __dir__)
+
+# Set up gems listed in the Gemfile.
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../Gemfile', __dir__)
+require 'bundler/setup' if File.exist?(ENV['BUNDLE_GEMFILE'])
+
+require "rails"
+# Pick the frameworks you want:
+require "active_model/railtie"
+require "active_job/railtie"
+# require "active_record/railtie"
+# require "active_storage/engine"
+require "action_controller/railtie"
+require "action_mailer/railtie"
+require "action_view/railtie"
+# require "action_cable/engine"
+require "sprockets/railtie"
+require "rails/test_unit/railtie"
+require 'rails/engine/commands'

data/config/initializers/appends.rb

@@ -0,0 +1,4 @@
+Workarea.append_partials(
+  "admin.settings_menu",
+  "workarea/admin/shared/circuit_breaker_link"
+)

data/config/initializers/workarea.rb

@@ -0,0 +1,11 @@
+Workarea.configure do |config|
+  config.circuit_breaker = ActiveSupport::Configurable::Configuration.new
+
+  config.circuit_breaker.circuits ||= {}
+  config.circuit_breaker.redis ||= {}
+  config.circuit_breaker.circuit_defaults ||= {
+    max_fails: 3,
+    window: 5.minutes,
+    break_for: 5.minutes
+  }
+end

data/config/locales/en.yml

@@ -0,0 +1,29 @@
+en:
+  workarea:
+    admin:
+      shared:
+        primary_nav:
+          circuit_breaker_title: Circuit Breakers
+      circuit_breaker:
+        flash_messages:
+          turned_on: "%{circuit} has been enabled"
+          turned_off: "%{circuit} has been disabled for %{break_for}"
+        break: Break
+        break_for: Break For
+        broken_until: Broken Until
+        circuit: Circuit
+        current_fails: Current Fails
+        enable: Enable
+        max_fails: Max Fails
+        message: Message
+        sentry: Sentry
+        timeline: Timeline
+        title: Circuit Breakers
+        window: Window
+        manually_broke_circuit: "%{user} manually broke circuit"
+        no_circuits: There are currently no circuits defined.
+        help_text: Circuits are developer configured code blocks that allow for processes to execute a defined fallback when a service fails within a given set of parameters.
+        help:
+          max_fails: Maximum times this circuit can fail before shutting off.
+          window: Timeframe in which a Max fails would trigger a circuit to shut off. Eg. five failures (max fails) in one minute (window) will trigger the circuit to shut off.
+          break_for: How long to turn the circuit off for when the max failures limit is reached within the window duration.

data/config/routes.rb

@@ -0,0 +1,10 @@
+Workarea::Admin::Engine.routes.draw do
+  scope '(:locale)', constraints: Workarea::I18n.routes_constraint do
+    resources :circuits, only: :index do
+      member do
+        post :enable
+        post :disable
+      end
+    end
+  end
+end

data/lib/workarea/circuit_breaker.rb

@@ -0,0 +1,96 @@
+require 'connection_pool'
+
+require 'workarea'
+require 'workarea/storefront'
+require 'workarea/admin'
+
+require 'workarea/circuit_breaker/engine'
+require 'workarea/circuit_breaker/version'
+require 'workarea/circuit_breaker/circuit'
+require 'workarea/circuit_breaker/failure_message'
+
+module Workarea
+  module CircuitBreaker
+    class CircuitBreakerError < RuntimeError; end
+    module ConnectionPoolLike
+      def with
+        yield self
+      end
+    end
+
+    unless ::Redis.instance_methods.include?(:with)
+      ::Redis.include(ConnectionPoolLike)
+    end
+
+    CIRCUIT_KEY_BASE = "workarea_circuits"
+
+    def self.config
+      Workarea.config.circuit_breaker
+    end
+
+    def self.circuit_defaults
+      config.circuit_defaults
+    end
+
+    def self.add_service(name, **options)
+      options = options.symbolize_keys
+      overwrite = options.delete(:overwrite) || false
+
+      if !overwrite && config.circuits.has_key?(name)
+        Rails.logger.warn "Overwriting existing service #{name} in CircuitBreaker"
+      end
+
+      config.circuits[name.to_s] = options
+    end
+
+    def self.[](circuit_name)
+      circuit_name = circuit_name.to_s
+      options = config.circuits[circuit_name] || {}
+      Circuit.new(circuit_name, **options)
+    end
+
+    def self.redis
+      @redis ||= if pool_options.present?
+        ConnectionPool.new(pool_options) do
+          Redis.new(url: Workarea::Configuration::Redis.persistent.to_url)
+        end
+      else
+        Redis.new(url: Workarea::Configuration::Redis.persistent.to_url)
+      end
+    end
+
+    # Capture errors thrown by code that is running in the circuit
+    # breaker. When `Sentry::Raven` is installed, send the error over to
+    # Sentry and return the event ID. Otherwise, just return `nil`. This
+    # will also return `nil` when Sentry is not configured to capture
+    # exceptions in the environment, such as when tests are running.
+    #
+    # @param [Exception] exception - Error thrown by the application
+    # @return [String] Event ID or `nil` when the exception was not
+    #                  captured by Sentry.
+    def self.capture_exception(exception)
+      event = if defined?(::Raven)
+        Raven.capture_exception(exception) || nil
+      else
+        Rails.logger.warn exception
+        nil
+      end
+
+      return if event.blank?
+
+      event.id
+    end
+
+    def self.freeze_config!
+      Workarea.config.circuit_breaker.freeze
+      Workarea.config.circuit_breaker.circuits.freeze
+      Workarea.config.circuit_breaker.circuit_defaults.freeze
+    end
+
+    private
+
+      def self.pool_options
+        config.redis.fetch(:pool, {})
+      end
+  end
+end

data/lib/workarea/circuit_breaker/circuit.rb

@@ -0,0 +1,137 @@
+module Workarea
+  module CircuitBreaker
+    class Circuit
+      attr_reader :name, :options
+
+      def initialize(name, **options)
+        @name = name.to_s
+        @options = CircuitBreaker.circuit_defaults.merge(options).symbolize_keys
+      end
+
+      # Checks if this circuit is currently, defaults to true
+      #
+      # @return [Boolean]
+      def healthy?
+        broken = CircuitBreaker.redis.with do |connection|
+          connection.get(redis_broken_key)
+        end
+
+        broken.nil?
+      rescue Timeout::Error => error
+        CircuitBreaker.capture_exception(error)
+        true
+      end
+
+      # Set the circuit to be healthy
+      #
+      # @return [nil]
+      def set_healthy!
+        CircuitBreaker.redis.with do |connection|
+          connection.pipelined do
+            connection.del(redis_broken_key)
+            connection.del(redis_failure_set_key)
+          end
+        end
+        nil
+      rescue Timeout::Error => error
+        CircuitBreaker.capture_exception(error)
+        nil
+      end
+
+      # Break this circuit for the configured break_for duration of this service
+      #
+      # @return nil
+      def break!
+        CircuitBreaker.redis.with do |connection|
+          connection.setex(redis_broken_key, break_for, break_for.from_now.to_i)
+        end
+        nil
+      rescue Timeout::Error => error
+        CircuitBreaker.capture_exception(error)
+        nil
+      end
+
+      # Wraps the black around a circuit.  If the circuit is healthy the block
+      # is executed and it's return value is returned.  If the black raises an error
+      # an event is added to the circuit timeline.  If a fallback is provided the
+      # fallback is invoked otherwise an error is raised
+      #
+      # @param [Symbol, Object] either a symbol of a method defined on the blocks binding receiver or anything that responds to call
+      def wrap(fallback: nil, &block)
+        if healthy?
+          begin
+            block.call
+          rescue => error
+            event_id = CircuitBreaker.capture_exception(error)
+            add_failure(event_id: event_id, message: "#{error.class}: #{error.message}")
+            fallback_or_error(block.binding.receiver, fallback: fallback, error: error)
+          end
+        else
+          fallback_or_error(block.binding.receiver, fallback: fallback, error: CircuitBreakerError)
+        end
+      end
+
+      def window
+        options[:window]
+      end
+
+      def break_for
+        options[:break_for]
+      end
+
+      def max_fails
+        options[:max_fails]
+      end
+
+      # @param [String] event_id Raven::Event#id
+      # @param [String] message - normally an error message
+      #
+      # @return [Int] number of events in the timeline
+      def add_failure(message:, event_id: nil)
+        CircuitBreaker.redis.with do |connection|
+          failure_message = FailureMessage.new(event_id: event_id, message: message)
+
+          _deleted, _added, length, _expiry_set = connection.pipelined do
+            connection.zremrangebyscore(redis_failure_set_key, 0, break_for.ago.to_i)
+            connection.zadd(redis_failure_set_key, Time.current.to_i, failure_message.to_s)
+            connection.zcard(redis_failure_set_key)
+            # expire as the time a circuit could be broken so info is avaiable whole length
+            # of broken time
+            connection.expire(redis_failure_set_key, break_for.to_i)
+          end
+
+          break! if length >= max_fails
+
+          length
+        end
+      rescue Timeout::Error => error
+        CircuitBreaker.capture_exception(error)
+        0
+      end
+
+      def redis_broken_key
+        "#{redis_key_base}:broken"
+      end
+
+      def redis_failure_set_key
+        "#{redis_key_base}:failures"
+      end
+
+      private
+
+        def fallback_or_error(receiver, fallback: nil, error: nil)
+          if fallback.blank?
+            raise error
+          elsif fallback.respond_to?(:call)
+            fallback.call
+          elsif receiver.respond_to?(fallback, true)
+            receiver.send(fallback)
+          end
+        end
+
+        def redis_key_base
+          "#{CircuitBreaker::CIRCUIT_KEY_BASE}:#{name.to_s.systemize}"
+        end
+    end
+  end
+end