freno-client 0.4.0 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: fe5c746dd4f13a74f1d005ef02525980186175a2
-  data.tar.gz: b77184aa9a5e0a98116e6444d61405cf2f0a5cc2
+  metadata.gz: bf297a41a22996c917599c6bc5835a8abb482d09
+  data.tar.gz: 1a45ae21c0e31b7fda7769b78c3fcc8d3d6b4bf5
 SHA512:
-  metadata.gz: e969f338ff696426546ea5c1906f8665b77ba8088aa1ed7146b8e9e4d701e42b779b8834ca92060e0c4bd322e9c7ccbaf4405bea7cde6abe0a7177a91c12fd24
-  data.tar.gz: 7a2f7982b3027e94a4bf02b340a8cf538f7f26cfd8cbd1916591de9083afec49ccd4a7752634ca85bdf29805ddb7855da01eade6d3cd254c8a21e0182eec77a8
+  metadata.gz: 3f6ab33f6c2008972cc3eea6e8611d1aca5e93c64d318211ee7e44d7066cc6cc73f38a97ea2d398e3738e6f35663d8c1dbc6bcd0fb6fb27dc0fe6e7502ea120c
+  data.tar.gz: 9bc8e0b6a8531768b77fc7b6b6b0edd99c9d9f1c0c0bc1d50264806ced07ba960de438a7f1f9f3c1c96309c79a4edf0ee95892a84ad610b0933b456d8d114cbe

data/.rubocop.yml

@@ -140,6 +140,18 @@ Metrics/ParameterLists:
 Metrics/PerceivedComplexity:
   Enabled: false
 
+Naming/AsciiIdentifiers:
+  Enabled: true
+
+Naming/ClassAndModuleCamelCase:
+  Enabled: true
+
+Naming/FileName:
+  Enabled: true
+
+Naming/MethodName:
+  Enabled: true
+
 Performance/CaseWhenSplat:
   Enabled: false
 
@@ -195,9 +207,6 @@ Security/Eval:
 Style/ArrayJoin:
   Enabled: true
 
-Style/AsciiIdentifiers:
-  Enabled: true
-
 Style/BeginBlock:
   Enabled: true
 
@@ -213,9 +222,6 @@ Style/CaseEquality:
 Style/CharacterLiteral:
   Enabled: true
 
-Style/ClassAndModuleCamelCase:
-  Enabled: true
-
 Style/ClassMethods:
   Enabled: true
 
@@ -231,9 +237,6 @@ Style/EndBlock:
 Layout/EndOfLine:
   Enabled: true
 
-Style/FileName:
-  Enabled: true
-
 Style/FlipFlop:
   Enabled: true
 
@@ -252,9 +255,6 @@ Style/MethodCallWithoutArgsParentheses:
 Style/MethodDefParentheses:
   Enabled: true
 
-Style/MethodName:
-  Enabled: true
-
 Style/MultilineIfThen:
   Enabled: true
 

data/CONTRIBUTING.md

@@ -11,7 +11,7 @@ Please note that this project is released with a [Contributor Code of Conduct][c
 
 ## Submitting a pull request
 
-0. [Fork][fork] and clone the freno-clientsitory
+0. [Fork][fork] and clone the freno-client repository
 0. Configure and install the dependencies: `script/bootstrap`
 0. Make sure the tests pass on your machine: `rake`
 0. Create a new branch: `git checkout -b my-branch-name`

data/Gemfile

@@ -3,5 +3,6 @@ source "https://rubygems.org"
 gemspec
 
 group :test do
+  gem "mocha"
   gem "rubocop", require: false
 end

data/README.md

@@ -1,6 +1,6 @@
 # Freno Client [![Build Status](https://travis-ci.org/github/freno-client.svg)](https://travis-ci.org/github/freno-client)
 
-A ruby client for [Freno](https://github.com/github/freno): the cooperative, highly available throttler service.
+A ruby client and throttling library for [Freno](https://github.com/github/freno): the cooperative, highly available throttler service.
 
 ## Current status
 
@@ -175,6 +175,183 @@ freno = Freno::Client.new(faraday) do |client|
 end
 ```
 
+### Throttler objects
+
+Apart from the operations above, freno-client comes with `Freno::Throttler`, a ruby library for throttling. You can use it in the following way:
+
+```ruby
+require "freno/throttler"
+
+client    = Freno::Client.new(faraday)
+throttler = Freno::Throttler.new(client: client, app: :my_app)
+context   = :my_cluster
+
+bid_data_set.each_slice(SLICE_SIZE) do |slice|
+  throttler.throttle(context) do
+    update(slice)
+  end
+end
+```
+
+In the above example, `Freno::Throttler#throttle(context, &block)` will check freno to determine whether is OK to proceed with the given block. If so, the block will be executed immediately, otherwise the throttler will sleep and try
+again.
+
+#### Throttler configuration
+
+```ruby
+module Freno
+  class Throttler
+
+    DEFAULT_WAIT_SECONDS = 0.5
+    DEFAULT_MAX_WAIT_SECONDS = 10
+
+    def initialize(client: nil,
+                    app: nil,
+                    mapper: Mapper::Identity,
+                    instrumenter: Instrumenter::Noop,
+                    circuit_breaker: CircuitBreaker::Noop,
+                    wait_seconds: DEFAULT_WAIT_SECONDS,
+                    max_wait_seconds: DEFAULT_MAX_WAIT_SECONDS
+
+
+      @client           = client
+      @app              = app
+      @mapper           = mapper
+      @instrumenter     = instrumenter
+      @circuit_breaker  = circuit_breaker
+      @wait_seconds     = wait_seconds
+      @max_wait_seconds = max_wait_seconds
+
+      yield self if block_given?
+
+      validate_args
+    end
+
+    ...
+  end
+end
+```
+
+A Throttler instance will make calls to freno on behalf of the given `app`,
+using the given `client` (an instance of `Freno::Client`).
+
+You optionally provide the time you want the throttler to sleep in case the check to freno fails, this is `wait_seconds`.
+
+If replication lags badly, you can control until when you want to keep sleeping
+and retrying the check by setting `max_wait_seconds`. When that times out, the throttle will raise a `Freno::Throttler::WaitedTooLong` error.
+
+#### Instrumenting the throttler
+
+You can also configure the throttler with an `instrumenter` collaborator to subscribe to events happening during the `throttle` call.
+
+An instrumenter is an object that responds to `instrument(event_name, payload = {})` to receive events from the throttler. One could use `ActiveSupport::Notifications` as an instrumenter and subscribe to "freno.*" events somewhere else in the application, or implement one like the following to push some metrics to a stats system.
+
+```ruby
+  class StatsInstrumenter
+
+    attr_reader :stats
+
+    def initialize(stats:)
+      @stats = stats
+    end
+
+    def instrument(event_name, payload)
+      method = event_name.sub("throttler.", "")
+      send(method, payload) if respond_to?(method)
+    end
+
+    def called(payload)
+      increment("throttler.called", tags: extract_tags(payload))
+    end
+
+    def waited(payload)
+      stats.histogram("throttler.waited", payload[:waited], tags: extract_tags(payload))
+    end
+
+    ...
+
+    def circuit_open(payload)
+      stats.increment("throttler.circuit_open", tags: extract_tags(payload))
+    end
+
+    private
+
+    def extract_tags(payload)
+      cluster_names = payload[:store_names] || []
+      cluster_tags = cluster_names.map{ |cluster_name "cluster:#{cluster_name}" }
+    end
+  end
+```
+
+#### Adding resiliency
+
+The throttler can also receive a `circuit_breaker` object to implement resiliency.
+
+With that information it receives, the circuit breaker determines whether or not to allow the next request. A circuit is said to be open when the next request is not allowed; and it's said to be closed when the next request is allowed
+
+If the throttler waited too long, or an unexpected error happened; the circuit breaker will receive a `failure`. If in contrast it succeeded, the circuit breaker will receive a `success` message.
+
+Once the circuit is open, the throttler will not try to throttle calls, an instead throw a `Freno::Throttler::CircuitOpen`
+
+The following is a simple per-process circuit breaker implementation:
+
+```ruby
+class MemoryCircuitBreaker
+
+  DEFAULT_CIRCUIT_RETRY_INTERVAL = 10
+
+  def initialize(circuit_retry_interval: DEFAULT_CIRCUIT_RETRY_INTERVAL)
+    @circuit_closed = true
+    @last_failure = nil
+    @circuit_retry_interval = circuit_retry_interval
+  end
+
+  def allow_request?
+    @circuit_closed || (Time.now - @last_failure) > @circuit_retry_interval
+  end
+
+  def success
+    @circuit_closed = true
+  end
+
+  def failure
+    @last_failure = Time.now
+    @circuit_closed = false
+  end
+end
+```
+
+#### Flexible throttling strategies
+
+The throttler uses a `mapper` to determine, based on the context provided to `#throttle`, the clusters which replication delay needs to be checked.
+
+By default the throttler uses `Mapper::Identity`, which expect the context to be the store name(s) to check:
+
+```ruby
+# will check my_cluster's health
+throttler.throttle(:my_cluster) { ... }
+# will check the health of cluster_a and cluster_b and throttle if any of them is not OK.
+throttler.throttle([:cluster_a, :cluster_b]) { ... }
+```
+
+You can create your own mapper, which is just an callable object (like a Proc, or any other object that responds to `call(context)`). The following is a mapper that knows how to throttle access to certain tables and shards.
+
+
+```ruby
+class ShardMapper
+  def call(context = {})
+    context.map do |table, shards|
+      DatabaseStructure.cluster_for(table, shards)
+    end
+  end
+end
+
+throttler = Freno::Throttler.new(client: freno, app: :my_app, mapper: ShardMapper.new)
+
+throttler.throttle(:users => [1,2,3], :repositories => 5) do
+  perform_writes
+end
+```
 
 ## Development
 
@@ -193,9 +370,7 @@ If you are the current maintainer of this gem:
 1. Ensure that tests are green: `bundle exec rake test`
 1. Bump gem version in `lib/freno/client/version.rb`
 1. Merge a PR to github/freno-client containing the changes in the version file
-1. Tag and push: `git tag vx.xx.xx; git push --tags`
-1. Build the gem: `gem build freno-client`
-1. Push to rubygems.org: `gem push freno-client-x.y.z.gem`
+1. Run `script/release`
 
 ## License
 

data/Rakefile

@@ -1,5 +1,26 @@
-require "bundler/gem_tasks"
+$LOAD_PATH.push File.expand_path("../lib", __FILE__)
 require "rake/testtask"
+require "freno/client/version"
+
+# gem install pkg/*.gem
+# gem uninstall freno-client freno-throttler
+desc "Build gem into the pkg directory"
+task :build do
+  FileUtils.rm_rf("pkg")
+  Dir["*.gemspec"].each do |gemspec|
+    system "gem build #{gemspec}"
+  end
+  FileUtils.mkdir_p("pkg")
+  FileUtils.mv(Dir["*.gem"], "pkg")
+end
+
+desc "Tags version, pushes to remote, and pushes gem"
+task release: :build do
+  sh "git", "tag", "v#{Freno::Client::VERSION}"
+  sh "git push origin master"
+  sh "git push origin v#{Freno::Client::VERSION}"
+  sh "ls pkg/*.gem | xargs -n 1 gem push"
+end
 
 Rake::TestTask.new(:test) do |t|
   t.libs << "test"

data/lib/freno/client/version.rb

@@ -1,5 +1,5 @@
 module Freno
   class Client
-    VERSION = "0.4.0"
+    VERSION = "0.6.0"
   end
 end

data/lib/freno/throttler.rb

@@ -0,0 +1,208 @@
+require "freno/client"
+require "freno/throttler/errors"
+require "freno/throttler/mapper"
+require "freno/throttler/instrumenter"
+require "freno/throttler/circuit_breaker"
+
+module Freno
+
+  # Freno::Throttler is the class responsible for throttling writes to a cluster
+  # or a set of clusters. Throttling means to slow down the pace at which write
+  # operations occur by checking with freno whether all the clusters affected by
+  # the operation are in good health before allowing it. If any of the clusters
+  # is not in good health, the throttler will wait some time and repeat the
+  # process.
+  #
+  # Examples:
+  #
+  # Let's use the following throttler, which uses Mapper::Identity implicitly.
+  # (see #initialze docs)
+  #
+  # ```
+  # throttler = Throttler.new(client: freno_client, app: :my_app)
+  # data.find_in_batches do |batch|
+  #   throttler.throttle([:mysqla, :mysqlb]) do
+  #     update(batch)
+  #   end
+  # end
+  # ```
+  #
+  # Before each call to `update(batch)` the throttler will call freno to
+  # check the health of the `mysqla` and `mysqlb` stores on behalf of :my_app;
+  # and sleep if any of the stores is not ok.
+  #
+  class Throttler
+
+    DEFAULT_WAIT_SECONDS = 0.5
+    DEFAULT_MAX_WAIT_SECONDS = 10
+
+    attr_accessor :client,
+                  :app,
+                  :mapper,
+                  :instrumenter,
+                  :circuit_breaker,
+                  :wait_seconds,
+                  :max_wait_seconds
+
+    # Initializes a new instance of the throttler
+    #
+    # In order to initialize a Throttler you need the following arguments:
+    #
+    #  - a `client`: a instance of Freno::Client
+    #
+    #  - an `app`: a symbol indicating the app-name for which Freno will respond
+    #    checks.
+    #
+    # Also, you can optionally provide the following named arguments:
+    #
+    #  - `:mapper`: An object that responds to `call(context)` and returns a
+    #     `Enumerable` of the store names for which we need to wait for
+    #     replication delay. By default this is the `IdentityMapper`, which will
+    #     check the stores given as context.
+    #
+    #     For example, if the `throttler` object used the default mapper:
+    #
+    #      ```
+    #      throttler.throttle(:mysqlc) do
+    #         update(batch)
+    #      end
+    #      ```
+    #
+    #  - `:instrumenter`: An object that responds to
+    #     `instrument(event_name, context = {}, &block)` that can be used to
+    #     add cross-cutting concerns like logging or stats to the throttler.
+    #
+    #     By default, the instrumenter is `Instrumenter::Noop`, which does
+    #     nothing but yielding the block it receives.
+    #
+    #  - `:circuit_breaker`: An object responding to `allow_request?`,
+    #     `success`, and `failure?`, compatible with `Resilient::CircuitBreaker`
+    #     (see https://github.com/jnunemaker/resilient).
+    #
+    #     By default, the circuit breaker is `CircuitBreaker::Noop`, which
+    #     always allows requests, and does not provide resiliency guarantees.
+    #
+    #  - `:wait_seconds`: A positive float indicating the number of seconds the
+    #     throttler will wait before checking again, in case some of the stores
+    #     didn't catch-up the last time they were check.
+    #
+    #  - `:max_wait_seconds`: A positive float indicating the maxium number of
+    #     seconds the throttler will wait in total for replicas to catch-up
+    #     before raising a `WaitedTooLong` error.
+    #
+    def initialize(client: nil,
+                    app: nil,
+                    mapper: Mapper::Identity,
+                    instrumenter: Instrumenter::Noop,
+                    circuit_breaker: CircuitBreaker::Noop,
+                    wait_seconds: DEFAULT_WAIT_SECONDS,
+                    max_wait_seconds: DEFAULT_MAX_WAIT_SECONDS)
+
+      @client           = client
+      @app              = app
+      @mapper           = mapper
+      @instrumenter     = instrumenter
+      @circuit_breaker  = circuit_breaker
+      @wait_seconds     = wait_seconds
+      @max_wait_seconds = max_wait_seconds
+
+      yield self if block_given?
+
+      validate_args
+    end
+
+    # This method receives a context to infer the set of stores that it needs to
+    # throttle writes to.
+    #
+    # With that information it asks freno whether all the stores are ok.
+    # In case they are, it executes the given block.
+    # Otherwise, it waits `wait_seconds` before trying again.
+    #
+    # In case the throttler has waited more than `max_wait_seconds`, it raises
+    # a `WaitedTooLong` error.
+    #
+    # In case there's an underlying Freno error, it raises a `ClientError`
+    # error.
+    #
+    # In case the circuit breaker is open, it raises a `CircuitOpen` error.
+    #
+    # this method is instrumented, the instrumenter will receive the following
+    # events:
+    #
+    # - "throttler.called" each time this method is called
+    # - "throttler.succeeded" when the stores were ok, before yielding the block
+    # - "throttler.waited" when the stores were not ok, after waiting
+    #   `wait_seconds`
+    # - "throttler.waited_too_long" when the stores were not ok, but the
+    #   thottler already waited at least `max_wait_seconds`, right before
+    #   raising `WaitedTooLong`
+    # - "throttler.freno_errored" when there was an error with freno, before
+    #   raising `ClientError`.
+    # - "throttler.circuit_open" when the circuit breaker does not allow the
+    #   next request, before raising `CircuitOpen`
+    #
+    def throttle(context = nil)
+      store_names = mapper.call(context)
+      instrument(:called, store_names: store_names)
+      waited = 0
+
+      while true do # rubocop:disable Lint/LiteralInCondition
+        unless circuit_breaker.allow_request?
+          instrument(:circuit_open, store_names: store_names, waited: waited)
+          raise CircuitOpen
+        end
+
+        if all_stores_ok?(store_names)
+          instrument(:succeeded, store_names: store_names, waited: waited)
+          circuit_breaker.success
+          return yield
+        end
+
+        wait
+        waited += wait_seconds
+        instrument(:waited, store_names: store_names, waited: waited, max: max_wait_seconds)
+
+        if waited > max_wait_seconds
+          instrument(:waited_too_long, store_names: store_names, waited: waited, max: max_wait_seconds)
+          circuit_breaker.failure
+          raise WaitedTooLong.new(waited_seconds: waited, max_wait_seconds: max_wait_seconds)
+        end
+      end
+    end
+
+    private
+
+    def validate_args
+      errors = []
+
+      %i(client app mapper instrumenter circuit_breaker
+        wait_seconds max_wait_seconds).each do |argument|
+        errors << "#{argument} must be provided" unless send(argument)
+      end
+
+      unless max_wait_seconds > wait_seconds
+        errors << "max_wait_seconds (#{max_wait_seconds}) has to be greather than wait_seconds (#{wait_seconds})"
+      end
+
+      raise ArgumentError.new(errors.join("\n")) if errors.any?
+    end
+
+    def all_stores_ok?(store_names)
+      store_names.all? do |store_name|
+        client.check?(app: app, store_name: store_name)
+      end
+    rescue Freno::Error => e
+      instrument(:freno_errored, store_names: store_names, error: e)
+      circuit_breaker.failure
+      raise ClientError.new(e)
+    end
+
+    def wait
+      sleep wait_seconds
+    end
+
+    def instrument(event_name, payload = {}, &block)
+      instrumenter.instrument("throttler.#{event_name}", payload, &block)
+    end
+  end
+end

data/lib/freno/throttler/circuit_breaker.rb

@@ -0,0 +1,39 @@
+module Freno
+  class Throttler
+
+    # A CircuitBreaker is the entry point of the pattern with same name.
+    # (see https://martinfowler.com/bliki/CircuitBreaker.html)
+    #
+    # Clients that use circuit breakers to add resiliency to their processes
+    # send `failure` or `sucess` messages to the CircuitBreaker depending on the
+    # results of the last requests made.
+    #
+    # With that information, the circuit breaker determines whether or not to
+    # allow the next request (`allow_request?`). A circuit is said to be open
+    # when the next request is not allowed; and it's said to be closed when the
+    # next request is allowed.
+    #
+    module CircuitBreaker
+
+      # The Noop circuit breaker is the `:circuit_breaker` used by default in
+      # the Throttler
+      #
+      # It always allows requests, and does nothing when given `success` or
+      # `failure` messages. For that reason it doesn't provide any resiliency
+      # guarantee.
+      #
+      # See https://github.com/jnunemaker/resilient for a real ruby implementation
+      # of the CircuitBreaker pattern.
+      #
+      class Noop
+        def self.allow_request?
+          true
+        end
+
+        def self.success; end
+
+        def self.failure; end
+      end
+    end
+  end
+end

data/lib/freno/throttler/errors.rb

@@ -0,0 +1,23 @@
+require "freno/client"
+
+module Freno
+  class Throttler
+
+    # Any throttler-related error.
+    class Error < Freno::Error; end
+
+    # Raised if the throttler has waited too long for replication delay
+    # to catch up.
+    class WaitedTooLong < Error
+      def initialize(waited_seconds: DEFAULT_WAIT_SECONDS, max_wait_seconds: DEFAULT_MAX_WAIT_SECONDS)
+        super("Waited #{waited_seconds} seconds. Max allowed was #{max_wait_seconds} seconds")
+      end
+    end
+
+    # Raised if the circuit breaker is open and didn't allow latest request
+    class CircuitOpen < Error; end
+
+    # Raised if the freno client errored.
+    class ClientError < Error; end
+  end
+end

data/lib/freno/throttler/instrumenter.rb

@@ -0,0 +1,27 @@
+module Freno
+  class Throttler
+
+    # An Instrumenter is an object that responds to
+    # `instrument(event_name, payload = {})` to receive events from the
+    # throttler.
+    #
+    # As an example, in a rails app one could use ActiveSupport::Notifications
+    # as an instrumenter and subscribe to the "freno.*" events somewhere else in
+    # the application.
+    #
+    module Instrumenter
+
+      # The Noop instrumenter is the `:instrumenter` used by default in the
+      # Throttler
+      #
+      # It does nothing but yielding the control to the block given if it is
+      # provided.
+      #
+      class Noop
+        def self.instrument(event_name, payload = {})
+          yield payload if block_given?
+        end
+      end
+    end
+  end
+end

data/lib/freno/throttler/mapper.rb

@@ -0,0 +1,45 @@
+module Freno
+  class Throttler
+
+    # A Mapper is any object that responds to `call`, by receiving a context
+    # object and returning a list of strings, each of which corresponds to the
+    # store_name that will be checked in freno.
+    #
+    # See https://github.com/github/freno/blob/master/doc/http.md#client-requests
+    # for more context.
+    #
+    # As an example we could use a mapper that will receive as a context a set
+    # of [table, shard_id] tuples, and could return the list of all the stores
+    # where that shards exist.
+    #
+    module Mapper
+
+      # The Identity mapper is the one used by default in the Throttler.
+      #
+      # It works by informing the throttler to check exact same stores that it
+      # receives as context, without any translation.
+      #
+      # Let's use the following throttler, which uses Mapper::Identity
+      # implicitly.
+      #
+      # ```ruby
+      # throttler = Throttler.new(client: freno_client, app: :my_app)
+      # data.find_in_batches do |batch|
+      #   throttler.throttle([:mysqla, :mysqlb]) do
+      #     update(batch)
+      #   end
+      # end
+      # ```
+      #
+      # Before each call to `update(batch)` the throttler will call freno to
+      # check the health of `mysqla` and `mysqlb`. And sleep if any of them is
+      # not ok.
+      #
+      class Identity
+        def self.call(context)
+          Array(context)
+        end
+      end
+    end
+  end
+end

data/script/release

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle exec rake release

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: freno-client
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.6.0
 platform: ruby
 authors:
 - Miguel Fernández
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-08-29 00:00:00.000000000 Z
+date: 2017-10-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -97,10 +97,16 @@ files:
 - lib/freno/client/requests/replication_delay.rb
 - lib/freno/client/result.rb
 - lib/freno/client/version.rb
+- lib/freno/throttler.rb
+- lib/freno/throttler/circuit_breaker.rb
+- lib/freno/throttler/errors.rb
+- lib/freno/throttler/instrumenter.rb
+- lib/freno/throttler/mapper.rb
 - script/bootstrap
 - script/cibuild
 - script/cibuild-lint
 - script/console
+- script/release
 homepage: https://github.com/github/freno-client
 licenses:
 - MIT