http-threshold 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: dd6259ff9cac77fc952d5099d7374634aff25038
+  data.tar.gz: 5055cd6cac2e8c06a26c91c34dc35d2a397de40b
+SHA512:
+  metadata.gz: 2accd9766182e7f04bc000851fc70107668a63c58f55f639e4ec0322c57da577cdbf20b1d0ea166e1da7b4c84bb4da77fcd0945ee1daecf66f93f80b6acfe414
+  data.tar.gz: 1aa7bd8035be0e7894eef358716a66c89c25c4d436b6692116b717e785c80dad165d0556c817ffe231244d38baf22a3a839ebbcf393f1ea20990dbdc2879f47c

data/.gitignore

@@ -0,0 +1,7 @@
+*~
+*.sqlite3
+.bundle
+pkg
+log
+
+*.gem

data/Gemfile

@@ -0,0 +1 @@
+gemspec

data/Gemfile.lock

@@ -0,0 +1,24 @@
+PATH
+  remote: .
+  specs:
+    threshold (0.0.2)
+      rack-attack (~> 5.0)
+      redlock (~> 0.2)
+
+GEM
+  specs:
+    rack (2.0.3)
+    rack-attack (5.0.1)
+      rack
+    redis (3.3.3)
+    redlock (0.2.1)
+      redis (>= 3.0.0, < 5.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  threshold!
+
+BUNDLED WITH
+   1.15.4

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2017 Ankun Yu
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,46 @@
+# Threshold
+
+A gem to control external API calls. Many API server has rate limit.e.g. Recurly: 1k requests/1minute, Shopify: 4 requests/second.
+To resolve external API call issue, I create this gem.
+
+Ideas are come from `rack-attach`, `redlock`. 
+This gem add extension to `Net::HTTPRequest`.
+Before you made request, check redis if out request reach the limit.
+If reach the limit, sleep and wait rack-attack expire the out rate limit count.
+
+
+## How to use
+
+`gem install threshold`
+
+```ruby
+
+# currently, throttle is based on host + port
+HttpThreshold.redis_url = "redis://localhost:6379/0"
+HttpThreshold::Client.set_throttle("your-store.shopify.com", limit: 4, period: 1.second)
+HttpThreshold::Client.set_throttle("localhost:3000", limit: 50, period: 1.minute)
+HttpThreshold::Client.set_throttle("google.com", limit: 5, period: 10.second)
+```
+Then when you run any http call like below
+```
+require 'httparty'
+HTTParty.get("https://www.google.com")#ok
+HTTParty.get("https://www.google.com")#ok
+HTTParty.get("https://www.google.com")#ok
+HTTParty.get("https://www.google.com")#ok
+HTTParty.get("https://www.google.com")#ok
+HTTParty.get("https://www.google.com")#will sleep until this 10 seconds past, and then get response
+
+```
+
+It support distributed system, because we use redis lock.
+
+## Note
+The best way to use this, is add threshold at background job, set a more strict limit in a period.
+So that it could leave enough rate resources for website.
+
+Background job sleep is OK. A user's request sleep long would cause problem.
+
+
+## Todo
+Add test

data/http-threshold.gemspec

@@ -0,0 +1,17 @@
+# encoding: UTF-8
+Gem::Specification.new do |s|
+  s.platform    = Gem::Platform::RUBY
+  s.name = "http-threshold"
+  s.version = "0.0.1"
+  s.date = "2017-11-15"
+  s.homepage = 'https://github.com/yakjuly/http-threshold'
+  s.summary = "A gem for to control external API rate,Could useful for Shopify, Recurly API etc"
+  s.author = 'Ankun Yu'
+  s.email = 'yakjuly@gmail.com'
+  s.files = `git ls-files`.split("\n").reject { |f| f.match(/^spec/) && !f.match(/^spec\/fixtures/) }
+  s.require_paths = "lib"
+  s.license = 'MIT'
+
+  s.add_dependency 'rack-attack', '~> 5.0'
+  s.add_dependency 'redlock', '~> 0.2'
+end

data/lib/http-threshold.rb

@@ -0,0 +1 @@
+require 'http_threshold'

data/lib/http_threshold.rb

@@ -0,0 +1,12 @@
+require 'http_threshold/cache'
+require 'http_threshold/throttle'
+require 'http_threshold/client'
+require 'http_threshold/request_holder'
+require 'pp'
+
+module HttpThreshold
+
+  mattr_accessor :redis_url
+  @@redis_url = nil
+
+end

data/lib/http_threshold/cache.rb

@@ -0,0 +1,17 @@
+require 'rack/attack'
+module HttpThreshold
+  class Cache < Rack::Attack::Cache
+    attr_accessor :prefix
+
+    def initialize
+      self.store = ::Rails.cache if defined?(::Rails.cache)
+      @prefix = 'threshold'
+    end
+
+    def read_count(unprefixed_key, period)
+      key, _ = key_and_expiry(unprefixed_key, period)
+      store.read(key).to_i
+    end
+
+  end
+end

data/lib/http_threshold/client.rb

@@ -0,0 +1,46 @@
+require 'redlock'
+
+module HttpThreshold
+  class Client
+
+    class << self
+      def throttles
+        @throttles  ||= {}
+      end
+
+      def cache
+        @cache ||= Cache.new
+      end
+
+      def lock_manager
+        @lock_manager ||= Redlock::Client.new([Redis.new(:url => HttpThreshold.redis_url)])
+      end
+
+      def incr_count(domain)
+        self.throttles[domain].try(:incr_count)
+      end
+
+      def set_throttle(domain, options = {})
+        self.throttles[domain] = HttpThreshold::Throttle.new(domain, options)
+      end
+
+      def sleep_until_allowed(host)
+        throttle = self.throttles[host]
+        if throttle
+          if !throttle.incr_count
+            puts "#{host} reach threshold limit"
+            sleep 0.25
+
+            while !throttle.incr_count
+              sleep 0.25
+            end
+          end
+          yield
+        else
+          yield
+        end
+      end
+    end
+
+  end
+end

data/lib/http_threshold/request_holder.rb

@@ -0,0 +1,15 @@
+module HttpThreshold
+  module RequestHold
+
+    def exec(sock, ver, path)
+      host = @header["host"].first
+      # here can stop the request
+      HttpThreshold::Client.sleep_until_allowed(host) do
+        super
+      end
+    end
+
+  end
+end
+
+Net::HTTPRequest.include HttpThreshold::RequestHold

data/lib/http_threshold/throttle.rb

@@ -0,0 +1,55 @@
+module HttpThreshold
+  class Throttle
+    MANDATORY_OPTIONS = [:limit, :period]
+
+    attr_reader :name, :limit, :period, :block, :type
+    def initialize(name, options)
+      @name = name
+      MANDATORY_OPTIONS.each do |opt|
+        raise ArgumentError.new("Must pass #{opt.inspect} option") unless options[opt]
+      end
+      @limit  = options[:limit]
+      @period = options[:period].to_i
+    end
+
+    def cache
+      HttpThreshold::Client.cache
+    end
+
+    def lock_manager
+      HttpThreshold::Client.lock_manager
+    end
+
+    # similar as method #[]
+    def incr_count
+      begin
+        result = lock_manager.lock!("lock:#{name}", 2000) do
+          if reach_limit?
+            false
+          else
+            cache.count(threshold_key, period)
+            true
+          end
+        end
+        result
+      rescue Redlock::LockError
+        sleep 0.25
+        retry
+      end
+    end
+
+    def reach_limit?
+      count = cache.read_count(threshold_key, period)
+      count >= limit
+    end
+
+    def status
+      count = cache.read_count(threshold_key, period)
+      "#{count}/#{limit}"
+    end
+
+    def threshold_key
+      "threshold:#{name}"
+    end
+  end
+end

metadata

@@ -0,0 +1,84 @@
+--- !ruby/object:Gem::Specification
+name: http-threshold
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Ankun Yu
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-11-15 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rack-attack
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: redlock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+description: 
+email: yakjuly@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- Gemfile.lock
+- LICENSE
+- README.md
+- http-threshold.gemspec
+- lib/http-threshold.rb
+- lib/http_threshold.rb
+- lib/http_threshold/cache.rb
+- lib/http_threshold/client.rb
+- lib/http_threshold/request_holder.rb
+- lib/http_threshold/throttle.rb
+homepage: https://github.com/yakjuly/http-threshold
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.8
+signing_key: 
+specification_version: 4
+summary: A gem for to control external API rate,Could useful for Shopify, Recurly
+  API etc
+test_files: []