ratelimit-ruby 0.1.6 → 0.1.7
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile +0 -5
- data/README.md +41 -9
- data/VERSION +1 -1
- data/lib/ratelimit/murmur3.rb +50 -0
- data/lib/ratelimit/noop_cache.rb +13 -0
- data/lib/ratelimit/toy_cache.rb +26 -0
- data/lib/ratelimit-ruby.rb +96 -15
- data/ratelimit-ruby.gemspec +6 -3
- metadata +5 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e2ff9d738826e547d7d64b087858282fe5b9a4d3
|
4
|
+
data.tar.gz: 39bbd78258a2805bec76eb852077f36e6ad3081b
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 6e2a19885fcd3952411fc44a80efbef3ffb888d013530c7dfd23e245a69f8a1dcfeb27d8259dfe8f4a0fd48e965594b49efc0a403d3d9530c8497607a872de5c
|
7
|
+
data.tar.gz: 4da6feb04c65f31a6ae53fb3f14ebc8f12c7bf0ea57b7b15fe972c39c4afaa0184b429c2df7264909d75525dc0feec9d634ecfe14a4c4a52ad2f0fd2ec8cb46c
|
data/Gemfile
CHANGED
@@ -1,10 +1,5 @@
|
|
1
1
|
source "https://rubygems.org"
|
2
|
-
# Add dependencies required to use your gem here.
|
3
|
-
# Example:
|
4
|
-
# gem "activesupport", ">= 2.3.5"
|
5
2
|
|
6
|
-
# Add dependencies to develop your gem here.
|
7
|
-
# Include everything needed to run rake, tests, features, etc.
|
8
3
|
gem 'faraday'
|
9
4
|
gem 'faraday_middleware'
|
10
5
|
|
data/README.md
CHANGED
@@ -18,24 +18,56 @@ See full documentation http://www.ratelim.it/documentation
|
|
18
18
|
|
19
19
|
## Supports
|
20
20
|
|
21
|
-
* RateLimits
|
21
|
+
* [RateLimits](http://www.ratelim.it/documentation/basic_rate_limits)
|
22
22
|
* Millions of individual limits sharing the same policies
|
23
|
-
* WebUI for tweaking limits
|
24
|
-
* Logging
|
25
|
-
* Semaphores
|
26
|
-
* Infinite retention
|
23
|
+
* WebUI for tweaking limits & feature flags
|
24
|
+
* Logging to help you debug
|
25
|
+
* [Concurrency](http://www.ratelim.it/documentation/concurrency) & Semaphores
|
26
|
+
* Infinite retention for [deduplication workflows](http://www.ratelim.it/documentation/once_and_only_once)
|
27
|
+
* [FeatureFlags](http://www.ratelim.it/documentation/feature_flags) as a Service
|
27
28
|
|
28
29
|
## Options and Defaults
|
29
30
|
```ruby
|
30
|
-
limiter = RateLimit::Limiter.new(
|
31
|
+
limiter = RateLimit::Limiter.new(
|
32
|
+
apikey: "ACCT_ID|APIKEY",
|
31
33
|
on_error: :log_and_pass, # :log_and_pass, :log_and_hit, :throw
|
32
|
-
logger: nil, # pass in your own logger here
|
33
|
-
debug: false #Faraday debugging
|
34
|
+
logger: nil, # pass in your own logger here. ie Rails.logger
|
35
|
+
debug: false, #Faraday debugging
|
36
|
+
stats: nil, # receives increment("it.ratelim.limitcheck", {:tags=>["policy_group:page_view", "pass:true"]})
|
37
|
+
shared_cache: nil, # Something that quacks like Rails.cache ideally memcached
|
38
|
+
# used to avoid hitting feature flag endpoint too much
|
39
|
+
in_process_cache: nil # Something like ActiveSupport::Cache::MemoryStore.new(size: 2.megabytes)
|
40
|
+
# used to memoize featureflags if used in tight loops
|
41
|
+
)
|
42
|
+
```
|
43
|
+
|
44
|
+
## Full Example with Feature Flags
|
45
|
+
```ruby
|
46
|
+
@limiter = RateLimit::Limiter.new(apikey: "",
|
47
|
+
shared_cache = Rails.cache,
|
48
|
+
logger = Rails.logger,
|
49
|
+
in_process_cahe = ActiveSupport::Cache::MemoryStore.new(size: 1.megabytes)
|
34
50
|
)
|
35
51
|
|
52
|
+
@limiter.create_limit("event:pageload", 1, RateLimIt::HOURLY_ROLLING)
|
53
|
+
@limiter.create_limit("event:activation", 1, RateLimIt::INFINITE)
|
36
54
|
|
37
|
-
```
|
38
55
|
|
56
|
+
def track_event(event, user_id)
|
57
|
+
if @limiter.feature_is_on_for?("Services::RateLimit", user_id)
|
58
|
+
return unless @limiter.pass?("event:#{event}:#{user_id}")
|
59
|
+
end
|
60
|
+
actually_track_event(event, user_id)
|
61
|
+
end
|
62
|
+
|
63
|
+
|
64
|
+
track_event("pageload:home_page", 1) # will track
|
65
|
+
track_event("pageload:home_page", 1) # will skip for the next hour
|
66
|
+
track_event("activation", 1) # will track
|
67
|
+
track_event("activation", 1) # will skip forever
|
68
|
+
|
69
|
+
|
70
|
+
```
|
39
71
|
|
40
72
|
## Contributing to ratelimit-ruby
|
41
73
|
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.1.
|
1
|
+
0.1.7
|
@@ -0,0 +1,50 @@
|
|
1
|
+
class Murmur3
|
2
|
+
## MurmurHash3 was written by Austin Appleby, and is placed in the public
|
3
|
+
## domain. The author hereby disclaims copyright to this source code.
|
4
|
+
|
5
|
+
MASK32 = 0xffffffff
|
6
|
+
|
7
|
+
def self.murmur3_32_rotl(x, r)
|
8
|
+
((x << r) | (x >> (32 - r))) & MASK32
|
9
|
+
end
|
10
|
+
|
11
|
+
|
12
|
+
def self.murmur3_32_fmix(h)
|
13
|
+
h &= MASK32
|
14
|
+
h ^= h >> 16
|
15
|
+
h = (h * 0x85ebca6b) & MASK32
|
16
|
+
h ^= h >> 13
|
17
|
+
h = (h * 0xc2b2ae35) & MASK32
|
18
|
+
h ^ (h >> 16)
|
19
|
+
end
|
20
|
+
|
21
|
+
def self.murmur3_32__mmix(k1)
|
22
|
+
k1 = (k1 * 0xcc9e2d51) & MASK32
|
23
|
+
k1 = murmur3_32_rotl(k1, 15)
|
24
|
+
(k1 * 0x1b873593) & MASK32
|
25
|
+
end
|
26
|
+
|
27
|
+
def self.murmur3_32(str, seed=0)
|
28
|
+
h1 = seed
|
29
|
+
numbers = str.unpack('V*C*')
|
30
|
+
tailn = str.length % 4
|
31
|
+
tail = numbers.slice!(numbers.size - tailn, tailn)
|
32
|
+
for k1 in numbers
|
33
|
+
h1 ^= murmur3_32__mmix(k1)
|
34
|
+
h1 = murmur3_32_rotl(h1, 13)
|
35
|
+
h1 = (h1*5 + 0xe6546b64) & MASK32
|
36
|
+
end
|
37
|
+
|
38
|
+
unless tail.empty?
|
39
|
+
k1 = 0
|
40
|
+
tail.reverse_each do |c1|
|
41
|
+
k1 = (k1 << 8) | c1
|
42
|
+
end
|
43
|
+
h1 ^= murmur3_32__mmix(k1)
|
44
|
+
end
|
45
|
+
|
46
|
+
h1 ^= str.length
|
47
|
+
murmur3_32_fmix(h1)
|
48
|
+
end
|
49
|
+
|
50
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
# Don't use me in prod
|
2
|
+
# Just a toy for testing
|
3
|
+
module RateLimit
|
4
|
+
class ToyCache
|
5
|
+
@@cache = {}
|
6
|
+
|
7
|
+
def fetch(name, opts, &block)
|
8
|
+
result = read(name)
|
9
|
+
|
10
|
+
return result unless result.nil?
|
11
|
+
|
12
|
+
r = yield
|
13
|
+
|
14
|
+
write(name, r)
|
15
|
+
read(name)
|
16
|
+
end
|
17
|
+
|
18
|
+
def write(name, value, opts=nil)
|
19
|
+
@@cache[name] = value
|
20
|
+
end
|
21
|
+
|
22
|
+
def read(name)
|
23
|
+
@@cache[name]
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
data/lib/ratelimit-ruby.rb
CHANGED
@@ -6,16 +6,25 @@ module RateLimit
|
|
6
6
|
end
|
7
7
|
|
8
8
|
class Limiter
|
9
|
-
def base_url(local)
|
10
|
-
local ? 'http://localhost:8080' : 'http://www.ratelim.it'
|
11
|
-
end
|
12
9
|
|
13
|
-
def initialize(apikey:,
|
10
|
+
def initialize(apikey:,
|
11
|
+
on_error: :log_and_pass,
|
12
|
+
logger: nil,
|
13
|
+
debug: false,
|
14
|
+
stats: nil, # receives increment("it.ratelim.limitcheck", {:tags=>["policy_group:page_view", "pass:true"]})
|
15
|
+
shared_cache: nil, # Something that quacks like Rails.cache ideally memcached
|
16
|
+
in_process_cache: nil, # ideally ActiveSupport::Cache::MemoryStore.new(size: 2.megabytes)
|
17
|
+
use_expiry_cache: true, # must have shared_cache defined
|
18
|
+
local: false # local development
|
19
|
+
)
|
20
|
+
@on_error = on_error
|
14
21
|
@logger = (logger || Logger.new($stdout)).tap do |log|
|
15
22
|
log.progname = "RateLimit"
|
16
23
|
end
|
17
24
|
@stats = (stats || NoopStats.new)
|
18
|
-
@
|
25
|
+
@shared_cache = (shared_cache || NoopCache.new)
|
26
|
+
@in_process_cache = (in_process_cache || NoopCache.new)
|
27
|
+
@use_expiry_cache = use_expiry_cache
|
19
28
|
@conn = Faraday.new(:url => self.base_url(local)) do |faraday|
|
20
29
|
faraday.request :json # form-encode POST params
|
21
30
|
faraday.response :logger if debug
|
@@ -23,8 +32,8 @@ module RateLimit
|
|
23
32
|
faraday.options[:timeout] = 5
|
24
33
|
faraday.adapter Faraday.default_adapter # make requests with Net::HTTP
|
25
34
|
end
|
26
|
-
(
|
27
|
-
@conn.basic_auth(
|
35
|
+
(@account_id, pass) = apikey.split("|")
|
36
|
+
@conn.basic_auth(@account_id, pass)
|
28
37
|
end
|
29
38
|
|
30
39
|
|
@@ -54,14 +63,29 @@ module RateLimit
|
|
54
63
|
end
|
55
64
|
|
56
65
|
def acquire(group, acquire_amount, allow_partial_response: false)
|
66
|
+
|
67
|
+
expiry_cache_key = "it.ratelim.expiry.#{group}"
|
68
|
+
if @use_expiry_cache
|
69
|
+
expiry = @shared_cache.read(expiry_cache_key)
|
70
|
+
if !expiry.nil? && Integer(expiry) > Time.now.utc.to_f * 1000
|
71
|
+
@stats.increment("it.ratelim.limitcheck.expirycache.hit", tags: [])
|
72
|
+
return OpenStruct.new(passed: false, amount: 0)
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
57
76
|
result = @conn.post '/api/v1/limitcheck', { acquireAmount: acquire_amount,
|
58
77
|
groups: [group],
|
59
78
|
allowPartialResponse: allow_partial_response }.to_json
|
60
79
|
handle_failure(result) unless result.success?
|
61
80
|
res =JSON.parse(result.body, object_class: OpenStruct)
|
62
81
|
res.amount ||= 0
|
82
|
+
|
63
83
|
@stats.increment("it.ratelim.limitcheck", tags: ["policy_group:#{res.policyGroup}", "pass:#{res.passed}"])
|
64
|
-
|
84
|
+
if @use_expiry_cache
|
85
|
+
reset = result.headers['X-Rate-Limit-Reset']
|
86
|
+
@shared_cache.write(expiry_cache_key, reset) unless reset.nil?
|
87
|
+
end
|
88
|
+
return res
|
65
89
|
rescue => e
|
66
90
|
handle_error(e)
|
67
91
|
end
|
@@ -80,7 +104,6 @@ module RateLimit
|
|
80
104
|
raise RateLimit::WaitExceeded
|
81
105
|
end
|
82
106
|
|
83
|
-
|
84
107
|
def return(limit_result)
|
85
108
|
result = @conn.post '/api/v1/limitreturn',
|
86
109
|
{ enforcedGroup: limit_result.enforcedGroup,
|
@@ -95,8 +118,42 @@ module RateLimit
|
|
95
118
|
end
|
96
119
|
|
97
120
|
def feature_is_on_for?(feature, lookup_key, attributes: [])
|
98
|
-
|
121
|
+
@stats.increment("it.ratelim.featureflag.on", tags: ["feature:#{feature}"])
|
122
|
+
|
123
|
+
cache_key = "it.ratelim.ff.#{feature}.#{lookup_key}.#{attributes}"
|
124
|
+
@in_process_cache.fetch(cache_key, expires_in: 60) do
|
125
|
+
next uncached_feature_is_on_for?(feature, lookup_key, attributes) if @shared_cache.class == NoopCache
|
126
|
+
|
127
|
+
feature_obj = get_feature(feature)
|
128
|
+
if feature_obj.nil?
|
129
|
+
next false
|
130
|
+
end
|
131
|
+
|
132
|
+
attributes << lookup_key if lookup_key
|
133
|
+
if (attributes & feature_obj.whitelisted).size > 0
|
134
|
+
next true
|
135
|
+
end
|
136
|
+
|
137
|
+
if lookup_key
|
138
|
+
next get_user_pct(feature, lookup_key) < feature_obj.pct
|
139
|
+
end
|
140
|
+
|
141
|
+
next feature_obj.pct > 0.999
|
142
|
+
end
|
143
|
+
end
|
144
|
+
|
145
|
+
def base_url(local)
|
146
|
+
local ? 'http://localhost:8080' : 'http://www.ratelim.it'
|
147
|
+
end
|
148
|
+
|
149
|
+
private
|
150
|
+
|
151
|
+
def uncached_feature_is_on_for?(feature, lookup_key, attributes)
|
152
|
+
to_send = {}
|
153
|
+
to_send[:lookupKey] = lookup_key unless lookup_key.nil?
|
154
|
+
to_send[:attributes] = attributes if attributes.any?
|
99
155
|
result = @conn.get "/api/v1/featureflags/#{feature}/on", to_send
|
156
|
+
@stats.increment("it.ratelim.featureflag.on.req", tags: ["success:#{result.success?}"])
|
100
157
|
if result.success?
|
101
158
|
result.body == "true"
|
102
159
|
else
|
@@ -104,7 +161,28 @@ module RateLimit
|
|
104
161
|
end
|
105
162
|
end
|
106
163
|
|
107
|
-
|
164
|
+
def get_feature(feature)
|
165
|
+
get_all_features[feature]
|
166
|
+
end
|
167
|
+
|
168
|
+
def get_all_features
|
169
|
+
@shared_cache.fetch("it.ratelim.get_all_features", expires_in: 60) do
|
170
|
+
result = @conn.get "/api/v1/featureflags"
|
171
|
+
@stats.increment("it.ratelim.featureflag.getall.req", tags: ["success:#{result.success?}"])
|
172
|
+
if result.success?
|
173
|
+
res =JSON.parse(result.body, object_class: OpenStruct)
|
174
|
+
Hash[res.map { |r| [r.feature, r] }]
|
175
|
+
else
|
176
|
+
@logger.error("failed to fetch feature flags #{result.status}")
|
177
|
+
{}
|
178
|
+
end
|
179
|
+
end
|
180
|
+
end
|
181
|
+
|
182
|
+
def get_user_pct(feature, lookup_key)
|
183
|
+
int_value = Murmur3.murmur3_32("#{@account_id}#{feature}#{lookup_key}")
|
184
|
+
int_value / 4294967294.0
|
185
|
+
end
|
108
186
|
|
109
187
|
def upsert(limit_definition, method)
|
110
188
|
to_send = { limit: limit_definition.limit,
|
@@ -128,10 +206,10 @@ module RateLimit
|
|
128
206
|
case @on_error
|
129
207
|
when :log_and_pass
|
130
208
|
@logger.warn("returned #{result.status}")
|
131
|
-
OpenStruct.new(passed: true)
|
209
|
+
OpenStruct.new(passed: true, amount: 0)
|
132
210
|
when :log_and_hit
|
133
211
|
@logger.warn("returned #{result.status}")
|
134
|
-
OpenStruct.new(passed: false)
|
212
|
+
OpenStruct.new(passed: false, amount: 0)
|
135
213
|
when :throw
|
136
214
|
raise "#{result.status} calling RateLim.it"
|
137
215
|
end
|
@@ -141,10 +219,10 @@ module RateLimit
|
|
141
219
|
case @on_error
|
142
220
|
when :log_and_pass
|
143
221
|
@logger.warn(e)
|
144
|
-
OpenStruct.new(passed: true)
|
222
|
+
OpenStruct.new(passed: true, amount: 0)
|
145
223
|
when :log_and_hit
|
146
224
|
@logger.warn(e)
|
147
|
-
OpenStruct.new(passed: false)
|
225
|
+
OpenStruct.new(passed: false, amount: 0)
|
148
226
|
when :throw
|
149
227
|
raise e
|
150
228
|
end
|
@@ -163,6 +241,7 @@ module RateLimit
|
|
163
241
|
raise "#{result.status} calling feature flag RateLim.it"
|
164
242
|
end
|
165
243
|
end
|
244
|
+
|
166
245
|
end
|
167
246
|
|
168
247
|
end
|
@@ -171,4 +250,6 @@ require 'faraday'
|
|
171
250
|
require 'faraday_middleware'
|
172
251
|
require 'logger'
|
173
252
|
require 'ratelimit/noop_stats'
|
253
|
+
require 'ratelimit/noop_cache'
|
254
|
+
require 'ratelimit/murmur3'
|
174
255
|
require 'ratelimit/limit_definition'
|
data/ratelimit-ruby.gemspec
CHANGED
@@ -2,16 +2,16 @@
|
|
2
2
|
# DO NOT EDIT THIS FILE DIRECTLY
|
3
3
|
# Instead, edit Juwelier::Tasks in Rakefile, and run 'rake gemspec'
|
4
4
|
# -*- encoding: utf-8 -*-
|
5
|
-
# stub: ratelimit-ruby 0.1.
|
5
|
+
# stub: ratelimit-ruby 0.1.7 ruby lib
|
6
6
|
|
7
7
|
Gem::Specification.new do |s|
|
8
8
|
s.name = "ratelimit-ruby"
|
9
|
-
s.version = "0.1.
|
9
|
+
s.version = "0.1.7"
|
10
10
|
|
11
11
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
12
12
|
s.require_paths = ["lib"]
|
13
13
|
s.authors = ["Jeff Dwyer"]
|
14
|
-
s.date = "2017-01-
|
14
|
+
s.date = "2017-01-26"
|
15
15
|
s.description = "rate limit your ruby"
|
16
16
|
s.email = "jdwyah@gmail.com"
|
17
17
|
s.extra_rdoc_files = [
|
@@ -30,7 +30,10 @@ Gem::Specification.new do |s|
|
|
30
30
|
"VERSION",
|
31
31
|
"lib/ratelimit-ruby.rb",
|
32
32
|
"lib/ratelimit/limit_definition.rb",
|
33
|
+
"lib/ratelimit/murmur3.rb",
|
34
|
+
"lib/ratelimit/noop_cache.rb",
|
33
35
|
"lib/ratelimit/noop_stats.rb",
|
36
|
+
"lib/ratelimit/toy_cache.rb",
|
34
37
|
"ratelimit-ruby.gemspec"
|
35
38
|
]
|
36
39
|
s.homepage = "http://github.com/jdwyah/ratelimit-ruby"
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: ratelimit-ruby
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.7
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Jeff Dwyer
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-01-
|
11
|
+
date: 2017-01-26 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: faraday
|
@@ -127,7 +127,10 @@ files:
|
|
127
127
|
- VERSION
|
128
128
|
- lib/ratelimit-ruby.rb
|
129
129
|
- lib/ratelimit/limit_definition.rb
|
130
|
+
- lib/ratelimit/murmur3.rb
|
131
|
+
- lib/ratelimit/noop_cache.rb
|
130
132
|
- lib/ratelimit/noop_stats.rb
|
133
|
+
- lib/ratelimit/toy_cache.rb
|
131
134
|
- ratelimit-ruby.gemspec
|
132
135
|
homepage: http://github.com/jdwyah/ratelimit-ruby
|
133
136
|
licenses:
|