em-throttled_queue 1.0.3 → 1.1.0
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +7 -0
- data/README.markdown +6 -6
- data/Rakefile +3 -21
- data/em-throttled_queue.gemspec +19 -60
- data/lib/em/throttled_queue/version.rb +15 -2
- data/lib/em/throttled_queue.rb +7 -3
- data/spec/throttled_queue.rb +21 -1
- metadata +55 -74
- data/Gemfile.lock +0 -22
data/.gitignore
ADDED
data/README.markdown
CHANGED
@@ -1,3 +1,9 @@
|
|
1
|
+
# THIS LIBRARY HAS BEEN ABANDONED!
|
2
|
+
|
3
|
+
If somebody wants to take on the project and make something useful of it, go on ahead. I, personally, will not be doing *any* more work on this.
|
4
|
+
|
5
|
+
---
|
6
|
+
|
1
7
|
# EM-Throttled_Queue is a throttled queue (surprise, surprise!)
|
2
8
|
[ThrottledQueue](http://rdoc.info/github/Burgestrand/em-throttled_queue/master/EventMachine/ThrottledQueue) is just like an [EM::Queue](http://rdoc.info/github/eventmachine/eventmachine/master/EventMachine/Queue), but will pop items off itself at a pace specified by you!
|
3
9
|
|
@@ -27,12 +33,6 @@ Usage
|
|
27
33
|
|
28
34
|
Gem is on rubygems.org, so installation is a mere `gem install em-throttled_queue`.
|
29
35
|
|
30
|
-
What problem does EM-Throttled_Queue solve?
|
31
|
-
-------------------------------------------
|
32
|
-
At [radiofy](http://radiofy.se/) we consume a lot of external services. As we are rebuilding our internal structures we will be using EventMachine for the consumption of said services. Some of them have an applied rate-limit that will punish you if you don’t restrain how many queries you execute.
|
33
|
-
|
34
|
-
As a result, we need to call our blocks of code within the allowed rate-limit. For example, the [Spotify Metadata API](http://developer.spotify.com/en/metadata-api/overview/) allow a maximum of 10 requests per second, and if exceeded will force you to wait 10 seconds (essentially losing 90 reqs/10 seconds). EM::ThrottledQueue will allow us to easily limit how many calls we make per second to their API.
|
35
|
-
|
36
36
|
How do I contribute?
|
37
37
|
--------------------
|
38
38
|
Fork, add tests (important!), add your code and send a pull request. If you wish to report an issue, please use the GitHub issue tracker. I can also be contacted by mail (visible on [my GitHub user page](http://github.com/Burgestrand)).
|
data/Rakefile
CHANGED
@@ -1,24 +1,7 @@
|
|
1
|
-
require 'bundler
|
2
|
-
|
3
|
-
require 'jeweler'
|
4
|
-
require './lib/em/throttled_queue/version'
|
5
|
-
Jeweler::Tasks.new do |gem|
|
6
|
-
gem.name = "em-throttled_queue"
|
7
|
-
gem.homepage = "http://github.com/Burgestrand/em-throttled_queue"
|
8
|
-
|
9
|
-
gem.license = "X11 License"
|
10
|
-
|
11
|
-
gem.summary = "A rate-limited Queue for EventMachine"
|
12
|
-
gem.authors = ["Kim Burgestrand"]
|
13
|
-
gem.email = "kim@burgestrand.se"
|
14
|
-
|
15
|
-
gem.version = EventMachine::ThrottledQueue::VERSION
|
16
|
-
gem.required_ruby_version = '~> 1.9'
|
17
|
-
end
|
18
|
-
Jeweler::RubygemsDotOrgTasks.new
|
1
|
+
require 'bundler'
|
2
|
+
Bundler::GemHelper.install_tasks
|
19
3
|
|
20
4
|
require 'yard'
|
21
|
-
require 'yard/rake/yardoc_task'
|
22
5
|
YARD::Rake::YardocTask.new
|
23
6
|
|
24
7
|
require 'rake/testtask'
|
@@ -26,5 +9,4 @@ Rake::TestTask.new(:spec) do |t|
|
|
26
9
|
t.pattern = 'spec/*.rb'
|
27
10
|
end
|
28
11
|
|
29
|
-
task :
|
30
|
-
task :default => :spec
|
12
|
+
task :default => :spec
|
data/em-throttled_queue.gemspec
CHANGED
@@ -1,66 +1,25 @@
|
|
1
|
-
# Generated by jeweler
|
2
|
-
# DO NOT EDIT THIS FILE DIRECTLY
|
3
|
-
# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
|
4
1
|
# -*- encoding: utf-8 -*-
|
2
|
+
require './lib/em/throttled_queue/version'
|
5
3
|
|
6
|
-
Gem::Specification.new do |
|
7
|
-
|
8
|
-
|
4
|
+
Gem::Specification.new do |gem|
|
5
|
+
gem.name = 'em-throttled_queue'
|
6
|
+
gem.summary = 'A rate-limited Queue for EventMachine'
|
7
|
+
gem.homepage = 'http://github.com/Burgestrand/em-throttled_queue'
|
8
|
+
gem.authors = ["Kim Burgestrand"]
|
9
|
+
gem.email = ['kim@burgestrand.se']
|
10
|
+
gem.license = 'X11 License'
|
9
11
|
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
s.extra_rdoc_files = [
|
15
|
-
"LICENSE",
|
16
|
-
"README.markdown"
|
17
|
-
]
|
18
|
-
s.files = [
|
19
|
-
".yardopts",
|
20
|
-
"Gemfile",
|
21
|
-
"Gemfile.lock",
|
22
|
-
"LICENSE",
|
23
|
-
"README.markdown",
|
24
|
-
"Rakefile",
|
25
|
-
"em-throttled_queue.gemspec",
|
26
|
-
"lib/em-throttled_queue.rb",
|
27
|
-
"lib/em/throttled_queue.rb",
|
28
|
-
"lib/em/throttled_queue/version.rb",
|
29
|
-
"lib/eventmachine/throttled_queue.rb",
|
30
|
-
"spec/throttled_queue.rb"
|
31
|
-
]
|
32
|
-
s.homepage = %q{http://github.com/Burgestrand/em-throttled_queue}
|
33
|
-
s.licenses = ["X11 License"]
|
34
|
-
s.require_paths = ["lib"]
|
35
|
-
s.required_ruby_version = Gem::Requirement.new("~> 1.9")
|
36
|
-
s.rubygems_version = %q{1.5.0}
|
37
|
-
s.summary = %q{A rate-limited Queue for EventMachine}
|
38
|
-
s.test_files = [
|
39
|
-
"spec/throttled_queue.rb"
|
40
|
-
]
|
12
|
+
gem.files = `git ls-files`.split("\n")
|
13
|
+
gem.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
14
|
+
gem.executables = []
|
15
|
+
gem.require_paths = ["lib"]
|
41
16
|
|
42
|
-
|
43
|
-
|
17
|
+
gem.version = EventMachine::ThrottledQueue::VERSION
|
18
|
+
gem.platform = Gem::Platform::RUBY
|
19
|
+
gem.required_ruby_version = '~> 1.9'
|
44
20
|
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
s.add_development_dependency(%q<yard>, [">= 0"])
|
50
|
-
s.add_development_dependency(%q<rdiscount>, [">= 0"])
|
51
|
-
else
|
52
|
-
s.add_dependency(%q<eventmachine>, [">= 0"])
|
53
|
-
s.add_dependency(%q<jeweler>, ["~> 1.5"])
|
54
|
-
s.add_dependency(%q<rake>, [">= 0"])
|
55
|
-
s.add_dependency(%q<yard>, [">= 0"])
|
56
|
-
s.add_dependency(%q<rdiscount>, [">= 0"])
|
57
|
-
end
|
58
|
-
else
|
59
|
-
s.add_dependency(%q<eventmachine>, [">= 0"])
|
60
|
-
s.add_dependency(%q<jeweler>, ["~> 1.5"])
|
61
|
-
s.add_dependency(%q<rake>, [">= 0"])
|
62
|
-
s.add_dependency(%q<yard>, [">= 0"])
|
63
|
-
s.add_dependency(%q<rdiscount>, [">= 0"])
|
64
|
-
end
|
21
|
+
gem.add_dependency 'eventmachine'
|
22
|
+
gem.add_development_dependency 'yard'
|
23
|
+
gem.add_development_dependency 'rdiscount'
|
24
|
+
gem.add_development_dependency 'bundler'
|
65
25
|
end
|
66
|
-
|
@@ -1,7 +1,20 @@
|
|
1
|
+
unless defined?(EventMachine)
|
2
|
+
module EventMachine
|
3
|
+
class Queue
|
4
|
+
# this is an ugly hack, really, and if you find
|
5
|
+
# a better solution that:
|
6
|
+
# - don’t require me to require eventmachine
|
7
|
+
# - don’t require me to keep version in two places
|
8
|
+
# - i like (:d)
|
9
|
+
# then, I will buy you an ice cream
|
10
|
+
end
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
1
14
|
module EventMachine
|
2
15
|
class ThrottledQueue < Queue
|
3
16
|
# Gem version, following Semantic Versioning since v1.0.2
|
4
17
|
# @see http://semver.org/
|
5
|
-
VERSION = [1,
|
18
|
+
VERSION = [1, 1, 0].join('.')
|
6
19
|
end
|
7
|
-
end
|
20
|
+
end
|
data/lib/em/throttled_queue.rb
CHANGED
@@ -15,11 +15,15 @@ module EventMachine
|
|
15
15
|
#
|
16
16
|
# EM::ThrottledQueue.new(10)
|
17
17
|
# # => allows maximum 10 deqs every second
|
18
|
+
#
|
19
|
+
# EM::ThrottledQueue.new(5, 10)
|
20
|
+
# # => allows maximum 5 deqs every 10 seconds
|
18
21
|
#
|
19
|
-
# @param [Fixnum] maximum of dequeues every
|
20
|
-
|
22
|
+
# @param [Fixnum] maximum of dequeues every +refresh_interval+
|
23
|
+
# @param [Fixnum] how many seconds between each refresh of tokens. Default: 1 (second)
|
24
|
+
def initialize(limit, refresh_interval = 1)
|
21
25
|
@credits = @limit = limit.to_i
|
22
|
-
@ticker = EM::add_periodic_timer(
|
26
|
+
@ticker = EM::add_periodic_timer(refresh_interval) do
|
23
27
|
@credits = @limit
|
24
28
|
scheduled_dequeue
|
25
29
|
end
|
data/spec/throttled_queue.rb
CHANGED
@@ -54,7 +54,27 @@ describe EM::ThrottledQueue do
|
|
54
54
|
|
55
55
|
ticker.call(ticker)
|
56
56
|
end
|
57
|
-
|
57
|
+
|
58
|
+
it "handles slower refresh rates than every second" do
|
59
|
+
deqs = 0
|
60
|
+
|
61
|
+
queue = EM::ThrottledQueue.new(1, 2) # 1 requests per 2 seconds, meaning 2 seconds between requests
|
62
|
+
queue.push(*(1..100))
|
63
|
+
queue.size.must_equal 100
|
64
|
+
|
65
|
+
EM::add_timer(1.5) do
|
66
|
+
deqs.must_equal 1
|
67
|
+
EM::stop
|
68
|
+
end
|
69
|
+
|
70
|
+
ticker = proc do |me|
|
71
|
+
queue.pop { deqs += 1 }
|
72
|
+
EM::next_tick { me.call(me) }
|
73
|
+
end
|
74
|
+
|
75
|
+
ticker.call(ticker)
|
76
|
+
end
|
77
|
+
|
58
78
|
it "should not build up credits over time" do
|
59
79
|
deqs = 0
|
60
80
|
|
metadata
CHANGED
@@ -1,86 +1,70 @@
|
|
1
|
-
--- !ruby/object:Gem::Specification
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
2
|
name: em-throttled_queue
|
3
|
-
version: !ruby/object:Gem::Version
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 1.1.0
|
4
5
|
prerelease:
|
5
|
-
version: 1.0.3
|
6
6
|
platform: ruby
|
7
|
-
authors:
|
7
|
+
authors:
|
8
8
|
- Kim Burgestrand
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
dependencies:
|
16
|
-
- !ruby/object:Gem::Dependency
|
12
|
+
date: 2011-07-13 00:00:00.000000000Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
17
15
|
name: eventmachine
|
18
|
-
requirement: &
|
16
|
+
requirement: &2157029440 !ruby/object:Gem::Requirement
|
19
17
|
none: false
|
20
|
-
requirements:
|
21
|
-
- -
|
22
|
-
- !ruby/object:Gem::Version
|
23
|
-
version:
|
18
|
+
requirements:
|
19
|
+
- - ! '>='
|
20
|
+
- !ruby/object:Gem::Version
|
21
|
+
version: '0'
|
24
22
|
type: :runtime
|
25
23
|
prerelease: false
|
26
|
-
version_requirements: *
|
27
|
-
- !ruby/object:Gem::Dependency
|
28
|
-
name:
|
29
|
-
requirement: &
|
30
|
-
none: false
|
31
|
-
requirements:
|
32
|
-
- - ~>
|
33
|
-
- !ruby/object:Gem::Version
|
34
|
-
version: "1.5"
|
35
|
-
type: :development
|
36
|
-
prerelease: false
|
37
|
-
version_requirements: *id002
|
38
|
-
- !ruby/object:Gem::Dependency
|
39
|
-
name: rake
|
40
|
-
requirement: &id003 !ruby/object:Gem::Requirement
|
24
|
+
version_requirements: *2157029440
|
25
|
+
- !ruby/object:Gem::Dependency
|
26
|
+
name: yard
|
27
|
+
requirement: &2157028880 !ruby/object:Gem::Requirement
|
41
28
|
none: false
|
42
|
-
requirements:
|
43
|
-
- -
|
44
|
-
- !ruby/object:Gem::Version
|
45
|
-
version:
|
29
|
+
requirements:
|
30
|
+
- - ! '>='
|
31
|
+
- !ruby/object:Gem::Version
|
32
|
+
version: '0'
|
46
33
|
type: :development
|
47
34
|
prerelease: false
|
48
|
-
version_requirements: *
|
49
|
-
- !ruby/object:Gem::Dependency
|
50
|
-
name:
|
51
|
-
requirement: &
|
35
|
+
version_requirements: *2157028880
|
36
|
+
- !ruby/object:Gem::Dependency
|
37
|
+
name: rdiscount
|
38
|
+
requirement: &2157028280 !ruby/object:Gem::Requirement
|
52
39
|
none: false
|
53
|
-
requirements:
|
54
|
-
- -
|
55
|
-
- !ruby/object:Gem::Version
|
56
|
-
version:
|
40
|
+
requirements:
|
41
|
+
- - ! '>='
|
42
|
+
- !ruby/object:Gem::Version
|
43
|
+
version: '0'
|
57
44
|
type: :development
|
58
45
|
prerelease: false
|
59
|
-
version_requirements: *
|
60
|
-
- !ruby/object:Gem::Dependency
|
61
|
-
name:
|
62
|
-
requirement: &
|
46
|
+
version_requirements: *2157028280
|
47
|
+
- !ruby/object:Gem::Dependency
|
48
|
+
name: bundler
|
49
|
+
requirement: &2157027500 !ruby/object:Gem::Requirement
|
63
50
|
none: false
|
64
|
-
requirements:
|
65
|
-
- -
|
66
|
-
- !ruby/object:Gem::Version
|
67
|
-
version:
|
51
|
+
requirements:
|
52
|
+
- - ! '>='
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
68
55
|
type: :development
|
69
56
|
prerelease: false
|
70
|
-
version_requirements: *
|
57
|
+
version_requirements: *2157027500
|
71
58
|
description:
|
72
|
-
email:
|
59
|
+
email:
|
60
|
+
- kim@burgestrand.se
|
73
61
|
executables: []
|
74
|
-
|
75
62
|
extensions: []
|
76
|
-
|
77
|
-
|
78
|
-
-
|
79
|
-
- README.markdown
|
80
|
-
files:
|
63
|
+
extra_rdoc_files: []
|
64
|
+
files:
|
65
|
+
- .gitignore
|
81
66
|
- .yardopts
|
82
67
|
- Gemfile
|
83
|
-
- Gemfile.lock
|
84
68
|
- LICENSE
|
85
69
|
- README.markdown
|
86
70
|
- Rakefile
|
@@ -90,33 +74,30 @@ files:
|
|
90
74
|
- lib/em/throttled_queue/version.rb
|
91
75
|
- lib/eventmachine/throttled_queue.rb
|
92
76
|
- spec/throttled_queue.rb
|
93
|
-
has_rdoc: true
|
94
77
|
homepage: http://github.com/Burgestrand/em-throttled_queue
|
95
|
-
licenses:
|
78
|
+
licenses:
|
96
79
|
- X11 License
|
97
80
|
post_install_message:
|
98
81
|
rdoc_options: []
|
99
|
-
|
100
|
-
require_paths:
|
82
|
+
require_paths:
|
101
83
|
- lib
|
102
|
-
required_ruby_version: !ruby/object:Gem::Requirement
|
84
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
103
85
|
none: false
|
104
|
-
requirements:
|
86
|
+
requirements:
|
105
87
|
- - ~>
|
106
|
-
- !ruby/object:Gem::Version
|
107
|
-
version:
|
108
|
-
required_rubygems_version: !ruby/object:Gem::Requirement
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '1.9'
|
90
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
109
91
|
none: false
|
110
|
-
requirements:
|
111
|
-
- -
|
112
|
-
- !ruby/object:Gem::Version
|
113
|
-
version:
|
92
|
+
requirements:
|
93
|
+
- - ! '>='
|
94
|
+
- !ruby/object:Gem::Version
|
95
|
+
version: '0'
|
114
96
|
requirements: []
|
115
|
-
|
116
97
|
rubyforge_project:
|
117
|
-
rubygems_version: 1.5
|
98
|
+
rubygems_version: 1.8.5
|
118
99
|
signing_key:
|
119
100
|
specification_version: 3
|
120
101
|
summary: A rate-limited Queue for EventMachine
|
121
|
-
test_files:
|
102
|
+
test_files:
|
122
103
|
- spec/throttled_queue.rb
|
data/Gemfile.lock
DELETED
@@ -1,22 +0,0 @@
|
|
1
|
-
GEM
|
2
|
-
remote: http://rubygems.org/
|
3
|
-
specs:
|
4
|
-
eventmachine (0.12.10)
|
5
|
-
git (1.2.5)
|
6
|
-
jeweler (1.5.2)
|
7
|
-
bundler (~> 1.0.0)
|
8
|
-
git (>= 1.2.5)
|
9
|
-
rake
|
10
|
-
rake (0.8.7)
|
11
|
-
rdiscount (1.6.8)
|
12
|
-
yard (0.6.4)
|
13
|
-
|
14
|
-
PLATFORMS
|
15
|
-
ruby
|
16
|
-
|
17
|
-
DEPENDENCIES
|
18
|
-
eventmachine
|
19
|
-
jeweler (~> 1.5)
|
20
|
-
rake
|
21
|
-
rdiscount
|
22
|
-
yard
|