fiberpool 0.0.1

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,6 @@
1
+ *.gem
2
+ .bundle
3
+ .rvmrc
4
+ .rspec
5
+ Gemfile.lock
6
+ pkg/*
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source "http://rubygems.org"
2
+
3
+ # Specify your gem's dependencies in fiberpool.gemspec
4
+ gemspec
@@ -0,0 +1,2 @@
1
+ require 'bundler'
2
+ Bundler::GemHelper.install_tasks
File without changes
@@ -0,0 +1,24 @@
1
+ # -*- encoding: utf-8 -*-
2
+ $:.push File.expand_path("../lib", __FILE__)
3
+ require "fiberpool/version"
4
+
5
+ Gem::Specification.new do |s|
6
+ s.name = "fiberpool"
7
+ s.version = FiberPool::VERSION
8
+ s.platform = Gem::Platform::RUBY
9
+ s.authors = ["Jon Rowe"]
10
+ s.email = ["hello@jonrowe.co.uk"]
11
+ s.homepage = "http://github.com/jonrowe/fiberpool"
12
+ s.summary = %q{A Fiberpool implementation for running tasks cooperatively}
13
+ s.description = %q{A Fiberpool implementation for running tasks cooperatively, allows throttling to max concurrency, best used with event machine and non blocking operations.}
14
+
15
+ s.rubyforge_project = "fiberpool"
16
+
17
+ s.files = `git ls-files`.split("\n")
18
+ s.test_files = `git ls-files -- {spec,features}/*`.split("\n")
19
+ s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
20
+ s.require_paths = ["lib"]
21
+
22
+ s.add_development_dependency 'rspec'
23
+ s.add_development_dependency 'autotest-standalone'
24
+ end
@@ -0,0 +1 @@
1
+ require 'fiberpool/fiberpool'
@@ -0,0 +1,64 @@
1
+ require 'fiber'
2
+ class FiberPool
3
+ attr_accessor :fibers, :pool_size, :pool_fiber
4
+
5
+ def initialize(pool_size=10)
6
+ self.pool_size = pool_size
7
+ self.fibers = []
8
+ self.pool_fiber = Fiber.current
9
+ end
10
+
11
+ def self.start(pool_size=10, finished_callback=nil, &block)
12
+ Fiber.new do
13
+ pool = FiberPool.new(pool_size)
14
+ yield pool
15
+ pool.drain
16
+ finished_callback.call() if finished_callback
17
+ end.resume
18
+ end
19
+
20
+ def add(&block)
21
+ fiber = Fiber.new do
22
+ f = Fiber.current
23
+ completion_callback = proc do
24
+ pool_fiber.transfer(f)
25
+ end
26
+ yield completion_callback
27
+ end
28
+ add_to_pool(fiber)
29
+ end
30
+
31
+ def add_to_pool(fiber)
32
+ wait_for_free_pool_space if over_capacity?
33
+ fibers << fiber
34
+ remove_fiber_from_pool fiber.resume
35
+ end
36
+
37
+ def wait_for_free_pool_space
38
+ remove_fiber_from_pool(wait_for_next_complete_fiber)
39
+ end
40
+
41
+ def wait_for_next_complete_fiber
42
+ Fiber.yield
43
+ end
44
+
45
+ def over_capacity?
46
+ fibers_in_use >= pool_size
47
+ end
48
+
49
+ def fibers_in_use
50
+ fibers.size
51
+ end
52
+
53
+ def fibers_left_to_process?
54
+ fibers_in_use > 0
55
+ end
56
+
57
+ def remove_fiber_from_pool(fiber)
58
+ fibers.delete(fiber)
59
+ end
60
+
61
+ def drain
62
+ wait_for_free_pool_space while fibers_left_to_process?
63
+ end
64
+ end
@@ -0,0 +1,3 @@
1
+ module FiberPool
2
+ VERSION = "0.0.1"
3
+ end
@@ -0,0 +1,227 @@
1
+ require 'spec_helper'
2
+
3
+ describe FiberPool do
4
+ let(:pool) { FiberPool.new }
5
+
6
+ describe ".start" do
7
+ let(:fiber) { mock "fiber", resume: nil }
8
+
9
+ before do
10
+ Fiber.stub(:new).and_return(fiber)
11
+ end
12
+
13
+ subject { FiberPool.start { |*args| @args = *args } }
14
+
15
+ it "should create a new fiber, which will become the pool fiber" do
16
+ Fiber.should_receive(:new).and_yield.and_return(fiber)
17
+ subject
18
+ end
19
+ it "should kick off the fiber" do
20
+ fiber.should_receive(:resume)
21
+ subject
22
+ end
23
+
24
+ context "within that fiber" do
25
+ let(:fake_pool) { mock "fake_pool", drain: nil }
26
+
27
+ before do
28
+ Fiber.stub(:new).and_yield.and_return(fiber)
29
+ FiberPool.stub(:new).and_return(fake_pool)
30
+ end
31
+
32
+ it "should create a new fiberpool" do
33
+ FiberPool.should_receive(:new).and_return(fake_pool)
34
+ subject
35
+ end
36
+ it "should yield the pool" do
37
+ subject
38
+ @args.should eq [fake_pool]
39
+ end
40
+ it "should drain the pool when done" do
41
+ fake_pool.should_receive(:drain)
42
+ subject
43
+ end
44
+
45
+ context "where pool size is specified" do
46
+ subject { FiberPool.start(20) { |args| 'lal block' } }
47
+
48
+ it "should create a new fiberpool of the correct size" do
49
+ FiberPool.should_receive(:new).with(20).and_return(fake_pool)
50
+ subject
51
+ end
52
+ end
53
+ context "were a callback is specified" do
54
+ let(:callback) { mock "callback" }
55
+
56
+ subject { FiberPool.start(10, callback) { |args| 'lal block' } }
57
+
58
+ it "should call the callback" do
59
+ callback.should_receive(:call)
60
+ subject
61
+ end
62
+ end
63
+ end
64
+ end
65
+
66
+ describe "#initialize" do
67
+ context "defaults" do
68
+ subject { pool }
69
+
70
+ its(:pool_size) { should eq 10 }
71
+ its(:fibers) { should eq [] }
72
+ its(:pool_fiber) { should = Fiber.current }
73
+ end
74
+ context "pool size specified" do
75
+ let(:pool_size) { mock "pool_size" }
76
+
77
+ subject { FiberPool.new pool_size }
78
+ its(:pool_size) { should eq pool_size }
79
+ end
80
+ end
81
+
82
+ describe "#add" do
83
+ let(:fiber) { mock "fiber" }
84
+ let(:completion_callback) { @callback }
85
+
86
+ before do
87
+ Fiber.stub(:new).and_yield.and_return(fiber)
88
+ pool.stub(:add_to_pool)
89
+ end
90
+
91
+ subject { pool.add { |callback| @callback = callback } }
92
+
93
+ it "should create a fiber" do
94
+ Fiber.should_receive(:new).and_yield.and_return(fiber)
95
+ subject
96
+ end
97
+ it "should yield a completion callback" do
98
+ subject
99
+ completion_callback.should be_a Proc
100
+ end
101
+ specify "the compleition callback should transfer the fiber back to the pool" do
102
+ Fiber.stub(:current).and_return(fiber)
103
+ subject
104
+ pool.pool_fiber.should_receive(:transfer).with(fiber)
105
+ completion_callback.call
106
+ end
107
+ it "should add the fiber to the pool" do
108
+ pool.should_receive(:add_to_pool).with(fiber)
109
+ subject
110
+ end
111
+ end
112
+
113
+ describe "#add_to_pool" do
114
+ let(:completed_fiber) { mock "completed_fiber" }
115
+ let(:fiber) { mock "fiber", resume: completed_fiber }
116
+
117
+ subject { pool.add_to_pool fiber }
118
+
119
+ context "where the pool is over capacity" do
120
+ before { pool.stub(:over_capacity?).and_return(true) }
121
+ it "should wait for free pool space" do
122
+ pool.should_receive(:wait_for_free_pool_space)
123
+ subject
124
+ end
125
+ end
126
+ it "should store the paused fiber in the pool" do
127
+ subject
128
+ pool.fibers.should == [fiber]
129
+ #note, normally the fiber would be removed at the end of the method
130
+ #but mocks are allowing this to be inspected
131
+ end
132
+ it "should kick off the fiber" do
133
+ fiber.should_receive(:resume).and_return(completed_fiber)
134
+ subject
135
+ end
136
+ it "should remove the fiber when finished" do
137
+ pool.should_receive(:remove_fiber_from_pool).with(completed_fiber)
138
+ subject
139
+ end
140
+ end
141
+
142
+ describe "#wait_for_free_pool_space" do
143
+ let(:completed_fiber) { mock "completed_fiber" }
144
+
145
+ before { pool.stub(:wait_for_next_complete_fiber).and_return(completed_fiber) }
146
+
147
+ subject { pool.wait_for_free_pool_space }
148
+
149
+ it "should wait for the next complete fiber" do
150
+ pool.should_receive(:wait_for_next_complete_fiber).and_return(completed_fiber)
151
+ subject
152
+ end
153
+ it "should remove that fiber from the pool" do
154
+ pool.should_receive(:remove_fiber_from_pool).with(completed_fiber)
155
+ subject
156
+ end
157
+ end
158
+
159
+ describe "#wait_for_next_complete_fiber" do
160
+ it "should yield back to the original calling context" do
161
+ Fiber.should_receive(:yield)
162
+ pool.wait_for_next_complete_fiber
163
+ end
164
+ end
165
+
166
+ describe "#over_capacity?" do
167
+ before { pool.fibers = [mock,mock,mock] }
168
+ subject { pool.over_capacity? }
169
+
170
+ context "where pool size equals fibers in use" do
171
+ before { pool.pool_size = pool.fibers.size }
172
+ it { should be_true }
173
+ end
174
+ context "where pool size is greater than fibers in use" do
175
+ before { pool.pool_size = pool.fibers.size + 1 }
176
+ it { should be_false }
177
+ end
178
+ context "where pool size is less than fibers in use" do
179
+ before { pool.pool_size = pool.fibers.size - 1 }
180
+ it { should be_true }
181
+ end
182
+ end
183
+
184
+ describe "#fibers_in_use" do
185
+ it "should return the size of the fibers array" do
186
+ pool.fibers = [mock,mock,mock]
187
+ pool.fibers_in_use.should eq 3
188
+ end
189
+ end
190
+
191
+ describe "#fibers_left_to_process" do
192
+ subject { pool.fibers_left_to_process? }
193
+
194
+ context "where there are fibers in use" do
195
+ before { pool.stub(:fibers_in_use).and_return(3) }
196
+ it { should be_true }
197
+ end
198
+ context "where there are no fibers left to process" do
199
+ before { pool.stub(:fibers_in_use).and_return(0) }
200
+ it { should be_false }
201
+ end
202
+ end
203
+
204
+ describe "#remove_fiber_from_pool" do
205
+ let(:fiber) { mock "fiber" }
206
+
207
+ before { pool.fibers = [fiber] }
208
+
209
+ it "should remove the fiber from the pool" do
210
+ pool.remove_fiber_from_pool fiber
211
+ pool.fibers.should == []
212
+ end
213
+ it "should do nothing if it isnt in the pool" do
214
+ pool.remove_fiber_from_pool mock("not in the pool")
215
+ pool.fibers.should == [fiber]
216
+ end
217
+ end
218
+
219
+ describe "#drain" do
220
+ before { pool.stub(:fibers_left_to_process?).and_return(true,true,false) }
221
+
222
+ it "should wait for free pool space while there are fibers left to process" do
223
+ pool.should_receive(:wait_for_free_pool_space).exactly(:twice)
224
+ pool.drain
225
+ end
226
+ end
227
+ end
@@ -0,0 +1 @@
1
+ require 'fiberpool'
metadata ADDED
@@ -0,0 +1,89 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: fiberpool
3
+ version: !ruby/object:Gem::Version
4
+ prerelease:
5
+ version: 0.0.1
6
+ platform: ruby
7
+ authors:
8
+ - Jon Rowe
9
+ autorequire:
10
+ bindir: bin
11
+ cert_chain: []
12
+
13
+ date: 2011-04-19 00:00:00 +01:00
14
+ default_executable:
15
+ dependencies:
16
+ - !ruby/object:Gem::Dependency
17
+ name: rspec
18
+ prerelease: false
19
+ requirement: &id001 !ruby/object:Gem::Requirement
20
+ none: false
21
+ requirements:
22
+ - - ">="
23
+ - !ruby/object:Gem::Version
24
+ version: "0"
25
+ type: :development
26
+ version_requirements: *id001
27
+ - !ruby/object:Gem::Dependency
28
+ name: autotest-standalone
29
+ prerelease: false
30
+ requirement: &id002 !ruby/object:Gem::Requirement
31
+ none: false
32
+ requirements:
33
+ - - ">="
34
+ - !ruby/object:Gem::Version
35
+ version: "0"
36
+ type: :development
37
+ version_requirements: *id002
38
+ description: A Fiberpool implementation for running tasks cooperatively, allows throttling to max concurrency, best used with event machine and non blocking operations.
39
+ email:
40
+ - hello@jonrowe.co.uk
41
+ executables: []
42
+
43
+ extensions: []
44
+
45
+ extra_rdoc_files: []
46
+
47
+ files:
48
+ - .gitignore
49
+ - Gemfile
50
+ - Rakefile
51
+ - features/support/env.rb
52
+ - fiberpool.gemspec
53
+ - lib/fiberpool.rb
54
+ - lib/fiberpool/fiberpool.rb
55
+ - lib/fiberpool/version.rb
56
+ - spec/fiberpool_spec.rb
57
+ - spec/spec_helper.rb
58
+ has_rdoc: true
59
+ homepage: http://github.com/jonrowe/fiberpool
60
+ licenses: []
61
+
62
+ post_install_message:
63
+ rdoc_options: []
64
+
65
+ require_paths:
66
+ - lib
67
+ required_ruby_version: !ruby/object:Gem::Requirement
68
+ none: false
69
+ requirements:
70
+ - - ">="
71
+ - !ruby/object:Gem::Version
72
+ version: "0"
73
+ required_rubygems_version: !ruby/object:Gem::Requirement
74
+ none: false
75
+ requirements:
76
+ - - ">="
77
+ - !ruby/object:Gem::Version
78
+ version: "0"
79
+ requirements: []
80
+
81
+ rubyforge_project: fiberpool
82
+ rubygems_version: 1.5.3
83
+ signing_key:
84
+ specification_version: 3
85
+ summary: A Fiberpool implementation for running tasks cooperatively
86
+ test_files:
87
+ - features/support/env.rb
88
+ - spec/fiberpool_spec.rb
89
+ - spec/spec_helper.rb