threadded_enumerator 0.0.0 → 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Gemfile +1 -0
- data/Gemfile.lock +2 -0
- data/VERSION +1 -1
- data/lib/threadded_enumerator.rb +86 -29
- data/spec/threadded_enumerator_spec.rb +88 -0
- data/threadded_enumerator.gemspec +4 -1
- metadata +23 -12
data/Gemfile
CHANGED
data/Gemfile.lock
CHANGED
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.0.
|
1
|
+
0.0.1
|
data/lib/threadded_enumerator.rb
CHANGED
@@ -1,21 +1,93 @@
|
|
1
1
|
require "tsafe"
|
2
|
+
require "timeout"
|
2
3
|
|
3
4
|
class Threadded_enumerator
|
5
|
+
@@threads = Tsafe::MonHash.new
|
6
|
+
|
7
|
+
#Kills waiting threads when 'Threadded_enumerator'-objects are garbage-collected. This makes ensures being executed, objects GC'ed and so on.
|
8
|
+
def self.finalizer(id)
|
9
|
+
begin
|
10
|
+
Timeout.timeout(3) do
|
11
|
+
thread = @@threads[id]
|
12
|
+
|
13
|
+
#The thread is not always started, if the loop is never called... The thread might not exist for this reason.
|
14
|
+
return nil if !thread
|
15
|
+
|
16
|
+
#Remove reference.
|
17
|
+
@@threads.delete(id)
|
18
|
+
|
19
|
+
#Thread is already dead - ignore.
|
20
|
+
return nil if !thread.alive?
|
21
|
+
|
22
|
+
#Kill thread to release references to objects within and make it execute any ensures within.
|
23
|
+
thread.kill
|
24
|
+
|
25
|
+
#Check that the thread is actually killed by joining - else the timeout would have no effect, since 'kill' doesnt block. If the thread is never killed, this will be properly be a memory leak scenario, which we report in stderr!
|
26
|
+
thread.join
|
27
|
+
|
28
|
+
#This will make all sleeps and thread-stops be ignored. Commented out until further... Maybe this is good?
|
29
|
+
#thread.run while thread.alive?
|
30
|
+
end
|
31
|
+
rescue Timeout::Error
|
32
|
+
$stderr.puts "Couldnt kill thread #{id} for 'Threadded_enumerator' - possible memory leak detected!"
|
33
|
+
rescue Exception => e
|
34
|
+
$stderr.puts "Error while killing 'Threadded_enumerator'-thread."
|
35
|
+
$stderr.puts e.inspect
|
36
|
+
$stderr.puts e.backtrace
|
37
|
+
raise e
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
#Starts a thread which fills the yielder. Its done by this method to allowed GC'ing of 'Threadded_enumerator'-objects.
|
42
|
+
def self.block_runner(args)
|
43
|
+
@@threads[args[:id]] = Thread.new do
|
44
|
+
args[:yielder].thread = Thread.current
|
45
|
+
|
46
|
+
begin
|
47
|
+
if args[:block]
|
48
|
+
args[:block].call(args[:yielder])
|
49
|
+
args[:yielder].done = true
|
50
|
+
elsif enum = args[:enum]
|
51
|
+
begin
|
52
|
+
loop do
|
53
|
+
args[:yielder] << enum.next
|
54
|
+
end
|
55
|
+
rescue StopIteration
|
56
|
+
#ignore.
|
57
|
+
end
|
58
|
+
else
|
59
|
+
raise "Dont know what to do?"
|
60
|
+
end
|
61
|
+
rescue => e
|
62
|
+
$stderr.puts e.inspect
|
63
|
+
$stderr.puts e.backtrace
|
64
|
+
ensure
|
65
|
+
args[:yielder].done = true
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
4
70
|
def initialize(args = {}, &block)
|
5
71
|
@args = {
|
6
|
-
:cache =>
|
7
|
-
:block => block
|
72
|
+
:cache => 0,
|
73
|
+
:block => block,
|
74
|
+
:id => self.__id__
|
8
75
|
}.merge(args)
|
9
76
|
|
10
77
|
@debug = @args[:debug]
|
11
78
|
@yielder = Threadded_enumerator::Yielder.new(@args)
|
79
|
+
|
80
|
+
#We use this to kill the block-thread, execute any ensures and release references to any objects within.
|
81
|
+
ObjectSpace.define_finalizer(self, Threadded_enumerator.method(:finalizer))
|
12
82
|
end
|
13
83
|
|
84
|
+
#Returns the next result.
|
14
85
|
def next
|
15
86
|
block_start if !@block_started
|
16
87
|
return @yielder.get_result
|
17
88
|
end
|
18
89
|
|
90
|
+
#Loops over each result and yields it or returns an enumerator if no block is given.
|
19
91
|
def each(&block)
|
20
92
|
enum = Enumerator.new do |yielder|
|
21
93
|
begin
|
@@ -42,57 +114,42 @@ class Threadded_enumerator
|
|
42
114
|
|
43
115
|
private
|
44
116
|
|
117
|
+
#Starts the thread that spawns the results.
|
45
118
|
def block_start
|
46
119
|
@block_started = true
|
47
120
|
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
begin
|
52
|
-
if @args[:block]
|
53
|
-
@args[:block].call(@yielder)
|
54
|
-
@yielder.done = true
|
55
|
-
elsif enum = @args[:enum]
|
56
|
-
begin
|
57
|
-
loop do
|
58
|
-
@yielder << enum.next
|
59
|
-
end
|
60
|
-
rescue StopIteration
|
61
|
-
#ignore.
|
62
|
-
end
|
63
|
-
else
|
64
|
-
raise "Dont know what to do?"
|
65
|
-
end
|
66
|
-
rescue => e
|
67
|
-
$stderr.puts e.inspect
|
68
|
-
$stderr.puts e.backtrace
|
69
|
-
ensure
|
70
|
-
@yielder.done = true
|
71
|
-
end
|
72
|
-
end
|
121
|
+
#It has to be done this way in order to allowed the GC'ing of the object. Else the thread would contain an alive reference to self.
|
122
|
+
Threadded_enumerator.block_runner(:id => self.__id__, :block => @args[:block], :enum => @args[:enum], :yielder => @yielder)
|
73
123
|
end
|
74
124
|
end
|
75
125
|
|
76
126
|
class Threadded_enumerator::Yielder
|
77
|
-
attr_accessor :done, :thread
|
127
|
+
attr_accessor :args, :done, :thread
|
78
128
|
|
79
129
|
def initialize(args)
|
80
130
|
@args = args
|
81
131
|
@done = false
|
82
132
|
@debug = @args[:debug]
|
83
133
|
@results = Tsafe::MonArray.new
|
134
|
+
@waiting_for_result = 0
|
84
135
|
end
|
85
136
|
|
137
|
+
#Adds a new result to the yielder.
|
86
138
|
def <<(res)
|
87
139
|
@results << res
|
140
|
+
@waiting_for_result -= 1
|
88
141
|
|
89
|
-
while @results.length >= @args[:cache]
|
142
|
+
while @results.length >= @args[:cache] and @waiting_for_result <= 0
|
90
143
|
print "Stopping thread - too many results (#{@results}).\n" if @debug
|
91
144
|
Thread.stop
|
92
145
|
end
|
93
146
|
end
|
94
147
|
|
148
|
+
#Returns the next result.
|
95
149
|
def get_result
|
150
|
+
#Increase count to control cache.
|
151
|
+
@waiting_for_result += 1
|
152
|
+
|
96
153
|
#Wait for results-thread to be spawned before continuing.
|
97
154
|
Thread.pass while !@thread
|
98
155
|
|
@@ -61,4 +61,92 @@ describe "ThreaddedEnumerator" do
|
|
61
61
|
print "i: #{i}\n" if debug
|
62
62
|
end
|
63
63
|
end
|
64
|
+
|
65
|
+
it "should not cache unless told to" do
|
66
|
+
$count = 0
|
67
|
+
debug = false
|
68
|
+
|
69
|
+
tenum = Threadded_enumerator.new do |yielder|
|
70
|
+
0.upto(100) do |i|
|
71
|
+
STDOUT.print "Te: #{i}\n" if debug
|
72
|
+
$count = i
|
73
|
+
yielder << i
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
0.upto(5) do
|
78
|
+
i = tenum.next
|
79
|
+
STDOUT.print "i: #{i}\n" if debug
|
80
|
+
end
|
81
|
+
|
82
|
+
raise "Expected count to be 5 but it wasnt: #{$count}" if $count != 5
|
83
|
+
end
|
84
|
+
|
85
|
+
it "should cache when told to" do
|
86
|
+
$count = 0
|
87
|
+
debug = false
|
88
|
+
|
89
|
+
tenum = Threadded_enumerator.new(:cache => 10) do |yielder|
|
90
|
+
0.upto(100) do |i|
|
91
|
+
STDOUT.print "Te: #{i}\n" if debug
|
92
|
+
$count = i
|
93
|
+
yielder << i
|
94
|
+
end
|
95
|
+
end
|
96
|
+
|
97
|
+
0.upto(5) do
|
98
|
+
i = tenum.next
|
99
|
+
STDOUT.print "i: #{i}\n" if debug
|
100
|
+
end
|
101
|
+
|
102
|
+
raise "Expected count to be 5 but it wasnt: #{$count}" if $count <= 5
|
103
|
+
end
|
104
|
+
|
105
|
+
it "should execute ensures and GC" do
|
106
|
+
require "wref"
|
107
|
+
$ensured = false #Used to check if ensures within 'Threadded_enumerator' are executed later.
|
108
|
+
debug = false
|
109
|
+
|
110
|
+
tenum = Threadded_enumerator.new(:cache => 1) do |yielder|
|
111
|
+
begin
|
112
|
+
someobj = "Kasper"
|
113
|
+
$someobj_wref = Wref.new(someobj) #Used to check GC for objects within 'Threadded_enumerator' later.
|
114
|
+
|
115
|
+
0.upto(100) do |i|
|
116
|
+
STDOUT.print "Ensure: #{i} (#{yielder.args[:id]})\n" if debug
|
117
|
+
yielder << i
|
118
|
+
end
|
119
|
+
ensure
|
120
|
+
STDOUT.print "Ensured!\n" if debug
|
121
|
+
$ensured = true #Used to check if ensures within 'Threadded_enumerator' are executed later.
|
122
|
+
end
|
123
|
+
end
|
124
|
+
|
125
|
+
$tenum_wref = Wref.new(tenum)
|
126
|
+
|
127
|
+
0.upto(5) do
|
128
|
+
tenum.next
|
129
|
+
end
|
130
|
+
|
131
|
+
raise "Expected ensure to be false but it wasnt: #{$ensured}" if $ensured != false
|
132
|
+
|
133
|
+
#Needs to create an object of same class in Ruby 1.9.2.
|
134
|
+
tenum = Threadded_enumerator.new(:cache => 1) do |yielder|
|
135
|
+
|
136
|
+
end
|
137
|
+
end
|
138
|
+
|
139
|
+
it "should execute ensures and GC (still)" do
|
140
|
+
someobj = "Johan" #Or else 'someobj' wont be GC'ed for some reason.
|
141
|
+
|
142
|
+
#Trigger GC-runs to collect everything (sleep and second GC makes it collect all!).
|
143
|
+
GC.start
|
144
|
+
sleep 0.001
|
145
|
+
GC.start
|
146
|
+
|
147
|
+
tenum = $tenum_wref.alive?
|
148
|
+
raise "Expected threadded enumerator to be GCed, but it wasnt." if tenum
|
149
|
+
raise "Expected ensured to be true but it wasnt: #{$ensured}" if !$ensured
|
150
|
+
raise "Expected 'someobj' to be GCed, but it wasnt." if $someobj_wref.alive?
|
151
|
+
end
|
64
152
|
end
|
@@ -5,7 +5,7 @@
|
|
5
5
|
|
6
6
|
Gem::Specification.new do |s|
|
7
7
|
s.name = %q{threadded_enumerator}
|
8
|
-
s.version = "0.0.
|
8
|
+
s.version = "0.0.1"
|
9
9
|
|
10
10
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
11
|
s.authors = ["Kasper Johansen"]
|
@@ -41,6 +41,7 @@ Gem::Specification.new do |s|
|
|
41
41
|
|
42
42
|
if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
|
43
43
|
s.add_runtime_dependency(%q<tsafe>, [">= 0"])
|
44
|
+
s.add_development_dependency(%q<wref>, [">= 0"])
|
44
45
|
s.add_development_dependency(%q<rspec>, ["~> 2.8.0"])
|
45
46
|
s.add_development_dependency(%q<rdoc>, ["~> 3.12"])
|
46
47
|
s.add_development_dependency(%q<bundler>, [">= 1.0.0"])
|
@@ -48,6 +49,7 @@ Gem::Specification.new do |s|
|
|
48
49
|
s.add_development_dependency(%q<rcov>, [">= 0"])
|
49
50
|
else
|
50
51
|
s.add_dependency(%q<tsafe>, [">= 0"])
|
52
|
+
s.add_dependency(%q<wref>, [">= 0"])
|
51
53
|
s.add_dependency(%q<rspec>, ["~> 2.8.0"])
|
52
54
|
s.add_dependency(%q<rdoc>, ["~> 3.12"])
|
53
55
|
s.add_dependency(%q<bundler>, [">= 1.0.0"])
|
@@ -56,6 +58,7 @@ Gem::Specification.new do |s|
|
|
56
58
|
end
|
57
59
|
else
|
58
60
|
s.add_dependency(%q<tsafe>, [">= 0"])
|
61
|
+
s.add_dependency(%q<wref>, [">= 0"])
|
59
62
|
s.add_dependency(%q<rspec>, ["~> 2.8.0"])
|
60
63
|
s.add_dependency(%q<rdoc>, ["~> 3.12"])
|
61
64
|
s.add_dependency(%q<bundler>, [">= 1.0.0"])
|
metadata
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
name: threadded_enumerator
|
3
3
|
version: !ruby/object:Gem::Version
|
4
4
|
prerelease:
|
5
|
-
version: 0.0.
|
5
|
+
version: 0.0.1
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
8
8
|
- Kasper Johansen
|
@@ -25,8 +25,19 @@ dependencies:
|
|
25
25
|
prerelease: false
|
26
26
|
version_requirements: *id001
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
|
-
name:
|
28
|
+
name: wref
|
29
29
|
requirement: &id002 !ruby/object:Gem::Requirement
|
30
|
+
none: false
|
31
|
+
requirements:
|
32
|
+
- - ">="
|
33
|
+
- !ruby/object:Gem::Version
|
34
|
+
version: "0"
|
35
|
+
type: :development
|
36
|
+
prerelease: false
|
37
|
+
version_requirements: *id002
|
38
|
+
- !ruby/object:Gem::Dependency
|
39
|
+
name: rspec
|
40
|
+
requirement: &id003 !ruby/object:Gem::Requirement
|
30
41
|
none: false
|
31
42
|
requirements:
|
32
43
|
- - ~>
|
@@ -34,10 +45,10 @@ dependencies:
|
|
34
45
|
version: 2.8.0
|
35
46
|
type: :development
|
36
47
|
prerelease: false
|
37
|
-
version_requirements: *
|
48
|
+
version_requirements: *id003
|
38
49
|
- !ruby/object:Gem::Dependency
|
39
50
|
name: rdoc
|
40
|
-
requirement: &
|
51
|
+
requirement: &id004 !ruby/object:Gem::Requirement
|
41
52
|
none: false
|
42
53
|
requirements:
|
43
54
|
- - ~>
|
@@ -45,10 +56,10 @@ dependencies:
|
|
45
56
|
version: "3.12"
|
46
57
|
type: :development
|
47
58
|
prerelease: false
|
48
|
-
version_requirements: *
|
59
|
+
version_requirements: *id004
|
49
60
|
- !ruby/object:Gem::Dependency
|
50
61
|
name: bundler
|
51
|
-
requirement: &
|
62
|
+
requirement: &id005 !ruby/object:Gem::Requirement
|
52
63
|
none: false
|
53
64
|
requirements:
|
54
65
|
- - ">="
|
@@ -56,10 +67,10 @@ dependencies:
|
|
56
67
|
version: 1.0.0
|
57
68
|
type: :development
|
58
69
|
prerelease: false
|
59
|
-
version_requirements: *
|
70
|
+
version_requirements: *id005
|
60
71
|
- !ruby/object:Gem::Dependency
|
61
72
|
name: jeweler
|
62
|
-
requirement: &
|
73
|
+
requirement: &id006 !ruby/object:Gem::Requirement
|
63
74
|
none: false
|
64
75
|
requirements:
|
65
76
|
- - ~>
|
@@ -67,10 +78,10 @@ dependencies:
|
|
67
78
|
version: 1.8.4
|
68
79
|
type: :development
|
69
80
|
prerelease: false
|
70
|
-
version_requirements: *
|
81
|
+
version_requirements: *id006
|
71
82
|
- !ruby/object:Gem::Dependency
|
72
83
|
name: rcov
|
73
|
-
requirement: &
|
84
|
+
requirement: &id007 !ruby/object:Gem::Requirement
|
74
85
|
none: false
|
75
86
|
requirements:
|
76
87
|
- - ">="
|
@@ -78,7 +89,7 @@ dependencies:
|
|
78
89
|
version: "0"
|
79
90
|
type: :development
|
80
91
|
prerelease: false
|
81
|
-
version_requirements: *
|
92
|
+
version_requirements: *id007
|
82
93
|
description: Enumerator using thread to avoid fiber-bugs.
|
83
94
|
email: k@spernj.org
|
84
95
|
executables: []
|
@@ -115,7 +126,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
115
126
|
requirements:
|
116
127
|
- - ">="
|
117
128
|
- !ruby/object:Gem::Version
|
118
|
-
hash:
|
129
|
+
hash: 2306409188715448666
|
119
130
|
segments:
|
120
131
|
- 0
|
121
132
|
version: "0"
|