threadded_enumerator 0.0.0 → 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/Gemfile CHANGED
@@ -8,6 +8,7 @@ gem "tsafe"
8
8
  # Add dependencies to develop your gem here.
9
9
  # Include everything needed to run rake, tests, features, etc.
10
10
  group :development do
11
+ gem "wref"
11
12
  gem "rspec", "~> 2.8.0"
12
13
  gem "rdoc", "~> 3.12"
13
14
  gem "bundler", ">= 1.0.0"
data/Gemfile.lock CHANGED
@@ -22,6 +22,7 @@ GEM
22
22
  diff-lcs (~> 1.1.2)
23
23
  rspec-mocks (2.8.0)
24
24
  tsafe (0.0.1)
25
+ wref (0.0.4)
25
26
 
26
27
  PLATFORMS
27
28
  ruby
@@ -33,3 +34,4 @@ DEPENDENCIES
33
34
  rdoc (~> 3.12)
34
35
  rspec (~> 2.8.0)
35
36
  tsafe
37
+ wref
data/VERSION CHANGED
@@ -1 +1 @@
1
- 0.0.0
1
+ 0.0.1
@@ -1,21 +1,93 @@
1
1
  require "tsafe"
2
+ require "timeout"
2
3
 
3
4
  class Threadded_enumerator
5
+ @@threads = Tsafe::MonHash.new
6
+
7
+ #Kills waiting threads when 'Threadded_enumerator'-objects are garbage-collected. This makes ensures being executed, objects GC'ed and so on.
8
+ def self.finalizer(id)
9
+ begin
10
+ Timeout.timeout(3) do
11
+ thread = @@threads[id]
12
+
13
+ #The thread is not always started, if the loop is never called... The thread might not exist for this reason.
14
+ return nil if !thread
15
+
16
+ #Remove reference.
17
+ @@threads.delete(id)
18
+
19
+ #Thread is already dead - ignore.
20
+ return nil if !thread.alive?
21
+
22
+ #Kill thread to release references to objects within and make it execute any ensures within.
23
+ thread.kill
24
+
25
+ #Check that the thread is actually killed by joining - else the timeout would have no effect, since 'kill' doesnt block. If the thread is never killed, this will be properly be a memory leak scenario, which we report in stderr!
26
+ thread.join
27
+
28
+ #This will make all sleeps and thread-stops be ignored. Commented out until further... Maybe this is good?
29
+ #thread.run while thread.alive?
30
+ end
31
+ rescue Timeout::Error
32
+ $stderr.puts "Couldnt kill thread #{id} for 'Threadded_enumerator' - possible memory leak detected!"
33
+ rescue Exception => e
34
+ $stderr.puts "Error while killing 'Threadded_enumerator'-thread."
35
+ $stderr.puts e.inspect
36
+ $stderr.puts e.backtrace
37
+ raise e
38
+ end
39
+ end
40
+
41
+ #Starts a thread which fills the yielder. Its done by this method to allowed GC'ing of 'Threadded_enumerator'-objects.
42
+ def self.block_runner(args)
43
+ @@threads[args[:id]] = Thread.new do
44
+ args[:yielder].thread = Thread.current
45
+
46
+ begin
47
+ if args[:block]
48
+ args[:block].call(args[:yielder])
49
+ args[:yielder].done = true
50
+ elsif enum = args[:enum]
51
+ begin
52
+ loop do
53
+ args[:yielder] << enum.next
54
+ end
55
+ rescue StopIteration
56
+ #ignore.
57
+ end
58
+ else
59
+ raise "Dont know what to do?"
60
+ end
61
+ rescue => e
62
+ $stderr.puts e.inspect
63
+ $stderr.puts e.backtrace
64
+ ensure
65
+ args[:yielder].done = true
66
+ end
67
+ end
68
+ end
69
+
4
70
  def initialize(args = {}, &block)
5
71
  @args = {
6
- :cache => 1,
7
- :block => block
72
+ :cache => 0,
73
+ :block => block,
74
+ :id => self.__id__
8
75
  }.merge(args)
9
76
 
10
77
  @debug = @args[:debug]
11
78
  @yielder = Threadded_enumerator::Yielder.new(@args)
79
+
80
+ #We use this to kill the block-thread, execute any ensures and release references to any objects within.
81
+ ObjectSpace.define_finalizer(self, Threadded_enumerator.method(:finalizer))
12
82
  end
13
83
 
84
+ #Returns the next result.
14
85
  def next
15
86
  block_start if !@block_started
16
87
  return @yielder.get_result
17
88
  end
18
89
 
90
+ #Loops over each result and yields it or returns an enumerator if no block is given.
19
91
  def each(&block)
20
92
  enum = Enumerator.new do |yielder|
21
93
  begin
@@ -42,57 +114,42 @@ class Threadded_enumerator
42
114
 
43
115
  private
44
116
 
117
+ #Starts the thread that spawns the results.
45
118
  def block_start
46
119
  @block_started = true
47
120
 
48
- Thread.new do
49
- @yielder.thread = Thread.current
50
-
51
- begin
52
- if @args[:block]
53
- @args[:block].call(@yielder)
54
- @yielder.done = true
55
- elsif enum = @args[:enum]
56
- begin
57
- loop do
58
- @yielder << enum.next
59
- end
60
- rescue StopIteration
61
- #ignore.
62
- end
63
- else
64
- raise "Dont know what to do?"
65
- end
66
- rescue => e
67
- $stderr.puts e.inspect
68
- $stderr.puts e.backtrace
69
- ensure
70
- @yielder.done = true
71
- end
72
- end
121
+ #It has to be done this way in order to allowed the GC'ing of the object. Else the thread would contain an alive reference to self.
122
+ Threadded_enumerator.block_runner(:id => self.__id__, :block => @args[:block], :enum => @args[:enum], :yielder => @yielder)
73
123
  end
74
124
  end
75
125
 
76
126
  class Threadded_enumerator::Yielder
77
- attr_accessor :done, :thread
127
+ attr_accessor :args, :done, :thread
78
128
 
79
129
  def initialize(args)
80
130
  @args = args
81
131
  @done = false
82
132
  @debug = @args[:debug]
83
133
  @results = Tsafe::MonArray.new
134
+ @waiting_for_result = 0
84
135
  end
85
136
 
137
+ #Adds a new result to the yielder.
86
138
  def <<(res)
87
139
  @results << res
140
+ @waiting_for_result -= 1
88
141
 
89
- while @results.length >= @args[:cache]
142
+ while @results.length >= @args[:cache] and @waiting_for_result <= 0
90
143
  print "Stopping thread - too many results (#{@results}).\n" if @debug
91
144
  Thread.stop
92
145
  end
93
146
  end
94
147
 
148
+ #Returns the next result.
95
149
  def get_result
150
+ #Increase count to control cache.
151
+ @waiting_for_result += 1
152
+
96
153
  #Wait for results-thread to be spawned before continuing.
97
154
  Thread.pass while !@thread
98
155
 
@@ -61,4 +61,92 @@ describe "ThreaddedEnumerator" do
61
61
  print "i: #{i}\n" if debug
62
62
  end
63
63
  end
64
+
65
+ it "should not cache unless told to" do
66
+ $count = 0
67
+ debug = false
68
+
69
+ tenum = Threadded_enumerator.new do |yielder|
70
+ 0.upto(100) do |i|
71
+ STDOUT.print "Te: #{i}\n" if debug
72
+ $count = i
73
+ yielder << i
74
+ end
75
+ end
76
+
77
+ 0.upto(5) do
78
+ i = tenum.next
79
+ STDOUT.print "i: #{i}\n" if debug
80
+ end
81
+
82
+ raise "Expected count to be 5 but it wasnt: #{$count}" if $count != 5
83
+ end
84
+
85
+ it "should cache when told to" do
86
+ $count = 0
87
+ debug = false
88
+
89
+ tenum = Threadded_enumerator.new(:cache => 10) do |yielder|
90
+ 0.upto(100) do |i|
91
+ STDOUT.print "Te: #{i}\n" if debug
92
+ $count = i
93
+ yielder << i
94
+ end
95
+ end
96
+
97
+ 0.upto(5) do
98
+ i = tenum.next
99
+ STDOUT.print "i: #{i}\n" if debug
100
+ end
101
+
102
+ raise "Expected count to be 5 but it wasnt: #{$count}" if $count <= 5
103
+ end
104
+
105
+ it "should execute ensures and GC" do
106
+ require "wref"
107
+ $ensured = false #Used to check if ensures within 'Threadded_enumerator' are executed later.
108
+ debug = false
109
+
110
+ tenum = Threadded_enumerator.new(:cache => 1) do |yielder|
111
+ begin
112
+ someobj = "Kasper"
113
+ $someobj_wref = Wref.new(someobj) #Used to check GC for objects within 'Threadded_enumerator' later.
114
+
115
+ 0.upto(100) do |i|
116
+ STDOUT.print "Ensure: #{i} (#{yielder.args[:id]})\n" if debug
117
+ yielder << i
118
+ end
119
+ ensure
120
+ STDOUT.print "Ensured!\n" if debug
121
+ $ensured = true #Used to check if ensures within 'Threadded_enumerator' are executed later.
122
+ end
123
+ end
124
+
125
+ $tenum_wref = Wref.new(tenum)
126
+
127
+ 0.upto(5) do
128
+ tenum.next
129
+ end
130
+
131
+ raise "Expected ensure to be false but it wasnt: #{$ensured}" if $ensured != false
132
+
133
+ #Needs to create an object of same class in Ruby 1.9.2.
134
+ tenum = Threadded_enumerator.new(:cache => 1) do |yielder|
135
+
136
+ end
137
+ end
138
+
139
+ it "should execute ensures and GC (still)" do
140
+ someobj = "Johan" #Or else 'someobj' wont be GC'ed for some reason.
141
+
142
+ #Trigger GC-runs to collect everything (sleep and second GC makes it collect all!).
143
+ GC.start
144
+ sleep 0.001
145
+ GC.start
146
+
147
+ tenum = $tenum_wref.alive?
148
+ raise "Expected threadded enumerator to be GCed, but it wasnt." if tenum
149
+ raise "Expected ensured to be true but it wasnt: #{$ensured}" if !$ensured
150
+ raise "Expected 'someobj' to be GCed, but it wasnt." if $someobj_wref.alive?
151
+ end
64
152
  end
@@ -5,7 +5,7 @@
5
5
 
6
6
  Gem::Specification.new do |s|
7
7
  s.name = %q{threadded_enumerator}
8
- s.version = "0.0.0"
8
+ s.version = "0.0.1"
9
9
 
10
10
  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
11
11
  s.authors = ["Kasper Johansen"]
@@ -41,6 +41,7 @@ Gem::Specification.new do |s|
41
41
 
42
42
  if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
43
43
  s.add_runtime_dependency(%q<tsafe>, [">= 0"])
44
+ s.add_development_dependency(%q<wref>, [">= 0"])
44
45
  s.add_development_dependency(%q<rspec>, ["~> 2.8.0"])
45
46
  s.add_development_dependency(%q<rdoc>, ["~> 3.12"])
46
47
  s.add_development_dependency(%q<bundler>, [">= 1.0.0"])
@@ -48,6 +49,7 @@ Gem::Specification.new do |s|
48
49
  s.add_development_dependency(%q<rcov>, [">= 0"])
49
50
  else
50
51
  s.add_dependency(%q<tsafe>, [">= 0"])
52
+ s.add_dependency(%q<wref>, [">= 0"])
51
53
  s.add_dependency(%q<rspec>, ["~> 2.8.0"])
52
54
  s.add_dependency(%q<rdoc>, ["~> 3.12"])
53
55
  s.add_dependency(%q<bundler>, [">= 1.0.0"])
@@ -56,6 +58,7 @@ Gem::Specification.new do |s|
56
58
  end
57
59
  else
58
60
  s.add_dependency(%q<tsafe>, [">= 0"])
61
+ s.add_dependency(%q<wref>, [">= 0"])
59
62
  s.add_dependency(%q<rspec>, ["~> 2.8.0"])
60
63
  s.add_dependency(%q<rdoc>, ["~> 3.12"])
61
64
  s.add_dependency(%q<bundler>, [">= 1.0.0"])
metadata CHANGED
@@ -2,7 +2,7 @@
2
2
  name: threadded_enumerator
3
3
  version: !ruby/object:Gem::Version
4
4
  prerelease:
5
- version: 0.0.0
5
+ version: 0.0.1
6
6
  platform: ruby
7
7
  authors:
8
8
  - Kasper Johansen
@@ -25,8 +25,19 @@ dependencies:
25
25
  prerelease: false
26
26
  version_requirements: *id001
27
27
  - !ruby/object:Gem::Dependency
28
- name: rspec
28
+ name: wref
29
29
  requirement: &id002 !ruby/object:Gem::Requirement
30
+ none: false
31
+ requirements:
32
+ - - ">="
33
+ - !ruby/object:Gem::Version
34
+ version: "0"
35
+ type: :development
36
+ prerelease: false
37
+ version_requirements: *id002
38
+ - !ruby/object:Gem::Dependency
39
+ name: rspec
40
+ requirement: &id003 !ruby/object:Gem::Requirement
30
41
  none: false
31
42
  requirements:
32
43
  - - ~>
@@ -34,10 +45,10 @@ dependencies:
34
45
  version: 2.8.0
35
46
  type: :development
36
47
  prerelease: false
37
- version_requirements: *id002
48
+ version_requirements: *id003
38
49
  - !ruby/object:Gem::Dependency
39
50
  name: rdoc
40
- requirement: &id003 !ruby/object:Gem::Requirement
51
+ requirement: &id004 !ruby/object:Gem::Requirement
41
52
  none: false
42
53
  requirements:
43
54
  - - ~>
@@ -45,10 +56,10 @@ dependencies:
45
56
  version: "3.12"
46
57
  type: :development
47
58
  prerelease: false
48
- version_requirements: *id003
59
+ version_requirements: *id004
49
60
  - !ruby/object:Gem::Dependency
50
61
  name: bundler
51
- requirement: &id004 !ruby/object:Gem::Requirement
62
+ requirement: &id005 !ruby/object:Gem::Requirement
52
63
  none: false
53
64
  requirements:
54
65
  - - ">="
@@ -56,10 +67,10 @@ dependencies:
56
67
  version: 1.0.0
57
68
  type: :development
58
69
  prerelease: false
59
- version_requirements: *id004
70
+ version_requirements: *id005
60
71
  - !ruby/object:Gem::Dependency
61
72
  name: jeweler
62
- requirement: &id005 !ruby/object:Gem::Requirement
73
+ requirement: &id006 !ruby/object:Gem::Requirement
63
74
  none: false
64
75
  requirements:
65
76
  - - ~>
@@ -67,10 +78,10 @@ dependencies:
67
78
  version: 1.8.4
68
79
  type: :development
69
80
  prerelease: false
70
- version_requirements: *id005
81
+ version_requirements: *id006
71
82
  - !ruby/object:Gem::Dependency
72
83
  name: rcov
73
- requirement: &id006 !ruby/object:Gem::Requirement
84
+ requirement: &id007 !ruby/object:Gem::Requirement
74
85
  none: false
75
86
  requirements:
76
87
  - - ">="
@@ -78,7 +89,7 @@ dependencies:
78
89
  version: "0"
79
90
  type: :development
80
91
  prerelease: false
81
- version_requirements: *id006
92
+ version_requirements: *id007
82
93
  description: Enumerator using thread to avoid fiber-bugs.
83
94
  email: k@spernj.org
84
95
  executables: []
@@ -115,7 +126,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
115
126
  requirements:
116
127
  - - ">="
117
128
  - !ruby/object:Gem::Version
118
- hash: -2474286237917417948
129
+ hash: 2306409188715448666
119
130
  segments:
120
131
  - 0
121
132
  version: "0"