multi_headed_greek_monster 0.0.1 → 0.0.2

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,38 @@
1
+ Yet another parallel processing utility.
2
+
3
+ # To run the test:
4
+ ```
5
+ ruby -Ilib test/multi_headed_greek_monster_test.rb
6
+ ```
7
+
8
+ # Goals:
9
+ * forks workers to run work, and then clean up after yourself
10
+ * no dependencies
11
+ * no threads
12
+ * everything in 1 file.
13
+
14
+ # Primary use case:
15
+ A migration that updates many rows.
16
+
17
+ ```ruby
18
+ #make 3 workers, use DRB to communicate over port 28371
19
+ monster = MultiHeadedGreekMonster.new(nil, 3, 28371) do |face, work|
20
+ #do you work on the given record
21
+ face.name = face.name + " improved"
22
+ face.save!
23
+ end
24
+ Face.find_in_batches do |batch_of_things|
25
+ batch_of_things.each do |thing|
26
+ #put 'thing' into the Q of work to be done
27
+ monster.feed(thing)
28
+ end
29
+ #wait for the Q to drain down to no more than 5 waiting items
30
+ monster.wait
31
+ end
32
+ #wait for the Q to drain to empty and all workers to finish. Then shut down the workers and DRB
33
+ monster.finish
34
+ ```
35
+
36
+ Copyright (c) 2008-2010 3M. All rights reserved. Released under the MIT license.
37
+
38
+ Authored by Jacob Burkhart.
@@ -1,5 +1,5 @@
1
1
  class MultiHeadedGreekMonster
2
-
2
+
3
3
  def initialize(progress = nil, worker_count = 3, on_port = 23121, &block)
4
4
  @action = block
5
5
  @on_port = on_port
@@ -8,11 +8,11 @@ class MultiHeadedGreekMonster
8
8
  start_service
9
9
  start_workers
10
10
  end
11
-
11
+
12
12
  def feed(thing)
13
13
  @service_manager.give(thing)
14
14
  end
15
-
15
+
16
16
  def wait(for_min_q_size = 5, &block)
17
17
  while(@service_manager.q_size > for_min_q_size)
18
18
  sleep(1)
@@ -21,26 +21,33 @@ class MultiHeadedGreekMonster
21
21
  end
22
22
  end
23
23
  end
24
-
24
+
25
25
  def finish
26
26
  @service_manager.done!
27
27
  while(!@service_manager.done?)
28
28
  sleep(1)
29
29
  end
30
+ results = @service_manager.results
30
31
  @worker_pids.each do |pid|
31
32
  Process.wait(pid)
32
33
  end
33
34
  Process.kill("KILL", @server_pid)
35
+ results
34
36
  end
35
-
37
+
36
38
  class ServiceManager
39
+ attr_accessor :results
40
+ def give(thing)
41
+ @things.unshift(thing)
42
+ end
37
43
  def initialize(progress)
38
44
  @progress = progress
39
45
  @things = []
40
46
  @done = false
47
+ @results = []
41
48
  end
42
- def give(thing)
43
- @things << thing
49
+ def result(thing)
50
+ @results << thing
44
51
  end
45
52
  def take
46
53
  @things && @things.pop
@@ -58,12 +65,12 @@ class MultiHeadedGreekMonster
58
65
  @progress.tick if @progress
59
66
  end
60
67
  end
61
-
68
+
62
69
  private
63
-
70
+
64
71
  def start_service
65
72
  require 'drb'
66
-
73
+
67
74
  @server_pid = fork do
68
75
  if defined?(ActiveRecord)
69
76
  ActiveRecord::Base.clear_all_connections!
@@ -76,7 +83,7 @@ class MultiHeadedGreekMonster
76
83
  @service_manager = DRbObject.new nil, "druby://localhost:#{@on_port}"
77
84
  sleep 0.2 # FIXME
78
85
  end
79
-
86
+
80
87
  def start_workers
81
88
  @worker_pids = []
82
89
  @worker_count.times do |i|
@@ -103,5 +110,5 @@ class MultiHeadedGreekMonster
103
110
  end
104
111
  end
105
112
  end
106
-
113
+
107
114
  end
@@ -19,19 +19,17 @@ end
19
19
  class MultiHeadedGreekMonsterTest < Test::Unit::TestCase
20
20
 
21
21
  def test_should_work
22
- 100.times{ Face.create(:size => rand.to_s) }
23
- # puts "accounts created"
22
+ 100.times{|x| Face.create(:name => "#{x}-#{rand}") }
24
23
  monster = MultiHeadedGreekMonster.new(nil, 3, 28371) do |face, work|
25
24
  puts "working on #{face.id} from #{Process.pid}"
26
- face.size = face.size + " improved"
25
+ face.name = face.name + " improved"
27
26
  face.save!
28
- puts "face renamed to " + face.size.inspect
27
+ work.result(face.name)
28
+ puts "face renamed to " + face.name.inspect
29
29
  end
30
30
  total = Face.all.size
31
- # progress = Progress.new(total, 20)
32
31
  Face.find_in_batches do |batch_of_things|
33
32
  batch_of_things.each do |thing|
34
- # progress.tick
35
33
  puts "feed #{thing.id}"
36
34
  monster.feed(thing)
37
35
  end
@@ -39,7 +37,8 @@ class MultiHeadedGreekMonsterTest < Test::Unit::TestCase
39
37
  monster.wait
40
38
  end
41
39
  puts "finishing..."
42
- monster.finish
40
+ results = monster.finish
41
+ assert_equal(results.size, total)
43
42
  end
44
43
 
45
44
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: multi_headed_greek_monster
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.1
4
+ version: 0.0.2
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2012-09-06 00:00:00.000000000 Z
12
+ date: 2013-04-29 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: cubbyhole
@@ -35,7 +35,7 @@ extra_rdoc_files: []
35
35
  files:
36
36
  - lib/multi_headed_greek_monster.rb
37
37
  - MIT-LICENSE
38
- - README.rdoc
38
+ - README.md
39
39
  - test/multi_headed_greek_monster_test.rb
40
40
  homepage: https://github.com/engineyard/multi_headed_greek_monster
41
41
  licenses: []
@@ -57,7 +57,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
57
57
  version: 1.3.6
58
58
  requirements: []
59
59
  rubyforge_project:
60
- rubygems_version: 1.8.24
60
+ rubygems_version: 1.8.25
61
61
  signing_key:
62
62
  specification_version: 3
63
63
  summary: parallelize stuff
@@ -1,9 +0,0 @@
1
- Yet another parallel processing utility. because I felt like it.
2
-
3
- To run the test:
4
- ruby -Ilib test/multi_headed_greek_monster_test.rb
5
-
6
-
7
- Copyright (c) 2008-2010 3M. All rights reserved. Released under the MIT license.
8
-
9
- Authored by Jacob Burkhart.