starfish 1.0.0 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README +33 -2
- data/doc/classes/MapReduce.src/M000001.html +1 -1
- data/doc/classes/MapReduce.src/M000002.html +1 -1
- data/doc/classes/MapReduce.src/M000003.html +1 -1
- data/doc/classes/MapReduce.src/M000004.html +1 -1
- data/doc/classes/MapReduce.src/M000005.html +1 -1
- data/doc/classes/MapReduce.src/M000006.html +1 -1
- data/doc/classes/MapReduce.src/M000007.html +1 -1
- data/doc/classes/MapReduce.src/M000008.html +1 -1
- data/doc/classes/MapReduce.src/M000009.html +1 -1
- data/doc/classes/MapReduce.src/M000010.html +1 -1
- data/doc/classes/MapReduce.src/M000011.html +1 -1
- data/doc/classes/MapReduce.src/M000012.html +1 -1
- data/doc/classes/MapReduce.src/M000013.html +1 -1
- data/doc/classes/MapReduce.src/M000014.html +1 -1
- data/doc/classes/MapReduce/ActiveRecord/Base.src/M000015.html +6 -14
- data/doc/classes/MapReduce/File.html +70 -0
- data/doc/classes/MapReduce/File.src/M000019.html +42 -0
- data/doc/classes/MapReduce/File/Client.html +177 -0
- data/doc/classes/MapReduce/File/Client.src/M000020.html +18 -0
- data/doc/classes/MapReduce/File/Client.src/M000021.html +20 -0
- data/doc/classes/MapReduce/File/Client.src/M000022.html +18 -0
- data/doc/classes/Starfish.html +61 -62
- data/doc/classes/Starfish.src/M000023.html +5 -9
- data/doc/classes/Starfish.src/M000024.html +4 -22
- data/doc/classes/Starfish.src/M000025.html +29 -19
- data/doc/classes/Starfish.src/M000026.html +64 -4
- data/doc/classes/Starfish.src/M000027.html +9 -8
- data/doc/classes/Starfish.src/M000028.html +23 -5
- data/doc/classes/Starfish.src/M000029.html +22 -5
- data/doc/classes/Starfish.src/M000030.html +5 -5
- data/doc/classes/Starfish.src/M000031.html +22 -0
- data/doc/classes/Starfish.src/{M000020.html → M000032.html} +5 -5
- data/doc/classes/Starfish.src/M000033.html +18 -0
- data/doc/classes/Starfish.src/M000034.html +18 -0
- data/doc/classes/Starfish/RingFinger.html +6 -6
- data/doc/classes/Starfish/RingFinger.src/{M000031.html → M000035.html} +1 -1
- data/doc/created.rid +1 -1
- data/doc/files/README.html +52 -6
- data/doc/files/lib/map_reduce/active_record_rb.html +1 -2
- data/doc/files/lib/map_reduce/file_rb.html +1 -1
- data/doc/files/lib/map_reduce_rb.html +1 -1
- data/doc/files/lib/starfish_rb.html +1 -1
- data/doc/fr_class_index.html +1 -0
- data/doc/fr_method_index.html +17 -13
- data/examples/map_reduce/file.rb +2 -3
- data/lib/map_reduce.rb +2 -1
- data/lib/map_reduce/active_record.rb +25 -19
- data/lib/map_reduce/file.rb +93 -0
- data/lib/starfish.rb +9 -2
- metadata +15 -7
- data/doc/classes/Starfish.src/M000019.html +0 -19
- data/doc/classes/Starfish.src/M000021.html +0 -45
- data/doc/classes/Starfish.src/M000022.html +0 -71
data/examples/map_reduce/file.rb
CHANGED
data/lib/map_reduce.rb
CHANGED
|
@@ -7,7 +7,8 @@ require 'map_reduce/array'
|
|
|
7
7
|
class MapReduceError < StandardError; end
|
|
8
8
|
class MapReduce
|
|
9
9
|
@@types = {
|
|
10
|
-
::ActiveRecord::Base => [::Array, String, NilClass]
|
|
10
|
+
::ActiveRecord::Base => [::Array, String, NilClass],
|
|
11
|
+
::File => String
|
|
11
12
|
}
|
|
12
13
|
|
|
13
14
|
attr_accessor :type, :input, :limit
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
require 'rubygems'
|
|
2
|
-
|
|
2
|
+
require_gem 'activerecord'
|
|
3
3
|
|
|
4
4
|
class MapReduce
|
|
5
5
|
module ActiveRecord
|
|
@@ -43,38 +43,44 @@ class MapReduce
|
|
|
43
43
|
end
|
|
44
44
|
|
|
45
45
|
def get_id
|
|
46
|
-
if @queue.empty? && @offset == 0
|
|
47
|
-
@total = type.count(input)
|
|
48
|
-
if not @rescan_when_complete
|
|
49
|
-
@queue_size ||= @total
|
|
50
|
-
end
|
|
51
|
-
end
|
|
52
|
-
|
|
53
46
|
t = Time.now
|
|
54
47
|
|
|
55
|
-
|
|
56
|
-
:locked_queue_wait
|
|
48
|
+
if @lock
|
|
49
|
+
return :locked_queue_wait
|
|
57
50
|
else
|
|
58
|
-
queue.shift
|
|
51
|
+
unless object_id = queue.shift
|
|
52
|
+
return :empty_queue_wait
|
|
53
|
+
end
|
|
59
54
|
end
|
|
60
55
|
|
|
61
56
|
@time_spent_grabbing_objects += (Time.now - t)
|
|
62
|
-
|
|
63
|
-
if object_id
|
|
64
|
-
@num_objects_grabbed += 1
|
|
65
|
-
end
|
|
57
|
+
@num_objects_grabbed += 1
|
|
66
58
|
|
|
67
59
|
return object_id
|
|
68
60
|
end
|
|
69
61
|
|
|
70
62
|
private
|
|
71
63
|
|
|
72
|
-
def
|
|
73
|
-
|
|
74
|
-
|
|
64
|
+
def set_total
|
|
65
|
+
@total = type.count(input)
|
|
66
|
+
unless @rescan_when_complete
|
|
67
|
+
@queue_size ||= @total
|
|
75
68
|
end
|
|
76
|
-
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
def queue
|
|
77
72
|
if @queue.empty?
|
|
73
|
+
case @offset
|
|
74
|
+
when 0
|
|
75
|
+
set_total
|
|
76
|
+
when @total
|
|
77
|
+
if @rescan_when_complete || @vigilant
|
|
78
|
+
set_total
|
|
79
|
+
else
|
|
80
|
+
exit
|
|
81
|
+
end
|
|
82
|
+
end
|
|
83
|
+
|
|
78
84
|
GC.start
|
|
79
85
|
|
|
80
86
|
@time_began = Time.now if @time_began == 0
|
data/lib/map_reduce/file.rb
CHANGED
|
@@ -1,4 +1,97 @@
|
|
|
1
1
|
class MapReduce
|
|
2
2
|
module File
|
|
3
|
+
attr_accessor :queue_size, :locked_queue_wait, :empty_queue_wait, :rescan_when_complete, :vigilant, :lines_per_client
|
|
4
|
+
|
|
5
|
+
class Client
|
|
6
|
+
include DRbUndumped
|
|
7
|
+
|
|
8
|
+
include Enumerable
|
|
9
|
+
|
|
10
|
+
def initialize(server_object)
|
|
11
|
+
@server_object = server_object
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def each
|
|
15
|
+
@server_object.limit.times do
|
|
16
|
+
yield @server_object.get_lines
|
|
17
|
+
end
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
def logger(*args)
|
|
21
|
+
@server_object.logger(*args)
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def get_lines
|
|
26
|
+
t = Time.now
|
|
27
|
+
|
|
28
|
+
lines = []
|
|
29
|
+
(@lines_per_client || 1).times do
|
|
30
|
+
begin
|
|
31
|
+
if @lock
|
|
32
|
+
sleep locked_queue_wait || 1
|
|
33
|
+
retry
|
|
34
|
+
else
|
|
35
|
+
if line = queue.shift
|
|
36
|
+
lines << line
|
|
37
|
+
else
|
|
38
|
+
sleep empty_queue_wait || 30
|
|
39
|
+
retry
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
rescue MapReduceError
|
|
43
|
+
retry
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
@time_spent_grabbing_objects += (Time.now - t)
|
|
48
|
+
@num_objects_grabbed += 1
|
|
49
|
+
|
|
50
|
+
return lines.join
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
private
|
|
54
|
+
|
|
55
|
+
def set_total
|
|
56
|
+
@total = ::File.size(input)
|
|
57
|
+
@queue_size ||= 1000
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
def queue
|
|
61
|
+
if @queue.empty?
|
|
62
|
+
case @offset
|
|
63
|
+
when 0
|
|
64
|
+
set_total
|
|
65
|
+
when @total
|
|
66
|
+
if @rescan_when_complete || @vigilant
|
|
67
|
+
set_total
|
|
68
|
+
else
|
|
69
|
+
exit
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
GC.start
|
|
74
|
+
|
|
75
|
+
@time_began = Time.now if @time_began == 0
|
|
76
|
+
@lock = true
|
|
77
|
+
t = Time.now
|
|
78
|
+
|
|
79
|
+
file = ::File.open(input)
|
|
80
|
+
file.seek(@offset)
|
|
81
|
+
@queue_size.times do
|
|
82
|
+
@queue << file.gets
|
|
83
|
+
end
|
|
84
|
+
@queue.compact!
|
|
85
|
+
|
|
86
|
+
@time_spent_grabbing_queues += (Time.now - t)
|
|
87
|
+
@num_queues_grabbed += 1
|
|
88
|
+
|
|
89
|
+
@offset += file.pos unless @queue.empty?
|
|
90
|
+
@offset = 0 if @offset == @total && @rescan_when_complete
|
|
91
|
+
@lock = false
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
@queue
|
|
95
|
+
end
|
|
3
96
|
end
|
|
4
97
|
end
|
data/lib/starfish.rb
CHANGED
|
@@ -11,7 +11,7 @@ DRb.start_service
|
|
|
11
11
|
|
|
12
12
|
class StarfishError < StandardError; end
|
|
13
13
|
class Starfish
|
|
14
|
-
VERSION = "1.
|
|
14
|
+
VERSION = "1.1.0"
|
|
15
15
|
|
|
16
16
|
@@server = false
|
|
17
17
|
@@client = false
|
|
@@ -72,7 +72,11 @@ class Starfish
|
|
|
72
72
|
negotiate
|
|
73
73
|
Timeout::timeout(5) { @server_object = @ring_server.read([:name, uniq.intern, nil, nil])[2] }
|
|
74
74
|
|
|
75
|
+
i = 0
|
|
75
76
|
loop do
|
|
77
|
+
i += 1
|
|
78
|
+
GC.start if i%1000 == 0
|
|
79
|
+
|
|
76
80
|
begin
|
|
77
81
|
@server_object.map_reduce?
|
|
78
82
|
rescue NoMethodError
|
|
@@ -88,6 +92,9 @@ class Starfish
|
|
|
88
92
|
define_method(:logger) do |*args|
|
|
89
93
|
$server_object._logger(*args)
|
|
90
94
|
end
|
|
95
|
+
define_method(:server) do
|
|
96
|
+
$server_object
|
|
97
|
+
end
|
|
91
98
|
end
|
|
92
99
|
|
|
93
100
|
map_reduce_client.each do |object|
|
|
@@ -130,7 +137,7 @@ class Starfish
|
|
|
130
137
|
end
|
|
131
138
|
|
|
132
139
|
def sanitize(object)
|
|
133
|
-
object.
|
|
140
|
+
object.extend DRbUndumped
|
|
134
141
|
|
|
135
142
|
@@log = case @@options[:log]
|
|
136
143
|
when String
|
metadata
CHANGED
|
@@ -3,8 +3,8 @@ rubygems_version: 0.8.11
|
|
|
3
3
|
specification_version: 1
|
|
4
4
|
name: starfish
|
|
5
5
|
version: !ruby/object:Gem::Version
|
|
6
|
-
version: 1.
|
|
7
|
-
date: 2006-08-
|
|
6
|
+
version: 1.1.0
|
|
7
|
+
date: 2006-08-18 00:00:00 -07:00
|
|
8
8
|
summary: Starfish is a utility to make distributed programming ridiculously easy.
|
|
9
9
|
require_paths:
|
|
10
10
|
- lib
|
|
@@ -57,7 +57,9 @@ files:
|
|
|
57
57
|
- doc/classes/MapReduce/ActiveRecord
|
|
58
58
|
- doc/classes/MapReduce/ActiveRecord.html
|
|
59
59
|
- doc/classes/MapReduce/Array.html
|
|
60
|
+
- doc/classes/MapReduce/File
|
|
60
61
|
- doc/classes/MapReduce/File.html
|
|
62
|
+
- doc/classes/MapReduce/File.src
|
|
61
63
|
- doc/classes/MapReduce/ActiveRecord/Base
|
|
62
64
|
- doc/classes/MapReduce/ActiveRecord/Base.html
|
|
63
65
|
- doc/classes/MapReduce/ActiveRecord/Base.src
|
|
@@ -67,6 +69,12 @@ files:
|
|
|
67
69
|
- doc/classes/MapReduce/ActiveRecord/Base/Client.src/M000017.html
|
|
68
70
|
- doc/classes/MapReduce/ActiveRecord/Base/Client.src/M000018.html
|
|
69
71
|
- doc/classes/MapReduce/ActiveRecord/Base.src/M000015.html
|
|
72
|
+
- doc/classes/MapReduce/File/Client.html
|
|
73
|
+
- doc/classes/MapReduce/File/Client.src
|
|
74
|
+
- doc/classes/MapReduce/File/Client.src/M000020.html
|
|
75
|
+
- doc/classes/MapReduce/File/Client.src/M000021.html
|
|
76
|
+
- doc/classes/MapReduce/File/Client.src/M000022.html
|
|
77
|
+
- doc/classes/MapReduce/File.src/M000019.html
|
|
70
78
|
- doc/classes/MapReduce.src/M000001.html
|
|
71
79
|
- doc/classes/MapReduce.src/M000002.html
|
|
72
80
|
- doc/classes/MapReduce.src/M000003.html
|
|
@@ -83,11 +91,7 @@ files:
|
|
|
83
91
|
- doc/classes/MapReduce.src/M000014.html
|
|
84
92
|
- doc/classes/Starfish/RingFinger.html
|
|
85
93
|
- doc/classes/Starfish/RingFinger.src
|
|
86
|
-
- doc/classes/Starfish/RingFinger.src/
|
|
87
|
-
- doc/classes/Starfish.src/M000019.html
|
|
88
|
-
- doc/classes/Starfish.src/M000020.html
|
|
89
|
-
- doc/classes/Starfish.src/M000021.html
|
|
90
|
-
- doc/classes/Starfish.src/M000022.html
|
|
94
|
+
- doc/classes/Starfish/RingFinger.src/M000035.html
|
|
91
95
|
- doc/classes/Starfish.src/M000023.html
|
|
92
96
|
- doc/classes/Starfish.src/M000024.html
|
|
93
97
|
- doc/classes/Starfish.src/M000025.html
|
|
@@ -96,6 +100,10 @@ files:
|
|
|
96
100
|
- doc/classes/Starfish.src/M000028.html
|
|
97
101
|
- doc/classes/Starfish.src/M000029.html
|
|
98
102
|
- doc/classes/Starfish.src/M000030.html
|
|
103
|
+
- doc/classes/Starfish.src/M000031.html
|
|
104
|
+
- doc/classes/Starfish.src/M000032.html
|
|
105
|
+
- doc/classes/Starfish.src/M000033.html
|
|
106
|
+
- doc/classes/Starfish.src/M000034.html
|
|
99
107
|
- doc/files/lib
|
|
100
108
|
- doc/files/README.html
|
|
101
109
|
- doc/files/lib/map_reduce
|
|
@@ -1,19 +0,0 @@
|
|
|
1
|
-
<?xml version="1.0" encoding="iso-8859-1"?>
|
|
2
|
-
<!DOCTYPE html
|
|
3
|
-
PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
|
|
4
|
-
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
|
|
5
|
-
|
|
6
|
-
<html>
|
|
7
|
-
<head>
|
|
8
|
-
<title>new (Starfish)</title>
|
|
9
|
-
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
|
|
10
|
-
<link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
|
|
11
|
-
</head>
|
|
12
|
-
<body class="standalone-code">
|
|
13
|
-
<pre><span class="ruby-comment cmt"># File lib/starfish.rb, line 26</span>
|
|
14
|
-
<span class="ruby-keyword kw">def</span> <span class="ruby-identifier">initialize</span>(<span class="ruby-identifier">uniq</span>=<span class="ruby-constant">ARGV</span>.<span class="ruby-identifier">first</span>)
|
|
15
|
-
<span class="ruby-ivar">@retry_count</span> = <span class="ruby-value">0</span>
|
|
16
|
-
<span class="ruby-ivar">@uniq</span> = <span class="ruby-identifier">uniq</span>
|
|
17
|
-
<span class="ruby-keyword kw">end</span></pre>
|
|
18
|
-
</body>
|
|
19
|
-
</html>
|
|
@@ -1,45 +0,0 @@
|
|
|
1
|
-
<?xml version="1.0" encoding="iso-8859-1"?>
|
|
2
|
-
<!DOCTYPE html
|
|
3
|
-
PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
|
|
4
|
-
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
|
|
5
|
-
|
|
6
|
-
<html>
|
|
7
|
-
<head>
|
|
8
|
-
<title>server (Starfish)</title>
|
|
9
|
-
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
|
|
10
|
-
<link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
|
|
11
|
-
</head>
|
|
12
|
-
<body class="standalone-code">
|
|
13
|
-
<pre><span class="ruby-comment cmt"># File lib/starfish.rb, line 35</span>
|
|
14
|
-
<span class="ruby-keyword kw">def</span> <span class="ruby-identifier">server</span>
|
|
15
|
-
<span class="ruby-keyword kw">unless</span> <span class="ruby-ivar">@@server</span>
|
|
16
|
-
<span class="ruby-identifier">$stderr</span>.<span class="ruby-identifier">puts</span> <span class="ruby-value str">"You must specify a server"</span>
|
|
17
|
-
<span class="ruby-identifier">exit</span>
|
|
18
|
-
<span class="ruby-keyword kw">end</span>
|
|
19
|
-
|
|
20
|
-
<span class="ruby-identifier">map_reduce</span> = <span class="ruby-constant">MapReduce</span>.<span class="ruby-identifier">new</span>
|
|
21
|
-
<span class="ruby-identifier">object</span> = <span class="ruby-ivar">@@server</span>.<span class="ruby-identifier">call</span>(<span class="ruby-identifier">map_reduce</span>)
|
|
22
|
-
|
|
23
|
-
<span class="ruby-keyword kw">if</span> <span class="ruby-identifier">map_reduce</span>.<span class="ruby-identifier">valid?</span>
|
|
24
|
-
<span class="ruby-identifier">object</span> = <span class="ruby-identifier">map_reduce</span>
|
|
25
|
-
<span class="ruby-keyword kw">end</span>
|
|
26
|
-
|
|
27
|
-
<span class="ruby-identifier">sanitize</span> <span class="ruby-identifier">object</span>
|
|
28
|
-
|
|
29
|
-
<span class="ruby-identifier">ts</span> = <span class="ruby-constant">Rinda</span><span class="ruby-operator">::</span><span class="ruby-constant">TupleSpace</span>.<span class="ruby-identifier">new</span>
|
|
30
|
-
<span class="ruby-keyword kw">begin</span>
|
|
31
|
-
<span class="ruby-identifier">ts</span>.<span class="ruby-identifier">write</span>([<span class="ruby-identifier">:name</span>, <span class="ruby-identifier">uniq</span>.<span class="ruby-identifier">intern</span>, <span class="ruby-identifier">object</span>, <span class="ruby-ivar">@uniq</span>])
|
|
32
|
-
<span class="ruby-identifier">ring_server</span> = <span class="ruby-constant">Rinda</span><span class="ruby-operator">::</span><span class="ruby-constant">RingServer</span>.<span class="ruby-identifier">new</span>(<span class="ruby-identifier">ts</span>)
|
|
33
|
-
<span class="ruby-keyword kw">rescue</span> <span class="ruby-constant">Errno</span><span class="ruby-operator">::</span><span class="ruby-constant">EADDRINUSE</span>
|
|
34
|
-
<span class="ruby-identifier">ts</span> = <span class="ruby-constant">RingFinger</span>.<span class="ruby-identifier">primary</span>
|
|
35
|
-
<span class="ruby-identifier">ts</span>.<span class="ruby-identifier">write</span>([<span class="ruby-identifier">:name</span>, <span class="ruby-identifier">uniq</span>.<span class="ruby-identifier">intern</span>, <span class="ruby-identifier">object</span>, <span class="ruby-ivar">@uniq</span>])
|
|
36
|
-
<span class="ruby-keyword kw">end</span>
|
|
37
|
-
|
|
38
|
-
<span class="ruby-constant">File</span>.<span class="ruby-identifier">open</span>(<span class="ruby-ivar">@@options</span>[<span class="ruby-identifier">:pid</span>] <span class="ruby-operator">||</span> <span class="ruby-node">"/tmp/starfish-#{uniq}.pid"</span>,<span class="ruby-value str">"w"</span>){<span class="ruby-operator">|</span><span class="ruby-identifier">f</span><span class="ruby-operator">|</span><span class="ruby-identifier">f</span><span class="ruby-operator"><<</span><span class="ruby-constant">Process</span>.<span class="ruby-identifier">pid</span>}
|
|
39
|
-
|
|
40
|
-
<span class="ruby-identifier">$stderr</span>.<span class="ruby-identifier">puts</span> <span class="ruby-node">"server started for #{object.inspect}"</span>
|
|
41
|
-
|
|
42
|
-
<span class="ruby-constant">DRb</span>.<span class="ruby-identifier">thread</span>.<span class="ruby-identifier">join</span>
|
|
43
|
-
<span class="ruby-keyword kw">end</span></pre>
|
|
44
|
-
</body>
|
|
45
|
-
</html>
|
|
@@ -1,71 +0,0 @@
|
|
|
1
|
-
<?xml version="1.0" encoding="iso-8859-1"?>
|
|
2
|
-
<!DOCTYPE html
|
|
3
|
-
PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
|
|
4
|
-
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
|
|
5
|
-
|
|
6
|
-
<html>
|
|
7
|
-
<head>
|
|
8
|
-
<title>client (Starfish)</title>
|
|
9
|
-
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
|
|
10
|
-
<link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
|
|
11
|
-
</head>
|
|
12
|
-
<body class="standalone-code">
|
|
13
|
-
<pre><span class="ruby-comment cmt"># File lib/starfish.rb, line 66</span>
|
|
14
|
-
<span class="ruby-keyword kw">def</span> <span class="ruby-identifier">client</span>
|
|
15
|
-
<span class="ruby-keyword kw">unless</span> <span class="ruby-ivar">@@client</span>
|
|
16
|
-
<span class="ruby-identifier">$stderr</span>.<span class="ruby-identifier">puts</span> <span class="ruby-value str">"You must specify a client"</span>
|
|
17
|
-
<span class="ruby-identifier">exit</span>
|
|
18
|
-
<span class="ruby-keyword kw">end</span>
|
|
19
|
-
|
|
20
|
-
<span class="ruby-identifier">negotiate</span>
|
|
21
|
-
<span class="ruby-constant">Timeout</span><span class="ruby-operator">::</span><span class="ruby-identifier">timeout</span>(<span class="ruby-value">5</span>) { <span class="ruby-ivar">@server_object</span> = <span class="ruby-ivar">@ring_server</span>.<span class="ruby-identifier">read</span>([<span class="ruby-identifier">:name</span>, <span class="ruby-identifier">uniq</span>.<span class="ruby-identifier">intern</span>, <span class="ruby-keyword kw">nil</span>, <span class="ruby-keyword kw">nil</span>])[<span class="ruby-value">2</span>] }
|
|
22
|
-
|
|
23
|
-
<span class="ruby-identifier">loop</span> <span class="ruby-keyword kw">do</span>
|
|
24
|
-
<span class="ruby-keyword kw">begin</span>
|
|
25
|
-
<span class="ruby-ivar">@server_object</span>.<span class="ruby-identifier">map_reduce?</span>
|
|
26
|
-
<span class="ruby-keyword kw">rescue</span> <span class="ruby-constant">NoMethodError</span>
|
|
27
|
-
<span class="ruby-ivar">@called</span> = <span class="ruby-ivar">@@client</span>.<span class="ruby-identifier">call</span>(<span class="ruby-ivar">@server_object</span>)
|
|
28
|
-
<span class="ruby-keyword kw">end</span>
|
|
29
|
-
|
|
30
|
-
<span class="ruby-keyword kw">unless</span> <span class="ruby-ivar">@called</span>
|
|
31
|
-
<span class="ruby-keyword kw">if</span> <span class="ruby-ivar">@server_object</span>.<span class="ruby-identifier">map_reduce?</span> <span class="ruby-operator">&&</span> <span class="ruby-ivar">@server_object</span>.<span class="ruby-identifier">valid?</span>
|
|
32
|
-
<span class="ruby-identifier">map_reduce_client</span> = <span class="ruby-identifier">eval</span>(<span class="ruby-node">"MapReduce::#{@server_object.base_type_to_s}::Client"</span>).<span class="ruby-identifier">new</span>(<span class="ruby-ivar">@server_object</span>)
|
|
33
|
-
|
|
34
|
-
<span class="ruby-identifier">$server_object</span> = <span class="ruby-ivar">@server_object</span>
|
|
35
|
-
<span class="ruby-constant">Object</span>.<span class="ruby-identifier">instance_eval</span> <span class="ruby-keyword kw">do</span>
|
|
36
|
-
<span class="ruby-identifier">define_method</span>(<span class="ruby-identifier">:logger</span>) <span class="ruby-keyword kw">do</span> <span class="ruby-operator">|</span><span class="ruby-operator">*</span><span class="ruby-identifier">args</span><span class="ruby-operator">|</span>
|
|
37
|
-
<span class="ruby-identifier">$server_object</span>.<span class="ruby-identifier">_logger</span>(<span class="ruby-operator">*</span><span class="ruby-identifier">args</span>)
|
|
38
|
-
<span class="ruby-keyword kw">end</span>
|
|
39
|
-
<span class="ruby-keyword kw">end</span>
|
|
40
|
-
|
|
41
|
-
<span class="ruby-identifier">map_reduce_client</span>.<span class="ruby-identifier">each</span> <span class="ruby-keyword kw">do</span> <span class="ruby-operator">|</span><span class="ruby-identifier">object</span><span class="ruby-operator">|</span>
|
|
42
|
-
<span class="ruby-identifier">t</span> = <span class="ruby-constant">Time</span>.<span class="ruby-identifier">now</span>
|
|
43
|
-
<span class="ruby-ivar">@@client</span>.<span class="ruby-identifier">call</span>(<span class="ruby-identifier">object</span>)
|
|
44
|
-
<span class="ruby-ivar">@server_object</span>.<span class="ruby-identifier">add_time_spent_processing_objects</span>(<span class="ruby-constant">Time</span>.<span class="ruby-identifier">now</span><span class="ruby-operator">-</span><span class="ruby-identifier">t</span>)
|
|
45
|
-
<span class="ruby-keyword kw">end</span>
|
|
46
|
-
<span class="ruby-keyword kw">else</span>
|
|
47
|
-
<span class="ruby-identifier">raise</span> <span class="ruby-constant">MapReduceError</span>, <span class="ruby-value str">"invalid map reduce server (possibly missing type or input)"</span>
|
|
48
|
-
<span class="ruby-keyword kw">end</span>
|
|
49
|
-
<span class="ruby-keyword kw">end</span>
|
|
50
|
-
<span class="ruby-keyword kw">end</span>
|
|
51
|
-
|
|
52
|
-
<span class="ruby-keyword kw">rescue</span> <span class="ruby-constant">Timeout</span><span class="ruby-operator">::</span><span class="ruby-constant">Error</span> =<span class="ruby-operator">></span> <span class="ruby-identifier">m</span>
|
|
53
|
-
<span class="ruby-identifier">spawn</span>
|
|
54
|
-
<span class="ruby-ivar">@retry_count</span> <span class="ruby-operator">+=</span> <span class="ruby-value">1</span>
|
|
55
|
-
<span class="ruby-keyword kw">if</span> <span class="ruby-ivar">@retry_count</span> <span class="ruby-operator"><=</span> <span class="ruby-value">5</span>
|
|
56
|
-
<span class="ruby-keyword kw">retry</span>
|
|
57
|
-
<span class="ruby-keyword kw">else</span>
|
|
58
|
-
<span class="ruby-identifier">raise</span> <span class="ruby-constant">Timeout</span><span class="ruby-operator">::</span><span class="ruby-constant">Error</span>, <span class="ruby-identifier">m</span>
|
|
59
|
-
<span class="ruby-keyword kw">end</span>
|
|
60
|
-
<span class="ruby-keyword kw">rescue</span> <span class="ruby-constant">DRb</span><span class="ruby-operator">::</span><span class="ruby-constant">DRbConnError</span> =<span class="ruby-operator">></span> <span class="ruby-identifier">m</span>
|
|
61
|
-
<span class="ruby-identifier">stop</span>
|
|
62
|
-
<span class="ruby-identifier">negotiate</span>
|
|
63
|
-
<span class="ruby-ivar">@retry_count</span> <span class="ruby-operator">+=</span> <span class="ruby-value">1</span>
|
|
64
|
-
<span class="ruby-keyword kw">if</span> <span class="ruby-ivar">@retry_count</span> <span class="ruby-operator"><=</span> <span class="ruby-value">5</span>
|
|
65
|
-
<span class="ruby-keyword kw">retry</span>
|
|
66
|
-
<span class="ruby-keyword kw">else</span>
|
|
67
|
-
<span class="ruby-identifier">raise</span> <span class="ruby-constant">DRb</span><span class="ruby-operator">::</span><span class="ruby-constant">DRbConnError</span>, <span class="ruby-identifier">m</span>
|
|
68
|
-
<span class="ruby-keyword kw">end</span>
|
|
69
|
-
<span class="ruby-keyword kw">end</span></pre>
|
|
70
|
-
</body>
|
|
71
|
-
</html>
|