starfish 1.1.2 → 1.1.3
Sign up to get free protection for your applications and to get access to all the features.
- data/doc/classes/MapReduce/ActiveRecord/Base/Client.src/M000016.html +1 -1
- data/doc/classes/MapReduce/ActiveRecord/Base/Client.src/M000017.html +1 -1
- data/doc/classes/MapReduce/ActiveRecord/Base/Client.src/M000018.html +1 -1
- data/doc/classes/MapReduce/ActiveRecord/Base.html +30 -0
- data/doc/classes/MapReduce/ActiveRecord/Base.src/M000015.html +1 -1
- data/doc/classes/MapReduce/File/Client.src/M000020.html +1 -1
- data/doc/classes/MapReduce/File/Client.src/M000021.html +1 -1
- data/doc/classes/MapReduce/File/Client.src/M000022.html +1 -1
- data/doc/classes/MapReduce/File.html +10 -0
- data/doc/classes/MapReduce/File.src/M000019.html +1 -1
- data/doc/classes/Starfish/RingFinger.src/M000035.html +1 -1
- data/doc/classes/Starfish.html +2 -2
- data/doc/classes/Starfish.src/M000026.html +23 -18
- data/doc/classes/Starfish.src/M000027.html +1 -1
- data/doc/classes/Starfish.src/M000028.html +1 -1
- data/doc/classes/Starfish.src/M000029.html +1 -1
- data/doc/classes/Starfish.src/M000030.html +1 -1
- data/doc/classes/Starfish.src/M000031.html +1 -1
- data/doc/classes/Starfish.src/M000032.html +1 -1
- data/doc/classes/Starfish.src/M000033.html +1 -1
- data/doc/classes/Starfish.src/M000034.html +1 -1
- data/doc/created.rid +1 -1
- data/doc/files/lib/map_reduce/active_record_rb.html +1 -1
- data/doc/files/lib/map_reduce/file_rb.html +1 -1
- data/doc/files/lib/map_reduce_rb.html +1 -1
- data/doc/files/lib/starfish_rb.html +1 -1
- data/lib/map_reduce/active_record.rb +16 -2
- data/lib/map_reduce/file.rb +15 -12
- data/lib/starfish.rb +25 -20
- metadata +2 -3
- data/NUL +0 -0
@@ -10,7 +10,7 @@
|
|
10
10
|
<link rel="stylesheet" href="../../../../.././rdoc-style.css" type="text/css" media="screen" />
|
11
11
|
</head>
|
12
12
|
<body class="standalone-code">
|
13
|
-
<pre><span class="ruby-comment cmt"># File lib/map_reduce/active_record.rb, line
|
13
|
+
<pre><span class="ruby-comment cmt"># File lib/map_reduce/active_record.rb, line 15</span>
|
14
14
|
<span class="ruby-keyword kw">def</span> <span class="ruby-identifier">initialize</span>(<span class="ruby-identifier">server_object</span>)
|
15
15
|
<span class="ruby-ivar">@server_object</span> = <span class="ruby-identifier">server_object</span>
|
16
16
|
<span class="ruby-ivar">@type</span> = <span class="ruby-identifier">eval</span>(<span class="ruby-identifier">server_object</span>.<span class="ruby-identifier">type_to_s</span>)
|
@@ -10,7 +10,7 @@
|
|
10
10
|
<link rel="stylesheet" href="../../../../.././rdoc-style.css" type="text/css" media="screen" />
|
11
11
|
</head>
|
12
12
|
<body class="standalone-code">
|
13
|
-
<pre><span class="ruby-comment cmt"># File lib/map_reduce/active_record.rb, line
|
13
|
+
<pre><span class="ruby-comment cmt"># File lib/map_reduce/active_record.rb, line 20</span>
|
14
14
|
<span class="ruby-keyword kw">def</span> <span class="ruby-identifier">each</span>
|
15
15
|
<span class="ruby-ivar">@server_object</span>.<span class="ruby-identifier">limit</span>.<span class="ruby-identifier">times</span> <span class="ruby-keyword kw">do</span>
|
16
16
|
<span class="ruby-keyword kw">yield</span> <span class="ruby-identifier">get_value_from</span>(<span class="ruby-ivar">@server_object</span>.<span class="ruby-identifier">get_id</span>)
|
@@ -10,7 +10,7 @@
|
|
10
10
|
<link rel="stylesheet" href="../../../../.././rdoc-style.css" type="text/css" media="screen" />
|
11
11
|
</head>
|
12
12
|
<body class="standalone-code">
|
13
|
-
<pre><span class="ruby-comment cmt"># File lib/map_reduce/active_record.rb, line
|
13
|
+
<pre><span class="ruby-comment cmt"># File lib/map_reduce/active_record.rb, line 26</span>
|
14
14
|
<span class="ruby-keyword kw">def</span> <span class="ruby-identifier">logger</span>(<span class="ruby-operator">*</span><span class="ruby-identifier">args</span>)
|
15
15
|
<span class="ruby-ivar">@server_object</span>.<span class="ruby-identifier">logger</span>(<span class="ruby-operator">*</span><span class="ruby-identifier">args</span>)
|
16
16
|
<span class="ruby-keyword kw">end</span></pre>
|
@@ -111,21 +111,51 @@
|
|
111
111
|
<td class="context-item-value"> [RW] </td>
|
112
112
|
<td class="context-item-desc"></td>
|
113
113
|
</tr>
|
114
|
+
<tr class="top-aligned-row context-row">
|
115
|
+
<td class="context-item-name">group</td>
|
116
|
+
<td class="context-item-value"> [RW] </td>
|
117
|
+
<td class="context-item-desc"></td>
|
118
|
+
</tr>
|
119
|
+
<tr class="top-aligned-row context-row">
|
120
|
+
<td class="context-item-name">include</td>
|
121
|
+
<td class="context-item-value"> [RW] </td>
|
122
|
+
<td class="context-item-desc"></td>
|
123
|
+
</tr>
|
124
|
+
<tr class="top-aligned-row context-row">
|
125
|
+
<td class="context-item-name">joins</td>
|
126
|
+
<td class="context-item-value"> [RW] </td>
|
127
|
+
<td class="context-item-desc"></td>
|
128
|
+
</tr>
|
114
129
|
<tr class="top-aligned-row context-row">
|
115
130
|
<td class="context-item-name">locked_queue_wait</td>
|
116
131
|
<td class="context-item-value"> [RW] </td>
|
117
132
|
<td class="context-item-desc"></td>
|
118
133
|
</tr>
|
134
|
+
<tr class="top-aligned-row context-row">
|
135
|
+
<td class="context-item-name">order</td>
|
136
|
+
<td class="context-item-value"> [RW] </td>
|
137
|
+
<td class="context-item-desc"></td>
|
138
|
+
</tr>
|
119
139
|
<tr class="top-aligned-row context-row">
|
120
140
|
<td class="context-item-name">queue_size</td>
|
121
141
|
<td class="context-item-value"> [RW] </td>
|
122
142
|
<td class="context-item-desc"></td>
|
123
143
|
</tr>
|
144
|
+
<tr class="top-aligned-row context-row">
|
145
|
+
<td class="context-item-name">readonly</td>
|
146
|
+
<td class="context-item-value"> [RW] </td>
|
147
|
+
<td class="context-item-desc"></td>
|
148
|
+
</tr>
|
124
149
|
<tr class="top-aligned-row context-row">
|
125
150
|
<td class="context-item-name">rescan_when_complete</td>
|
126
151
|
<td class="context-item-value"> [RW] </td>
|
127
152
|
<td class="context-item-desc"></td>
|
128
153
|
</tr>
|
154
|
+
<tr class="top-aligned-row context-row">
|
155
|
+
<td class="context-item-name">select</td>
|
156
|
+
<td class="context-item-value"> [RW] </td>
|
157
|
+
<td class="context-item-desc"></td>
|
158
|
+
</tr>
|
129
159
|
<tr class="top-aligned-row context-row">
|
130
160
|
<td class="context-item-name">vigilant</td>
|
131
161
|
<td class="context-item-value"> [RW] </td>
|
@@ -10,7 +10,7 @@
|
|
10
10
|
<link rel="stylesheet" href="../../../.././rdoc-style.css" type="text/css" media="screen" />
|
11
11
|
</head>
|
12
12
|
<body class="standalone-code">
|
13
|
-
<pre><span class="ruby-comment cmt"># File lib/map_reduce/active_record.rb, line
|
13
|
+
<pre><span class="ruby-comment cmt"># File lib/map_reduce/active_record.rb, line 49</span>
|
14
14
|
<span class="ruby-keyword kw">def</span> <span class="ruby-identifier">get_id</span>
|
15
15
|
<span class="ruby-identifier">t</span> = <span class="ruby-constant">Time</span>.<span class="ruby-identifier">now</span>
|
16
16
|
|
@@ -10,7 +10,7 @@
|
|
10
10
|
<link rel="stylesheet" href="../../../.././rdoc-style.css" type="text/css" media="screen" />
|
11
11
|
</head>
|
12
12
|
<body class="standalone-code">
|
13
|
-
<pre><span class="ruby-comment cmt"># File lib/map_reduce/file.rb, line
|
13
|
+
<pre><span class="ruby-comment cmt"># File lib/map_reduce/file.rb, line 11</span>
|
14
14
|
<span class="ruby-keyword kw">def</span> <span class="ruby-identifier">initialize</span>(<span class="ruby-identifier">server_object</span>)
|
15
15
|
<span class="ruby-ivar">@server_object</span> = <span class="ruby-identifier">server_object</span>
|
16
16
|
<span class="ruby-keyword kw">end</span></pre>
|
@@ -10,7 +10,7 @@
|
|
10
10
|
<link rel="stylesheet" href="../../../.././rdoc-style.css" type="text/css" media="screen" />
|
11
11
|
</head>
|
12
12
|
<body class="standalone-code">
|
13
|
-
<pre><span class="ruby-comment cmt"># File lib/map_reduce/file.rb, line
|
13
|
+
<pre><span class="ruby-comment cmt"># File lib/map_reduce/file.rb, line 15</span>
|
14
14
|
<span class="ruby-keyword kw">def</span> <span class="ruby-identifier">each</span>
|
15
15
|
<span class="ruby-ivar">@server_object</span>.<span class="ruby-identifier">limit</span>.<span class="ruby-identifier">times</span> <span class="ruby-keyword kw">do</span>
|
16
16
|
<span class="ruby-keyword kw">yield</span> <span class="ruby-ivar">@server_object</span>.<span class="ruby-identifier">get_lines</span>
|
@@ -10,7 +10,7 @@
|
|
10
10
|
<link rel="stylesheet" href="../../../.././rdoc-style.css" type="text/css" media="screen" />
|
11
11
|
</head>
|
12
12
|
<body class="standalone-code">
|
13
|
-
<pre><span class="ruby-comment cmt"># File lib/map_reduce/file.rb, line
|
13
|
+
<pre><span class="ruby-comment cmt"># File lib/map_reduce/file.rb, line 21</span>
|
14
14
|
<span class="ruby-keyword kw">def</span> <span class="ruby-identifier">logger</span>(<span class="ruby-operator">*</span><span class="ruby-identifier">args</span>)
|
15
15
|
<span class="ruby-ivar">@server_object</span>.<span class="ruby-identifier">logger</span>(<span class="ruby-operator">*</span><span class="ruby-identifier">args</span>)
|
16
16
|
<span class="ruby-keyword kw">end</span></pre>
|
@@ -121,6 +121,11 @@
|
|
121
121
|
<td class="context-item-value"> [RW] </td>
|
122
122
|
<td class="context-item-desc"></td>
|
123
123
|
</tr>
|
124
|
+
<tr class="top-aligned-row context-row">
|
125
|
+
<td class="context-item-name">offset</td>
|
126
|
+
<td class="context-item-value"> [RW] </td>
|
127
|
+
<td class="context-item-desc"></td>
|
128
|
+
</tr>
|
124
129
|
<tr class="top-aligned-row context-row">
|
125
130
|
<td class="context-item-name">queue_size</td>
|
126
131
|
<td class="context-item-value"> [RW] </td>
|
@@ -131,6 +136,11 @@
|
|
131
136
|
<td class="context-item-value"> [RW] </td>
|
132
137
|
<td class="context-item-desc"></td>
|
133
138
|
</tr>
|
139
|
+
<tr class="top-aligned-row context-row">
|
140
|
+
<td class="context-item-name">total</td>
|
141
|
+
<td class="context-item-value"> [R] </td>
|
142
|
+
<td class="context-item-desc"></td>
|
143
|
+
</tr>
|
134
144
|
<tr class="top-aligned-row context-row">
|
135
145
|
<td class="context-item-name">vigilant</td>
|
136
146
|
<td class="context-item-value"> [RW] </td>
|
@@ -10,7 +10,7 @@
|
|
10
10
|
<link rel="stylesheet" href="../../.././rdoc-style.css" type="text/css" media="screen" />
|
11
11
|
</head>
|
12
12
|
<body class="standalone-code">
|
13
|
-
<pre><span class="ruby-comment cmt"># File lib/map_reduce/file.rb, line
|
13
|
+
<pre><span class="ruby-comment cmt"># File lib/map_reduce/file.rb, line 26</span>
|
14
14
|
<span class="ruby-keyword kw">def</span> <span class="ruby-identifier">get_lines</span>
|
15
15
|
<span class="ruby-identifier">t</span> = <span class="ruby-constant">Time</span>.<span class="ruby-identifier">now</span>
|
16
16
|
|
@@ -10,7 +10,7 @@
|
|
10
10
|
<link rel="stylesheet" href="../../.././rdoc-style.css" type="text/css" media="screen" />
|
11
11
|
</head>
|
12
12
|
<body class="standalone-code">
|
13
|
-
<pre><span class="ruby-comment cmt"># File lib/starfish.rb, line
|
13
|
+
<pre><span class="ruby-comment cmt"># File lib/starfish.rb, line 184</span>
|
14
14
|
<span class="ruby-keyword kw">def</span> <span class="ruby-constant">RingFinger</span>.<span class="ruby-identifier">finger</span>
|
15
15
|
<span class="ruby-ivar">@@finger</span> = <span class="ruby-keyword kw">self</span>.<span class="ruby-identifier">new</span>
|
16
16
|
<span class="ruby-ivar">@@finger</span>.<span class="ruby-identifier">lookup_ring_any</span>
|
data/doc/classes/Starfish.html
CHANGED
@@ -130,12 +130,12 @@
|
|
130
130
|
<tr class="top-aligned-row context-row">
|
131
131
|
<td class="context-item-name">VERSION</td>
|
132
132
|
<td>=</td>
|
133
|
-
<td class="context-item-value">"1.1.
|
133
|
+
<td class="context-item-value">"1.1.3"</td>
|
134
134
|
</tr>
|
135
135
|
<tr class="top-aligned-row context-row">
|
136
136
|
<td class="context-item-name">NULL</td>
|
137
137
|
<td>=</td>
|
138
|
-
<td class="context-item-value">RUBY_PLATFORM =~ /mswin/ ? 'NUL' : '/dev/null'</td>
|
138
|
+
<td class="context-item-value">(RUBY_PLATFORM =~ /mswin/) ? 'NUL' : '/dev/null'</td>
|
139
139
|
</tr>
|
140
140
|
</table>
|
141
141
|
</div>
|
@@ -30,29 +30,34 @@
|
|
30
30
|
<span class="ruby-keyword kw">rescue</span> <span class="ruby-constant">NoMethodError</span>
|
31
31
|
<span class="ruby-ivar">@called</span> = <span class="ruby-ivar">@@client</span>.<span class="ruby-identifier">call</span>(<span class="ruby-ivar">@server_object</span>)
|
32
32
|
<span class="ruby-keyword kw">end</span>
|
33
|
-
|
34
|
-
<span class="ruby-keyword kw">
|
35
|
-
<span class="ruby-keyword kw">
|
36
|
-
<span class="ruby-
|
33
|
+
|
34
|
+
<span class="ruby-keyword kw">begin</span>
|
35
|
+
<span class="ruby-keyword kw">unless</span> <span class="ruby-ivar">@called</span>
|
36
|
+
<span class="ruby-keyword kw">if</span> <span class="ruby-ivar">@server_object</span>.<span class="ruby-identifier">map_reduce?</span> <span class="ruby-operator">&&</span> <span class="ruby-ivar">@server_object</span>.<span class="ruby-identifier">valid?</span>
|
37
|
+
<span class="ruby-identifier">map_reduce_client</span> = <span class="ruby-identifier">eval</span>(<span class="ruby-node">"MapReduce::#{@server_object.base_type_to_s}::Client"</span>).<span class="ruby-identifier">new</span>(<span class="ruby-ivar">@server_object</span>)
|
37
38
|
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
39
|
+
<span class="ruby-identifier">$server_object</span> = <span class="ruby-ivar">@server_object</span>
|
40
|
+
<span class="ruby-constant">Object</span>.<span class="ruby-identifier">instance_eval</span> <span class="ruby-keyword kw">do</span>
|
41
|
+
<span class="ruby-identifier">define_method</span>(<span class="ruby-identifier">:logger</span>) <span class="ruby-keyword kw">do</span> <span class="ruby-operator">|</span><span class="ruby-operator">*</span><span class="ruby-identifier">args</span><span class="ruby-operator">|</span>
|
42
|
+
<span class="ruby-identifier">$server_object</span>.<span class="ruby-identifier">_logger</span>(<span class="ruby-operator">*</span><span class="ruby-identifier">args</span>)
|
43
|
+
<span class="ruby-keyword kw">end</span>
|
44
|
+
<span class="ruby-identifier">define_method</span>(<span class="ruby-identifier">:server</span>) <span class="ruby-keyword kw">do</span>
|
45
|
+
<span class="ruby-identifier">$server_object</span>
|
46
|
+
<span class="ruby-keyword kw">end</span>
|
45
47
|
<span class="ruby-keyword kw">end</span>
|
46
|
-
<span class="ruby-keyword kw">end</span>
|
47
48
|
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
49
|
+
<span class="ruby-identifier">map_reduce_client</span>.<span class="ruby-identifier">each</span> <span class="ruby-keyword kw">do</span> <span class="ruby-operator">|</span><span class="ruby-identifier">object</span><span class="ruby-operator">|</span>
|
50
|
+
<span class="ruby-identifier">t</span> = <span class="ruby-constant">Time</span>.<span class="ruby-identifier">now</span>
|
51
|
+
<span class="ruby-constant">Timeout</span><span class="ruby-operator">::</span><span class="ruby-identifier">timeout</span>(<span class="ruby-ivar">@@options</span>[<span class="ruby-identifier">:timeout</span>] <span class="ruby-operator">||</span> <span class="ruby-value">60</span>) <span class="ruby-keyword kw">do</span>
|
52
|
+
<span class="ruby-ivar">@@client</span>.<span class="ruby-identifier">call</span>(<span class="ruby-identifier">object</span>)
|
53
|
+
<span class="ruby-keyword kw">end</span>
|
54
|
+
<span class="ruby-ivar">@server_object</span>.<span class="ruby-identifier">add_time_spent_processing_objects</span>(<span class="ruby-constant">Time</span>.<span class="ruby-identifier">now</span><span class="ruby-operator">-</span><span class="ruby-identifier">t</span>)
|
55
|
+
<span class="ruby-keyword kw">end</span>
|
56
|
+
<span class="ruby-keyword kw">else</span>
|
57
|
+
<span class="ruby-identifier">raise</span> <span class="ruby-constant">MapReduceError</span>, <span class="ruby-value str">"invalid map reduce server (possibly missing type or input)"</span>
|
52
58
|
<span class="ruby-keyword kw">end</span>
|
53
|
-
<span class="ruby-keyword kw">else</span>
|
54
|
-
<span class="ruby-identifier">raise</span> <span class="ruby-constant">MapReduceError</span>, <span class="ruby-value str">"invalid map reduce server (possibly missing type or input)"</span>
|
55
59
|
<span class="ruby-keyword kw">end</span>
|
60
|
+
<span class="ruby-keyword kw">rescue</span> <span class="ruby-constant">NoMethodError</span>
|
56
61
|
<span class="ruby-keyword kw">end</span>
|
57
62
|
<span class="ruby-keyword kw">end</span>
|
58
63
|
|
@@ -10,7 +10,7 @@
|
|
10
10
|
<link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
|
11
11
|
</head>
|
12
12
|
<body class="standalone-code">
|
13
|
-
<pre><span class="ruby-comment cmt"># File lib/starfish.rb, line
|
13
|
+
<pre><span class="ruby-comment cmt"># File lib/starfish.rb, line 148</span>
|
14
14
|
<span class="ruby-keyword kw">def</span> <span class="ruby-identifier">stats</span>
|
15
15
|
<span class="ruby-identifier">negotiate</span>
|
16
16
|
<span class="ruby-constant">Timeout</span><span class="ruby-operator">::</span><span class="ruby-identifier">timeout</span>(<span class="ruby-value">5</span>) { <span class="ruby-ivar">@server_object</span> = <span class="ruby-ivar">@ring_server</span>.<span class="ruby-identifier">read</span>([<span class="ruby-identifier">:name</span>, <span class="ruby-identifier">uniq</span>.<span class="ruby-identifier">intern</span>, <span class="ruby-keyword kw">nil</span>, <span class="ruby-keyword kw">nil</span>])[<span class="ruby-value">2</span>] }
|
@@ -10,7 +10,7 @@
|
|
10
10
|
<link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
|
11
11
|
</head>
|
12
12
|
<body class="standalone-code">
|
13
|
-
<pre><span class="ruby-comment cmt"># File lib/starfish.rb, line
|
13
|
+
<pre><span class="ruby-comment cmt"># File lib/starfish.rb, line 157</span>
|
14
14
|
<span class="ruby-keyword kw">def</span> <span class="ruby-identifier">sanitize</span>(<span class="ruby-identifier">object</span>)
|
15
15
|
<span class="ruby-identifier">object</span>.<span class="ruby-identifier">extend</span> <span class="ruby-constant">DRbUndumped</span>
|
16
16
|
|
@@ -10,7 +10,7 @@
|
|
10
10
|
<link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
|
11
11
|
</head>
|
12
12
|
<body class="standalone-code">
|
13
|
-
<pre><span class="ruby-comment cmt"># File lib/starfish.rb, line
|
13
|
+
<pre><span class="ruby-comment cmt"># File lib/starfish.rb, line 179</span>
|
14
14
|
<span class="ruby-keyword kw">def</span> <span class="ruby-identifier">negotiate</span><span class="ruby-identifier">negotiate</span><span class="ruby-identifier">negotiate</span>
|
15
15
|
<span class="ruby-keyword kw">begin</span>
|
16
16
|
<span class="ruby-ivar">@ring_server</span> = <span class="ruby-constant">RingFinger</span>.<span class="ruby-identifier">primary</span>
|
@@ -10,7 +10,7 @@
|
|
10
10
|
<link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
|
11
11
|
</head>
|
12
12
|
<body class="standalone-code">
|
13
|
-
<pre><span class="ruby-comment cmt"># File lib/starfish.rb, line
|
13
|
+
<pre><span class="ruby-comment cmt"># File lib/starfish.rb, line 200</span>
|
14
14
|
<span class="ruby-keyword kw">def</span> <span class="ruby-identifier">spawn</span>
|
15
15
|
<span class="ruby-ivar">@started</span> <span class="ruby-operator">||=</span> <span class="ruby-constant">STARFISH_FORK</span>.<span class="ruby-identifier">call</span> { <span class="ruby-identifier">system</span>(<span class="ruby-node">"ruby #{File.dirname(__FILE__)}/../bin/starfish #{@uniq} server > #{NULL}"</span>) }
|
16
16
|
<span class="ruby-keyword kw">end</span></pre>
|
@@ -10,7 +10,7 @@
|
|
10
10
|
<link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
|
11
11
|
</head>
|
12
12
|
<body class="standalone-code">
|
13
|
-
<pre><span class="ruby-comment cmt"># File lib/starfish.rb, line
|
13
|
+
<pre><span class="ruby-comment cmt"># File lib/starfish.rb, line 204</span>
|
14
14
|
<span class="ruby-keyword kw">def</span> <span class="ruby-identifier">stop</span>
|
15
15
|
<span class="ruby-identifier">puts</span> <span class="ruby-value str">"stopping the server"</span>
|
16
16
|
<span class="ruby-constant">Process</span>.<span class="ruby-identifier">kill</span>(<span class="ruby-value str">"SIGHUP"</span>, <span class="ruby-constant">IO</span>.<span class="ruby-identifier">read</span>(<span class="ruby-ivar">@@options</span>[<span class="ruby-identifier">:pid</span>] <span class="ruby-operator">||</span> <span class="ruby-node">"#{Dir.tmpdir}/starfish-#{uniq}.pid"</span>).<span class="ruby-identifier">to_i</span>)
|
@@ -10,7 +10,7 @@
|
|
10
10
|
<link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
|
11
11
|
</head>
|
12
12
|
<body class="standalone-code">
|
13
|
-
<pre><span class="ruby-comment cmt"># File lib/starfish.rb, line
|
13
|
+
<pre><span class="ruby-comment cmt"># File lib/starfish.rb, line 213</span>
|
14
14
|
<span class="ruby-keyword kw">def</span> <span class="ruby-identifier">server=</span>(<span class="ruby-identifier">server</span>)
|
15
15
|
<span class="ruby-ivar">@@server</span> = <span class="ruby-identifier">server</span>
|
16
16
|
<span class="ruby-keyword kw">end</span></pre>
|
@@ -10,7 +10,7 @@
|
|
10
10
|
<link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
|
11
11
|
</head>
|
12
12
|
<body class="standalone-code">
|
13
|
-
<pre><span class="ruby-comment cmt"># File lib/starfish.rb, line
|
13
|
+
<pre><span class="ruby-comment cmt"># File lib/starfish.rb, line 217</span>
|
14
14
|
<span class="ruby-keyword kw">def</span> <span class="ruby-identifier">client=</span>(<span class="ruby-identifier">client</span>)
|
15
15
|
<span class="ruby-ivar">@@client</span> = <span class="ruby-identifier">client</span>
|
16
16
|
<span class="ruby-keyword kw">end</span></pre>
|
@@ -10,7 +10,7 @@
|
|
10
10
|
<link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
|
11
11
|
</head>
|
12
12
|
<body class="standalone-code">
|
13
|
-
<pre><span class="ruby-comment cmt"># File lib/starfish.rb, line
|
13
|
+
<pre><span class="ruby-comment cmt"># File lib/starfish.rb, line 221</span>
|
14
14
|
<span class="ruby-keyword kw">def</span> <span class="ruby-identifier">options=</span>(<span class="ruby-identifier">options</span>={})
|
15
15
|
<span class="ruby-ivar">@@options</span>.<span class="ruby-identifier">update</span>(<span class="ruby-identifier">options</span>)
|
16
16
|
<span class="ruby-keyword kw">end</span></pre>
|
data/doc/created.rid
CHANGED
@@ -1 +1 @@
|
|
1
|
-
Tue
|
1
|
+
Tue Sep 19 17:18:16 PDT 2006
|
@@ -4,6 +4,7 @@ require_gem 'activerecord'
|
|
4
4
|
class MapReduce
|
5
5
|
module ActiveRecord
|
6
6
|
module Base
|
7
|
+
attr_accessor :order, :group, :joins, :include, :select, :readonly
|
7
8
|
attr_accessor :queue_size, :locked_queue_wait, :empty_queue_wait, :rescan_when_complete, :vigilant
|
8
9
|
|
9
10
|
class Client
|
@@ -37,7 +38,10 @@ class MapReduce
|
|
37
38
|
sleep @server_object.empty_queue_wait || 30
|
38
39
|
get_value_from(@server_object.get_id)
|
39
40
|
else
|
40
|
-
@type.find(object_id
|
41
|
+
@type.find(object_id,
|
42
|
+
:include => @server_object.include,
|
43
|
+
:readonly => @server_object.readonly
|
44
|
+
)
|
41
45
|
end
|
42
46
|
end
|
43
47
|
end
|
@@ -92,7 +96,17 @@ private
|
|
92
96
|
@lock = true
|
93
97
|
t = Time.now
|
94
98
|
|
95
|
-
@queue = type.find(:all,
|
99
|
+
@queue = type.find(:all,
|
100
|
+
:conditions => input,
|
101
|
+
:limit => @queue_size,
|
102
|
+
:offset => @offset,
|
103
|
+
:order => @order,
|
104
|
+
:group => @group,
|
105
|
+
:joins => @joins,
|
106
|
+
:include => @include,
|
107
|
+
:select => @select,
|
108
|
+
:readonly => @readonly
|
109
|
+
).map{|object|object.id}
|
96
110
|
|
97
111
|
@time_spent_grabbing_queues += (Time.now - t)
|
98
112
|
@num_queues_grabbed += 1
|
data/lib/map_reduce/file.rb
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
class MapReduce
|
2
2
|
module File
|
3
|
-
attr_accessor :queue_size, :locked_queue_wait, :empty_queue_wait, :rescan_when_complete, :vigilant, :lines_per_client
|
3
|
+
attr_accessor :offset, :queue_size, :locked_queue_wait, :empty_queue_wait, :rescan_when_complete, :vigilant, :lines_per_client
|
4
|
+
attr_reader :total
|
4
5
|
|
5
6
|
class Client
|
6
7
|
include DRbUndumped
|
@@ -62,15 +63,17 @@ private
|
|
62
63
|
case @offset
|
63
64
|
when 0
|
64
65
|
set_total
|
65
|
-
|
66
|
-
if @
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
66
|
+
else
|
67
|
+
if @offset >= @total
|
68
|
+
if @rescan_when_complete || @vigilant
|
69
|
+
set_total
|
70
|
+
else
|
71
|
+
begin
|
72
|
+
self.finished
|
73
|
+
rescue NameError
|
74
|
+
ensure
|
75
|
+
exit
|
76
|
+
end
|
74
77
|
end
|
75
78
|
end
|
76
79
|
end
|
@@ -84,14 +87,14 @@ private
|
|
84
87
|
file = ::File.open(input)
|
85
88
|
file.seek(@offset)
|
86
89
|
@queue_size.times do
|
87
|
-
@queue << file.gets
|
90
|
+
@queue << file.gets unless file.pos >= @total
|
88
91
|
end
|
89
92
|
@queue.compact!
|
90
93
|
|
91
94
|
@time_spent_grabbing_queues += (Time.now - t)
|
92
95
|
@num_queues_grabbed += 1
|
93
96
|
|
94
|
-
@offset
|
97
|
+
@offset = file.pos unless @queue.empty?
|
95
98
|
@offset = 0 if @offset == @total && @rescan_when_complete
|
96
99
|
@lock = false
|
97
100
|
end
|
data/lib/starfish.rb
CHANGED
@@ -23,8 +23,8 @@ end
|
|
23
23
|
|
24
24
|
class StarfishError < StandardError; end
|
25
25
|
class Starfish
|
26
|
-
VERSION = "1.1.
|
27
|
-
NULL = RUBY_PLATFORM =~ /mswin/ ? 'NUL' : '/dev/null'
|
26
|
+
VERSION = "1.1.3"
|
27
|
+
NULL = (RUBY_PLATFORM =~ /mswin/) ? 'NUL' : '/dev/null'
|
28
28
|
|
29
29
|
@@server = false
|
30
30
|
@@client = false
|
@@ -95,29 +95,34 @@ class Starfish
|
|
95
95
|
rescue NoMethodError
|
96
96
|
@called = @@client.call(@server_object)
|
97
97
|
end
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
98
|
+
|
99
|
+
begin
|
100
|
+
unless @called
|
101
|
+
if @server_object.map_reduce? && @server_object.valid?
|
102
|
+
map_reduce_client = eval("MapReduce::#{@server_object.base_type_to_s}::Client").new(@server_object)
|
102
103
|
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
104
|
+
$server_object = @server_object
|
105
|
+
Object.instance_eval do
|
106
|
+
define_method(:logger) do |*args|
|
107
|
+
$server_object._logger(*args)
|
108
|
+
end
|
109
|
+
define_method(:server) do
|
110
|
+
$server_object
|
111
|
+
end
|
110
112
|
end
|
111
|
-
end
|
112
113
|
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
114
|
+
map_reduce_client.each do |object|
|
115
|
+
t = Time.now
|
116
|
+
Timeout::timeout(@@options[:timeout] || 60) do
|
117
|
+
@@client.call(object)
|
118
|
+
end
|
119
|
+
@server_object.add_time_spent_processing_objects(Time.now-t)
|
120
|
+
end
|
121
|
+
else
|
122
|
+
raise MapReduceError, "invalid map reduce server (possibly missing type or input)"
|
117
123
|
end
|
118
|
-
else
|
119
|
-
raise MapReduceError, "invalid map reduce server (possibly missing type or input)"
|
120
124
|
end
|
125
|
+
rescue NoMethodError
|
121
126
|
end
|
122
127
|
end
|
123
128
|
|
metadata
CHANGED
@@ -3,8 +3,8 @@ rubygems_version: 0.8.11
|
|
3
3
|
specification_version: 1
|
4
4
|
name: starfish
|
5
5
|
version: !ruby/object:Gem::Version
|
6
|
-
version: 1.1.
|
7
|
-
date: 2006-
|
6
|
+
version: 1.1.3
|
7
|
+
date: 2006-09-19 00:00:00 -07:00
|
8
8
|
summary: Starfish is a utility to make distributed programming ridiculously easy.
|
9
9
|
require_paths:
|
10
10
|
- lib
|
@@ -36,7 +36,6 @@ files:
|
|
36
36
|
- lib/map_reduce/file.rb
|
37
37
|
- bin/starfish
|
38
38
|
- LICENSE
|
39
|
-
- NUL
|
40
39
|
- Rakefile
|
41
40
|
- README
|
42
41
|
- doc/classes
|
data/NUL
DELETED
File without changes
|