starfish 1.0.0 → 1.1.0
Sign up to get free protection for your applications and to get access to all the features.
- data/README +33 -2
- data/doc/classes/MapReduce.src/M000001.html +1 -1
- data/doc/classes/MapReduce.src/M000002.html +1 -1
- data/doc/classes/MapReduce.src/M000003.html +1 -1
- data/doc/classes/MapReduce.src/M000004.html +1 -1
- data/doc/classes/MapReduce.src/M000005.html +1 -1
- data/doc/classes/MapReduce.src/M000006.html +1 -1
- data/doc/classes/MapReduce.src/M000007.html +1 -1
- data/doc/classes/MapReduce.src/M000008.html +1 -1
- data/doc/classes/MapReduce.src/M000009.html +1 -1
- data/doc/classes/MapReduce.src/M000010.html +1 -1
- data/doc/classes/MapReduce.src/M000011.html +1 -1
- data/doc/classes/MapReduce.src/M000012.html +1 -1
- data/doc/classes/MapReduce.src/M000013.html +1 -1
- data/doc/classes/MapReduce.src/M000014.html +1 -1
- data/doc/classes/MapReduce/ActiveRecord/Base.src/M000015.html +6 -14
- data/doc/classes/MapReduce/File.html +70 -0
- data/doc/classes/MapReduce/File.src/M000019.html +42 -0
- data/doc/classes/MapReduce/File/Client.html +177 -0
- data/doc/classes/MapReduce/File/Client.src/M000020.html +18 -0
- data/doc/classes/MapReduce/File/Client.src/M000021.html +20 -0
- data/doc/classes/MapReduce/File/Client.src/M000022.html +18 -0
- data/doc/classes/Starfish.html +61 -62
- data/doc/classes/Starfish.src/M000023.html +5 -9
- data/doc/classes/Starfish.src/M000024.html +4 -22
- data/doc/classes/Starfish.src/M000025.html +29 -19
- data/doc/classes/Starfish.src/M000026.html +64 -4
- data/doc/classes/Starfish.src/M000027.html +9 -8
- data/doc/classes/Starfish.src/M000028.html +23 -5
- data/doc/classes/Starfish.src/M000029.html +22 -5
- data/doc/classes/Starfish.src/M000030.html +5 -5
- data/doc/classes/Starfish.src/M000031.html +22 -0
- data/doc/classes/Starfish.src/{M000020.html → M000032.html} +5 -5
- data/doc/classes/Starfish.src/M000033.html +18 -0
- data/doc/classes/Starfish.src/M000034.html +18 -0
- data/doc/classes/Starfish/RingFinger.html +6 -6
- data/doc/classes/Starfish/RingFinger.src/{M000031.html → M000035.html} +1 -1
- data/doc/created.rid +1 -1
- data/doc/files/README.html +52 -6
- data/doc/files/lib/map_reduce/active_record_rb.html +1 -2
- data/doc/files/lib/map_reduce/file_rb.html +1 -1
- data/doc/files/lib/map_reduce_rb.html +1 -1
- data/doc/files/lib/starfish_rb.html +1 -1
- data/doc/fr_class_index.html +1 -0
- data/doc/fr_method_index.html +17 -13
- data/examples/map_reduce/file.rb +2 -3
- data/lib/map_reduce.rb +2 -1
- data/lib/map_reduce/active_record.rb +25 -19
- data/lib/map_reduce/file.rb +93 -0
- data/lib/starfish.rb +9 -2
- metadata +15 -7
- data/doc/classes/Starfish.src/M000019.html +0 -19
- data/doc/classes/Starfish.src/M000021.html +0 -45
- data/doc/classes/Starfish.src/M000022.html +0 -71
@@ -5,18 +5,19 @@
|
|
5
5
|
|
6
6
|
<html>
|
7
7
|
<head>
|
8
|
-
<title>
|
8
|
+
<title>stats (Starfish)</title>
|
9
9
|
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
|
10
10
|
<link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
|
11
11
|
</head>
|
12
12
|
<body class="standalone-code">
|
13
|
-
<pre><span class="ruby-comment cmt"># File lib/starfish.rb, line
|
14
|
-
<span class="ruby-keyword kw">def</span> <span class="ruby-identifier">
|
15
|
-
<span class="ruby-identifier">
|
16
|
-
<span class="ruby-constant">
|
17
|
-
|
18
|
-
<span class="ruby-identifier">puts</span> <span class="ruby-
|
19
|
-
|
13
|
+
<pre><span class="ruby-comment cmt"># File lib/starfish.rb, line 130</span>
|
14
|
+
<span class="ruby-keyword kw">def</span> <span class="ruby-identifier">stats</span>
|
15
|
+
<span class="ruby-identifier">negotiate</span>
|
16
|
+
<span class="ruby-constant">Timeout</span><span class="ruby-operator">::</span><span class="ruby-identifier">timeout</span>(<span class="ruby-value">5</span>) { <span class="ruby-ivar">@server_object</span> = <span class="ruby-ivar">@ring_server</span>.<span class="ruby-identifier">read</span>([<span class="ruby-identifier">:name</span>, <span class="ruby-identifier">uniq</span>.<span class="ruby-identifier">intern</span>, <span class="ruby-keyword kw">nil</span>, <span class="ruby-keyword kw">nil</span>])[<span class="ruby-value">2</span>] }
|
17
|
+
|
18
|
+
<span class="ruby-identifier">puts</span> <span class="ruby-ivar">@server_object</span>.<span class="ruby-identifier">stats</span>.<span class="ruby-identifier">to_yaml</span>
|
19
|
+
<span class="ruby-keyword kw">rescue</span> <span class="ruby-constant">NoMethodError</span>
|
20
|
+
<span class="ruby-identifier">$stderr</span>.<span class="ruby-identifier">puts</span> <span class="ruby-value str">"The stats method is not defined for your server"</span>
|
20
21
|
<span class="ruby-keyword kw">end</span></pre>
|
21
22
|
</body>
|
22
23
|
</html>
|
@@ -5,14 +5,32 @@
|
|
5
5
|
|
6
6
|
<html>
|
7
7
|
<head>
|
8
|
-
<title>
|
8
|
+
<title>sanitize (Starfish)</title>
|
9
9
|
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
|
10
10
|
<link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
|
11
11
|
</head>
|
12
12
|
<body class="standalone-code">
|
13
|
-
<pre><span class="ruby-comment cmt"># File lib/starfish.rb, line
|
14
|
-
|
15
|
-
|
16
|
-
|
13
|
+
<pre><span class="ruby-comment cmt"># File lib/starfish.rb, line 139</span>
|
14
|
+
<span class="ruby-keyword kw">def</span> <span class="ruby-identifier">sanitize</span>(<span class="ruby-identifier">object</span>)
|
15
|
+
<span class="ruby-identifier">object</span>.<span class="ruby-identifier">extend</span> <span class="ruby-constant">DRbUndumped</span>
|
16
|
+
|
17
|
+
<span class="ruby-ivar">@@log</span> = <span class="ruby-keyword kw">case</span> <span class="ruby-ivar">@@options</span>[<span class="ruby-identifier">:log</span>]
|
18
|
+
<span class="ruby-keyword kw">when</span> <span class="ruby-constant">String</span>
|
19
|
+
<span class="ruby-constant">Logger</span>.<span class="ruby-identifier">new</span>(<span class="ruby-ivar">@@options</span>[<span class="ruby-identifier">:log</span>])
|
20
|
+
<span class="ruby-keyword kw">when</span> <span class="ruby-constant">Class</span>
|
21
|
+
<span class="ruby-ivar">@@options</span>[<span class="ruby-identifier">:log</span>].<span class="ruby-identifier">new</span>
|
22
|
+
<span class="ruby-keyword kw">when</span> <span class="ruby-keyword kw">nil</span>, <span class="ruby-keyword kw">false</span>
|
23
|
+
<span class="ruby-constant">Logger</span>.<span class="ruby-identifier">new</span>(<span class="ruby-value str">"/dev/null"</span>)
|
24
|
+
<span class="ruby-keyword kw">else</span>
|
25
|
+
<span class="ruby-ivar">@@options</span>[<span class="ruby-identifier">:log</span>]
|
26
|
+
<span class="ruby-keyword kw">end</span>
|
27
|
+
|
28
|
+
<span class="ruby-keyword kw">def</span> <span class="ruby-identifier">object</span>.<span class="ruby-identifier">logger</span>
|
29
|
+
<span class="ruby-ivar">@logger</span> <span class="ruby-operator">||=</span> <span class="ruby-ivar">@@log</span>
|
30
|
+
<span class="ruby-keyword kw">end</span>
|
31
|
+
<span class="ruby-keyword kw">def</span> <span class="ruby-identifier">object</span>.<span class="ruby-identifier">_logger</span>
|
32
|
+
<span class="ruby-ivar">@logger</span> <span class="ruby-operator">||=</span> <span class="ruby-ivar">@@log</span>
|
33
|
+
<span class="ruby-keyword kw">end</span>
|
34
|
+
<span class="ruby-keyword kw">end</span></pre>
|
17
35
|
</body>
|
18
36
|
</html>
|
@@ -5,14 +5,31 @@
|
|
5
5
|
|
6
6
|
<html>
|
7
7
|
<head>
|
8
|
-
<title>
|
8
|
+
<title>negotiate (Starfish)</title>
|
9
9
|
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
|
10
10
|
<link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
|
11
11
|
</head>
|
12
12
|
<body class="standalone-code">
|
13
|
-
<pre><span class="ruby-comment cmt"># File lib/starfish.rb, line
|
14
|
-
|
15
|
-
|
16
|
-
|
13
|
+
<pre><span class="ruby-comment cmt"># File lib/starfish.rb, line 161</span>
|
14
|
+
<span class="ruby-keyword kw">def</span> <span class="ruby-identifier">negotiate</span><span class="ruby-identifier">negotiate</span><span class="ruby-identifier">negotiate</span>
|
15
|
+
<span class="ruby-keyword kw">begin</span>
|
16
|
+
<span class="ruby-ivar">@ring_server</span> = <span class="ruby-constant">RingFinger</span>.<span class="ruby-identifier">primary</span>
|
17
|
+
<span class="ruby-keyword kw">rescue</span> <span class="ruby-constant">RuntimeError</span> =<span class="ruby-operator">></span> <span class="ruby-identifier">m</span>
|
18
|
+
<span class="ruby-comment cmt"># allow multiple un-cached calls to RingFinger.finger</span>
|
19
|
+
<span class="ruby-keyword kw">def</span> <span class="ruby-constant">RingFinger</span>.<span class="ruby-identifier">finger</span>
|
20
|
+
<span class="ruby-ivar">@@finger</span> = <span class="ruby-keyword kw">self</span>.<span class="ruby-identifier">new</span>
|
21
|
+
<span class="ruby-ivar">@@finger</span>.<span class="ruby-identifier">lookup_ring_any</span>
|
22
|
+
<span class="ruby-ivar">@@finger</span>
|
23
|
+
<span class="ruby-keyword kw">end</span>
|
24
|
+
|
25
|
+
<span class="ruby-identifier">spawn</span>
|
26
|
+
<span class="ruby-ivar">@retry_count</span> <span class="ruby-operator">+=</span> <span class="ruby-value">1</span>
|
27
|
+
<span class="ruby-keyword kw">if</span> <span class="ruby-ivar">@retry_count</span> <span class="ruby-operator"><=</span> <span class="ruby-value">5</span>
|
28
|
+
<span class="ruby-keyword kw">retry</span>
|
29
|
+
<span class="ruby-keyword kw">else</span>
|
30
|
+
<span class="ruby-identifier">raise</span> <span class="ruby-constant">RuntimeError</span>, <span class="ruby-identifier">m</span>
|
31
|
+
<span class="ruby-keyword kw">end</span>
|
32
|
+
<span class="ruby-keyword kw">end</span>
|
33
|
+
<span class="ruby-keyword kw">end</span></pre>
|
17
34
|
</body>
|
18
35
|
</html>
|
@@ -5,14 +5,14 @@
|
|
5
5
|
|
6
6
|
<html>
|
7
7
|
<head>
|
8
|
-
<title>
|
8
|
+
<title>spawn (Starfish)</title>
|
9
9
|
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
|
10
10
|
<link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
|
11
11
|
</head>
|
12
12
|
<body class="standalone-code">
|
13
|
-
<pre><span class="ruby-comment cmt"># File lib/starfish.rb, line
|
14
|
-
|
15
|
-
|
16
|
-
|
13
|
+
<pre><span class="ruby-comment cmt"># File lib/starfish.rb, line 182</span>
|
14
|
+
<span class="ruby-keyword kw">def</span> <span class="ruby-identifier">spawn</span>
|
15
|
+
<span class="ruby-ivar">@started</span> <span class="ruby-operator">||=</span> <span class="ruby-identifier">fork</span> { <span class="ruby-identifier">system</span>(<span class="ruby-node">"ruby #{File.dirname(__FILE__)}/../bin/starfish #{@uniq} server > /dev/null"</span>) }
|
16
|
+
<span class="ruby-keyword kw">end</span></pre>
|
17
17
|
</body>
|
18
18
|
</html>
|
@@ -0,0 +1,22 @@
|
|
1
|
+
<?xml version="1.0" encoding="iso-8859-1"?>
|
2
|
+
<!DOCTYPE html
|
3
|
+
PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
|
4
|
+
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
|
5
|
+
|
6
|
+
<html>
|
7
|
+
<head>
|
8
|
+
<title>stop (Starfish)</title>
|
9
|
+
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
|
10
|
+
<link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
|
11
|
+
</head>
|
12
|
+
<body class="standalone-code">
|
13
|
+
<pre><span class="ruby-comment cmt"># File lib/starfish.rb, line 186</span>
|
14
|
+
<span class="ruby-keyword kw">def</span> <span class="ruby-identifier">stop</span>
|
15
|
+
<span class="ruby-identifier">puts</span> <span class="ruby-value str">"stopping the server"</span>
|
16
|
+
<span class="ruby-constant">Process</span>.<span class="ruby-identifier">kill</span>(<span class="ruby-value str">"SIGHUP"</span>, <span class="ruby-constant">IO</span>.<span class="ruby-identifier">read</span>(<span class="ruby-ivar">@@options</span>[<span class="ruby-identifier">:pid</span>] <span class="ruby-operator">||</span> <span class="ruby-node">"/tmp/starfish-#{uniq}.pid"</span>).<span class="ruby-identifier">to_i</span>)
|
17
|
+
<span class="ruby-keyword kw">rescue</span> <span class="ruby-constant">Errno</span><span class="ruby-operator">::</span><span class="ruby-constant">ENOENT</span>
|
18
|
+
<span class="ruby-identifier">puts</span> <span class="ruby-value str">"Fatal error, please kill all starfish processes manually and try again"</span>
|
19
|
+
<span class="ruby-identifier">system</span>(<span class="ruby-value str">"ps auxww|grep starfish"</span>)
|
20
|
+
<span class="ruby-keyword kw">end</span></pre>
|
21
|
+
</body>
|
22
|
+
</html>
|
@@ -5,14 +5,14 @@
|
|
5
5
|
|
6
6
|
<html>
|
7
7
|
<head>
|
8
|
-
<title>
|
8
|
+
<title>server= (Starfish)</title>
|
9
9
|
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
|
10
10
|
<link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
|
11
11
|
</head>
|
12
12
|
<body class="standalone-code">
|
13
|
-
<pre><span class="ruby-comment cmt"># File lib/starfish.rb, line
|
14
|
-
|
15
|
-
|
16
|
-
|
13
|
+
<pre><span class="ruby-comment cmt"># File lib/starfish.rb, line 195</span>
|
14
|
+
<span class="ruby-keyword kw">def</span> <span class="ruby-identifier">server=</span>(<span class="ruby-identifier">server</span>)
|
15
|
+
<span class="ruby-ivar">@@server</span> = <span class="ruby-identifier">server</span>
|
16
|
+
<span class="ruby-keyword kw">end</span></pre>
|
17
17
|
</body>
|
18
18
|
</html>
|
@@ -0,0 +1,18 @@
|
|
1
|
+
<?xml version="1.0" encoding="iso-8859-1"?>
|
2
|
+
<!DOCTYPE html
|
3
|
+
PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
|
4
|
+
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
|
5
|
+
|
6
|
+
<html>
|
7
|
+
<head>
|
8
|
+
<title>client= (Starfish)</title>
|
9
|
+
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
|
10
|
+
<link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
|
11
|
+
</head>
|
12
|
+
<body class="standalone-code">
|
13
|
+
<pre><span class="ruby-comment cmt"># File lib/starfish.rb, line 199</span>
|
14
|
+
<span class="ruby-keyword kw">def</span> <span class="ruby-identifier">client=</span>(<span class="ruby-identifier">client</span>)
|
15
|
+
<span class="ruby-ivar">@@client</span> = <span class="ruby-identifier">client</span>
|
16
|
+
<span class="ruby-keyword kw">end</span></pre>
|
17
|
+
</body>
|
18
|
+
</html>
|
@@ -0,0 +1,18 @@
|
|
1
|
+
<?xml version="1.0" encoding="iso-8859-1"?>
|
2
|
+
<!DOCTYPE html
|
3
|
+
PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
|
4
|
+
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
|
5
|
+
|
6
|
+
<html>
|
7
|
+
<head>
|
8
|
+
<title>options= (Starfish)</title>
|
9
|
+
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
|
10
|
+
<link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
|
11
|
+
</head>
|
12
|
+
<body class="standalone-code">
|
13
|
+
<pre><span class="ruby-comment cmt"># File lib/starfish.rb, line 203</span>
|
14
|
+
<span class="ruby-keyword kw">def</span> <span class="ruby-identifier">options=</span>(<span class="ruby-identifier">options</span>={})
|
15
|
+
<span class="ruby-ivar">@@options</span>.<span class="ruby-identifier">update</span>(<span class="ruby-identifier">options</span>)
|
16
|
+
<span class="ruby-keyword kw">end</span></pre>
|
17
|
+
</body>
|
18
|
+
</html>
|
@@ -76,7 +76,7 @@
|
|
76
76
|
<h3 class="section-bar">Methods</h3>
|
77
77
|
|
78
78
|
<div class="name-list">
|
79
|
-
<a href="#
|
79
|
+
<a href="#M000035">finger</a>
|
80
80
|
</div>
|
81
81
|
</div>
|
82
82
|
|
@@ -98,12 +98,12 @@
|
|
98
98
|
<div id="methods">
|
99
99
|
<h3 class="section-bar">Public Class methods</h3>
|
100
100
|
|
101
|
-
<div id="method-
|
102
|
-
<a name="
|
101
|
+
<div id="method-M000035" class="method-detail">
|
102
|
+
<a name="M000035"></a>
|
103
103
|
|
104
104
|
<div class="method-heading">
|
105
|
-
<a href="RingFinger.src/
|
106
|
-
onclick="popupCode('RingFinger.src/
|
105
|
+
<a href="RingFinger.src/M000035.html" target="Code" class="method-signature"
|
106
|
+
onclick="popupCode('RingFinger.src/M000035.html');return false;">
|
107
107
|
<span class="method-name">finger</span><span class="method-args">()</span>
|
108
108
|
</a>
|
109
109
|
</div>
|
@@ -111,7 +111,7 @@
|
|
111
111
|
<div class="method-description">
|
112
112
|
<p>
|
113
113
|
allow multiple un-cached calls to <a
|
114
|
-
href="RingFinger.html#
|
114
|
+
href="RingFinger.html#M000035">RingFinger.finger</a>
|
115
115
|
</p>
|
116
116
|
</div>
|
117
117
|
</div>
|
@@ -10,7 +10,7 @@
|
|
10
10
|
<link rel="stylesheet" href="../../.././rdoc-style.css" type="text/css" media="screen" />
|
11
11
|
</head>
|
12
12
|
<body class="standalone-code">
|
13
|
-
<pre><span class="ruby-comment cmt"># File lib/starfish.rb, line
|
13
|
+
<pre><span class="ruby-comment cmt"># File lib/starfish.rb, line 166</span>
|
14
14
|
<span class="ruby-keyword kw">def</span> <span class="ruby-constant">RingFinger</span>.<span class="ruby-identifier">finger</span>
|
15
15
|
<span class="ruby-ivar">@@finger</span> = <span class="ruby-keyword kw">self</span>.<span class="ruby-identifier">new</span>
|
16
16
|
<span class="ruby-ivar">@@finger</span>.<span class="ruby-identifier">lookup_ring_any</span>
|
data/doc/created.rid
CHANGED
@@ -1 +1 @@
|
|
1
|
-
|
1
|
+
Fri Aug 18 23:00:34 PDT 2006
|
data/doc/files/README.html
CHANGED
@@ -56,7 +56,7 @@
|
|
56
56
|
</tr>
|
57
57
|
<tr class="top-aligned-row">
|
58
58
|
<td><strong>Last Update:</strong></td>
|
59
|
-
<td>
|
59
|
+
<td>Fri Aug 18 22:59:00 PDT 2006</td>
|
60
60
|
</tr>
|
61
61
|
</table>
|
62
62
|
</div>
|
@@ -71,10 +71,8 @@
|
|
71
71
|
<div id="description">
|
72
72
|
<h2>Welcome to <a href="../classes/Starfish.html">Starfish</a></h2>
|
73
73
|
<p>
|
74
|
-
<a href="../classes/Starfish.html">Starfish</a> is a utility
|
75
|
-
|
76
|
-
href="http://mog.com">mog.com</a>) to make distributed programming
|
77
|
-
ridiculously easy.
|
74
|
+
<a href="../classes/Starfish.html">Starfish</a> is a utility to make
|
75
|
+
distributed programming ridiculously easy.
|
78
76
|
</p>
|
79
77
|
<h2>Download</h2>
|
80
78
|
<ul>
|
@@ -89,6 +87,17 @@ href="http://rubyforge.org/projects/starfish">rubyforge.org/projects/starfish</a
|
|
89
87
|
|
90
88
|
</li>
|
91
89
|
</ul>
|
90
|
+
<h2>Background</h2>
|
91
|
+
<p>
|
92
|
+
<a href="../classes/Starfish.html">Starfish</a> was born by abstracting
|
93
|
+
code from a highly popular website (<a href="http://mog.com">mog.com</a>).
|
94
|
+
I had written distributed programming code for various tasks including
|
95
|
+
database corruption cleanup as well as real-time indexing requirements and
|
96
|
+
decided that nobody should have to write as much code as I did to do those
|
97
|
+
tasks. The general idea of distributed programming is vastly underused by
|
98
|
+
most programmers and <a href="../classes/Starfish.html">Starfish</a> hides
|
99
|
+
everything you don’t have to worry about.
|
100
|
+
</p>
|
92
101
|
<h2>Usage</h2>
|
93
102
|
<p>
|
94
103
|
Make a file and define the server and client. What follows is a simple
|
@@ -146,7 +155,7 @@ The :log parameter is intelligent and can take various other formats
|
|
146
155
|
including nil for no logging or instances of any kind of logger class you
|
147
156
|
prefer to use.
|
148
157
|
</p>
|
149
|
-
<h2><a href="../classes/MapReduce.html">MapReduce</a
|
158
|
+
<h2><a href="../classes/MapReduce.html">MapReduce</a> with ActiveRecord</h2>
|
150
159
|
<p>
|
151
160
|
The crown jewel of <a href="../classes/Starfish.html">Starfish</a> is
|
152
161
|
actually a library built on <a href="../classes/Starfish.html">Starfish</a>
|
@@ -287,6 +296,43 @@ And you will be returned a YAML hash of statistics.
|
|
287
296
|
time_spent_grabbing_queues: 0.090328
|
288
297
|
num_objects_grabbed: 10
|
289
298
|
</pre>
|
299
|
+
<h2><a href="../classes/MapReduce.html">MapReduce</a> with Files</h2>
|
300
|
+
<p>
|
301
|
+
You can also divide and conquer with files. The idea is that you have a
|
302
|
+
large file and want to process line-by-line as quickly as possible. It is
|
303
|
+
simple to accomplish this with <a
|
304
|
+
href="../classes/Starfish.html">Starfish</a>.
|
305
|
+
</p>
|
306
|
+
<pre>
|
307
|
+
server do |map_reduce|
|
308
|
+
map_reduce.type = File
|
309
|
+
map_reduce.input = "/tmp/big_log_file"
|
310
|
+
end
|
311
|
+
|
312
|
+
client do |line|
|
313
|
+
if line =~ /some_regex/
|
314
|
+
logger.info(line)
|
315
|
+
end
|
316
|
+
end
|
317
|
+
</pre>
|
318
|
+
<p>
|
319
|
+
Simply replacing the type and input lets you process your file in a
|
320
|
+
distributed way. Like when using ActiveRecord, you also have options:
|
321
|
+
</p>
|
322
|
+
<pre>
|
323
|
+
server do |map_reduce|
|
324
|
+
map_reduce.type = File
|
325
|
+
map_reduce.input = "/tmp/big_log_file"
|
326
|
+
map_reduce.queue_size = 1000 # how many lines of the file to buffer at a time
|
327
|
+
map_reduce.lines_per_client = 100 # how many lines each client will process at a time
|
328
|
+
map_reduce.rescan_when_complete = true
|
329
|
+
end
|
330
|
+
</pre>
|
331
|
+
<p>
|
332
|
+
For a file, rescan_when_complete goes back to the beginning of the file
|
333
|
+
when you have finished processing it. You can also use the vigilant option
|
334
|
+
to wait for new data to be added to the file.
|
335
|
+
</p>
|
290
336
|
<h2>Examples</h2>
|
291
337
|
<p>
|
292
338
|
See the examples/ directory.
|
@@ -56,7 +56,7 @@
|
|
56
56
|
</tr>
|
57
57
|
<tr class="top-aligned-row">
|
58
58
|
<td><strong>Last Update:</strong></td>
|
59
|
-
<td>
|
59
|
+
<td>Fri Aug 18 17:01:38 PDT 2006</td>
|
60
60
|
</tr>
|
61
61
|
</table>
|
62
62
|
</div>
|
@@ -74,7 +74,6 @@
|
|
74
74
|
|
75
75
|
<div class="name-list">
|
76
76
|
rubygems
|
77
|
-
active_record
|
78
77
|
</div>
|
79
78
|
</div>
|
80
79
|
|
data/doc/fr_class_index.html
CHANGED
@@ -26,6 +26,7 @@
|
|
26
26
|
<a href="classes/MapReduce/ActiveRecord/Base/Client.html">MapReduce::ActiveRecord::Base::Client</a><br />
|
27
27
|
<a href="classes/MapReduce/Array.html">MapReduce::Array</a><br />
|
28
28
|
<a href="classes/MapReduce/File.html">MapReduce::File</a><br />
|
29
|
+
<a href="classes/MapReduce/File/Client.html">MapReduce::File::Client</a><br />
|
29
30
|
<a href="classes/MapReduceError.html">MapReduceError</a><br />
|
30
31
|
<a href="classes/Starfish.html">Starfish</a><br />
|
31
32
|
<a href="classes/Starfish/RingFinger.html">Starfish::RingFinger</a><br />
|
data/doc/fr_method_index.html
CHANGED
@@ -23,31 +23,35 @@
|
|
23
23
|
<a href="classes/MapReduce.html#M000002">add_time_spent_processing_objects (MapReduce)</a><br />
|
24
24
|
<a href="classes/MapReduce.html#M000014">base_type (MapReduce)</a><br />
|
25
25
|
<a href="classes/MapReduce.html#M000005">base_type_to_s (MapReduce)</a><br />
|
26
|
-
<a href="classes/Starfish.html#
|
27
|
-
<a href="classes/Starfish.html#
|
26
|
+
<a href="classes/Starfish.html#M000026">client (Starfish)</a><br />
|
27
|
+
<a href="classes/Starfish.html#M000033">client= (Starfish)</a><br />
|
28
|
+
<a href="classes/MapReduce/File/Client.html#M000021">each (MapReduce::File::Client)</a><br />
|
28
29
|
<a href="classes/MapReduce/ActiveRecord/Base/Client.html#M000017">each (MapReduce::ActiveRecord::Base::Client)</a><br />
|
29
|
-
<a href="classes/Starfish/RingFinger.html#
|
30
|
+
<a href="classes/Starfish/RingFinger.html#M000035">finger (Starfish::RingFinger)</a><br />
|
30
31
|
<a href="classes/MapReduce/ActiveRecord/Base.html#M000015">get_id (MapReduce::ActiveRecord::Base)</a><br />
|
32
|
+
<a href="classes/MapReduce/File.html#M000019">get_lines (MapReduce::File)</a><br />
|
33
|
+
<a href="classes/MapReduce/File/Client.html#M000022">logger (MapReduce::File::Client)</a><br />
|
31
34
|
<a href="classes/MapReduce/ActiveRecord/Base/Client.html#M000018">logger (MapReduce::ActiveRecord::Base::Client)</a><br />
|
32
35
|
<a href="classes/MapReduce.html#M000013">map_reduce? (MapReduce)</a><br />
|
33
36
|
<a href="classes/MapReduce.html#M000011">method_missing (MapReduce)</a><br />
|
34
|
-
<a href="classes/Starfish.html#
|
37
|
+
<a href="classes/Starfish.html#M000029">negotiate (Starfish)</a><br />
|
35
38
|
<a href="classes/MapReduce/ActiveRecord/Base/Client.html#M000016">new (MapReduce::ActiveRecord::Base::Client)</a><br />
|
39
|
+
<a href="classes/MapReduce/File/Client.html#M000020">new (MapReduce::File::Client)</a><br />
|
40
|
+
<a href="classes/Starfish.html#M000023">new (Starfish)</a><br />
|
36
41
|
<a href="classes/MapReduce.html#M000001">new (MapReduce)</a><br />
|
37
|
-
<a href="classes/Starfish.html#
|
38
|
-
<a href="classes/Starfish.html#M000030">options= (Starfish)</a><br />
|
42
|
+
<a href="classes/Starfish.html#M000034">options= (Starfish)</a><br />
|
39
43
|
<a href="classes/MapReduce.html#M000012">raise_if_invalid! (MapReduce)</a><br />
|
40
|
-
<a href="classes/Starfish.html#
|
41
|
-
<a href="classes/Starfish.html#
|
42
|
-
<a href="classes/Starfish.html#
|
43
|
-
<a href="classes/Starfish.html#
|
44
|
+
<a href="classes/Starfish.html#M000028">sanitize (Starfish)</a><br />
|
45
|
+
<a href="classes/Starfish.html#M000025">server (Starfish)</a><br />
|
46
|
+
<a href="classes/Starfish.html#M000032">server= (Starfish)</a><br />
|
47
|
+
<a href="classes/Starfish.html#M000030">spawn (Starfish)</a><br />
|
44
48
|
<a href="classes/MapReduce.html#M000007">spool= (MapReduce)</a><br />
|
49
|
+
<a href="classes/Starfish.html#M000027">stats (Starfish)</a><br />
|
45
50
|
<a href="classes/MapReduce.html#M000003">stats (MapReduce)</a><br />
|
46
|
-
<a href="classes/Starfish.html#
|
47
|
-
<a href="classes/Starfish.html#M000027">stop (Starfish)</a><br />
|
51
|
+
<a href="classes/Starfish.html#M000031">stop (Starfish)</a><br />
|
48
52
|
<a href="classes/MapReduce.html#M000004">type= (MapReduce)</a><br />
|
49
53
|
<a href="classes/MapReduce.html#M000006">type_to_s (MapReduce)</a><br />
|
50
|
-
<a href="classes/Starfish.html#
|
54
|
+
<a href="classes/Starfish.html#M000024">uniq (Starfish)</a><br />
|
51
55
|
<a href="classes/MapReduce.html#M000008">valid? (MapReduce)</a><br />
|
52
56
|
<a href="classes/MapReduce.html#M000010">valid_input? (MapReduce)</a><br />
|
53
57
|
<a href="classes/MapReduce.html#M000009">valid_type? (MapReduce)</a><br />
|