starfish 1.0.0 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README +33 -2
- data/doc/classes/MapReduce.src/M000001.html +1 -1
- data/doc/classes/MapReduce.src/M000002.html +1 -1
- data/doc/classes/MapReduce.src/M000003.html +1 -1
- data/doc/classes/MapReduce.src/M000004.html +1 -1
- data/doc/classes/MapReduce.src/M000005.html +1 -1
- data/doc/classes/MapReduce.src/M000006.html +1 -1
- data/doc/classes/MapReduce.src/M000007.html +1 -1
- data/doc/classes/MapReduce.src/M000008.html +1 -1
- data/doc/classes/MapReduce.src/M000009.html +1 -1
- data/doc/classes/MapReduce.src/M000010.html +1 -1
- data/doc/classes/MapReduce.src/M000011.html +1 -1
- data/doc/classes/MapReduce.src/M000012.html +1 -1
- data/doc/classes/MapReduce.src/M000013.html +1 -1
- data/doc/classes/MapReduce.src/M000014.html +1 -1
- data/doc/classes/MapReduce/ActiveRecord/Base.src/M000015.html +6 -14
- data/doc/classes/MapReduce/File.html +70 -0
- data/doc/classes/MapReduce/File.src/M000019.html +42 -0
- data/doc/classes/MapReduce/File/Client.html +177 -0
- data/doc/classes/MapReduce/File/Client.src/M000020.html +18 -0
- data/doc/classes/MapReduce/File/Client.src/M000021.html +20 -0
- data/doc/classes/MapReduce/File/Client.src/M000022.html +18 -0
- data/doc/classes/Starfish.html +61 -62
- data/doc/classes/Starfish.src/M000023.html +5 -9
- data/doc/classes/Starfish.src/M000024.html +4 -22
- data/doc/classes/Starfish.src/M000025.html +29 -19
- data/doc/classes/Starfish.src/M000026.html +64 -4
- data/doc/classes/Starfish.src/M000027.html +9 -8
- data/doc/classes/Starfish.src/M000028.html +23 -5
- data/doc/classes/Starfish.src/M000029.html +22 -5
- data/doc/classes/Starfish.src/M000030.html +5 -5
- data/doc/classes/Starfish.src/M000031.html +22 -0
- data/doc/classes/Starfish.src/{M000020.html → M000032.html} +5 -5
- data/doc/classes/Starfish.src/M000033.html +18 -0
- data/doc/classes/Starfish.src/M000034.html +18 -0
- data/doc/classes/Starfish/RingFinger.html +6 -6
- data/doc/classes/Starfish/RingFinger.src/{M000031.html → M000035.html} +1 -1
- data/doc/created.rid +1 -1
- data/doc/files/README.html +52 -6
- data/doc/files/lib/map_reduce/active_record_rb.html +1 -2
- data/doc/files/lib/map_reduce/file_rb.html +1 -1
- data/doc/files/lib/map_reduce_rb.html +1 -1
- data/doc/files/lib/starfish_rb.html +1 -1
- data/doc/fr_class_index.html +1 -0
- data/doc/fr_method_index.html +17 -13
- data/examples/map_reduce/file.rb +2 -3
- data/lib/map_reduce.rb +2 -1
- data/lib/map_reduce/active_record.rb +25 -19
- data/lib/map_reduce/file.rb +93 -0
- data/lib/starfish.rb +9 -2
- metadata +15 -7
- data/doc/classes/Starfish.src/M000019.html +0 -19
- data/doc/classes/Starfish.src/M000021.html +0 -45
- data/doc/classes/Starfish.src/M000022.html +0 -71
@@ -5,18 +5,19 @@
|
|
5
5
|
|
6
6
|
<html>
|
7
7
|
<head>
|
8
|
-
<title>
|
8
|
+
<title>stats (Starfish)</title>
|
9
9
|
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
|
10
10
|
<link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
|
11
11
|
</head>
|
12
12
|
<body class="standalone-code">
|
13
|
-
<pre><span class="ruby-comment cmt"># File lib/starfish.rb, line
|
14
|
-
<span class="ruby-keyword kw">def</span> <span class="ruby-identifier">
|
15
|
-
<span class="ruby-identifier">
|
16
|
-
<span class="ruby-constant">
|
17
|
-
|
18
|
-
<span class="ruby-identifier">puts</span> <span class="ruby-
|
19
|
-
|
13
|
+
<pre><span class="ruby-comment cmt"># File lib/starfish.rb, line 130</span>
|
14
|
+
<span class="ruby-keyword kw">def</span> <span class="ruby-identifier">stats</span>
|
15
|
+
<span class="ruby-identifier">negotiate</span>
|
16
|
+
<span class="ruby-constant">Timeout</span><span class="ruby-operator">::</span><span class="ruby-identifier">timeout</span>(<span class="ruby-value">5</span>) { <span class="ruby-ivar">@server_object</span> = <span class="ruby-ivar">@ring_server</span>.<span class="ruby-identifier">read</span>([<span class="ruby-identifier">:name</span>, <span class="ruby-identifier">uniq</span>.<span class="ruby-identifier">intern</span>, <span class="ruby-keyword kw">nil</span>, <span class="ruby-keyword kw">nil</span>])[<span class="ruby-value">2</span>] }
|
17
|
+
|
18
|
+
<span class="ruby-identifier">puts</span> <span class="ruby-ivar">@server_object</span>.<span class="ruby-identifier">stats</span>.<span class="ruby-identifier">to_yaml</span>
|
19
|
+
<span class="ruby-keyword kw">rescue</span> <span class="ruby-constant">NoMethodError</span>
|
20
|
+
<span class="ruby-identifier">$stderr</span>.<span class="ruby-identifier">puts</span> <span class="ruby-value str">"The stats method is not defined for your server"</span>
|
20
21
|
<span class="ruby-keyword kw">end</span></pre>
|
21
22
|
</body>
|
22
23
|
</html>
|
@@ -5,14 +5,32 @@
|
|
5
5
|
|
6
6
|
<html>
|
7
7
|
<head>
|
8
|
-
<title>
|
8
|
+
<title>sanitize (Starfish)</title>
|
9
9
|
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
|
10
10
|
<link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
|
11
11
|
</head>
|
12
12
|
<body class="standalone-code">
|
13
|
-
<pre><span class="ruby-comment cmt"># File lib/starfish.rb, line
|
14
|
-
|
15
|
-
|
16
|
-
|
13
|
+
<pre><span class="ruby-comment cmt"># File lib/starfish.rb, line 139</span>
|
14
|
+
<span class="ruby-keyword kw">def</span> <span class="ruby-identifier">sanitize</span>(<span class="ruby-identifier">object</span>)
|
15
|
+
<span class="ruby-identifier">object</span>.<span class="ruby-identifier">extend</span> <span class="ruby-constant">DRbUndumped</span>
|
16
|
+
|
17
|
+
<span class="ruby-ivar">@@log</span> = <span class="ruby-keyword kw">case</span> <span class="ruby-ivar">@@options</span>[<span class="ruby-identifier">:log</span>]
|
18
|
+
<span class="ruby-keyword kw">when</span> <span class="ruby-constant">String</span>
|
19
|
+
<span class="ruby-constant">Logger</span>.<span class="ruby-identifier">new</span>(<span class="ruby-ivar">@@options</span>[<span class="ruby-identifier">:log</span>])
|
20
|
+
<span class="ruby-keyword kw">when</span> <span class="ruby-constant">Class</span>
|
21
|
+
<span class="ruby-ivar">@@options</span>[<span class="ruby-identifier">:log</span>].<span class="ruby-identifier">new</span>
|
22
|
+
<span class="ruby-keyword kw">when</span> <span class="ruby-keyword kw">nil</span>, <span class="ruby-keyword kw">false</span>
|
23
|
+
<span class="ruby-constant">Logger</span>.<span class="ruby-identifier">new</span>(<span class="ruby-value str">"/dev/null"</span>)
|
24
|
+
<span class="ruby-keyword kw">else</span>
|
25
|
+
<span class="ruby-ivar">@@options</span>[<span class="ruby-identifier">:log</span>]
|
26
|
+
<span class="ruby-keyword kw">end</span>
|
27
|
+
|
28
|
+
<span class="ruby-keyword kw">def</span> <span class="ruby-identifier">object</span>.<span class="ruby-identifier">logger</span>
|
29
|
+
<span class="ruby-ivar">@logger</span> <span class="ruby-operator">||=</span> <span class="ruby-ivar">@@log</span>
|
30
|
+
<span class="ruby-keyword kw">end</span>
|
31
|
+
<span class="ruby-keyword kw">def</span> <span class="ruby-identifier">object</span>.<span class="ruby-identifier">_logger</span>
|
32
|
+
<span class="ruby-ivar">@logger</span> <span class="ruby-operator">||=</span> <span class="ruby-ivar">@@log</span>
|
33
|
+
<span class="ruby-keyword kw">end</span>
|
34
|
+
<span class="ruby-keyword kw">end</span></pre>
|
17
35
|
</body>
|
18
36
|
</html>
|
@@ -5,14 +5,31 @@
|
|
5
5
|
|
6
6
|
<html>
|
7
7
|
<head>
|
8
|
-
<title>
|
8
|
+
<title>negotiate (Starfish)</title>
|
9
9
|
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
|
10
10
|
<link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
|
11
11
|
</head>
|
12
12
|
<body class="standalone-code">
|
13
|
-
<pre><span class="ruby-comment cmt"># File lib/starfish.rb, line
|
14
|
-
|
15
|
-
|
16
|
-
|
13
|
+
<pre><span class="ruby-comment cmt"># File lib/starfish.rb, line 161</span>
|
14
|
+
<span class="ruby-keyword kw">def</span> <span class="ruby-identifier">negotiate</span><span class="ruby-identifier">negotiate</span><span class="ruby-identifier">negotiate</span>
|
15
|
+
<span class="ruby-keyword kw">begin</span>
|
16
|
+
<span class="ruby-ivar">@ring_server</span> = <span class="ruby-constant">RingFinger</span>.<span class="ruby-identifier">primary</span>
|
17
|
+
<span class="ruby-keyword kw">rescue</span> <span class="ruby-constant">RuntimeError</span> =<span class="ruby-operator">></span> <span class="ruby-identifier">m</span>
|
18
|
+
<span class="ruby-comment cmt"># allow multiple un-cached calls to RingFinger.finger</span>
|
19
|
+
<span class="ruby-keyword kw">def</span> <span class="ruby-constant">RingFinger</span>.<span class="ruby-identifier">finger</span>
|
20
|
+
<span class="ruby-ivar">@@finger</span> = <span class="ruby-keyword kw">self</span>.<span class="ruby-identifier">new</span>
|
21
|
+
<span class="ruby-ivar">@@finger</span>.<span class="ruby-identifier">lookup_ring_any</span>
|
22
|
+
<span class="ruby-ivar">@@finger</span>
|
23
|
+
<span class="ruby-keyword kw">end</span>
|
24
|
+
|
25
|
+
<span class="ruby-identifier">spawn</span>
|
26
|
+
<span class="ruby-ivar">@retry_count</span> <span class="ruby-operator">+=</span> <span class="ruby-value">1</span>
|
27
|
+
<span class="ruby-keyword kw">if</span> <span class="ruby-ivar">@retry_count</span> <span class="ruby-operator"><=</span> <span class="ruby-value">5</span>
|
28
|
+
<span class="ruby-keyword kw">retry</span>
|
29
|
+
<span class="ruby-keyword kw">else</span>
|
30
|
+
<span class="ruby-identifier">raise</span> <span class="ruby-constant">RuntimeError</span>, <span class="ruby-identifier">m</span>
|
31
|
+
<span class="ruby-keyword kw">end</span>
|
32
|
+
<span class="ruby-keyword kw">end</span>
|
33
|
+
<span class="ruby-keyword kw">end</span></pre>
|
17
34
|
</body>
|
18
35
|
</html>
|
@@ -5,14 +5,14 @@
|
|
5
5
|
|
6
6
|
<html>
|
7
7
|
<head>
|
8
|
-
<title>
|
8
|
+
<title>spawn (Starfish)</title>
|
9
9
|
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
|
10
10
|
<link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
|
11
11
|
</head>
|
12
12
|
<body class="standalone-code">
|
13
|
-
<pre><span class="ruby-comment cmt"># File lib/starfish.rb, line
|
14
|
-
|
15
|
-
|
16
|
-
|
13
|
+
<pre><span class="ruby-comment cmt"># File lib/starfish.rb, line 182</span>
|
14
|
+
<span class="ruby-keyword kw">def</span> <span class="ruby-identifier">spawn</span>
|
15
|
+
<span class="ruby-ivar">@started</span> <span class="ruby-operator">||=</span> <span class="ruby-identifier">fork</span> { <span class="ruby-identifier">system</span>(<span class="ruby-node">"ruby #{File.dirname(__FILE__)}/../bin/starfish #{@uniq} server > /dev/null"</span>) }
|
16
|
+
<span class="ruby-keyword kw">end</span></pre>
|
17
17
|
</body>
|
18
18
|
</html>
|
@@ -0,0 +1,22 @@
|
|
1
|
+
<?xml version="1.0" encoding="iso-8859-1"?>
|
2
|
+
<!DOCTYPE html
|
3
|
+
PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
|
4
|
+
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
|
5
|
+
|
6
|
+
<html>
|
7
|
+
<head>
|
8
|
+
<title>stop (Starfish)</title>
|
9
|
+
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
|
10
|
+
<link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
|
11
|
+
</head>
|
12
|
+
<body class="standalone-code">
|
13
|
+
<pre><span class="ruby-comment cmt"># File lib/starfish.rb, line 186</span>
|
14
|
+
<span class="ruby-keyword kw">def</span> <span class="ruby-identifier">stop</span>
|
15
|
+
<span class="ruby-identifier">puts</span> <span class="ruby-value str">"stopping the server"</span>
|
16
|
+
<span class="ruby-constant">Process</span>.<span class="ruby-identifier">kill</span>(<span class="ruby-value str">"SIGHUP"</span>, <span class="ruby-constant">IO</span>.<span class="ruby-identifier">read</span>(<span class="ruby-ivar">@@options</span>[<span class="ruby-identifier">:pid</span>] <span class="ruby-operator">||</span> <span class="ruby-node">"/tmp/starfish-#{uniq}.pid"</span>).<span class="ruby-identifier">to_i</span>)
|
17
|
+
<span class="ruby-keyword kw">rescue</span> <span class="ruby-constant">Errno</span><span class="ruby-operator">::</span><span class="ruby-constant">ENOENT</span>
|
18
|
+
<span class="ruby-identifier">puts</span> <span class="ruby-value str">"Fatal error, please kill all starfish processes manually and try again"</span>
|
19
|
+
<span class="ruby-identifier">system</span>(<span class="ruby-value str">"ps auxww|grep starfish"</span>)
|
20
|
+
<span class="ruby-keyword kw">end</span></pre>
|
21
|
+
</body>
|
22
|
+
</html>
|
@@ -5,14 +5,14 @@
|
|
5
5
|
|
6
6
|
<html>
|
7
7
|
<head>
|
8
|
-
<title>
|
8
|
+
<title>server= (Starfish)</title>
|
9
9
|
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
|
10
10
|
<link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
|
11
11
|
</head>
|
12
12
|
<body class="standalone-code">
|
13
|
-
<pre><span class="ruby-comment cmt"># File lib/starfish.rb, line
|
14
|
-
|
15
|
-
|
16
|
-
|
13
|
+
<pre><span class="ruby-comment cmt"># File lib/starfish.rb, line 195</span>
|
14
|
+
<span class="ruby-keyword kw">def</span> <span class="ruby-identifier">server=</span>(<span class="ruby-identifier">server</span>)
|
15
|
+
<span class="ruby-ivar">@@server</span> = <span class="ruby-identifier">server</span>
|
16
|
+
<span class="ruby-keyword kw">end</span></pre>
|
17
17
|
</body>
|
18
18
|
</html>
|
@@ -0,0 +1,18 @@
|
|
1
|
+
<?xml version="1.0" encoding="iso-8859-1"?>
|
2
|
+
<!DOCTYPE html
|
3
|
+
PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
|
4
|
+
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
|
5
|
+
|
6
|
+
<html>
|
7
|
+
<head>
|
8
|
+
<title>client= (Starfish)</title>
|
9
|
+
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
|
10
|
+
<link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
|
11
|
+
</head>
|
12
|
+
<body class="standalone-code">
|
13
|
+
<pre><span class="ruby-comment cmt"># File lib/starfish.rb, line 199</span>
|
14
|
+
<span class="ruby-keyword kw">def</span> <span class="ruby-identifier">client=</span>(<span class="ruby-identifier">client</span>)
|
15
|
+
<span class="ruby-ivar">@@client</span> = <span class="ruby-identifier">client</span>
|
16
|
+
<span class="ruby-keyword kw">end</span></pre>
|
17
|
+
</body>
|
18
|
+
</html>
|
@@ -0,0 +1,18 @@
|
|
1
|
+
<?xml version="1.0" encoding="iso-8859-1"?>
|
2
|
+
<!DOCTYPE html
|
3
|
+
PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
|
4
|
+
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
|
5
|
+
|
6
|
+
<html>
|
7
|
+
<head>
|
8
|
+
<title>options= (Starfish)</title>
|
9
|
+
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
|
10
|
+
<link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
|
11
|
+
</head>
|
12
|
+
<body class="standalone-code">
|
13
|
+
<pre><span class="ruby-comment cmt"># File lib/starfish.rb, line 203</span>
|
14
|
+
<span class="ruby-keyword kw">def</span> <span class="ruby-identifier">options=</span>(<span class="ruby-identifier">options</span>={})
|
15
|
+
<span class="ruby-ivar">@@options</span>.<span class="ruby-identifier">update</span>(<span class="ruby-identifier">options</span>)
|
16
|
+
<span class="ruby-keyword kw">end</span></pre>
|
17
|
+
</body>
|
18
|
+
</html>
|
@@ -76,7 +76,7 @@
|
|
76
76
|
<h3 class="section-bar">Methods</h3>
|
77
77
|
|
78
78
|
<div class="name-list">
|
79
|
-
<a href="#
|
79
|
+
<a href="#M000035">finger</a>
|
80
80
|
</div>
|
81
81
|
</div>
|
82
82
|
|
@@ -98,12 +98,12 @@
|
|
98
98
|
<div id="methods">
|
99
99
|
<h3 class="section-bar">Public Class methods</h3>
|
100
100
|
|
101
|
-
<div id="method-
|
102
|
-
<a name="
|
101
|
+
<div id="method-M000035" class="method-detail">
|
102
|
+
<a name="M000035"></a>
|
103
103
|
|
104
104
|
<div class="method-heading">
|
105
|
-
<a href="RingFinger.src/
|
106
|
-
onclick="popupCode('RingFinger.src/
|
105
|
+
<a href="RingFinger.src/M000035.html" target="Code" class="method-signature"
|
106
|
+
onclick="popupCode('RingFinger.src/M000035.html');return false;">
|
107
107
|
<span class="method-name">finger</span><span class="method-args">()</span>
|
108
108
|
</a>
|
109
109
|
</div>
|
@@ -111,7 +111,7 @@
|
|
111
111
|
<div class="method-description">
|
112
112
|
<p>
|
113
113
|
allow multiple un-cached calls to <a
|
114
|
-
href="RingFinger.html#
|
114
|
+
href="RingFinger.html#M000035">RingFinger.finger</a>
|
115
115
|
</p>
|
116
116
|
</div>
|
117
117
|
</div>
|
@@ -10,7 +10,7 @@
|
|
10
10
|
<link rel="stylesheet" href="../../.././rdoc-style.css" type="text/css" media="screen" />
|
11
11
|
</head>
|
12
12
|
<body class="standalone-code">
|
13
|
-
<pre><span class="ruby-comment cmt"># File lib/starfish.rb, line
|
13
|
+
<pre><span class="ruby-comment cmt"># File lib/starfish.rb, line 166</span>
|
14
14
|
<span class="ruby-keyword kw">def</span> <span class="ruby-constant">RingFinger</span>.<span class="ruby-identifier">finger</span>
|
15
15
|
<span class="ruby-ivar">@@finger</span> = <span class="ruby-keyword kw">self</span>.<span class="ruby-identifier">new</span>
|
16
16
|
<span class="ruby-ivar">@@finger</span>.<span class="ruby-identifier">lookup_ring_any</span>
|
data/doc/created.rid
CHANGED
@@ -1 +1 @@
|
|
1
|
-
|
1
|
+
Fri Aug 18 23:00:34 PDT 2006
|
data/doc/files/README.html
CHANGED
@@ -56,7 +56,7 @@
|
|
56
56
|
</tr>
|
57
57
|
<tr class="top-aligned-row">
|
58
58
|
<td><strong>Last Update:</strong></td>
|
59
|
-
<td>
|
59
|
+
<td>Fri Aug 18 22:59:00 PDT 2006</td>
|
60
60
|
</tr>
|
61
61
|
</table>
|
62
62
|
</div>
|
@@ -71,10 +71,8 @@
|
|
71
71
|
<div id="description">
|
72
72
|
<h2>Welcome to <a href="../classes/Starfish.html">Starfish</a></h2>
|
73
73
|
<p>
|
74
|
-
<a href="../classes/Starfish.html">Starfish</a> is a utility
|
75
|
-
|
76
|
-
href="http://mog.com">mog.com</a>) to make distributed programming
|
77
|
-
ridiculously easy.
|
74
|
+
<a href="../classes/Starfish.html">Starfish</a> is a utility to make
|
75
|
+
distributed programming ridiculously easy.
|
78
76
|
</p>
|
79
77
|
<h2>Download</h2>
|
80
78
|
<ul>
|
@@ -89,6 +87,17 @@ href="http://rubyforge.org/projects/starfish">rubyforge.org/projects/starfish</a
|
|
89
87
|
|
90
88
|
</li>
|
91
89
|
</ul>
|
90
|
+
<h2>Background</h2>
|
91
|
+
<p>
|
92
|
+
<a href="../classes/Starfish.html">Starfish</a> was born by abstracting
|
93
|
+
code from a highly popular website (<a href="http://mog.com">mog.com</a>).
|
94
|
+
I had written distributed programming code for various tasks including
|
95
|
+
database corruption cleanup as well as real-time indexing requirements and
|
96
|
+
decided that nobody should have to write as much code as I did to do those
|
97
|
+
tasks. The general idea of distributed programming is vastly underused by
|
98
|
+
most programmers and <a href="../classes/Starfish.html">Starfish</a> hides
|
99
|
+
everything you don’t have to worry about.
|
100
|
+
</p>
|
92
101
|
<h2>Usage</h2>
|
93
102
|
<p>
|
94
103
|
Make a file and define the server and client. What follows is a simple
|
@@ -146,7 +155,7 @@ The :log parameter is intelligent and can take various other formats
|
|
146
155
|
including nil for no logging or instances of any kind of logger class you
|
147
156
|
prefer to use.
|
148
157
|
</p>
|
149
|
-
<h2><a href="../classes/MapReduce.html">MapReduce</a
|
158
|
+
<h2><a href="../classes/MapReduce.html">MapReduce</a> with ActiveRecord</h2>
|
150
159
|
<p>
|
151
160
|
The crown jewel of <a href="../classes/Starfish.html">Starfish</a> is
|
152
161
|
actually a library built on <a href="../classes/Starfish.html">Starfish</a>
|
@@ -287,6 +296,43 @@ And you will be returned a YAML hash of statistics.
|
|
287
296
|
time_spent_grabbing_queues: 0.090328
|
288
297
|
num_objects_grabbed: 10
|
289
298
|
</pre>
|
299
|
+
<h2><a href="../classes/MapReduce.html">MapReduce</a> with Files</h2>
|
300
|
+
<p>
|
301
|
+
You can also divide and conquer with files. The idea is that you have a
|
302
|
+
large file and want to process line-by-line as quickly as possible. It is
|
303
|
+
simple to accomplish this with <a
|
304
|
+
href="../classes/Starfish.html">Starfish</a>.
|
305
|
+
</p>
|
306
|
+
<pre>
|
307
|
+
server do |map_reduce|
|
308
|
+
map_reduce.type = File
|
309
|
+
map_reduce.input = "/tmp/big_log_file"
|
310
|
+
end
|
311
|
+
|
312
|
+
client do |line|
|
313
|
+
if line =~ /some_regex/
|
314
|
+
logger.info(line)
|
315
|
+
end
|
316
|
+
end
|
317
|
+
</pre>
|
318
|
+
<p>
|
319
|
+
Simply replacing the type and input lets you process your file in a
|
320
|
+
distributed way. Like when using ActiveRecord, you also have options:
|
321
|
+
</p>
|
322
|
+
<pre>
|
323
|
+
server do |map_reduce|
|
324
|
+
map_reduce.type = File
|
325
|
+
map_reduce.input = "/tmp/big_log_file"
|
326
|
+
map_reduce.queue_size = 1000 # how many lines of the file to buffer at a time
|
327
|
+
map_reduce.lines_per_client = 100 # how many lines each client will process at a time
|
328
|
+
map_reduce.rescan_when_complete = true
|
329
|
+
end
|
330
|
+
</pre>
|
331
|
+
<p>
|
332
|
+
For a file, rescan_when_complete goes back to the beginning of the file
|
333
|
+
when you have finished processing it. You can also use the vigilant option
|
334
|
+
to wait for new data to be added to the file.
|
335
|
+
</p>
|
290
336
|
<h2>Examples</h2>
|
291
337
|
<p>
|
292
338
|
See the examples/ directory.
|
@@ -56,7 +56,7 @@
|
|
56
56
|
</tr>
|
57
57
|
<tr class="top-aligned-row">
|
58
58
|
<td><strong>Last Update:</strong></td>
|
59
|
-
<td>
|
59
|
+
<td>Fri Aug 18 17:01:38 PDT 2006</td>
|
60
60
|
</tr>
|
61
61
|
</table>
|
62
62
|
</div>
|
@@ -74,7 +74,6 @@
|
|
74
74
|
|
75
75
|
<div class="name-list">
|
76
76
|
rubygems
|
77
|
-
active_record
|
78
77
|
</div>
|
79
78
|
</div>
|
80
79
|
|
data/doc/fr_class_index.html
CHANGED
@@ -26,6 +26,7 @@
|
|
26
26
|
<a href="classes/MapReduce/ActiveRecord/Base/Client.html">MapReduce::ActiveRecord::Base::Client</a><br />
|
27
27
|
<a href="classes/MapReduce/Array.html">MapReduce::Array</a><br />
|
28
28
|
<a href="classes/MapReduce/File.html">MapReduce::File</a><br />
|
29
|
+
<a href="classes/MapReduce/File/Client.html">MapReduce::File::Client</a><br />
|
29
30
|
<a href="classes/MapReduceError.html">MapReduceError</a><br />
|
30
31
|
<a href="classes/Starfish.html">Starfish</a><br />
|
31
32
|
<a href="classes/Starfish/RingFinger.html">Starfish::RingFinger</a><br />
|
data/doc/fr_method_index.html
CHANGED
@@ -23,31 +23,35 @@
|
|
23
23
|
<a href="classes/MapReduce.html#M000002">add_time_spent_processing_objects (MapReduce)</a><br />
|
24
24
|
<a href="classes/MapReduce.html#M000014">base_type (MapReduce)</a><br />
|
25
25
|
<a href="classes/MapReduce.html#M000005">base_type_to_s (MapReduce)</a><br />
|
26
|
-
<a href="classes/Starfish.html#
|
27
|
-
<a href="classes/Starfish.html#
|
26
|
+
<a href="classes/Starfish.html#M000026">client (Starfish)</a><br />
|
27
|
+
<a href="classes/Starfish.html#M000033">client= (Starfish)</a><br />
|
28
|
+
<a href="classes/MapReduce/File/Client.html#M000021">each (MapReduce::File::Client)</a><br />
|
28
29
|
<a href="classes/MapReduce/ActiveRecord/Base/Client.html#M000017">each (MapReduce::ActiveRecord::Base::Client)</a><br />
|
29
|
-
<a href="classes/Starfish/RingFinger.html#
|
30
|
+
<a href="classes/Starfish/RingFinger.html#M000035">finger (Starfish::RingFinger)</a><br />
|
30
31
|
<a href="classes/MapReduce/ActiveRecord/Base.html#M000015">get_id (MapReduce::ActiveRecord::Base)</a><br />
|
32
|
+
<a href="classes/MapReduce/File.html#M000019">get_lines (MapReduce::File)</a><br />
|
33
|
+
<a href="classes/MapReduce/File/Client.html#M000022">logger (MapReduce::File::Client)</a><br />
|
31
34
|
<a href="classes/MapReduce/ActiveRecord/Base/Client.html#M000018">logger (MapReduce::ActiveRecord::Base::Client)</a><br />
|
32
35
|
<a href="classes/MapReduce.html#M000013">map_reduce? (MapReduce)</a><br />
|
33
36
|
<a href="classes/MapReduce.html#M000011">method_missing (MapReduce)</a><br />
|
34
|
-
<a href="classes/Starfish.html#
|
37
|
+
<a href="classes/Starfish.html#M000029">negotiate (Starfish)</a><br />
|
35
38
|
<a href="classes/MapReduce/ActiveRecord/Base/Client.html#M000016">new (MapReduce::ActiveRecord::Base::Client)</a><br />
|
39
|
+
<a href="classes/MapReduce/File/Client.html#M000020">new (MapReduce::File::Client)</a><br />
|
40
|
+
<a href="classes/Starfish.html#M000023">new (Starfish)</a><br />
|
36
41
|
<a href="classes/MapReduce.html#M000001">new (MapReduce)</a><br />
|
37
|
-
<a href="classes/Starfish.html#
|
38
|
-
<a href="classes/Starfish.html#M000030">options= (Starfish)</a><br />
|
42
|
+
<a href="classes/Starfish.html#M000034">options= (Starfish)</a><br />
|
39
43
|
<a href="classes/MapReduce.html#M000012">raise_if_invalid! (MapReduce)</a><br />
|
40
|
-
<a href="classes/Starfish.html#
|
41
|
-
<a href="classes/Starfish.html#
|
42
|
-
<a href="classes/Starfish.html#
|
43
|
-
<a href="classes/Starfish.html#
|
44
|
+
<a href="classes/Starfish.html#M000028">sanitize (Starfish)</a><br />
|
45
|
+
<a href="classes/Starfish.html#M000025">server (Starfish)</a><br />
|
46
|
+
<a href="classes/Starfish.html#M000032">server= (Starfish)</a><br />
|
47
|
+
<a href="classes/Starfish.html#M000030">spawn (Starfish)</a><br />
|
44
48
|
<a href="classes/MapReduce.html#M000007">spool= (MapReduce)</a><br />
|
49
|
+
<a href="classes/Starfish.html#M000027">stats (Starfish)</a><br />
|
45
50
|
<a href="classes/MapReduce.html#M000003">stats (MapReduce)</a><br />
|
46
|
-
<a href="classes/Starfish.html#
|
47
|
-
<a href="classes/Starfish.html#M000027">stop (Starfish)</a><br />
|
51
|
+
<a href="classes/Starfish.html#M000031">stop (Starfish)</a><br />
|
48
52
|
<a href="classes/MapReduce.html#M000004">type= (MapReduce)</a><br />
|
49
53
|
<a href="classes/MapReduce.html#M000006">type_to_s (MapReduce)</a><br />
|
50
|
-
<a href="classes/Starfish.html#
|
54
|
+
<a href="classes/Starfish.html#M000024">uniq (Starfish)</a><br />
|
51
55
|
<a href="classes/MapReduce.html#M000008">valid? (MapReduce)</a><br />
|
52
56
|
<a href="classes/MapReduce.html#M000010">valid_input? (MapReduce)</a><br />
|
53
57
|
<a href="classes/MapReduce.html#M000009">valid_type? (MapReduce)</a><br />
|