starfish 1.0.0 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. data/README +33 -2
  2. data/doc/classes/MapReduce.src/M000001.html +1 -1
  3. data/doc/classes/MapReduce.src/M000002.html +1 -1
  4. data/doc/classes/MapReduce.src/M000003.html +1 -1
  5. data/doc/classes/MapReduce.src/M000004.html +1 -1
  6. data/doc/classes/MapReduce.src/M000005.html +1 -1
  7. data/doc/classes/MapReduce.src/M000006.html +1 -1
  8. data/doc/classes/MapReduce.src/M000007.html +1 -1
  9. data/doc/classes/MapReduce.src/M000008.html +1 -1
  10. data/doc/classes/MapReduce.src/M000009.html +1 -1
  11. data/doc/classes/MapReduce.src/M000010.html +1 -1
  12. data/doc/classes/MapReduce.src/M000011.html +1 -1
  13. data/doc/classes/MapReduce.src/M000012.html +1 -1
  14. data/doc/classes/MapReduce.src/M000013.html +1 -1
  15. data/doc/classes/MapReduce.src/M000014.html +1 -1
  16. data/doc/classes/MapReduce/ActiveRecord/Base.src/M000015.html +6 -14
  17. data/doc/classes/MapReduce/File.html +70 -0
  18. data/doc/classes/MapReduce/File.src/M000019.html +42 -0
  19. data/doc/classes/MapReduce/File/Client.html +177 -0
  20. data/doc/classes/MapReduce/File/Client.src/M000020.html +18 -0
  21. data/doc/classes/MapReduce/File/Client.src/M000021.html +20 -0
  22. data/doc/classes/MapReduce/File/Client.src/M000022.html +18 -0
  23. data/doc/classes/Starfish.html +61 -62
  24. data/doc/classes/Starfish.src/M000023.html +5 -9
  25. data/doc/classes/Starfish.src/M000024.html +4 -22
  26. data/doc/classes/Starfish.src/M000025.html +29 -19
  27. data/doc/classes/Starfish.src/M000026.html +64 -4
  28. data/doc/classes/Starfish.src/M000027.html +9 -8
  29. data/doc/classes/Starfish.src/M000028.html +23 -5
  30. data/doc/classes/Starfish.src/M000029.html +22 -5
  31. data/doc/classes/Starfish.src/M000030.html +5 -5
  32. data/doc/classes/Starfish.src/M000031.html +22 -0
  33. data/doc/classes/Starfish.src/{M000020.html → M000032.html} +5 -5
  34. data/doc/classes/Starfish.src/M000033.html +18 -0
  35. data/doc/classes/Starfish.src/M000034.html +18 -0
  36. data/doc/classes/Starfish/RingFinger.html +6 -6
  37. data/doc/classes/Starfish/RingFinger.src/{M000031.html → M000035.html} +1 -1
  38. data/doc/created.rid +1 -1
  39. data/doc/files/README.html +52 -6
  40. data/doc/files/lib/map_reduce/active_record_rb.html +1 -2
  41. data/doc/files/lib/map_reduce/file_rb.html +1 -1
  42. data/doc/files/lib/map_reduce_rb.html +1 -1
  43. data/doc/files/lib/starfish_rb.html +1 -1
  44. data/doc/fr_class_index.html +1 -0
  45. data/doc/fr_method_index.html +17 -13
  46. data/examples/map_reduce/file.rb +2 -3
  47. data/lib/map_reduce.rb +2 -1
  48. data/lib/map_reduce/active_record.rb +25 -19
  49. data/lib/map_reduce/file.rb +93 -0
  50. data/lib/starfish.rb +9 -2
  51. metadata +15 -7
  52. data/doc/classes/Starfish.src/M000019.html +0 -19
  53. data/doc/classes/Starfish.src/M000021.html +0 -45
  54. data/doc/classes/Starfish.src/M000022.html +0 -71
data/doc/classes/Starfish.src/M000027.html CHANGED
@@ -5,18 +5,19 @@
5
5
 
6
6
  <html>
7
7
  <head>
8
- <title>stop (Starfish)</title>
8
+ <title>stats (Starfish)</title>
9
9
  <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
10
10
  <link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
11
11
  </head>
12
12
  <body class="standalone-code">
13
- <pre><span class="ruby-comment cmt"># File lib/starfish.rb, line 179</span>
14
- <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">stop</span>
15
- <span class="ruby-identifier">puts</span> <span class="ruby-value str">&quot;stopping the server&quot;</span>
16
- <span class="ruby-constant">Process</span>.<span class="ruby-identifier">kill</span>(<span class="ruby-value str">&quot;SIGHUP&quot;</span>, <span class="ruby-constant">IO</span>.<span class="ruby-identifier">read</span>(<span class="ruby-ivar">@@options</span>[<span class="ruby-identifier">:pid</span>] <span class="ruby-operator">||</span> <span class="ruby-node">&quot;/tmp/starfish-#{uniq}.pid&quot;</span>).<span class="ruby-identifier">to_i</span>)
17
- <span class="ruby-keyword kw">rescue</span> <span class="ruby-constant">Errno</span><span class="ruby-operator">::</span><span class="ruby-constant">ENOENT</span>
18
- <span class="ruby-identifier">puts</span> <span class="ruby-value str">&quot;Fatal error, please kill all starfish processes manually and try again&quot;</span>
19
- <span class="ruby-identifier">system</span>(<span class="ruby-value str">&quot;ps auxww|grep starfish&quot;</span>)
13
+ <pre><span class="ruby-comment cmt"># File lib/starfish.rb, line 130</span>
14
+ <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">stats</span>
15
+ <span class="ruby-identifier">negotiate</span>
16
+ <span class="ruby-constant">Timeout</span><span class="ruby-operator">::</span><span class="ruby-identifier">timeout</span>(<span class="ruby-value">5</span>) { <span class="ruby-ivar">@server_object</span> = <span class="ruby-ivar">@ring_server</span>.<span class="ruby-identifier">read</span>([<span class="ruby-identifier">:name</span>, <span class="ruby-identifier">uniq</span>.<span class="ruby-identifier">intern</span>, <span class="ruby-keyword kw">nil</span>, <span class="ruby-keyword kw">nil</span>])[<span class="ruby-value">2</span>] }
17
+
18
+ <span class="ruby-identifier">puts</span> <span class="ruby-ivar">@server_object</span>.<span class="ruby-identifier">stats</span>.<span class="ruby-identifier">to_yaml</span>
19
+ <span class="ruby-keyword kw">rescue</span> <span class="ruby-constant">NoMethodError</span>
20
+ <span class="ruby-identifier">$stderr</span>.<span class="ruby-identifier">puts</span> <span class="ruby-value str">&quot;The stats method is not defined for your server&quot;</span>
20
21
  <span class="ruby-keyword kw">end</span></pre>
21
22
  </body>
22
23
  </html>
data/doc/classes/Starfish.src/M000028.html CHANGED
@@ -5,14 +5,32 @@
5
5
 
6
6
  <html>
7
7
  <head>
8
- <title>server= (Starfish)</title>
8
+ <title>sanitize (Starfish)</title>
9
9
  <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
10
10
  <link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
11
11
  </head>
12
12
  <body class="standalone-code">
13
- <pre><span class="ruby-comment cmt"># File lib/starfish.rb, line 188</span>
14
- <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">server=</span>(<span class="ruby-identifier">server</span>)
15
- <span class="ruby-ivar">@@server</span> = <span class="ruby-identifier">server</span>
16
- <span class="ruby-keyword kw">end</span></pre>
13
+ <pre><span class="ruby-comment cmt"># File lib/starfish.rb, line 139</span>
14
+ <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">sanitize</span>(<span class="ruby-identifier">object</span>)
15
+ <span class="ruby-identifier">object</span>.<span class="ruby-identifier">extend</span> <span class="ruby-constant">DRbUndumped</span>
16
+
17
+ <span class="ruby-ivar">@@log</span> = <span class="ruby-keyword kw">case</span> <span class="ruby-ivar">@@options</span>[<span class="ruby-identifier">:log</span>]
18
+ <span class="ruby-keyword kw">when</span> <span class="ruby-constant">String</span>
19
+ <span class="ruby-constant">Logger</span>.<span class="ruby-identifier">new</span>(<span class="ruby-ivar">@@options</span>[<span class="ruby-identifier">:log</span>])
20
+ <span class="ruby-keyword kw">when</span> <span class="ruby-constant">Class</span>
21
+ <span class="ruby-ivar">@@options</span>[<span class="ruby-identifier">:log</span>].<span class="ruby-identifier">new</span>
22
+ <span class="ruby-keyword kw">when</span> <span class="ruby-keyword kw">nil</span>, <span class="ruby-keyword kw">false</span>
23
+ <span class="ruby-constant">Logger</span>.<span class="ruby-identifier">new</span>(<span class="ruby-value str">&quot;/dev/null&quot;</span>)
24
+ <span class="ruby-keyword kw">else</span>
25
+ <span class="ruby-ivar">@@options</span>[<span class="ruby-identifier">:log</span>]
26
+ <span class="ruby-keyword kw">end</span>
27
+
28
+ <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">object</span>.<span class="ruby-identifier">logger</span>
29
+ <span class="ruby-ivar">@logger</span> <span class="ruby-operator">||=</span> <span class="ruby-ivar">@@log</span>
30
+ <span class="ruby-keyword kw">end</span>
31
+ <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">object</span>.<span class="ruby-identifier">_logger</span>
32
+ <span class="ruby-ivar">@logger</span> <span class="ruby-operator">||=</span> <span class="ruby-ivar">@@log</span>
33
+ <span class="ruby-keyword kw">end</span>
34
+ <span class="ruby-keyword kw">end</span></pre>
17
35
  </body>
18
36
  </html>
data/doc/classes/Starfish.src/M000029.html CHANGED
@@ -5,14 +5,31 @@
5
5
 
6
6
  <html>
7
7
  <head>
8
- <title>client= (Starfish)</title>
8
+ <title>negotiate (Starfish)</title>
9
9
  <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
10
10
  <link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
11
11
  </head>
12
12
  <body class="standalone-code">
13
- <pre><span class="ruby-comment cmt"># File lib/starfish.rb, line 192</span>
14
- <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">client=</span>(<span class="ruby-identifier">client</span>)
15
- <span class="ruby-ivar">@@client</span> = <span class="ruby-identifier">client</span>
16
- <span class="ruby-keyword kw">end</span></pre>
13
+ <pre><span class="ruby-comment cmt"># File lib/starfish.rb, line 161</span>
14
+ <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">negotiate</span><span class="ruby-identifier">negotiate</span><span class="ruby-identifier">negotiate</span>
15
+ <span class="ruby-keyword kw">begin</span>
16
+ <span class="ruby-ivar">@ring_server</span> = <span class="ruby-constant">RingFinger</span>.<span class="ruby-identifier">primary</span>
17
+ <span class="ruby-keyword kw">rescue</span> <span class="ruby-constant">RuntimeError</span> =<span class="ruby-operator">&gt;</span> <span class="ruby-identifier">m</span>
18
+ <span class="ruby-comment cmt"># allow multiple un-cached calls to RingFinger.finger</span>
19
+ <span class="ruby-keyword kw">def</span> <span class="ruby-constant">RingFinger</span>.<span class="ruby-identifier">finger</span>
20
+ <span class="ruby-ivar">@@finger</span> = <span class="ruby-keyword kw">self</span>.<span class="ruby-identifier">new</span>
21
+ <span class="ruby-ivar">@@finger</span>.<span class="ruby-identifier">lookup_ring_any</span>
22
+ <span class="ruby-ivar">@@finger</span>
23
+ <span class="ruby-keyword kw">end</span>
24
+
25
+ <span class="ruby-identifier">spawn</span>
26
+ <span class="ruby-ivar">@retry_count</span> <span class="ruby-operator">+=</span> <span class="ruby-value">1</span>
27
+ <span class="ruby-keyword kw">if</span> <span class="ruby-ivar">@retry_count</span> <span class="ruby-operator">&lt;=</span> <span class="ruby-value">5</span>
28
+ <span class="ruby-keyword kw">retry</span>
29
+ <span class="ruby-keyword kw">else</span>
30
+ <span class="ruby-identifier">raise</span> <span class="ruby-constant">RuntimeError</span>, <span class="ruby-identifier">m</span>
31
+ <span class="ruby-keyword kw">end</span>
32
+ <span class="ruby-keyword kw">end</span>
33
+ <span class="ruby-keyword kw">end</span></pre>
17
34
  </body>
18
35
  </html>
data/doc/classes/Starfish.src/M000030.html CHANGED
@@ -5,14 +5,14 @@
5
5
 
6
6
  <html>
7
7
  <head>
8
- <title>options= (Starfish)</title>
8
+ <title>spawn (Starfish)</title>
9
9
  <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
10
10
  <link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
11
11
  </head>
12
12
  <body class="standalone-code">
13
- <pre><span class="ruby-comment cmt"># File lib/starfish.rb, line 196</span>
14
- <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">options=</span>(<span class="ruby-identifier">options</span>={})
15
- <span class="ruby-ivar">@@options</span>.<span class="ruby-identifier">update</span>(<span class="ruby-identifier">options</span>)
16
- <span class="ruby-keyword kw">end</span></pre>
13
+ <pre><span class="ruby-comment cmt"># File lib/starfish.rb, line 182</span>
14
+ <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">spawn</span>
15
+ <span class="ruby-ivar">@started</span> <span class="ruby-operator">||=</span> <span class="ruby-identifier">fork</span> { <span class="ruby-identifier">system</span>(<span class="ruby-node">&quot;ruby #{File.dirname(__FILE__)}/../bin/starfish #{@uniq} server &gt; /dev/null&quot;</span>) }
16
+ <span class="ruby-keyword kw">end</span></pre>
17
17
  </body>
18
18
  </html>
data/doc/classes/Starfish.src/M000031.html ADDED
@@ -0,0 +1,22 @@
1
+ <?xml version="1.0" encoding="iso-8859-1"?>
2
+ <!DOCTYPE html
3
+ PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
4
+ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
5
+
6
+ <html>
7
+ <head>
8
+ <title>stop (Starfish)</title>
9
+ <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
10
+ <link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
11
+ </head>
12
+ <body class="standalone-code">
13
+ <pre><span class="ruby-comment cmt"># File lib/starfish.rb, line 186</span>
14
+ <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">stop</span>
15
+ <span class="ruby-identifier">puts</span> <span class="ruby-value str">&quot;stopping the server&quot;</span>
16
+ <span class="ruby-constant">Process</span>.<span class="ruby-identifier">kill</span>(<span class="ruby-value str">&quot;SIGHUP&quot;</span>, <span class="ruby-constant">IO</span>.<span class="ruby-identifier">read</span>(<span class="ruby-ivar">@@options</span>[<span class="ruby-identifier">:pid</span>] <span class="ruby-operator">||</span> <span class="ruby-node">&quot;/tmp/starfish-#{uniq}.pid&quot;</span>).<span class="ruby-identifier">to_i</span>)
17
+ <span class="ruby-keyword kw">rescue</span> <span class="ruby-constant">Errno</span><span class="ruby-operator">::</span><span class="ruby-constant">ENOENT</span>
18
+ <span class="ruby-identifier">puts</span> <span class="ruby-value str">&quot;Fatal error, please kill all starfish processes manually and try again&quot;</span>
19
+ <span class="ruby-identifier">system</span>(<span class="ruby-value str">&quot;ps auxww|grep starfish&quot;</span>)
20
+ <span class="ruby-keyword kw">end</span></pre>
21
+ </body>
22
+ </html>
data/doc/classes/Starfish.src/{M000020.html → M000032.html} RENAMED
@@ -5,14 +5,14 @@
5
5
 
6
6
  <html>
7
7
  <head>
8
- <title>uniq (Starfish)</title>
8
+ <title>server= (Starfish)</title>
9
9
  <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
10
10
  <link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
11
11
  </head>
12
12
  <body class="standalone-code">
13
- <pre><span class="ruby-comment cmt"># File lib/starfish.rb, line 31</span>
14
- <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">uniq</span>
15
- <span class="ruby-constant">MD5</span>.<span class="ruby-identifier">new</span>(<span class="ruby-ivar">@uniq</span>).<span class="ruby-identifier">to_s</span>
16
- <span class="ruby-keyword kw">end</span></pre>
13
+ <pre><span class="ruby-comment cmt"># File lib/starfish.rb, line 195</span>
14
+ <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">server=</span>(<span class="ruby-identifier">server</span>)
15
+ <span class="ruby-ivar">@@server</span> = <span class="ruby-identifier">server</span>
16
+ <span class="ruby-keyword kw">end</span></pre>
17
17
  </body>
18
18
  </html>
data/doc/classes/Starfish.src/M000033.html ADDED
@@ -0,0 +1,18 @@
1
+ <?xml version="1.0" encoding="iso-8859-1"?>
2
+ <!DOCTYPE html
3
+ PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
4
+ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
5
+
6
+ <html>
7
+ <head>
8
+ <title>client= (Starfish)</title>
9
+ <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
10
+ <link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
11
+ </head>
12
+ <body class="standalone-code">
13
+ <pre><span class="ruby-comment cmt"># File lib/starfish.rb, line 199</span>
14
+ <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">client=</span>(<span class="ruby-identifier">client</span>)
15
+ <span class="ruby-ivar">@@client</span> = <span class="ruby-identifier">client</span>
16
+ <span class="ruby-keyword kw">end</span></pre>
17
+ </body>
18
+ </html>
data/doc/classes/Starfish.src/M000034.html ADDED
@@ -0,0 +1,18 @@
1
+ <?xml version="1.0" encoding="iso-8859-1"?>
2
+ <!DOCTYPE html
3
+ PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
4
+ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
5
+
6
+ <html>
7
+ <head>
8
+ <title>options= (Starfish)</title>
9
+ <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
10
+ <link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
11
+ </head>
12
+ <body class="standalone-code">
13
+ <pre><span class="ruby-comment cmt"># File lib/starfish.rb, line 203</span>
14
+ <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">options=</span>(<span class="ruby-identifier">options</span>={})
15
+ <span class="ruby-ivar">@@options</span>.<span class="ruby-identifier">update</span>(<span class="ruby-identifier">options</span>)
16
+ <span class="ruby-keyword kw">end</span></pre>
17
+ </body>
18
+ </html>
data/doc/classes/Starfish/RingFinger.html CHANGED
@@ -76,7 +76,7 @@
76
76
  <h3 class="section-bar">Methods</h3>
77
77
 
78
78
  <div class="name-list">
79
- <a href="#M000031">finger</a>&nbsp;&nbsp;
79
+ <a href="#M000035">finger</a>&nbsp;&nbsp;
80
80
  </div>
81
81
  </div>
82
82
 
@@ -98,12 +98,12 @@
98
98
  <div id="methods">
99
99
  <h3 class="section-bar">Public Class methods</h3>
100
100
 
101
- <div id="method-M000031" class="method-detail">
102
- <a name="M000031"></a>
101
+ <div id="method-M000035" class="method-detail">
102
+ <a name="M000035"></a>
103
103
 
104
104
  <div class="method-heading">
105
- <a href="RingFinger.src/M000031.html" target="Code" class="method-signature"
106
- onclick="popupCode('RingFinger.src/M000031.html');return false;">
105
+ <a href="RingFinger.src/M000035.html" target="Code" class="method-signature"
106
+ onclick="popupCode('RingFinger.src/M000035.html');return false;">
107
107
  <span class="method-name">finger</span><span class="method-args">()</span>
108
108
  </a>
109
109
  </div>
@@ -111,7 +111,7 @@
111
111
  <div class="method-description">
112
112
  <p>
113
113
  allow multiple un-cached calls to <a
114
- href="RingFinger.html#M000031">RingFinger.finger</a>
114
+ href="RingFinger.html#M000035">RingFinger.finger</a>
115
115
  </p>
116
116
  </div>
117
117
  </div>
data/doc/classes/Starfish/RingFinger.src/{M000031.html → M000035.html} RENAMED
@@ -10,7 +10,7 @@
10
10
  <link rel="stylesheet" href="../../.././rdoc-style.css" type="text/css" media="screen" />
11
11
  </head>
12
12
  <body class="standalone-code">
13
- <pre><span class="ruby-comment cmt"># File lib/starfish.rb, line 159</span>
13
+ <pre><span class="ruby-comment cmt"># File lib/starfish.rb, line 166</span>
14
14
  <span class="ruby-keyword kw">def</span> <span class="ruby-constant">RingFinger</span>.<span class="ruby-identifier">finger</span>
15
15
  <span class="ruby-ivar">@@finger</span> = <span class="ruby-keyword kw">self</span>.<span class="ruby-identifier">new</span>
16
16
  <span class="ruby-ivar">@@finger</span>.<span class="ruby-identifier">lookup_ring_any</span>
data/doc/created.rid CHANGED
@@ -1 +1 @@
1
- Wed Aug 16 15:52:22 PDT 2006
1
+ Fri Aug 18 23:00:34 PDT 2006
data/doc/files/README.html CHANGED
@@ -56,7 +56,7 @@
56
56
  </tr>
57
57
  <tr class="top-aligned-row">
58
58
  <td><strong>Last Update:</strong></td>
59
- <td>Wed Aug 16 15:40:05 PDT 2006</td>
59
+ <td>Fri Aug 18 22:59:00 PDT 2006</td>
60
60
  </tr>
61
61
  </table>
62
62
  </div>
@@ -71,10 +71,8 @@
71
71
  <div id="description">
72
72
  <h2>Welcome to <a href="../classes/Starfish.html">Starfish</a></h2>
73
73
  <p>
74
- <a href="../classes/Starfish.html">Starfish</a> is a utility born from
75
- abstracting code from a highly popular website (<a
76
- href="http://mog.com">mog.com</a>) to make distributed programming
77
- ridiculously easy.
74
+ <a href="../classes/Starfish.html">Starfish</a> is a utility to make
75
+ distributed programming ridiculously easy.
78
76
  </p>
79
77
  <h2>Download</h2>
80
78
  <ul>
@@ -89,6 +87,17 @@ href="http://rubyforge.org/projects/starfish">rubyforge.org/projects/starfish</a
89
87
 
90
88
  </li>
91
89
  </ul>
90
+ <h2>Background</h2>
91
+ <p>
92
+ <a href="../classes/Starfish.html">Starfish</a> was born by abstracting
93
+ code from a highly popular website (<a href="http://mog.com">mog.com</a>).
94
+ I had written distributed programming code for various tasks including
95
+ database corruption cleanup as well as real-time indexing requirements and
96
+ decided that nobody should have to write as much code as I did to do those
97
+ tasks. The general idea of distributed programming is vastly underused by
98
+ most programmers and <a href="../classes/Starfish.html">Starfish</a> hides
99
+ everything you don&#8217;t have to worry about.
100
+ </p>
92
101
  <h2>Usage</h2>
93
102
  <p>
94
103
  Make a file and define the server and client. What follows is a simple
@@ -146,7 +155,7 @@ The :log parameter is intelligent and can take various other formats
146
155
  including nil for no logging or instances of any kind of logger class you
147
156
  prefer to use.
148
157
  </p>
149
- <h2><a href="../classes/MapReduce.html">MapReduce</a></h2>
158
+ <h2><a href="../classes/MapReduce.html">MapReduce</a> with ActiveRecord</h2>
150
159
  <p>
151
160
  The crown jewel of <a href="../classes/Starfish.html">Starfish</a> is
152
161
  actually a library built on <a href="../classes/Starfish.html">Starfish</a>
@@ -287,6 +296,43 @@ And you will be returned a YAML hash of statistics.
287
296
  time_spent_grabbing_queues: 0.090328
288
297
  num_objects_grabbed: 10
289
298
  </pre>
299
+ <h2><a href="../classes/MapReduce.html">MapReduce</a> with Files</h2>
300
+ <p>
301
+ You can also divide and conquer with files. The idea is that you have a
302
+ large file and want to process line-by-line as quickly as possible. It is
303
+ simple to accomplish this with <a
304
+ href="../classes/Starfish.html">Starfish</a>.
305
+ </p>
306
+ <pre>
307
+ server do |map_reduce|
308
+ map_reduce.type = File
309
+ map_reduce.input = &quot;/tmp/big_log_file&quot;
310
+ end
311
+
312
+ client do |line|
313
+ if line =~ /some_regex/
314
+ logger.info(line)
315
+ end
316
+ end
317
+ </pre>
318
+ <p>
319
+ Simply replacing the type and input lets you process your file in a
320
+ distributed way. Like when using ActiveRecord, you also have options:
321
+ </p>
322
+ <pre>
323
+ server do |map_reduce|
324
+ map_reduce.type = File
325
+ map_reduce.input = &quot;/tmp/big_log_file&quot;
326
+ map_reduce.queue_size = 1000 # how many lines of the file to buffer at a time
327
+ map_reduce.lines_per_client = 100 # how many lines each client will process at a time
328
+ map_reduce.rescan_when_complete = true
329
+ end
330
+ </pre>
331
+ <p>
332
+ For a file, rescan_when_complete goes back to the beginning of the file
333
+ when you have finished processing it. You can also use the vigilant option
334
+ to wait for new data to be added to the file.
335
+ </p>
290
336
  <h2>Examples</h2>
291
337
  <p>
292
338
  See the examples/ directory.
data/doc/files/lib/map_reduce/active_record_rb.html CHANGED
@@ -56,7 +56,7 @@
56
56
  </tr>
57
57
  <tr class="top-aligned-row">
58
58
  <td><strong>Last Update:</strong></td>
59
- <td>Wed Aug 16 14:38:32 PDT 2006</td>
59
+ <td>Fri Aug 18 17:01:38 PDT 2006</td>
60
60
  </tr>
61
61
  </table>
62
62
  </div>
@@ -74,7 +74,6 @@
74
74
 
75
75
  <div class="name-list">
76
76
  rubygems&nbsp;&nbsp;
77
- active_record&nbsp;&nbsp;
78
77
  </div>
79
78
  </div>
80
79
 
data/doc/files/lib/map_reduce/file_rb.html CHANGED
@@ -56,7 +56,7 @@
56
56
  </tr>
57
57
  <tr class="top-aligned-row">
58
58
  <td><strong>Last Update:</strong></td>
59
- <td>Tue Aug 15 22:11:58 PDT 2006</td>
59
+ <td>Fri Aug 18 22:57:08 PDT 2006</td>
60
60
  </tr>
61
61
  </table>
62
62
  </div>
data/doc/files/lib/map_reduce_rb.html CHANGED
@@ -56,7 +56,7 @@
56
56
  </tr>
57
57
  <tr class="top-aligned-row">
58
58
  <td><strong>Last Update:</strong></td>
59
- <td>Wed Aug 16 15:01:45 PDT 2006</td>
59
+ <td>Fri Aug 18 21:33:28 PDT 2006</td>
60
60
  </tr>
61
61
  </table>
62
62
  </div>
data/doc/files/lib/starfish_rb.html CHANGED
@@ -56,7 +56,7 @@
56
56
  </tr>
57
57
  <tr class="top-aligned-row">
58
58
  <td><strong>Last Update:</strong></td>
59
- <td>Wed Aug 16 15:01:29 PDT 2006</td>
59
+ <td>Fri Aug 18 22:57:15 PDT 2006</td>
60
60
  </tr>
61
61
  </table>
62
62
  </div>
data/doc/fr_class_index.html CHANGED
@@ -26,6 +26,7 @@
26
26
  <a href="classes/MapReduce/ActiveRecord/Base/Client.html">MapReduce::ActiveRecord::Base::Client</a><br />
27
27
  <a href="classes/MapReduce/Array.html">MapReduce::Array</a><br />
28
28
  <a href="classes/MapReduce/File.html">MapReduce::File</a><br />
29
+ <a href="classes/MapReduce/File/Client.html">MapReduce::File::Client</a><br />
29
30
  <a href="classes/MapReduceError.html">MapReduceError</a><br />
30
31
  <a href="classes/Starfish.html">Starfish</a><br />
31
32
  <a href="classes/Starfish/RingFinger.html">Starfish::RingFinger</a><br />
data/doc/fr_method_index.html CHANGED
@@ -23,31 +23,35 @@
23
23
  <a href="classes/MapReduce.html#M000002">add_time_spent_processing_objects (MapReduce)</a><br />
24
24
  <a href="classes/MapReduce.html#M000014">base_type (MapReduce)</a><br />
25
25
  <a href="classes/MapReduce.html#M000005">base_type_to_s (MapReduce)</a><br />
26
- <a href="classes/Starfish.html#M000022">client (Starfish)</a><br />
27
- <a href="classes/Starfish.html#M000029">client= (Starfish)</a><br />
26
+ <a href="classes/Starfish.html#M000026">client (Starfish)</a><br />
27
+ <a href="classes/Starfish.html#M000033">client= (Starfish)</a><br />
28
+ <a href="classes/MapReduce/File/Client.html#M000021">each (MapReduce::File::Client)</a><br />
28
29
  <a href="classes/MapReduce/ActiveRecord/Base/Client.html#M000017">each (MapReduce::ActiveRecord::Base::Client)</a><br />
29
- <a href="classes/Starfish/RingFinger.html#M000031">finger (Starfish::RingFinger)</a><br />
30
+ <a href="classes/Starfish/RingFinger.html#M000035">finger (Starfish::RingFinger)</a><br />
30
31
  <a href="classes/MapReduce/ActiveRecord/Base.html#M000015">get_id (MapReduce::ActiveRecord::Base)</a><br />
32
+ <a href="classes/MapReduce/File.html#M000019">get_lines (MapReduce::File)</a><br />
33
+ <a href="classes/MapReduce/File/Client.html#M000022">logger (MapReduce::File::Client)</a><br />
31
34
  <a href="classes/MapReduce/ActiveRecord/Base/Client.html#M000018">logger (MapReduce::ActiveRecord::Base::Client)</a><br />
32
35
  <a href="classes/MapReduce.html#M000013">map_reduce? (MapReduce)</a><br />
33
36
  <a href="classes/MapReduce.html#M000011">method_missing (MapReduce)</a><br />
34
- <a href="classes/Starfish.html#M000025">negotiate (Starfish)</a><br />
37
+ <a href="classes/Starfish.html#M000029">negotiate (Starfish)</a><br />
35
38
  <a href="classes/MapReduce/ActiveRecord/Base/Client.html#M000016">new (MapReduce::ActiveRecord::Base::Client)</a><br />
39
+ <a href="classes/MapReduce/File/Client.html#M000020">new (MapReduce::File::Client)</a><br />
40
+ <a href="classes/Starfish.html#M000023">new (Starfish)</a><br />
36
41
  <a href="classes/MapReduce.html#M000001">new (MapReduce)</a><br />
37
- <a href="classes/Starfish.html#M000019">new (Starfish)</a><br />
38
- <a href="classes/Starfish.html#M000030">options= (Starfish)</a><br />
42
+ <a href="classes/Starfish.html#M000034">options= (Starfish)</a><br />
39
43
  <a href="classes/MapReduce.html#M000012">raise_if_invalid! (MapReduce)</a><br />
40
- <a href="classes/Starfish.html#M000024">sanitize (Starfish)</a><br />
41
- <a href="classes/Starfish.html#M000021">server (Starfish)</a><br />
42
- <a href="classes/Starfish.html#M000028">server= (Starfish)</a><br />
43
- <a href="classes/Starfish.html#M000026">spawn (Starfish)</a><br />
44
+ <a href="classes/Starfish.html#M000028">sanitize (Starfish)</a><br />
45
+ <a href="classes/Starfish.html#M000025">server (Starfish)</a><br />
46
+ <a href="classes/Starfish.html#M000032">server= (Starfish)</a><br />
47
+ <a href="classes/Starfish.html#M000030">spawn (Starfish)</a><br />
44
48
  <a href="classes/MapReduce.html#M000007">spool= (MapReduce)</a><br />
49
+ <a href="classes/Starfish.html#M000027">stats (Starfish)</a><br />
45
50
  <a href="classes/MapReduce.html#M000003">stats (MapReduce)</a><br />
46
- <a href="classes/Starfish.html#M000023">stats (Starfish)</a><br />
47
- <a href="classes/Starfish.html#M000027">stop (Starfish)</a><br />
51
+ <a href="classes/Starfish.html#M000031">stop (Starfish)</a><br />
48
52
  <a href="classes/MapReduce.html#M000004">type= (MapReduce)</a><br />
49
53
  <a href="classes/MapReduce.html#M000006">type_to_s (MapReduce)</a><br />
50
- <a href="classes/Starfish.html#M000020">uniq (Starfish)</a><br />
54
+ <a href="classes/Starfish.html#M000024">uniq (Starfish)</a><br />
51
55
  <a href="classes/MapReduce.html#M000008">valid? (MapReduce)</a><br />
52
56
  <a href="classes/MapReduce.html#M000010">valid_input? (MapReduce)</a><br />
53
57
  <a href="classes/MapReduce.html#M000009">valid_type? (MapReduce)</a><br />