starfish 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/LICENSE +10 -0
- data/README +136 -0
- data/Rakefile +89 -0
- data/bin/starfish +27 -0
- data/doc/classes/MapReduce/ActiveRecord/Base/Client.html +177 -0
- data/doc/classes/MapReduce/ActiveRecord/Base/Client.src/M000016.html +19 -0
- data/doc/classes/MapReduce/ActiveRecord/Base/Client.src/M000017.html +20 -0
- data/doc/classes/MapReduce/ActiveRecord/Base/Client.src/M000018.html +18 -0
- data/doc/classes/MapReduce/ActiveRecord/Base.html +170 -0
- data/doc/classes/MapReduce/ActiveRecord/Base.src/M000015.html +39 -0
- data/doc/classes/MapReduce/ActiveRecord.html +111 -0
- data/doc/classes/MapReduce/Array.html +105 -0
- data/doc/classes/MapReduce/File.html +105 -0
- data/doc/classes/MapReduce.html +395 -0
- data/doc/classes/MapReduce.src/M000001.html +31 -0
- data/doc/classes/MapReduce.src/M000002.html +18 -0
- data/doc/classes/MapReduce.src/M000003.html +25 -0
- data/doc/classes/MapReduce.src/M000004.html +21 -0
- data/doc/classes/MapReduce.src/M000005.html +18 -0
- data/doc/classes/MapReduce.src/M000006.html +18 -0
- data/doc/classes/MapReduce.src/M000007.html +19 -0
- data/doc/classes/MapReduce.src/M000008.html +18 -0
- data/doc/classes/MapReduce.src/M000009.html +19 -0
- data/doc/classes/MapReduce.src/M000010.html +23 -0
- data/doc/classes/MapReduce.src/M000011.html +24 -0
- data/doc/classes/MapReduce.src/M000012.html +23 -0
- data/doc/classes/MapReduce.src/M000013.html +18 -0
- data/doc/classes/MapReduce.src/M000014.html +26 -0
- data/doc/classes/MapReduceError.html +111 -0
- data/doc/classes/Starfish/RingFinger.html +131 -0
- data/doc/classes/Starfish/RingFinger.src/M000031.html +20 -0
- data/doc/classes/Starfish.html +357 -0
- data/doc/classes/Starfish.src/M000019.html +19 -0
- data/doc/classes/Starfish.src/M000020.html +18 -0
- data/doc/classes/Starfish.src/M000021.html +45 -0
- data/doc/classes/Starfish.src/M000022.html +71 -0
- data/doc/classes/Starfish.src/M000023.html +23 -0
- data/doc/classes/Starfish.src/M000024.html +36 -0
- data/doc/classes/Starfish.src/M000025.html +35 -0
- data/doc/classes/Starfish.src/M000026.html +18 -0
- data/doc/classes/Starfish.src/M000027.html +22 -0
- data/doc/classes/Starfish.src/M000028.html +18 -0
- data/doc/classes/Starfish.src/M000029.html +18 -0
- data/doc/classes/Starfish.src/M000030.html +18 -0
- data/doc/classes/StarfishError.html +111 -0
- data/doc/created.rid +1 -0
- data/doc/files/README.html +336 -0
- data/doc/files/lib/map_reduce/active_record_rb.html +109 -0
- data/doc/files/lib/map_reduce/array_rb.html +101 -0
- data/doc/files/lib/map_reduce/file_rb.html +101 -0
- data/doc/files/lib/map_reduce_rb.html +111 -0
- data/doc/files/lib/starfish_rb.html +114 -0
- data/doc/fr_class_index.html +36 -0
- data/doc/fr_file_index.html +32 -0
- data/doc/fr_method_index.html +57 -0
- data/doc/index.html +24 -0
- data/doc/rdoc-style.css +208 -0
- data/examples/foo.rb +20 -0
- data/examples/map_reduce/active_record.rb +10 -0
- data/examples/map_reduce/file.rb +11 -0
- data/lib/map_reduce/active_record.rb +98 -0
- data/lib/map_reduce/array.rb +4 -0
- data/lib/map_reduce/file.rb +4 -0
- data/lib/map_reduce.rb +121 -0
- data/lib/starfish.rb +200 -0
- metadata +124 -0
@@ -0,0 +1,18 @@
|
|
1
|
+
<?xml version="1.0" encoding="iso-8859-1"?>
|
2
|
+
<!DOCTYPE html
|
3
|
+
PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
|
4
|
+
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
|
5
|
+
|
6
|
+
<html>
|
7
|
+
<head>
|
8
|
+
<title>spawn (Starfish)</title>
|
9
|
+
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
|
10
|
+
<link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
|
11
|
+
</head>
|
12
|
+
<body class="standalone-code">
|
13
|
+
<pre><span class="ruby-comment cmt"># File lib/starfish.rb, line 175</span>
|
14
|
+
<span class="ruby-keyword kw">def</span> <span class="ruby-identifier">spawn</span>
|
15
|
+
<span class="ruby-ivar">@started</span> <span class="ruby-operator">||=</span> <span class="ruby-identifier">fork</span> { <span class="ruby-identifier">system</span>(<span class="ruby-node">"ruby #{File.dirname(__FILE__)}/../bin/starfish #{@uniq} server > /dev/null"</span>) }
|
16
|
+
<span class="ruby-keyword kw">end</span></pre>
|
17
|
+
</body>
|
18
|
+
</html>
|
@@ -0,0 +1,22 @@
|
|
1
|
+
<?xml version="1.0" encoding="iso-8859-1"?>
|
2
|
+
<!DOCTYPE html
|
3
|
+
PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
|
4
|
+
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
|
5
|
+
|
6
|
+
<html>
|
7
|
+
<head>
|
8
|
+
<title>stop (Starfish)</title>
|
9
|
+
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
|
10
|
+
<link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
|
11
|
+
</head>
|
12
|
+
<body class="standalone-code">
|
13
|
+
<pre><span class="ruby-comment cmt"># File lib/starfish.rb, line 179</span>
|
14
|
+
<span class="ruby-keyword kw">def</span> <span class="ruby-identifier">stop</span>
|
15
|
+
<span class="ruby-identifier">puts</span> <span class="ruby-value str">"stopping the server"</span>
|
16
|
+
<span class="ruby-constant">Process</span>.<span class="ruby-identifier">kill</span>(<span class="ruby-value str">"SIGHUP"</span>, <span class="ruby-constant">IO</span>.<span class="ruby-identifier">read</span>(<span class="ruby-ivar">@@options</span>[<span class="ruby-identifier">:pid</span>] <span class="ruby-operator">||</span> <span class="ruby-node">"/tmp/starfish-#{uniq}.pid"</span>).<span class="ruby-identifier">to_i</span>)
|
17
|
+
<span class="ruby-keyword kw">rescue</span> <span class="ruby-constant">Errno</span><span class="ruby-operator">::</span><span class="ruby-constant">ENOENT</span>
|
18
|
+
<span class="ruby-identifier">puts</span> <span class="ruby-value str">"Fatal error, please kill all starfish processes manually and try again"</span>
|
19
|
+
<span class="ruby-identifier">system</span>(<span class="ruby-value str">"ps auxww|grep starfish"</span>)
|
20
|
+
<span class="ruby-keyword kw">end</span></pre>
|
21
|
+
</body>
|
22
|
+
</html>
|
@@ -0,0 +1,18 @@
|
|
1
|
+
<?xml version="1.0" encoding="iso-8859-1"?>
|
2
|
+
<!DOCTYPE html
|
3
|
+
PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
|
4
|
+
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
|
5
|
+
|
6
|
+
<html>
|
7
|
+
<head>
|
8
|
+
<title>server= (Starfish)</title>
|
9
|
+
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
|
10
|
+
<link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
|
11
|
+
</head>
|
12
|
+
<body class="standalone-code">
|
13
|
+
<pre><span class="ruby-comment cmt"># File lib/starfish.rb, line 188</span>
|
14
|
+
<span class="ruby-keyword kw">def</span> <span class="ruby-identifier">server=</span>(<span class="ruby-identifier">server</span>)
|
15
|
+
<span class="ruby-ivar">@@server</span> = <span class="ruby-identifier">server</span>
|
16
|
+
<span class="ruby-keyword kw">end</span></pre>
|
17
|
+
</body>
|
18
|
+
</html>
|
@@ -0,0 +1,18 @@
|
|
1
|
+
<?xml version="1.0" encoding="iso-8859-1"?>
|
2
|
+
<!DOCTYPE html
|
3
|
+
PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
|
4
|
+
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
|
5
|
+
|
6
|
+
<html>
|
7
|
+
<head>
|
8
|
+
<title>client= (Starfish)</title>
|
9
|
+
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
|
10
|
+
<link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
|
11
|
+
</head>
|
12
|
+
<body class="standalone-code">
|
13
|
+
<pre><span class="ruby-comment cmt"># File lib/starfish.rb, line 192</span>
|
14
|
+
<span class="ruby-keyword kw">def</span> <span class="ruby-identifier">client=</span>(<span class="ruby-identifier">client</span>)
|
15
|
+
<span class="ruby-ivar">@@client</span> = <span class="ruby-identifier">client</span>
|
16
|
+
<span class="ruby-keyword kw">end</span></pre>
|
17
|
+
</body>
|
18
|
+
</html>
|
@@ -0,0 +1,18 @@
|
|
1
|
+
<?xml version="1.0" encoding="iso-8859-1"?>
|
2
|
+
<!DOCTYPE html
|
3
|
+
PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
|
4
|
+
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
|
5
|
+
|
6
|
+
<html>
|
7
|
+
<head>
|
8
|
+
<title>options= (Starfish)</title>
|
9
|
+
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
|
10
|
+
<link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
|
11
|
+
</head>
|
12
|
+
<body class="standalone-code">
|
13
|
+
<pre><span class="ruby-comment cmt"># File lib/starfish.rb, line 196</span>
|
14
|
+
<span class="ruby-keyword kw">def</span> <span class="ruby-identifier">options=</span>(<span class="ruby-identifier">options</span>={})
|
15
|
+
<span class="ruby-ivar">@@options</span>.<span class="ruby-identifier">update</span>(<span class="ruby-identifier">options</span>)
|
16
|
+
<span class="ruby-keyword kw">end</span></pre>
|
17
|
+
</body>
|
18
|
+
</html>
|
@@ -0,0 +1,111 @@
|
|
1
|
+
<?xml version="1.0" encoding="iso-8859-1"?>
|
2
|
+
<!DOCTYPE html
|
3
|
+
PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
|
4
|
+
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
|
5
|
+
|
6
|
+
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
|
7
|
+
<head>
|
8
|
+
<title>Class: StarfishError</title>
|
9
|
+
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
|
10
|
+
<meta http-equiv="Content-Script-Type" content="text/javascript" />
|
11
|
+
<link rel="stylesheet" href=".././rdoc-style.css" type="text/css" media="screen" />
|
12
|
+
<script type="text/javascript">
|
13
|
+
// <![CDATA[
|
14
|
+
|
15
|
+
function popupCode( url ) {
|
16
|
+
window.open(url, "Code", "resizable=yes,scrollbars=yes,toolbar=no,status=no,height=150,width=400")
|
17
|
+
}
|
18
|
+
|
19
|
+
function toggleCode( id ) {
|
20
|
+
if ( document.getElementById )
|
21
|
+
elem = document.getElementById( id );
|
22
|
+
else if ( document.all )
|
23
|
+
elem = eval( "document.all." + id );
|
24
|
+
else
|
25
|
+
return false;
|
26
|
+
|
27
|
+
elemStyle = elem.style;
|
28
|
+
|
29
|
+
if ( elemStyle.display != "block" ) {
|
30
|
+
elemStyle.display = "block"
|
31
|
+
} else {
|
32
|
+
elemStyle.display = "none"
|
33
|
+
}
|
34
|
+
|
35
|
+
return true;
|
36
|
+
}
|
37
|
+
|
38
|
+
// Make codeblocks hidden by default
|
39
|
+
document.writeln( "<style type=\"text/css\">div.method-source-code { display: none }</style>" )
|
40
|
+
|
41
|
+
// ]]>
|
42
|
+
</script>
|
43
|
+
|
44
|
+
</head>
|
45
|
+
<body>
|
46
|
+
|
47
|
+
|
48
|
+
|
49
|
+
<div id="classHeader">
|
50
|
+
<table class="header-table">
|
51
|
+
<tr class="top-aligned-row">
|
52
|
+
<td><strong>Class</strong></td>
|
53
|
+
<td class="class-name-in-header">StarfishError</td>
|
54
|
+
</tr>
|
55
|
+
<tr class="top-aligned-row">
|
56
|
+
<td><strong>In:</strong></td>
|
57
|
+
<td>
|
58
|
+
<a href="../files/lib/starfish_rb.html">
|
59
|
+
lib/starfish.rb
|
60
|
+
</a>
|
61
|
+
<br />
|
62
|
+
</td>
|
63
|
+
</tr>
|
64
|
+
|
65
|
+
<tr class="top-aligned-row">
|
66
|
+
<td><strong>Parent:</strong></td>
|
67
|
+
<td>
|
68
|
+
StandardError
|
69
|
+
</td>
|
70
|
+
</tr>
|
71
|
+
</table>
|
72
|
+
</div>
|
73
|
+
<!-- banner header -->
|
74
|
+
|
75
|
+
<div id="bodyContent">
|
76
|
+
|
77
|
+
|
78
|
+
|
79
|
+
<div id="contextContent">
|
80
|
+
|
81
|
+
|
82
|
+
|
83
|
+
</div>
|
84
|
+
|
85
|
+
|
86
|
+
</div>
|
87
|
+
|
88
|
+
|
89
|
+
<!-- if includes -->
|
90
|
+
|
91
|
+
<div id="section">
|
92
|
+
|
93
|
+
|
94
|
+
|
95
|
+
|
96
|
+
|
97
|
+
|
98
|
+
|
99
|
+
|
100
|
+
<!-- if method_list -->
|
101
|
+
|
102
|
+
|
103
|
+
</div>
|
104
|
+
|
105
|
+
|
106
|
+
<div id="validator-badges">
|
107
|
+
<p><small><a href="http://validator.w3.org/check/referer">[Validate]</a></small></p>
|
108
|
+
</div>
|
109
|
+
|
110
|
+
</body>
|
111
|
+
</html>
|
data/doc/created.rid
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
Wed Aug 16 15:52:22 PDT 2006
|
@@ -0,0 +1,336 @@
|
|
1
|
+
<?xml version="1.0" encoding="iso-8859-1"?>
|
2
|
+
<!DOCTYPE html
|
3
|
+
PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
|
4
|
+
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
|
5
|
+
|
6
|
+
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
|
7
|
+
<head>
|
8
|
+
<title>File: README</title>
|
9
|
+
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
|
10
|
+
<meta http-equiv="Content-Script-Type" content="text/javascript" />
|
11
|
+
<link rel="stylesheet" href=".././rdoc-style.css" type="text/css" media="screen" />
|
12
|
+
<script type="text/javascript">
|
13
|
+
// <![CDATA[
|
14
|
+
|
15
|
+
function popupCode( url ) {
|
16
|
+
window.open(url, "Code", "resizable=yes,scrollbars=yes,toolbar=no,status=no,height=150,width=400")
|
17
|
+
}
|
18
|
+
|
19
|
+
function toggleCode( id ) {
|
20
|
+
if ( document.getElementById )
|
21
|
+
elem = document.getElementById( id );
|
22
|
+
else if ( document.all )
|
23
|
+
elem = eval( "document.all." + id );
|
24
|
+
else
|
25
|
+
return false;
|
26
|
+
|
27
|
+
elemStyle = elem.style;
|
28
|
+
|
29
|
+
if ( elemStyle.display != "block" ) {
|
30
|
+
elemStyle.display = "block"
|
31
|
+
} else {
|
32
|
+
elemStyle.display = "none"
|
33
|
+
}
|
34
|
+
|
35
|
+
return true;
|
36
|
+
}
|
37
|
+
|
38
|
+
// Make codeblocks hidden by default
|
39
|
+
document.writeln( "<style type=\"text/css\">div.method-source-code { display: none }</style>" )
|
40
|
+
|
41
|
+
// ]]>
|
42
|
+
</script>
|
43
|
+
|
44
|
+
</head>
|
45
|
+
<body>
|
46
|
+
|
47
|
+
|
48
|
+
|
49
|
+
<div id="fileHeader">
|
50
|
+
<h1>README</h1>
|
51
|
+
<table class="header-table">
|
52
|
+
<tr class="top-aligned-row">
|
53
|
+
<td><strong>Path:</strong></td>
|
54
|
+
<td>README
|
55
|
+
</td>
|
56
|
+
</tr>
|
57
|
+
<tr class="top-aligned-row">
|
58
|
+
<td><strong>Last Update:</strong></td>
|
59
|
+
<td>Wed Aug 16 15:40:05 PDT 2006</td>
|
60
|
+
</tr>
|
61
|
+
</table>
|
62
|
+
</div>
|
63
|
+
<!-- banner header -->
|
64
|
+
|
65
|
+
<div id="bodyContent">
|
66
|
+
|
67
|
+
|
68
|
+
|
69
|
+
<div id="contextContent">
|
70
|
+
|
71
|
+
<div id="description">
|
72
|
+
<h2>Welcome to <a href="../classes/Starfish.html">Starfish</a></h2>
|
73
|
+
<p>
|
74
|
+
<a href="../classes/Starfish.html">Starfish</a> is a utility born from
|
75
|
+
abstracting code from a highly popular website (<a
|
76
|
+
href="http://mog.com">mog.com</a>) to make distributed programming
|
77
|
+
ridiculously easy.
|
78
|
+
</p>
|
79
|
+
<h2>Download</h2>
|
80
|
+
<ul>
|
81
|
+
<li>gem install starfish
|
82
|
+
|
83
|
+
</li>
|
84
|
+
<li><a
|
85
|
+
href="http://rubyforge.org/projects/starfish">rubyforge.org/projects/starfish</a>
|
86
|
+
|
87
|
+
</li>
|
88
|
+
<li>svn co svn://rubyforge.org//var/svn/starfish
|
89
|
+
|
90
|
+
</li>
|
91
|
+
</ul>
|
92
|
+
<h2>Usage</h2>
|
93
|
+
<p>
|
94
|
+
Make a file and define the server and client. What follows is a simple
|
95
|
+
example:
|
96
|
+
</p>
|
97
|
+
<pre>
|
98
|
+
# foo.rb
|
99
|
+
class Foo
|
100
|
+
attr_reader :i
|
101
|
+
|
102
|
+
def initialize
|
103
|
+
@i = 0
|
104
|
+
end
|
105
|
+
|
106
|
+
def inc
|
107
|
+
logger.info "YAY it incremented by 1 up to #{@i}"
|
108
|
+
@i += 1
|
109
|
+
end
|
110
|
+
end
|
111
|
+
|
112
|
+
server do |object|
|
113
|
+
object = Foo.new
|
114
|
+
end
|
115
|
+
|
116
|
+
client do |object|
|
117
|
+
object.inc
|
118
|
+
end
|
119
|
+
</pre>
|
120
|
+
<p>
|
121
|
+
Notice that you need not specify any network code, nor learn a foreign API
|
122
|
+
like DRb or Rinda. You need not even define logger, starfish takes care of
|
123
|
+
it for you at runtime. You just tell it how to serve an object and how to
|
124
|
+
process the object. To run this code, simply run:
|
125
|
+
</p>
|
126
|
+
<pre>
|
127
|
+
starfish foo.rb
|
128
|
+
</pre>
|
129
|
+
<p>
|
130
|
+
This starts a server AND a client to start processing that object. The
|
131
|
+
client runs in a continuous loop. If you call starfish foo.rb subsequent
|
132
|
+
times from either the local machine or any machine on the local network,
|
133
|
+
more clients will start processing the server concurrently.
|
134
|
+
</p>
|
135
|
+
<p>
|
136
|
+
By default, the server will start a log file at /tmp/foo.rb.log. You can
|
137
|
+
override this behaviour.
|
138
|
+
</p>
|
139
|
+
<pre>
|
140
|
+
server :log => "/var/log/foo.log" do |object|
|
141
|
+
object = Foo.new
|
142
|
+
end
|
143
|
+
</pre>
|
144
|
+
<p>
|
145
|
+
The :log parameter is intelligent and can take various other formats
|
146
|
+
including nil for no logging or instances of any kind of logger class you
|
147
|
+
prefer to use.
|
148
|
+
</p>
|
149
|
+
<h2><a href="../classes/MapReduce.html">MapReduce</a></h2>
|
150
|
+
<p>
|
151
|
+
The crown jewel of <a href="../classes/Starfish.html">Starfish</a> is
|
152
|
+
actually a library built on <a href="../classes/Starfish.html">Starfish</a>
|
153
|
+
called <a href="../classes/MapReduce.html">MapReduce</a>. Inspired by
|
154
|
+
Google’s <a href="../classes/MapReduce.html">MapReduce</a> (<a
|
155
|
+
href="http://en.wikipedia.org/wiki/MapReduce">en.wikipedia.org/wiki/MapReduce</a>),
|
156
|
+
it automates the process of divide and conquer for large data sets.
|
157
|
+
Currently, <a href="../classes/Starfish.html">Starfish</a>’s
|
158
|
+
implementation only works with the ActiveRecord ORM from Ruby on Rails. The
|
159
|
+
idea is that you have a very large data set that would be unmanageable to
|
160
|
+
process otherwise. For example, let’s say your database table has
|
161
|
+
30GB of information in it. It would be very difficult to keep 30GB of
|
162
|
+
records in RAM at one time; even if you processed them serially, it would
|
163
|
+
take a very long time. The idea is to have many clients grabbing chunks of
|
164
|
+
the database at a time making much faster work, even on a single processor
|
165
|
+
machine, than would otherwise be feasible.
|
166
|
+
</p>
|
167
|
+
<p>
|
168
|
+
Here is the basic code that will get you up and running with <a
|
169
|
+
href="../classes/MapReduce.html">MapReduce</a> in <a
|
170
|
+
href="../classes/Starfish.html">Starfish</a>.
|
171
|
+
</p>
|
172
|
+
<pre>
|
173
|
+
# item.rb
|
174
|
+
ActiveRecord::Base.establish_connection(
|
175
|
+
:adapter => "mysql",
|
176
|
+
:host => "localhost",
|
177
|
+
:username => "root",
|
178
|
+
:password => "",
|
179
|
+
:database => "some_database"
|
180
|
+
)
|
181
|
+
|
182
|
+
class Item < ActiveRecord::Base; end
|
183
|
+
|
184
|
+
server do |map_reduce|
|
185
|
+
map_reduce.type = Item
|
186
|
+
end
|
187
|
+
|
188
|
+
client do |item|
|
189
|
+
logger.info item.id
|
190
|
+
end
|
191
|
+
</pre>
|
192
|
+
<p>
|
193
|
+
Just like the previous example, just run:
|
194
|
+
</p>
|
195
|
+
<pre>
|
196
|
+
starfish item.rb
|
197
|
+
</pre>
|
198
|
+
<p>
|
199
|
+
<a href="../classes/Starfish.html">Starfish</a> takes care of the rest. The
|
200
|
+
code above does the following:
|
201
|
+
</p>
|
202
|
+
<ul>
|
203
|
+
<li>The server grabs all the items via: Item.find(:all)
|
204
|
+
|
205
|
+
</li>
|
206
|
+
<li>Each of the clients grab an item from the collection
|
207
|
+
|
208
|
+
</li>
|
209
|
+
<li>When there are no more items to be grabbed, everything shuts down
|
210
|
+
|
211
|
+
</li>
|
212
|
+
</ul>
|
213
|
+
<p>
|
214
|
+
However there are cases when you might want some fancier behaviour.
|
215
|
+
</p>
|
216
|
+
<pre>
|
217
|
+
server do |map_reduce|
|
218
|
+
map_reduce.type = Item
|
219
|
+
map_reduce.conditions = ["some_important_flag = ?", 1]
|
220
|
+
map_reduce.vigilant = true
|
221
|
+
map_reduce.queue_size = 1000
|
222
|
+
end
|
223
|
+
</pre>
|
224
|
+
<p>
|
225
|
+
Here are three new options: conditions, vigilant, and queue_size.
|
226
|
+
Conditions simply limits what is grabbed via ActiveRecord. It is equivalent
|
227
|
+
to saying: Item.find(:all, :conditions => ["some_important_flag =
|
228
|
+
?", 1]). Vigilant simply keeps the clients alive and when
|
229
|
+
Item.find(:all) runs out of items, it will vigilantly poll for newly
|
230
|
+
created items periodically. Queue size specifies the number of items
|
231
|
+
buffered in the server queue. In the case of the 30GB database table, you
|
232
|
+
do not want to grab all 30GB at once, instead you grab 1000 at a time as
|
233
|
+
needed by your clients.
|
234
|
+
</p>
|
235
|
+
<p>
|
236
|
+
Another options you have when you use the vigilant option is
|
237
|
+
map_reduce.empty_queue_wait which specifies how long to wait between polls
|
238
|
+
for new information.
|
239
|
+
</p>
|
240
|
+
<p>
|
241
|
+
There is yet a third main run-mode for <a
|
242
|
+
href="../classes/MapReduce.html">MapReduce</a> under ActiveRecord.
|
243
|
+
</p>
|
244
|
+
<pre>
|
245
|
+
server do |map_reduce|
|
246
|
+
map_reduce.type = Item
|
247
|
+
map_reduce.rescan_when_complete = true
|
248
|
+
end
|
249
|
+
</pre>
|
250
|
+
<p>
|
251
|
+
The events follow like this when the rescan_when_complete option is
|
252
|
+
enabled:
|
253
|
+
</p>
|
254
|
+
<ul>
|
255
|
+
<li>The server grabs all the items via: Item.find(:all)
|
256
|
+
|
257
|
+
</li>
|
258
|
+
<li>Each of the clients grab an item from the collection
|
259
|
+
|
260
|
+
</li>
|
261
|
+
<li>When there are no more items to be grabbed, the queue is re-filled via
|
262
|
+
Item.find(:all) and the process starts again
|
263
|
+
|
264
|
+
</li>
|
265
|
+
</ul>
|
266
|
+
<p>
|
267
|
+
If you need to continuously check over data for integrity, this option will
|
268
|
+
meet your demands.
|
269
|
+
</p>
|
270
|
+
<p>
|
271
|
+
Another bonus you get while using <a
|
272
|
+
href="../classes/MapReduce.html">MapReduce</a> is automatic statistics.
|
273
|
+
Simply call:
|
274
|
+
</p>
|
275
|
+
<pre>
|
276
|
+
starfish item.rb stats
|
277
|
+
</pre>
|
278
|
+
<p>
|
279
|
+
And you will be returned a YAML hash of statistics.
|
280
|
+
</p>
|
281
|
+
<pre>
|
282
|
+
---
|
283
|
+
time_spent_grabbing_objects: 0.090328
|
284
|
+
time_spent_processing_objects: 0.023431
|
285
|
+
time_began: 2006-08-16 09:30:32.926399 -07:00
|
286
|
+
num_queues_grabbed: 5
|
287
|
+
time_spent_grabbing_queues: 0.090328
|
288
|
+
num_objects_grabbed: 10
|
289
|
+
</pre>
|
290
|
+
<h2>Examples</h2>
|
291
|
+
<p>
|
292
|
+
See the examples/ directory.
|
293
|
+
</p>
|
294
|
+
<h2>Authors</h2>
|
295
|
+
<ul>
|
296
|
+
<li>Lucas Carlson from MOG (<a href="mailto:lucas@rufy.com">lucas@rufy.com</a>)
|
297
|
+
- <a href="http://mog.com">mog.com</a>
|
298
|
+
|
299
|
+
</li>
|
300
|
+
</ul>
|
301
|
+
<p>
|
302
|
+
This library is released under the terms of the BSD.
|
303
|
+
</p>
|
304
|
+
|
305
|
+
</div>
|
306
|
+
|
307
|
+
|
308
|
+
</div>
|
309
|
+
|
310
|
+
|
311
|
+
</div>
|
312
|
+
|
313
|
+
|
314
|
+
<!-- if includes -->
|
315
|
+
|
316
|
+
<div id="section">
|
317
|
+
|
318
|
+
|
319
|
+
|
320
|
+
|
321
|
+
|
322
|
+
|
323
|
+
|
324
|
+
|
325
|
+
<!-- if method_list -->
|
326
|
+
|
327
|
+
|
328
|
+
</div>
|
329
|
+
|
330
|
+
|
331
|
+
<div id="validator-badges">
|
332
|
+
<p><small><a href="http://validator.w3.org/check/referer">[Validate]</a></small></p>
|
333
|
+
</div>
|
334
|
+
|
335
|
+
</body>
|
336
|
+
</html>
|
@@ -0,0 +1,109 @@
|
|
1
|
+
<?xml version="1.0" encoding="iso-8859-1"?>
|
2
|
+
<!DOCTYPE html
|
3
|
+
PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
|
4
|
+
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
|
5
|
+
|
6
|
+
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
|
7
|
+
<head>
|
8
|
+
<title>File: active_record.rb</title>
|
9
|
+
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
|
10
|
+
<meta http-equiv="Content-Script-Type" content="text/javascript" />
|
11
|
+
<link rel="stylesheet" href="../../.././rdoc-style.css" type="text/css" media="screen" />
|
12
|
+
<script type="text/javascript">
|
13
|
+
// <![CDATA[
|
14
|
+
|
15
|
+
function popupCode( url ) {
|
16
|
+
window.open(url, "Code", "resizable=yes,scrollbars=yes,toolbar=no,status=no,height=150,width=400")
|
17
|
+
}
|
18
|
+
|
19
|
+
function toggleCode( id ) {
|
20
|
+
if ( document.getElementById )
|
21
|
+
elem = document.getElementById( id );
|
22
|
+
else if ( document.all )
|
23
|
+
elem = eval( "document.all." + id );
|
24
|
+
else
|
25
|
+
return false;
|
26
|
+
|
27
|
+
elemStyle = elem.style;
|
28
|
+
|
29
|
+
if ( elemStyle.display != "block" ) {
|
30
|
+
elemStyle.display = "block"
|
31
|
+
} else {
|
32
|
+
elemStyle.display = "none"
|
33
|
+
}
|
34
|
+
|
35
|
+
return true;
|
36
|
+
}
|
37
|
+
|
38
|
+
// Make codeblocks hidden by default
|
39
|
+
document.writeln( "<style type=\"text/css\">div.method-source-code { display: none }</style>" )
|
40
|
+
|
41
|
+
// ]]>
|
42
|
+
</script>
|
43
|
+
|
44
|
+
</head>
|
45
|
+
<body>
|
46
|
+
|
47
|
+
|
48
|
+
|
49
|
+
<div id="fileHeader">
|
50
|
+
<h1>active_record.rb</h1>
|
51
|
+
<table class="header-table">
|
52
|
+
<tr class="top-aligned-row">
|
53
|
+
<td><strong>Path:</strong></td>
|
54
|
+
<td>lib/map_reduce/active_record.rb
|
55
|
+
</td>
|
56
|
+
</tr>
|
57
|
+
<tr class="top-aligned-row">
|
58
|
+
<td><strong>Last Update:</strong></td>
|
59
|
+
<td>Wed Aug 16 14:38:32 PDT 2006</td>
|
60
|
+
</tr>
|
61
|
+
</table>
|
62
|
+
</div>
|
63
|
+
<!-- banner header -->
|
64
|
+
|
65
|
+
<div id="bodyContent">
|
66
|
+
|
67
|
+
|
68
|
+
|
69
|
+
<div id="contextContent">
|
70
|
+
|
71
|
+
|
72
|
+
<div id="requires-list">
|
73
|
+
<h3 class="section-bar">Required files</h3>
|
74
|
+
|
75
|
+
<div class="name-list">
|
76
|
+
rubygems
|
77
|
+
active_record
|
78
|
+
</div>
|
79
|
+
</div>
|
80
|
+
|
81
|
+
</div>
|
82
|
+
|
83
|
+
|
84
|
+
</div>
|
85
|
+
|
86
|
+
|
87
|
+
<!-- if includes -->
|
88
|
+
|
89
|
+
<div id="section">
|
90
|
+
|
91
|
+
|
92
|
+
|
93
|
+
|
94
|
+
|
95
|
+
|
96
|
+
|
97
|
+
|
98
|
+
<!-- if method_list -->
|
99
|
+
|
100
|
+
|
101
|
+
</div>
|
102
|
+
|
103
|
+
|
104
|
+
<div id="validator-badges">
|
105
|
+
<p><small><a href="http://validator.w3.org/check/referer">[Validate]</a></small></p>
|
106
|
+
</div>
|
107
|
+
|
108
|
+
</body>
|
109
|
+
</html>
|