sip 0.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. data/Gemfile +2 -0
  2. data/LICENSE +674 -0
  3. data/README.rdoc +32 -0
  4. data/Rakefile +21 -0
  5. data/bin/sip +83 -0
  6. data/bin/transpart +114 -0
  7. data/docs/classes/Sip.html +169 -0
  8. data/docs/classes/Sip/CmdOpts.html +179 -0
  9. data/docs/classes/Sip/Config.html +362 -0
  10. data/docs/classes/Sip/DBBase.html +368 -0
  11. data/docs/classes/Sip/HadoopException.html +111 -0
  12. data/docs/classes/Sip/Hive.html +295 -0
  13. data/docs/classes/Sip/HiveQueryException.html +111 -0
  14. data/docs/classes/Sip/ImportScriptExecutionError.html +111 -0
  15. data/docs/classes/Sip/MySQLSipper.html +273 -0
  16. data/docs/classes/Sip/NoSuchColumn.html +111 -0
  17. data/docs/classes/Sip/NoSuchTable.html +111 -0
  18. data/docs/classes/Sip/PastFailureException.html +111 -0
  19. data/docs/classes/Sip/Sipper.html +454 -0
  20. data/docs/classes/Sip/UnsupportedDatabaseType.html +111 -0
  21. data/docs/classes/Sip/Utils.html +269 -0
  22. data/docs/classes/Struct.html +146 -0
  23. data/docs/created.rid +1 -0
  24. data/docs/files/README_rdoc.html +174 -0
  25. data/docs/files/lib/sip/cmdopts_rb.html +101 -0
  26. data/docs/files/lib/sip/config_rb.html +108 -0
  27. data/docs/files/lib/sip/databases/dbbase_rb.html +108 -0
  28. data/docs/files/lib/sip/databases/mysql_rb.html +108 -0
  29. data/docs/files/lib/sip/exceptions_rb.html +101 -0
  30. data/docs/files/lib/sip/extensions_rb.html +101 -0
  31. data/docs/files/lib/sip/hive_rb.html +101 -0
  32. data/docs/files/lib/sip/sipper_rb.html +101 -0
  33. data/docs/files/lib/sip/utils_rb.html +110 -0
  34. data/docs/files/lib/sip/version_rb.html +101 -0
  35. data/docs/files/lib/sip_rb.html +117 -0
  36. data/docs/fr_class_index.html +42 -0
  37. data/docs/fr_file_index.html +38 -0
  38. data/docs/fr_method_index.html +72 -0
  39. data/docs/index.html +24 -0
  40. data/docs/rdoc-style.css +208 -0
  41. data/lib/sip.rb +10 -0
  42. data/lib/sip/cmdopts.rb +20 -0
  43. data/lib/sip/config.rb +80 -0
  44. data/lib/sip/databases/dbbase.rb +56 -0
  45. data/lib/sip/databases/mysql.rb +52 -0
  46. data/lib/sip/exceptions.rb +9 -0
  47. data/lib/sip/extensions.rb +5 -0
  48. data/lib/sip/hive.rb +62 -0
  49. data/lib/sip/sipper.rb +118 -0
  50. data/lib/sip/templates/export.sh +73 -0
  51. data/lib/sip/utils.rb +58 -0
  52. data/lib/sip/version.rb +3 -0
  53. data/test/database_interaction_test.rb +7 -0
  54. data/test/hive_test.rb +28 -0
  55. data/test/sipper_test.rb +25 -0
  56. metadata +125 -0
data/docs/classes/Sip/NoSuchTable.html ADDED
@@ -0,0 +1,111 @@
1
+ <?xml version="1.0" encoding="iso-8859-1"?>
2
+ <!DOCTYPE html
3
+ PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
4
+ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
5
+
6
+ <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
7
+ <head>
8
+ <title>Class: Sip::NoSuchTable</title>
9
+ <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
10
+ <meta http-equiv="Content-Script-Type" content="text/javascript" />
11
+ <link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
12
+ <script type="text/javascript">
13
+ // <![CDATA[
14
+
15
+ function popupCode( url ) {
16
+ window.open(url, "Code", "resizable=yes,scrollbars=yes,toolbar=no,status=no,height=150,width=400")
17
+ }
18
+
19
+ function toggleCode( id ) {
20
+ if ( document.getElementById )
21
+ elem = document.getElementById( id );
22
+ else if ( document.all )
23
+ elem = eval( "document.all." + id );
24
+ else
25
+ return false;
26
+
27
+ elemStyle = elem.style;
28
+
29
+ if ( elemStyle.display != "block" ) {
30
+ elemStyle.display = "block"
31
+ } else {
32
+ elemStyle.display = "none"
33
+ }
34
+
35
+ return true;
36
+ }
37
+
38
+ // Make codeblocks hidden by default
39
+ document.writeln( "<style type=\"text/css\">div.method-source-code { display: none }</style>" )
40
+
41
+ // ]]>
42
+ </script>
43
+
44
+ </head>
45
+ <body>
46
+
47
+
48
+
49
+ <div id="classHeader">
50
+ <table class="header-table">
51
+ <tr class="top-aligned-row">
52
+ <td><strong>Class</strong></td>
53
+ <td class="class-name-in-header">Sip::NoSuchTable</td>
54
+ </tr>
55
+ <tr class="top-aligned-row">
56
+ <td><strong>In:</strong></td>
57
+ <td>
58
+ <a href="../../files/lib/sip/exceptions_rb.html">
59
+ lib/sip/exceptions.rb
60
+ </a>
61
+ <br />
62
+ </td>
63
+ </tr>
64
+
65
+ <tr class="top-aligned-row">
66
+ <td><strong>Parent:</strong></td>
67
+ <td>
68
+ StandardError
69
+ </td>
70
+ </tr>
71
+ </table>
72
+ </div>
73
+ <!-- banner header -->
74
+
75
+ <div id="bodyContent">
76
+
77
+
78
+
79
+ <div id="contextContent">
80
+
81
+
82
+
83
+ </div>
84
+
85
+
86
+ </div>
87
+
88
+
89
+ <!-- if includes -->
90
+
91
+ <div id="section">
92
+
93
+
94
+
95
+
96
+
97
+
98
+
99
+
100
+ <!-- if method_list -->
101
+
102
+
103
+ </div>
104
+
105
+
106
+ <div id="validator-badges">
107
+ <p><small><a href="http://validator.w3.org/check/referer">[Validate]</a></small></p>
108
+ </div>
109
+
110
+ </body>
111
+ </html>
data/docs/classes/Sip/PastFailureException.html ADDED
@@ -0,0 +1,111 @@
1
+ <?xml version="1.0" encoding="iso-8859-1"?>
2
+ <!DOCTYPE html
3
+ PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
4
+ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
5
+
6
+ <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
7
+ <head>
8
+ <title>Class: Sip::PastFailureException</title>
9
+ <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
10
+ <meta http-equiv="Content-Script-Type" content="text/javascript" />
11
+ <link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
12
+ <script type="text/javascript">
13
+ // <![CDATA[
14
+
15
+ function popupCode( url ) {
16
+ window.open(url, "Code", "resizable=yes,scrollbars=yes,toolbar=no,status=no,height=150,width=400")
17
+ }
18
+
19
+ function toggleCode( id ) {
20
+ if ( document.getElementById )
21
+ elem = document.getElementById( id );
22
+ else if ( document.all )
23
+ elem = eval( "document.all." + id );
24
+ else
25
+ return false;
26
+
27
+ elemStyle = elem.style;
28
+
29
+ if ( elemStyle.display != "block" ) {
30
+ elemStyle.display = "block"
31
+ } else {
32
+ elemStyle.display = "none"
33
+ }
34
+
35
+ return true;
36
+ }
37
+
38
+ // Make codeblocks hidden by default
39
+ document.writeln( "<style type=\"text/css\">div.method-source-code { display: none }</style>" )
40
+
41
+ // ]]>
42
+ </script>
43
+
44
+ </head>
45
+ <body>
46
+
47
+
48
+
49
+ <div id="classHeader">
50
+ <table class="header-table">
51
+ <tr class="top-aligned-row">
52
+ <td><strong>Class</strong></td>
53
+ <td class="class-name-in-header">Sip::PastFailureException</td>
54
+ </tr>
55
+ <tr class="top-aligned-row">
56
+ <td><strong>In:</strong></td>
57
+ <td>
58
+ <a href="../../files/lib/sip/exceptions_rb.html">
59
+ lib/sip/exceptions.rb
60
+ </a>
61
+ <br />
62
+ </td>
63
+ </tr>
64
+
65
+ <tr class="top-aligned-row">
66
+ <td><strong>Parent:</strong></td>
67
+ <td>
68
+ StandardError
69
+ </td>
70
+ </tr>
71
+ </table>
72
+ </div>
73
+ <!-- banner header -->
74
+
75
+ <div id="bodyContent">
76
+
77
+
78
+
79
+ <div id="contextContent">
80
+
81
+
82
+
83
+ </div>
84
+
85
+
86
+ </div>
87
+
88
+
89
+ <!-- if includes -->
90
+
91
+ <div id="section">
92
+
93
+
94
+
95
+
96
+
97
+
98
+
99
+
100
+ <!-- if method_list -->
101
+
102
+
103
+ </div>
104
+
105
+
106
+ <div id="validator-badges">
107
+ <p><small><a href="http://validator.w3.org/check/referer">[Validate]</a></small></p>
108
+ </div>
109
+
110
+ </body>
111
+ </html>
data/docs/classes/Sip/Sipper.html ADDED
@@ -0,0 +1,454 @@
1
+ <?xml version="1.0" encoding="iso-8859-1"?>
2
+ <!DOCTYPE html
3
+ PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
4
+ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
5
+
6
+ <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
7
+ <head>
8
+ <title>Class: Sip::Sipper</title>
9
+ <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
10
+ <meta http-equiv="Content-Script-Type" content="text/javascript" />
11
+ <link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
12
+ <script type="text/javascript">
13
+ // <![CDATA[
14
+
15
+ function popupCode( url ) {
16
+ window.open(url, "Code", "resizable=yes,scrollbars=yes,toolbar=no,status=no,height=150,width=400")
17
+ }
18
+
19
+ function toggleCode( id ) {
20
+ if ( document.getElementById )
21
+ elem = document.getElementById( id );
22
+ else if ( document.all )
23
+ elem = eval( "document.all." + id );
24
+ else
25
+ return false;
26
+
27
+ elemStyle = elem.style;
28
+
29
+ if ( elemStyle.display != "block" ) {
30
+ elemStyle.display = "block"
31
+ } else {
32
+ elemStyle.display = "none"
33
+ }
34
+
35
+ return true;
36
+ }
37
+
38
+ // Make codeblocks hidden by default
39
+ document.writeln( "<style type=\"text/css\">div.method-source-code { display: none }</style>" )
40
+
41
+ // ]]>
42
+ </script>
43
+
44
+ </head>
45
+ <body>
46
+
47
+
48
+
49
+ <div id="classHeader">
50
+ <table class="header-table">
51
+ <tr class="top-aligned-row">
52
+ <td><strong>Class</strong></td>
53
+ <td class="class-name-in-header">Sip::Sipper</td>
54
+ </tr>
55
+ <tr class="top-aligned-row">
56
+ <td><strong>In:</strong></td>
57
+ <td>
58
+ <a href="../../files/lib/sip/sipper_rb.html">
59
+ lib/sip/sipper.rb
60
+ </a>
61
+ <br />
62
+ </td>
63
+ </tr>
64
+
65
+ <tr class="top-aligned-row">
66
+ <td><strong>Parent:</strong></td>
67
+ <td>
68
+ Object
69
+ </td>
70
+ </tr>
71
+ </table>
72
+ </div>
73
+ <!-- banner header -->
74
+
75
+ <div id="bodyContent">
76
+
77
+
78
+
79
+ <div id="contextContent">
80
+
81
+
82
+
83
+ </div>
84
+
85
+ <div id="method-list">
86
+ <h3 class="section-bar">Methods</h3>
87
+
88
+ <div class="name-list">
89
+ <a href="#M000026">create_script</a>&nbsp;&nbsp;
90
+ <a href="#M000024">create_script_without_index</a>&nbsp;&nbsp;
91
+ <a href="#M000023">create_scripts</a>&nbsp;&nbsp;
92
+ <a href="#M000025">create_scripts_with_index</a>&nbsp;&nbsp;
93
+ <a href="#M000027">generate_transpart_options</a>&nbsp;&nbsp;
94
+ <a href="#M000028">get_even_split</a>&nbsp;&nbsp;
95
+ <a href="#M000022">log</a>&nbsp;&nbsp;
96
+ <a href="#M000021">new</a>&nbsp;&nbsp;
97
+ <a href="#M000030">run_hive_import</a>&nbsp;&nbsp;
98
+ <a href="#M000029">run_scripts</a>&nbsp;&nbsp;
99
+ </div>
100
+ </div>
101
+
102
+ </div>
103
+
104
+
105
+ <!-- if includes -->
106
+
107
+ <div id="section">
108
+
109
+
110
+
111
+
112
+
113
+ <div id="attribute-list">
114
+ <h3 class="section-bar">Attributes</h3>
115
+
116
+ <div class="name-list">
117
+ <table>
118
+ <tr class="top-aligned-row context-row">
119
+ <td class="context-item-name">config</td>
120
+ <td class="context-item-value">&nbsp;[R]&nbsp;</td>
121
+ <td class="context-item-desc"></td>
122
+ </tr>
123
+ </table>
124
+ </div>
125
+ </div>
126
+
127
+
128
+
129
+ <!-- if method_list -->
130
+ <div id="methods">
131
+ <h3 class="section-bar">Public Class methods</h3>
132
+
133
+ <div id="method-M000021" class="method-detail">
134
+ <a name="M000021"></a>
135
+
136
+ <div class="method-heading">
137
+ <a href="#M000021" class="method-signature">
138
+ <span class="method-name">new</span><span class="method-args">(config)</span>
139
+ </a>
140
+ </div>
141
+
142
+ <div class="method-description">
143
+ <p><a class="source-toggle" href="#"
144
+ onclick="toggleCode('M000021-source');return false;">[Source]</a></p>
145
+ <div class="method-source-code" id="M000021-source">
146
+ <pre>
147
+ <span class="ruby-comment cmt"># File lib/sip/sipper.rb, line 5</span>
148
+ <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">initialize</span>(<span class="ruby-identifier">config</span>)
149
+ <span class="ruby-ivar">@config</span> = <span class="ruby-identifier">config</span>
150
+ <span class="ruby-identifier">slavefile</span> = <span class="ruby-constant">File</span>.<span class="ruby-identifier">join</span>(<span class="ruby-constant">ENV</span>[<span class="ruby-value str">'HADOOP_HOME'</span>], <span class="ruby-value str">'conf'</span>, <span class="ruby-value str">'slaves'</span>)
151
+ <span class="ruby-identifier">log</span> <span class="ruby-node">&quot;Reading slaves from file #{slavefile}...&quot;</span>
152
+ <span class="ruby-keyword kw">begin</span>
153
+ <span class="ruby-identifier">open</span>(<span class="ruby-identifier">slavefile</span>, <span class="ruby-value str">'r'</span>) { <span class="ruby-operator">|</span><span class="ruby-identifier">f</span><span class="ruby-operator">|</span> <span class="ruby-ivar">@slaves</span> = <span class="ruby-identifier">f</span>.<span class="ruby-identifier">read</span>.<span class="ruby-identifier">split</span>(<span class="ruby-value str">&quot;\n&quot;</span>) }
154
+ <span class="ruby-keyword kw">rescue</span>
155
+ <span class="ruby-identifier">raise</span> <span class="ruby-constant">HadoopException</span>, <span class="ruby-node">&quot;Could not read \&quot;#{slavefile}\&quot;. Is your HADOOP_HOME environment variable correct?&quot;</span>
156
+ <span class="ruby-keyword kw">end</span>
157
+ <span class="ruby-constant">Utils</span><span class="ruby-operator">::</span><span class="ruby-identifier">sanity_check</span>(<span class="ruby-ivar">@config</span>)
158
+ <span class="ruby-keyword kw">end</span>
159
+ </pre>
160
+ </div>
161
+ </div>
162
+ </div>
163
+
164
+ <h3 class="section-bar">Public Instance methods</h3>
165
+
166
+ <div id="method-M000026" class="method-detail">
167
+ <a name="M000026"></a>
168
+
169
+ <div class="method-heading">
170
+ <a href="#M000026" class="method-signature">
171
+ <span class="method-name">create_script</span><span class="method-args">(slavename, index, dbconf, tableconf, db, max, method)</span>
172
+ </a>
173
+ </div>
174
+
175
+ <div class="method-description">
176
+ <p><a class="source-toggle" href="#"
177
+ onclick="toggleCode('M000026-source');return false;">[Source]</a></p>
178
+ <div class="method-source-code" id="M000026-source">
179
+ <pre>
180
+ <span class="ruby-comment cmt"># File lib/sip/sipper.rb, line 68</span>
181
+ <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">create_script</span>(<span class="ruby-identifier">slavename</span>, <span class="ruby-identifier">index</span>, <span class="ruby-identifier">dbconf</span>, <span class="ruby-identifier">tableconf</span>, <span class="ruby-identifier">db</span>, <span class="ruby-identifier">max</span>, <span class="ruby-identifier">method</span>)
182
+ <span class="ruby-keyword kw">if</span> <span class="ruby-identifier">method</span> <span class="ruby-operator">==</span> <span class="ruby-identifier">:append</span>
183
+ <span class="ruby-identifier">first</span>, <span class="ruby-identifier">last</span> = <span class="ruby-identifier">get_even_split</span>(<span class="ruby-identifier">tableconf</span>[<span class="ruby-value str">'incremental_index_value'</span>]<span class="ruby-operator">+</span><span class="ruby-value">1</span>, <span class="ruby-identifier">max</span>, <span class="ruby-identifier">index</span>, <span class="ruby-ivar">@slaves</span>.<span class="ruby-identifier">length</span>)
184
+ <span class="ruby-keyword kw">else</span>
185
+ <span class="ruby-identifier">first</span>, <span class="ruby-identifier">last</span> = <span class="ruby-value">1</span>, <span class="ruby-identifier">max</span>
186
+ <span class="ruby-keyword kw">end</span>
187
+
188
+ <span class="ruby-identifier">log</span> <span class="ruby-node">&quot;Importing #{first} &lt;= #{tableconf['incremental_index']} &lt;= #{last} from table #{dbconf['dbname']}.#{tableconf['tablename']}&quot;</span>
189
+ <span class="ruby-identifier">select</span> = <span class="ruby-identifier">db</span>.<span class="ruby-identifier">generate_command</span> <span class="ruby-identifier">tableconf</span>, <span class="ruby-identifier">first</span>, <span class="ruby-identifier">last</span>
190
+ <span class="ruby-identifier">transpart_opts</span> = <span class="ruby-identifier">generate_transpart_options</span>(<span class="ruby-identifier">tableconf</span>, <span class="ruby-identifier">db</span>)
191
+ <span class="ruby-constant">Utils</span><span class="ruby-operator">::</span><span class="ruby-identifier">write_script</span> <span class="ruby-keyword kw">self</span>, <span class="ruby-identifier">slavename</span>, <span class="ruby-identifier">select</span>, <span class="ruby-identifier">dbconf</span>[<span class="ruby-value str">'dbname'</span>], <span class="ruby-identifier">tableconf</span>[<span class="ruby-value str">'tablename'</span>], <span class="ruby-identifier">transpart_opts</span>
192
+ <span class="ruby-keyword kw">end</span>
193
+ </pre>
194
+ </div>
195
+ </div>
196
+ </div>
197
+
198
+ <div id="method-M000024" class="method-detail">
199
+ <a name="M000024"></a>
200
+
201
+ <div class="method-heading">
202
+ <a href="#M000024" class="method-signature">
203
+ <span class="method-name">create_script_without_index</span><span class="method-args">(dbconf, tableconf, db)</span>
204
+ </a>
205
+ </div>
206
+
207
+ <div class="method-description">
208
+ <p>
209
+ this is the case where there is no primary key index, so the whole table
210
+ will need to be imported
211
+ </p>
212
+ <p><a class="source-toggle" href="#"
213
+ onclick="toggleCode('M000024-source');return false;">[Source]</a></p>
214
+ <div class="method-source-code" id="M000024-source">
215
+ <pre>
216
+ <span class="ruby-comment cmt"># File lib/sip/sipper.rb, line 40</span>
217
+ <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">create_script_without_index</span>(<span class="ruby-identifier">dbconf</span>, <span class="ruby-identifier">tableconf</span>, <span class="ruby-identifier">db</span>)
218
+ <span class="ruby-constant">Hive</span><span class="ruby-operator">::</span><span class="ruby-identifier">run_hsql_create_table</span> <span class="ruby-keyword kw">self</span>, <span class="ruby-identifier">db</span>, <span class="ruby-identifier">tableconf</span>, <span class="ruby-identifier">:overwrite</span>
219
+
220
+ <span class="ruby-identifier">log</span> <span class="ruby-node">&quot;Importing all rows from table #{dbconf['dbname']}.#{tableconf['tablename']}&quot;</span>
221
+ <span class="ruby-identifier">select</span> = <span class="ruby-identifier">db</span>.<span class="ruby-identifier">generate_command</span> <span class="ruby-identifier">tableconf</span>
222
+ <span class="ruby-identifier">transpart_opts</span> = <span class="ruby-identifier">generate_transpart_options</span>(<span class="ruby-identifier">tableconf</span>, <span class="ruby-identifier">db</span>)
223
+ <span class="ruby-ivar">@scripts</span> <span class="ruby-operator">&lt;&lt;</span> <span class="ruby-constant">Utils</span><span class="ruby-operator">::</span><span class="ruby-identifier">write_script</span>(<span class="ruby-keyword kw">self</span>, <span class="ruby-ivar">@slaves</span>.<span class="ruby-identifier">first</span>, <span class="ruby-identifier">select</span>, <span class="ruby-identifier">dbconf</span>[<span class="ruby-value str">'dbname'</span>], <span class="ruby-identifier">tableconf</span>[<span class="ruby-value str">'tablename'</span>], <span class="ruby-identifier">transpart_opts</span>)
224
+ <span class="ruby-keyword kw">end</span>
225
+ </pre>
226
+ </div>
227
+ </div>
228
+ </div>
229
+
230
+ <div id="method-M000023" class="method-detail">
231
+ <a name="M000023"></a>
232
+
233
+ <div class="method-heading">
234
+ <a href="#M000023" class="method-signature">
235
+ <span class="method-name">create_scripts</span><span class="method-args">(dbconf, tableconf)</span>
236
+ </a>
237
+ </div>
238
+
239
+ <div class="method-description">
240
+ <p>
241
+ return number of scripts created
242
+ </p>
243
+ <p><a class="source-toggle" href="#"
244
+ onclick="toggleCode('M000023-source');return false;">[Source]</a></p>
245
+ <div class="method-source-code" id="M000023-source">
246
+ <pre>
247
+ <span class="ruby-comment cmt"># File lib/sip/sipper.rb, line 22</span>
248
+ <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">create_scripts</span>(<span class="ruby-identifier">dbconf</span>, <span class="ruby-identifier">tableconf</span>)
249
+ <span class="ruby-ivar">@scripts</span> = []
250
+ <span class="ruby-identifier">db</span> = <span class="ruby-constant">DBBase</span>.<span class="ruby-identifier">make_interface</span> <span class="ruby-identifier">dbconf</span>[<span class="ruby-value str">'type'</span>], <span class="ruby-identifier">dbconf</span>, <span class="ruby-keyword kw">self</span>
251
+
252
+ <span class="ruby-comment cmt"># set default columns if necessary</span>
253
+ <span class="ruby-identifier">tableconf</span>[<span class="ruby-value str">'columns'</span>] = <span class="ruby-identifier">db</span>.<span class="ruby-identifier">columns</span>(<span class="ruby-identifier">tableconf</span>[<span class="ruby-value str">'tablename'</span>]).<span class="ruby-identifier">map</span> { <span class="ruby-operator">|</span><span class="ruby-identifier">c</span><span class="ruby-operator">|</span> <span class="ruby-identifier">c</span>.<span class="ruby-identifier">first</span> } <span class="ruby-keyword kw">if</span> <span class="ruby-identifier">tableconf</span>[<span class="ruby-value str">'columns'</span>].<span class="ruby-identifier">nil?</span>
254
+
255
+ <span class="ruby-keyword kw">if</span> <span class="ruby-identifier">tableconf</span>[<span class="ruby-value str">'incremental_index'</span>].<span class="ruby-identifier">nil?</span>
256
+ <span class="ruby-identifier">create_script_without_index</span>(<span class="ruby-identifier">dbconf</span>, <span class="ruby-identifier">tableconf</span>, <span class="ruby-identifier">db</span>)
257
+ <span class="ruby-keyword kw">else</span>
258
+ <span class="ruby-identifier">create_scripts_with_index</span>(<span class="ruby-identifier">dbconf</span>, <span class="ruby-identifier">tableconf</span>, <span class="ruby-identifier">db</span>)
259
+ <span class="ruby-keyword kw">end</span>
260
+
261
+ <span class="ruby-ivar">@scripts</span>.<span class="ruby-identifier">length</span>
262
+ <span class="ruby-keyword kw">end</span>
263
+ </pre>
264
+ </div>
265
+ </div>
266
+ </div>
267
+
268
+ <div id="method-M000025" class="method-detail">
269
+ <a name="M000025"></a>
270
+
271
+ <div class="method-heading">
272
+ <a href="#M000025" class="method-signature">
273
+ <span class="method-name">create_scripts_with_index</span><span class="method-args">(dbconf, tableconf, db)</span>
274
+ </a>
275
+ </div>
276
+
277
+ <div class="method-description">
278
+ <p><a class="source-toggle" href="#"
279
+ onclick="toggleCode('M000025-source');return false;">[Source]</a></p>
280
+ <div class="method-source-code" id="M000025-source">
281
+ <pre>
282
+ <span class="ruby-comment cmt"># File lib/sip/sipper.rb, line 49</span>
283
+ <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">create_scripts_with_index</span>(<span class="ruby-identifier">dbconf</span>, <span class="ruby-identifier">tableconf</span>, <span class="ruby-identifier">db</span>)
284
+ <span class="ruby-identifier">max</span> = <span class="ruby-identifier">db</span>.<span class="ruby-identifier">get_column_max</span> <span class="ruby-identifier">tableconf</span>[<span class="ruby-value str">'tablename'</span>], <span class="ruby-identifier">tableconf</span>[<span class="ruby-value str">'incremental_index'</span>]
285
+
286
+ <span class="ruby-identifier">method</span> = (<span class="ruby-identifier">tableconf</span>[<span class="ruby-value str">'method'</span>] <span class="ruby-operator">==</span> <span class="ruby-value str">&quot;append&quot;</span> <span class="ruby-keyword kw">and</span> <span class="ruby-keyword kw">not</span> <span class="ruby-ivar">@config</span>[<span class="ruby-identifier">:overwrite</span>]) <span class="ruby-operator">?</span> <span class="ruby-identifier">:append</span> <span class="ruby-operator">:</span> <span class="ruby-identifier">:overwrite</span>
287
+ <span class="ruby-keyword kw">if</span> <span class="ruby-identifier">method</span> <span class="ruby-operator">==</span> <span class="ruby-identifier">:append</span> <span class="ruby-keyword kw">and</span> <span class="ruby-identifier">max</span> <span class="ruby-operator">==</span> <span class="ruby-identifier">tableconf</span>[<span class="ruby-value str">'incremental_index_value'</span>]
288
+ <span class="ruby-identifier">log</span> <span class="ruby-node">&quot;Ignoring #{dbconf['dbname']}.#{tableconf['tablename']} - already up to date&quot;</span>
289
+ <span class="ruby-keyword kw">else</span>
290
+ <span class="ruby-constant">Hive</span><span class="ruby-operator">::</span><span class="ruby-identifier">run_hsql_create_table</span> <span class="ruby-keyword kw">self</span>, <span class="ruby-identifier">db</span>, <span class="ruby-identifier">tableconf</span>, <span class="ruby-identifier">method</span>
291
+ <span class="ruby-ivar">@slaves</span>.<span class="ruby-identifier">each_with_index</span> { <span class="ruby-operator">|</span><span class="ruby-identifier">slavename</span>, <span class="ruby-identifier">index</span><span class="ruby-operator">|</span>
292
+ <span class="ruby-ivar">@scripts</span> <span class="ruby-operator">&lt;&lt;</span> <span class="ruby-identifier">create_script</span>(<span class="ruby-identifier">slavename</span>, <span class="ruby-identifier">index</span>, <span class="ruby-identifier">dbconf</span>, <span class="ruby-identifier">tableconf</span>, <span class="ruby-identifier">db</span>, <span class="ruby-identifier">max</span>, <span class="ruby-identifier">method</span>)
293
+ }
294
+ <span class="ruby-identifier">db</span>.<span class="ruby-identifier">close</span>
295
+ <span class="ruby-keyword kw">end</span>
296
+
297
+ <span class="ruby-comment cmt"># update incremental index value if method in conf is append, regardless of whether or </span>
298
+ <span class="ruby-comment cmt"># not this is a forced overwrite</span>
299
+ <span class="ruby-identifier">tableconf</span>[<span class="ruby-value str">'incremental_index_value'</span>] = <span class="ruby-identifier">max</span> <span class="ruby-keyword kw">if</span> <span class="ruby-identifier">tableconf</span>[<span class="ruby-value str">'method'</span>] <span class="ruby-operator">==</span> <span class="ruby-value str">&quot;append&quot;</span>
300
+ <span class="ruby-keyword kw">end</span>
301
+ </pre>
302
+ </div>
303
+ </div>
304
+ </div>
305
+
306
+ <div id="method-M000027" class="method-detail">
307
+ <a name="M000027"></a>
308
+
309
+ <div class="method-heading">
310
+ <a href="#M000027" class="method-signature">
311
+ <span class="method-name">generate_transpart_options</span><span class="method-args">(tableconf, db)</span>
312
+ </a>
313
+ </div>
314
+
315
+ <div class="method-description">
316
+ <p><a class="source-toggle" href="#"
317
+ onclick="toggleCode('M000027-source');return false;">[Source]</a></p>
318
+ <div class="method-source-code" id="M000027-source">
319
+ <pre>
320
+ <span class="ruby-comment cmt"># File lib/sip/sipper.rb, line 81</span>
321
+ <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">generate_transpart_options</span>(<span class="ruby-identifier">tableconf</span>, <span class="ruby-identifier">db</span>)
322
+ <span class="ruby-identifier">opts</span> = <span class="ruby-constant">CmdOpts</span>.<span class="ruby-identifier">new</span>
323
+ <span class="ruby-identifier">opts</span>[<span class="ruby-value str">'c'</span>] = <span class="ruby-identifier">db</span>.<span class="ruby-identifier">order_column_list</span>(<span class="ruby-identifier">tableconf</span>[<span class="ruby-value str">'tablename'</span>], <span class="ruby-identifier">tableconf</span>[<span class="ruby-value str">'columns'</span>]).<span class="ruby-identifier">join</span>(<span class="ruby-value str">','</span>)
324
+ <span class="ruby-identifier">opts</span>[<span class="ruby-value str">'p'</span>] = <span class="ruby-identifier">tableconf</span>[<span class="ruby-value str">'partition_by'</span>] <span class="ruby-keyword kw">if</span> <span class="ruby-identifier">tableconf</span>.<span class="ruby-identifier">has_key?</span> <span class="ruby-value str">&quot;partition_by&quot;</span>
325
+ <span class="ruby-keyword kw">if</span> <span class="ruby-identifier">tableconf</span>.<span class="ruby-identifier">has_key?</span> <span class="ruby-value str">'transformations'</span> <span class="ruby-keyword kw">and</span> <span class="ruby-identifier">tableconf</span>[<span class="ruby-value str">'transformations'</span>].<span class="ruby-identifier">length</span> <span class="ruby-operator">&gt;</span> <span class="ruby-value">0</span>
326
+ <span class="ruby-identifier">opts</span>[<span class="ruby-value str">'t'</span>] = <span class="ruby-identifier">tableconf</span>[<span class="ruby-value str">'transformations'</span>].<span class="ruby-identifier">map</span> { <span class="ruby-operator">|</span><span class="ruby-identifier">k</span>,<span class="ruby-identifier">v</span><span class="ruby-operator">|</span> <span class="ruby-node">&quot;#{k}:#{v}&quot;</span> }.<span class="ruby-identifier">join</span>(<span class="ruby-value str">','</span>)
327
+ <span class="ruby-keyword kw">end</span>
328
+ <span class="ruby-identifier">opts</span>[<span class="ruby-value str">'H'</span>] = <span class="ruby-identifier">tableconf</span>[<span class="ruby-value str">'hive_table_name'</span>]
329
+ <span class="ruby-identifier">opts</span>[<span class="ruby-value str">'o'</span>] = <span class="ruby-constant">File</span>.<span class="ruby-identifier">join</span>(<span class="ruby-ivar">@config</span>[<span class="ruby-value str">'tmpdir'</span>], <span class="ruby-value str">'partitions'</span>)
330
+ <span class="ruby-identifier">opts</span>.<span class="ruby-identifier">set</span>(<span class="ruby-value str">'d'</span>) <span class="ruby-keyword kw">if</span> <span class="ruby-ivar">@config</span>[<span class="ruby-identifier">:debug</span>]
331
+ <span class="ruby-identifier">opts</span>
332
+ <span class="ruby-keyword kw">end</span>
333
+ </pre>
334
+ </div>
335
+ </div>
336
+ </div>
337
+
338
+ <div id="method-M000028" class="method-detail">
339
+ <a name="M000028"></a>
340
+
341
+ <div class="method-heading">
342
+ <a href="#M000028" class="method-signature">
343
+ <span class="method-name">get_even_split</span><span class="method-args">(min, max, index, count)</span>
344
+ </a>
345
+ </div>
346
+
347
+ <div class="method-description">
348
+ <p><a class="source-toggle" href="#"
349
+ onclick="toggleCode('M000028-source');return false;">[Source]</a></p>
350
+ <div class="method-source-code" id="M000028-source">
351
+ <pre>
352
+ <span class="ruby-comment cmt"># File lib/sip/sipper.rb, line 94</span>
353
+ <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">get_even_split</span>(<span class="ruby-identifier">min</span>, <span class="ruby-identifier">max</span>, <span class="ruby-identifier">index</span>, <span class="ruby-identifier">count</span>)
354
+ <span class="ruby-identifier">size</span> = ((<span class="ruby-identifier">max</span> <span class="ruby-operator">-</span> <span class="ruby-identifier">min</span> <span class="ruby-operator">+</span> <span class="ruby-value">1</span>).<span class="ruby-identifier">to_f</span> <span class="ruby-operator">/</span> <span class="ruby-identifier">count</span>.<span class="ruby-identifier">to_f</span>).<span class="ruby-identifier">ceil</span>
355
+ <span class="ruby-identifier">first</span> = (<span class="ruby-identifier">size</span> <span class="ruby-operator">*</span> <span class="ruby-identifier">index</span>) <span class="ruby-operator">+</span> <span class="ruby-identifier">min</span>
356
+ <span class="ruby-identifier">last</span> = (<span class="ruby-identifier">size</span> <span class="ruby-operator">*</span> (<span class="ruby-identifier">index</span><span class="ruby-operator">+</span><span class="ruby-value">1</span>)) <span class="ruby-operator">+</span> <span class="ruby-identifier">min</span> <span class="ruby-operator">-</span> <span class="ruby-value">1</span>
357
+ [<span class="ruby-identifier">first</span>, [<span class="ruby-identifier">max</span>, <span class="ruby-identifier">last</span>].<span class="ruby-identifier">min</span>]
358
+ <span class="ruby-keyword kw">end</span>
359
+ </pre>
360
+ </div>
361
+ </div>
362
+ </div>
363
+
364
+ <div id="method-M000022" class="method-detail">
365
+ <a name="M000022"></a>
366
+
367
+ <div class="method-heading">
368
+ <a href="#M000022" class="method-signature">
369
+ <span class="method-name">log</span><span class="method-args">(msg)</span>
370
+ </a>
371
+ </div>
372
+
373
+ <div class="method-description">
374
+ <p><a class="source-toggle" href="#"
375
+ onclick="toggleCode('M000022-source');return false;">[Source]</a></p>
376
+ <div class="method-source-code" id="M000022-source">
377
+ <pre>
378
+ <span class="ruby-comment cmt"># File lib/sip/sipper.rb, line 17</span>
379
+ <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">log</span>(<span class="ruby-identifier">msg</span>)
380
+ <span class="ruby-identifier">puts</span> <span class="ruby-node">&quot;#{Utils::hostname} #{Time.now.strftime '%Y-%m-%d %H:%M:%S'}: #{msg}&quot;</span> <span class="ruby-keyword kw">if</span> <span class="ruby-ivar">@config</span>[<span class="ruby-identifier">:debug</span>]
381
+ <span class="ruby-keyword kw">end</span>
382
+ </pre>
383
+ </div>
384
+ </div>
385
+ </div>
386
+
387
+ <div id="method-M000030" class="method-detail">
388
+ <a name="M000030"></a>
389
+
390
+ <div class="method-heading">
391
+ <a href="#M000030" class="method-signature">
392
+ <span class="method-name">run_hive_import</span><span class="method-args">(tableconf)</span>
393
+ </a>
394
+ </div>
395
+
396
+ <div class="method-description">
397
+ <p><a class="source-toggle" href="#"
398
+ onclick="toggleCode('M000030-source');return false;">[Source]</a></p>
399
+ <div class="method-source-code" id="M000030-source">
400
+ <pre>
401
+ <span class="ruby-comment cmt"># File lib/sip/sipper.rb, line 114</span>
402
+ <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">run_hive_import</span>(<span class="ruby-identifier">tableconf</span>)
403
+ <span class="ruby-constant">Hive</span>.<span class="ruby-identifier">run_import</span> <span class="ruby-keyword kw">self</span>, <span class="ruby-identifier">tableconf</span>
404
+ <span class="ruby-keyword kw">end</span>
405
+ </pre>
406
+ </div>
407
+ </div>
408
+ </div>
409
+
410
+ <div id="method-M000029" class="method-detail">
411
+ <a name="M000029"></a>
412
+
413
+ <div class="method-heading">
414
+ <a href="#M000029" class="method-signature">
415
+ <span class="method-name">run_scripts</span><span class="method-args">()</span>
416
+ </a>
417
+ </div>
418
+
419
+ <div class="method-description">
420
+ <p><a class="source-toggle" href="#"
421
+ onclick="toggleCode('M000029-source');return false;">[Source]</a></p>
422
+ <div class="method-source-code" id="M000029-source">
423
+ <pre>
424
+ <span class="ruby-comment cmt"># File lib/sip/sipper.rb, line 101</span>
425
+ <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">run_scripts</span>
426
+ <span class="ruby-identifier">pids</span> = {}
427
+ <span class="ruby-ivar">@scripts</span>.<span class="ruby-identifier">each</span> { <span class="ruby-operator">|</span><span class="ruby-identifier">script</span><span class="ruby-operator">|</span>
428
+ <span class="ruby-identifier">log</span> <span class="ruby-node">&quot;Running #{script}...&quot;</span>
429
+ <span class="ruby-identifier">pid</span> = <span class="ruby-identifier">fork</span> { <span class="ruby-constant">Kernel</span>.<span class="ruby-identifier">exit</span> <span class="ruby-identifier">system</span>(<span class="ruby-node">&quot;sh #{script}&quot;</span>) }
430
+ <span class="ruby-identifier">pids</span>[<span class="ruby-identifier">pid</span>] = <span class="ruby-identifier">script</span>
431
+ <span class="ruby-identifier">sleep</span> <span class="ruby-value">1</span>
432
+ }
433
+ <span class="ruby-constant">Process</span>.<span class="ruby-identifier">waitall</span>.<span class="ruby-identifier">map</span> { <span class="ruby-operator">|</span><span class="ruby-identifier">r</span><span class="ruby-operator">|</span> <span class="ruby-identifier">r</span>.<span class="ruby-identifier">last</span> }.<span class="ruby-identifier">each</span> { <span class="ruby-operator">|</span><span class="ruby-identifier">status</span><span class="ruby-operator">|</span>
434
+ <span class="ruby-identifier">raise</span> <span class="ruby-constant">ImportScriptExecutionError</span>, <span class="ruby-node">&quot;Error runing script '#{pids[status.pid]}'&quot;</span> <span class="ruby-keyword kw">if</span> <span class="ruby-identifier">status</span>.<span class="ruby-identifier">exitstatus</span> <span class="ruby-operator">!=</span> <span class="ruby-value">0</span>
435
+ }
436
+ <span class="ruby-keyword kw">end</span>
437
+ </pre>
438
+ </div>
439
+ </div>
440
+ </div>
441
+
442
+
443
+ </div>
444
+
445
+
446
+ </div>
447
+
448
+
449
+ <div id="validator-badges">
450
+ <p><small><a href="http://validator.w3.org/check/referer">[Validate]</a></small></p>
451
+ </div>
452
+
453
+ </body>
454
+ </html>