sip 0.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (56) hide show
  1. data/Gemfile +2 -0
  2. data/LICENSE +674 -0
  3. data/README.rdoc +32 -0
  4. data/Rakefile +21 -0
  5. data/bin/sip +83 -0
  6. data/bin/transpart +114 -0
  7. data/docs/classes/Sip.html +169 -0
  8. data/docs/classes/Sip/CmdOpts.html +179 -0
  9. data/docs/classes/Sip/Config.html +362 -0
  10. data/docs/classes/Sip/DBBase.html +368 -0
  11. data/docs/classes/Sip/HadoopException.html +111 -0
  12. data/docs/classes/Sip/Hive.html +295 -0
  13. data/docs/classes/Sip/HiveQueryException.html +111 -0
  14. data/docs/classes/Sip/ImportScriptExecutionError.html +111 -0
  15. data/docs/classes/Sip/MySQLSipper.html +273 -0
  16. data/docs/classes/Sip/NoSuchColumn.html +111 -0
  17. data/docs/classes/Sip/NoSuchTable.html +111 -0
  18. data/docs/classes/Sip/PastFailureException.html +111 -0
  19. data/docs/classes/Sip/Sipper.html +454 -0
  20. data/docs/classes/Sip/UnsupportedDatabaseType.html +111 -0
  21. data/docs/classes/Sip/Utils.html +269 -0
  22. data/docs/classes/Struct.html +146 -0
  23. data/docs/created.rid +1 -0
  24. data/docs/files/README_rdoc.html +174 -0
  25. data/docs/files/lib/sip/cmdopts_rb.html +101 -0
  26. data/docs/files/lib/sip/config_rb.html +108 -0
  27. data/docs/files/lib/sip/databases/dbbase_rb.html +108 -0
  28. data/docs/files/lib/sip/databases/mysql_rb.html +108 -0
  29. data/docs/files/lib/sip/exceptions_rb.html +101 -0
  30. data/docs/files/lib/sip/extensions_rb.html +101 -0
  31. data/docs/files/lib/sip/hive_rb.html +101 -0
  32. data/docs/files/lib/sip/sipper_rb.html +101 -0
  33. data/docs/files/lib/sip/utils_rb.html +110 -0
  34. data/docs/files/lib/sip/version_rb.html +101 -0
  35. data/docs/files/lib/sip_rb.html +117 -0
  36. data/docs/fr_class_index.html +42 -0
  37. data/docs/fr_file_index.html +38 -0
  38. data/docs/fr_method_index.html +72 -0
  39. data/docs/index.html +24 -0
  40. data/docs/rdoc-style.css +208 -0
  41. data/lib/sip.rb +10 -0
  42. data/lib/sip/cmdopts.rb +20 -0
  43. data/lib/sip/config.rb +80 -0
  44. data/lib/sip/databases/dbbase.rb +56 -0
  45. data/lib/sip/databases/mysql.rb +52 -0
  46. data/lib/sip/exceptions.rb +9 -0
  47. data/lib/sip/extensions.rb +5 -0
  48. data/lib/sip/hive.rb +62 -0
  49. data/lib/sip/sipper.rb +118 -0
  50. data/lib/sip/templates/export.sh +73 -0
  51. data/lib/sip/utils.rb +58 -0
  52. data/lib/sip/version.rb +3 -0
  53. data/test/database_interaction_test.rb +7 -0
  54. data/test/hive_test.rb +28 -0
  55. data/test/sipper_test.rb +25 -0
  56. metadata +125 -0
data/docs/classes/Sip/NoSuchTable.html ADDED
@@ -0,0 +1,111 @@
1
+ <?xml version="1.0" encoding="iso-8859-1"?>
2
+ <!DOCTYPE html
3
+ PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
4
+ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
5
+
6
+ <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
7
+ <head>
8
+ <title>Class: Sip::NoSuchTable</title>
9
+ <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
10
+ <meta http-equiv="Content-Script-Type" content="text/javascript" />
11
+ <link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
12
+ <script type="text/javascript">
13
+ // <![CDATA[
14
+
15
+ function popupCode( url ) {
16
+ window.open(url, "Code", "resizable=yes,scrollbars=yes,toolbar=no,status=no,height=150,width=400")
17
+ }
18
+
19
+ function toggleCode( id ) {
20
+ if ( document.getElementById )
21
+ elem = document.getElementById( id );
22
+ else if ( document.all )
23
+ elem = eval( "document.all." + id );
24
+ else
25
+ return false;
26
+
27
+ elemStyle = elem.style;
28
+
29
+ if ( elemStyle.display != "block" ) {
30
+ elemStyle.display = "block"
31
+ } else {
32
+ elemStyle.display = "none"
33
+ }
34
+
35
+ return true;
36
+ }
37
+
38
+ // Make codeblocks hidden by default
39
+ document.writeln( "<style type=\"text/css\">div.method-source-code { display: none }</style>" )
40
+
41
+ // ]]>
42
+ </script>
43
+
44
+ </head>
45
+ <body>
46
+
47
+
48
+
49
+ <div id="classHeader">
50
+ <table class="header-table">
51
+ <tr class="top-aligned-row">
52
+ <td><strong>Class</strong></td>
53
+ <td class="class-name-in-header">Sip::NoSuchTable</td>
54
+ </tr>
55
+ <tr class="top-aligned-row">
56
+ <td><strong>In:</strong></td>
57
+ <td>
58
+ <a href="../../files/lib/sip/exceptions_rb.html">
59
+ lib/sip/exceptions.rb
60
+ </a>
61
+ <br />
62
+ </td>
63
+ </tr>
64
+
65
+ <tr class="top-aligned-row">
66
+ <td><strong>Parent:</strong></td>
67
+ <td>
68
+ StandardError
69
+ </td>
70
+ </tr>
71
+ </table>
72
+ </div>
73
+ <!-- banner header -->
74
+
75
+ <div id="bodyContent">
76
+
77
+
78
+
79
+ <div id="contextContent">
80
+
81
+
82
+
83
+ </div>
84
+
85
+
86
+ </div>
87
+
88
+
89
+ <!-- if includes -->
90
+
91
+ <div id="section">
92
+
93
+
94
+
95
+
96
+
97
+
98
+
99
+
100
+ <!-- if method_list -->
101
+
102
+
103
+ </div>
104
+
105
+
106
+ <div id="validator-badges">
107
+ <p><small><a href="http://validator.w3.org/check/referer">[Validate]</a></small></p>
108
+ </div>
109
+
110
+ </body>
111
+ </html>
data/docs/classes/Sip/PastFailureException.html ADDED
@@ -0,0 +1,111 @@
1
+ <?xml version="1.0" encoding="iso-8859-1"?>
2
+ <!DOCTYPE html
3
+ PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
4
+ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
5
+
6
+ <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
7
+ <head>
8
+ <title>Class: Sip::PastFailureException</title>
9
+ <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
10
+ <meta http-equiv="Content-Script-Type" content="text/javascript" />
11
+ <link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
12
+ <script type="text/javascript">
13
+ // <![CDATA[
14
+
15
+ function popupCode( url ) {
16
+ window.open(url, "Code", "resizable=yes,scrollbars=yes,toolbar=no,status=no,height=150,width=400")
17
+ }
18
+
19
+ function toggleCode( id ) {
20
+ if ( document.getElementById )
21
+ elem = document.getElementById( id );
22
+ else if ( document.all )
23
+ elem = eval( "document.all." + id );
24
+ else
25
+ return false;
26
+
27
+ elemStyle = elem.style;
28
+
29
+ if ( elemStyle.display != "block" ) {
30
+ elemStyle.display = "block"
31
+ } else {
32
+ elemStyle.display = "none"
33
+ }
34
+
35
+ return true;
36
+ }
37
+
38
+ // Make codeblocks hidden by default
39
+ document.writeln( "<style type=\"text/css\">div.method-source-code { display: none }</style>" )
40
+
41
+ // ]]>
42
+ </script>
43
+
44
+ </head>
45
+ <body>
46
+
47
+
48
+
49
+ <div id="classHeader">
50
+ <table class="header-table">
51
+ <tr class="top-aligned-row">
52
+ <td><strong>Class</strong></td>
53
+ <td class="class-name-in-header">Sip::PastFailureException</td>
54
+ </tr>
55
+ <tr class="top-aligned-row">
56
+ <td><strong>In:</strong></td>
57
+ <td>
58
+ <a href="../../files/lib/sip/exceptions_rb.html">
59
+ lib/sip/exceptions.rb
60
+ </a>
61
+ <br />
62
+ </td>
63
+ </tr>
64
+
65
+ <tr class="top-aligned-row">
66
+ <td><strong>Parent:</strong></td>
67
+ <td>
68
+ StandardError
69
+ </td>
70
+ </tr>
71
+ </table>
72
+ </div>
73
+ <!-- banner header -->
74
+
75
+ <div id="bodyContent">
76
+
77
+
78
+
79
+ <div id="contextContent">
80
+
81
+
82
+
83
+ </div>
84
+
85
+
86
+ </div>
87
+
88
+
89
+ <!-- if includes -->
90
+
91
+ <div id="section">
92
+
93
+
94
+
95
+
96
+
97
+
98
+
99
+
100
+ <!-- if method_list -->
101
+
102
+
103
+ </div>
104
+
105
+
106
+ <div id="validator-badges">
107
+ <p><small><a href="http://validator.w3.org/check/referer">[Validate]</a></small></p>
108
+ </div>
109
+
110
+ </body>
111
+ </html>
data/docs/classes/Sip/Sipper.html ADDED
@@ -0,0 +1,454 @@
1
+ <?xml version="1.0" encoding="iso-8859-1"?>
2
+ <!DOCTYPE html
3
+ PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
4
+ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
5
+
6
+ <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
7
+ <head>
8
+ <title>Class: Sip::Sipper</title>
9
+ <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
10
+ <meta http-equiv="Content-Script-Type" content="text/javascript" />
11
+ <link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
12
+ <script type="text/javascript">
13
+ // <![CDATA[
14
+
15
+ function popupCode( url ) {
16
+ window.open(url, "Code", "resizable=yes,scrollbars=yes,toolbar=no,status=no,height=150,width=400")
17
+ }
18
+
19
+ function toggleCode( id ) {
20
+ if ( document.getElementById )
21
+ elem = document.getElementById( id );
22
+ else if ( document.all )
23
+ elem = eval( "document.all." + id );
24
+ else
25
+ return false;
26
+
27
+ elemStyle = elem.style;
28
+
29
+ if ( elemStyle.display != "block" ) {
30
+ elemStyle.display = "block"
31
+ } else {
32
+ elemStyle.display = "none"
33
+ }
34
+
35
+ return true;
36
+ }
37
+
38
+ // Make codeblocks hidden by default
39
+ document.writeln( "<style type=\"text/css\">div.method-source-code { display: none }</style>" )
40
+
41
+ // ]]>
42
+ </script>
43
+
44
+ </head>
45
+ <body>
46
+
47
+
48
+
49
+ <div id="classHeader">
50
+ <table class="header-table">
51
+ <tr class="top-aligned-row">
52
+ <td><strong>Class</strong></td>
53
+ <td class="class-name-in-header">Sip::Sipper</td>
54
+ </tr>
55
+ <tr class="top-aligned-row">
56
+ <td><strong>In:</strong></td>
57
+ <td>
58
+ <a href="../../files/lib/sip/sipper_rb.html">
59
+ lib/sip/sipper.rb
60
+ </a>
61
+ <br />
62
+ </td>
63
+ </tr>
64
+
65
+ <tr class="top-aligned-row">
66
+ <td><strong>Parent:</strong></td>
67
+ <td>
68
+ Object
69
+ </td>
70
+ </tr>
71
+ </table>
72
+ </div>
73
+ <!-- banner header -->
74
+
75
+ <div id="bodyContent">
76
+
77
+
78
+
79
+ <div id="contextContent">
80
+
81
+
82
+
83
+ </div>
84
+
85
+ <div id="method-list">
86
+ <h3 class="section-bar">Methods</h3>
87
+
88
+ <div class="name-list">
89
+ <a href="#M000026">create_script</a>&nbsp;&nbsp;
90
+ <a href="#M000024">create_script_without_index</a>&nbsp;&nbsp;
91
+ <a href="#M000023">create_scripts</a>&nbsp;&nbsp;
92
+ <a href="#M000025">create_scripts_with_index</a>&nbsp;&nbsp;
93
+ <a href="#M000027">generate_transpart_options</a>&nbsp;&nbsp;
94
+ <a href="#M000028">get_even_split</a>&nbsp;&nbsp;
95
+ <a href="#M000022">log</a>&nbsp;&nbsp;
96
+ <a href="#M000021">new</a>&nbsp;&nbsp;
97
+ <a href="#M000030">run_hive_import</a>&nbsp;&nbsp;
98
+ <a href="#M000029">run_scripts</a>&nbsp;&nbsp;
99
+ </div>
100
+ </div>
101
+
102
+ </div>
103
+
104
+
105
+ <!-- if includes -->
106
+
107
+ <div id="section">
108
+
109
+
110
+
111
+
112
+
113
+ <div id="attribute-list">
114
+ <h3 class="section-bar">Attributes</h3>
115
+
116
+ <div class="name-list">
117
+ <table>
118
+ <tr class="top-aligned-row context-row">
119
+ <td class="context-item-name">config</td>
120
+ <td class="context-item-value">&nbsp;[R]&nbsp;</td>
121
+ <td class="context-item-desc"></td>
122
+ </tr>
123
+ </table>
124
+ </div>
125
+ </div>
126
+
127
+
128
+
129
+ <!-- if method_list -->
130
+ <div id="methods">
131
+ <h3 class="section-bar">Public Class methods</h3>
132
+
133
+ <div id="method-M000021" class="method-detail">
134
+ <a name="M000021"></a>
135
+
136
+ <div class="method-heading">
137
+ <a href="#M000021" class="method-signature">
138
+ <span class="method-name">new</span><span class="method-args">(config)</span>
139
+ </a>
140
+ </div>
141
+
142
+ <div class="method-description">
143
+ <p><a class="source-toggle" href="#"
144
+ onclick="toggleCode('M000021-source');return false;">[Source]</a></p>
145
+ <div class="method-source-code" id="M000021-source">
146
+ <pre>
147
+ <span class="ruby-comment cmt"># File lib/sip/sipper.rb, line 5</span>
148
+ <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">initialize</span>(<span class="ruby-identifier">config</span>)
149
+ <span class="ruby-ivar">@config</span> = <span class="ruby-identifier">config</span>
150
+ <span class="ruby-identifier">slavefile</span> = <span class="ruby-constant">File</span>.<span class="ruby-identifier">join</span>(<span class="ruby-constant">ENV</span>[<span class="ruby-value str">'HADOOP_HOME'</span>], <span class="ruby-value str">'conf'</span>, <span class="ruby-value str">'slaves'</span>)
151
+ <span class="ruby-identifier">log</span> <span class="ruby-node">&quot;Reading slaves from file #{slavefile}...&quot;</span>
152
+ <span class="ruby-keyword kw">begin</span>
153
+ <span class="ruby-identifier">open</span>(<span class="ruby-identifier">slavefile</span>, <span class="ruby-value str">'r'</span>) { <span class="ruby-operator">|</span><span class="ruby-identifier">f</span><span class="ruby-operator">|</span> <span class="ruby-ivar">@slaves</span> = <span class="ruby-identifier">f</span>.<span class="ruby-identifier">read</span>.<span class="ruby-identifier">split</span>(<span class="ruby-value str">&quot;\n&quot;</span>) }
154
+ <span class="ruby-keyword kw">rescue</span>
155
+ <span class="ruby-identifier">raise</span> <span class="ruby-constant">HadoopException</span>, <span class="ruby-node">&quot;Could not read \&quot;#{slavefile}\&quot;. Is your HADOOP_HOME environment variable correct?&quot;</span>
156
+ <span class="ruby-keyword kw">end</span>
157
+ <span class="ruby-constant">Utils</span><span class="ruby-operator">::</span><span class="ruby-identifier">sanity_check</span>(<span class="ruby-ivar">@config</span>)
158
+ <span class="ruby-keyword kw">end</span>
159
+ </pre>
160
+ </div>
161
+ </div>
162
+ </div>
163
+
164
+ <h3 class="section-bar">Public Instance methods</h3>
165
+
166
+ <div id="method-M000026" class="method-detail">
167
+ <a name="M000026"></a>
168
+
169
+ <div class="method-heading">
170
+ <a href="#M000026" class="method-signature">
171
+ <span class="method-name">create_script</span><span class="method-args">(slavename, index, dbconf, tableconf, db, max, method)</span>
172
+ </a>
173
+ </div>
174
+
175
+ <div class="method-description">
176
+ <p><a class="source-toggle" href="#"
177
+ onclick="toggleCode('M000026-source');return false;">[Source]</a></p>
178
+ <div class="method-source-code" id="M000026-source">
179
+ <pre>
180
+ <span class="ruby-comment cmt"># File lib/sip/sipper.rb, line 68</span>
181
+ <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">create_script</span>(<span class="ruby-identifier">slavename</span>, <span class="ruby-identifier">index</span>, <span class="ruby-identifier">dbconf</span>, <span class="ruby-identifier">tableconf</span>, <span class="ruby-identifier">db</span>, <span class="ruby-identifier">max</span>, <span class="ruby-identifier">method</span>)
182
+ <span class="ruby-keyword kw">if</span> <span class="ruby-identifier">method</span> <span class="ruby-operator">==</span> <span class="ruby-identifier">:append</span>
183
+ <span class="ruby-identifier">first</span>, <span class="ruby-identifier">last</span> = <span class="ruby-identifier">get_even_split</span>(<span class="ruby-identifier">tableconf</span>[<span class="ruby-value str">'incremental_index_value'</span>]<span class="ruby-operator">+</span><span class="ruby-value">1</span>, <span class="ruby-identifier">max</span>, <span class="ruby-identifier">index</span>, <span class="ruby-ivar">@slaves</span>.<span class="ruby-identifier">length</span>)
184
+ <span class="ruby-keyword kw">else</span>
185
+ <span class="ruby-identifier">first</span>, <span class="ruby-identifier">last</span> = <span class="ruby-value">1</span>, <span class="ruby-identifier">max</span>
186
+ <span class="ruby-keyword kw">end</span>
187
+
188
+ <span class="ruby-identifier">log</span> <span class="ruby-node">&quot;Importing #{first} &lt;= #{tableconf['incremental_index']} &lt;= #{last} from table #{dbconf['dbname']}.#{tableconf['tablename']}&quot;</span>
189
+ <span class="ruby-identifier">select</span> = <span class="ruby-identifier">db</span>.<span class="ruby-identifier">generate_command</span> <span class="ruby-identifier">tableconf</span>, <span class="ruby-identifier">first</span>, <span class="ruby-identifier">last</span>
190
+ <span class="ruby-identifier">transpart_opts</span> = <span class="ruby-identifier">generate_transpart_options</span>(<span class="ruby-identifier">tableconf</span>, <span class="ruby-identifier">db</span>)
191
+ <span class="ruby-constant">Utils</span><span class="ruby-operator">::</span><span class="ruby-identifier">write_script</span> <span class="ruby-keyword kw">self</span>, <span class="ruby-identifier">slavename</span>, <span class="ruby-identifier">select</span>, <span class="ruby-identifier">dbconf</span>[<span class="ruby-value str">'dbname'</span>], <span class="ruby-identifier">tableconf</span>[<span class="ruby-value str">'tablename'</span>], <span class="ruby-identifier">transpart_opts</span>
192
+ <span class="ruby-keyword kw">end</span>
193
+ </pre>
194
+ </div>
195
+ </div>
196
+ </div>
197
+
198
+ <div id="method-M000024" class="method-detail">
199
+ <a name="M000024"></a>
200
+
201
+ <div class="method-heading">
202
+ <a href="#M000024" class="method-signature">
203
+ <span class="method-name">create_script_without_index</span><span class="method-args">(dbconf, tableconf, db)</span>
204
+ </a>
205
+ </div>
206
+
207
+ <div class="method-description">
208
+ <p>
209
+ this is the case where there is no primary key index, so the whole table
210
+ will need to be imported
211
+ </p>
212
+ <p><a class="source-toggle" href="#"
213
+ onclick="toggleCode('M000024-source');return false;">[Source]</a></p>
214
+ <div class="method-source-code" id="M000024-source">
215
+ <pre>
216
+ <span class="ruby-comment cmt"># File lib/sip/sipper.rb, line 40</span>
217
+ <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">create_script_without_index</span>(<span class="ruby-identifier">dbconf</span>, <span class="ruby-identifier">tableconf</span>, <span class="ruby-identifier">db</span>)
218
+ <span class="ruby-constant">Hive</span><span class="ruby-operator">::</span><span class="ruby-identifier">run_hsql_create_table</span> <span class="ruby-keyword kw">self</span>, <span class="ruby-identifier">db</span>, <span class="ruby-identifier">tableconf</span>, <span class="ruby-identifier">:overwrite</span>
219
+
220
+ <span class="ruby-identifier">log</span> <span class="ruby-node">&quot;Importing all rows from table #{dbconf['dbname']}.#{tableconf['tablename']}&quot;</span>
221
+ <span class="ruby-identifier">select</span> = <span class="ruby-identifier">db</span>.<span class="ruby-identifier">generate_command</span> <span class="ruby-identifier">tableconf</span>
222
+ <span class="ruby-identifier">transpart_opts</span> = <span class="ruby-identifier">generate_transpart_options</span>(<span class="ruby-identifier">tableconf</span>, <span class="ruby-identifier">db</span>)
223
+ <span class="ruby-ivar">@scripts</span> <span class="ruby-operator">&lt;&lt;</span> <span class="ruby-constant">Utils</span><span class="ruby-operator">::</span><span class="ruby-identifier">write_script</span>(<span class="ruby-keyword kw">self</span>, <span class="ruby-ivar">@slaves</span>.<span class="ruby-identifier">first</span>, <span class="ruby-identifier">select</span>, <span class="ruby-identifier">dbconf</span>[<span class="ruby-value str">'dbname'</span>], <span class="ruby-identifier">tableconf</span>[<span class="ruby-value str">'tablename'</span>], <span class="ruby-identifier">transpart_opts</span>)
224
+ <span class="ruby-keyword kw">end</span>
225
+ </pre>
226
+ </div>
227
+ </div>
228
+ </div>
229
+
230
+ <div id="method-M000023" class="method-detail">
231
+ <a name="M000023"></a>
232
+
233
+ <div class="method-heading">
234
+ <a href="#M000023" class="method-signature">
235
+ <span class="method-name">create_scripts</span><span class="method-args">(dbconf, tableconf)</span>
236
+ </a>
237
+ </div>
238
+
239
+ <div class="method-description">
240
+ <p>
241
+ return number of scripts created
242
+ </p>
243
+ <p><a class="source-toggle" href="#"
244
+ onclick="toggleCode('M000023-source');return false;">[Source]</a></p>
245
+ <div class="method-source-code" id="M000023-source">
246
+ <pre>
247
+ <span class="ruby-comment cmt"># File lib/sip/sipper.rb, line 22</span>
248
+ <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">create_scripts</span>(<span class="ruby-identifier">dbconf</span>, <span class="ruby-identifier">tableconf</span>)
249
+ <span class="ruby-ivar">@scripts</span> = []
250
+ <span class="ruby-identifier">db</span> = <span class="ruby-constant">DBBase</span>.<span class="ruby-identifier">make_interface</span> <span class="ruby-identifier">dbconf</span>[<span class="ruby-value str">'type'</span>], <span class="ruby-identifier">dbconf</span>, <span class="ruby-keyword kw">self</span>
251
+
252
+ <span class="ruby-comment cmt"># set default columns if necessary</span>
253
+ <span class="ruby-identifier">tableconf</span>[<span class="ruby-value str">'columns'</span>] = <span class="ruby-identifier">db</span>.<span class="ruby-identifier">columns</span>(<span class="ruby-identifier">tableconf</span>[<span class="ruby-value str">'tablename'</span>]).<span class="ruby-identifier">map</span> { <span class="ruby-operator">|</span><span class="ruby-identifier">c</span><span class="ruby-operator">|</span> <span class="ruby-identifier">c</span>.<span class="ruby-identifier">first</span> } <span class="ruby-keyword kw">if</span> <span class="ruby-identifier">tableconf</span>[<span class="ruby-value str">'columns'</span>].<span class="ruby-identifier">nil?</span>
254
+
255
+ <span class="ruby-keyword kw">if</span> <span class="ruby-identifier">tableconf</span>[<span class="ruby-value str">'incremental_index'</span>].<span class="ruby-identifier">nil?</span>
256
+ <span class="ruby-identifier">create_script_without_index</span>(<span class="ruby-identifier">dbconf</span>, <span class="ruby-identifier">tableconf</span>, <span class="ruby-identifier">db</span>)
257
+ <span class="ruby-keyword kw">else</span>
258
+ <span class="ruby-identifier">create_scripts_with_index</span>(<span class="ruby-identifier">dbconf</span>, <span class="ruby-identifier">tableconf</span>, <span class="ruby-identifier">db</span>)
259
+ <span class="ruby-keyword kw">end</span>
260
+
261
+ <span class="ruby-ivar">@scripts</span>.<span class="ruby-identifier">length</span>
262
+ <span class="ruby-keyword kw">end</span>
263
+ </pre>
264
+ </div>
265
+ </div>
266
+ </div>
267
+
268
+ <div id="method-M000025" class="method-detail">
269
+ <a name="M000025"></a>
270
+
271
+ <div class="method-heading">
272
+ <a href="#M000025" class="method-signature">
273
+ <span class="method-name">create_scripts_with_index</span><span class="method-args">(dbconf, tableconf, db)</span>
274
+ </a>
275
+ </div>
276
+
277
+ <div class="method-description">
278
+ <p><a class="source-toggle" href="#"
279
+ onclick="toggleCode('M000025-source');return false;">[Source]</a></p>
280
+ <div class="method-source-code" id="M000025-source">
281
+ <pre>
282
+ <span class="ruby-comment cmt"># File lib/sip/sipper.rb, line 49</span>
283
+ <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">create_scripts_with_index</span>(<span class="ruby-identifier">dbconf</span>, <span class="ruby-identifier">tableconf</span>, <span class="ruby-identifier">db</span>)
284
+ <span class="ruby-identifier">max</span> = <span class="ruby-identifier">db</span>.<span class="ruby-identifier">get_column_max</span> <span class="ruby-identifier">tableconf</span>[<span class="ruby-value str">'tablename'</span>], <span class="ruby-identifier">tableconf</span>[<span class="ruby-value str">'incremental_index'</span>]
285
+
286
+ <span class="ruby-identifier">method</span> = (<span class="ruby-identifier">tableconf</span>[<span class="ruby-value str">'method'</span>] <span class="ruby-operator">==</span> <span class="ruby-value str">&quot;append&quot;</span> <span class="ruby-keyword kw">and</span> <span class="ruby-keyword kw">not</span> <span class="ruby-ivar">@config</span>[<span class="ruby-identifier">:overwrite</span>]) <span class="ruby-operator">?</span> <span class="ruby-identifier">:append</span> <span class="ruby-operator">:</span> <span class="ruby-identifier">:overwrite</span>
287
+ <span class="ruby-keyword kw">if</span> <span class="ruby-identifier">method</span> <span class="ruby-operator">==</span> <span class="ruby-identifier">:append</span> <span class="ruby-keyword kw">and</span> <span class="ruby-identifier">max</span> <span class="ruby-operator">==</span> <span class="ruby-identifier">tableconf</span>[<span class="ruby-value str">'incremental_index_value'</span>]
288
+ <span class="ruby-identifier">log</span> <span class="ruby-node">&quot;Ignoring #{dbconf['dbname']}.#{tableconf['tablename']} - already up to date&quot;</span>
289
+ <span class="ruby-keyword kw">else</span>
290
+ <span class="ruby-constant">Hive</span><span class="ruby-operator">::</span><span class="ruby-identifier">run_hsql_create_table</span> <span class="ruby-keyword kw">self</span>, <span class="ruby-identifier">db</span>, <span class="ruby-identifier">tableconf</span>, <span class="ruby-identifier">method</span>
291
+ <span class="ruby-ivar">@slaves</span>.<span class="ruby-identifier">each_with_index</span> { <span class="ruby-operator">|</span><span class="ruby-identifier">slavename</span>, <span class="ruby-identifier">index</span><span class="ruby-operator">|</span>
292
+ <span class="ruby-ivar">@scripts</span> <span class="ruby-operator">&lt;&lt;</span> <span class="ruby-identifier">create_script</span>(<span class="ruby-identifier">slavename</span>, <span class="ruby-identifier">index</span>, <span class="ruby-identifier">dbconf</span>, <span class="ruby-identifier">tableconf</span>, <span class="ruby-identifier">db</span>, <span class="ruby-identifier">max</span>, <span class="ruby-identifier">method</span>)
293
+ }
294
+ <span class="ruby-identifier">db</span>.<span class="ruby-identifier">close</span>
295
+ <span class="ruby-keyword kw">end</span>
296
+
297
+ <span class="ruby-comment cmt"># update incremental index value if method in conf is append, regardless of whether or </span>
298
+ <span class="ruby-comment cmt"># not this is a forced overwrite</span>
299
+ <span class="ruby-identifier">tableconf</span>[<span class="ruby-value str">'incremental_index_value'</span>] = <span class="ruby-identifier">max</span> <span class="ruby-keyword kw">if</span> <span class="ruby-identifier">tableconf</span>[<span class="ruby-value str">'method'</span>] <span class="ruby-operator">==</span> <span class="ruby-value str">&quot;append&quot;</span>
300
+ <span class="ruby-keyword kw">end</span>
301
+ </pre>
302
+ </div>
303
+ </div>
304
+ </div>
305
+
306
+ <div id="method-M000027" class="method-detail">
307
+ <a name="M000027"></a>
308
+
309
+ <div class="method-heading">
310
+ <a href="#M000027" class="method-signature">
311
+ <span class="method-name">generate_transpart_options</span><span class="method-args">(tableconf, db)</span>
312
+ </a>
313
+ </div>
314
+
315
+ <div class="method-description">
316
+ <p><a class="source-toggle" href="#"
317
+ onclick="toggleCode('M000027-source');return false;">[Source]</a></p>
318
+ <div class="method-source-code" id="M000027-source">
319
+ <pre>
320
+ <span class="ruby-comment cmt"># File lib/sip/sipper.rb, line 81</span>
321
+ <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">generate_transpart_options</span>(<span class="ruby-identifier">tableconf</span>, <span class="ruby-identifier">db</span>)
322
+ <span class="ruby-identifier">opts</span> = <span class="ruby-constant">CmdOpts</span>.<span class="ruby-identifier">new</span>
323
+ <span class="ruby-identifier">opts</span>[<span class="ruby-value str">'c'</span>] = <span class="ruby-identifier">db</span>.<span class="ruby-identifier">order_column_list</span>(<span class="ruby-identifier">tableconf</span>[<span class="ruby-value str">'tablename'</span>], <span class="ruby-identifier">tableconf</span>[<span class="ruby-value str">'columns'</span>]).<span class="ruby-identifier">join</span>(<span class="ruby-value str">','</span>)
324
+ <span class="ruby-identifier">opts</span>[<span class="ruby-value str">'p'</span>] = <span class="ruby-identifier">tableconf</span>[<span class="ruby-value str">'partition_by'</span>] <span class="ruby-keyword kw">if</span> <span class="ruby-identifier">tableconf</span>.<span class="ruby-identifier">has_key?</span> <span class="ruby-value str">&quot;partition_by&quot;</span>
325
+ <span class="ruby-keyword kw">if</span> <span class="ruby-identifier">tableconf</span>.<span class="ruby-identifier">has_key?</span> <span class="ruby-value str">'transformations'</span> <span class="ruby-keyword kw">and</span> <span class="ruby-identifier">tableconf</span>[<span class="ruby-value str">'transformations'</span>].<span class="ruby-identifier">length</span> <span class="ruby-operator">&gt;</span> <span class="ruby-value">0</span>
326
+ <span class="ruby-identifier">opts</span>[<span class="ruby-value str">'t'</span>] = <span class="ruby-identifier">tableconf</span>[<span class="ruby-value str">'transformations'</span>].<span class="ruby-identifier">map</span> { <span class="ruby-operator">|</span><span class="ruby-identifier">k</span>,<span class="ruby-identifier">v</span><span class="ruby-operator">|</span> <span class="ruby-node">&quot;#{k}:#{v}&quot;</span> }.<span class="ruby-identifier">join</span>(<span class="ruby-value str">','</span>)
327
+ <span class="ruby-keyword kw">end</span>
328
+ <span class="ruby-identifier">opts</span>[<span class="ruby-value str">'H'</span>] = <span class="ruby-identifier">tableconf</span>[<span class="ruby-value str">'hive_table_name'</span>]
329
+ <span class="ruby-identifier">opts</span>[<span class="ruby-value str">'o'</span>] = <span class="ruby-constant">File</span>.<span class="ruby-identifier">join</span>(<span class="ruby-ivar">@config</span>[<span class="ruby-value str">'tmpdir'</span>], <span class="ruby-value str">'partitions'</span>)
330
+ <span class="ruby-identifier">opts</span>.<span class="ruby-identifier">set</span>(<span class="ruby-value str">'d'</span>) <span class="ruby-keyword kw">if</span> <span class="ruby-ivar">@config</span>[<span class="ruby-identifier">:debug</span>]
331
+ <span class="ruby-identifier">opts</span>
332
+ <span class="ruby-keyword kw">end</span>
333
+ </pre>
334
+ </div>
335
+ </div>
336
+ </div>
337
+
338
+ <div id="method-M000028" class="method-detail">
339
+ <a name="M000028"></a>
340
+
341
+ <div class="method-heading">
342
+ <a href="#M000028" class="method-signature">
343
+ <span class="method-name">get_even_split</span><span class="method-args">(min, max, index, count)</span>
344
+ </a>
345
+ </div>
346
+
347
+ <div class="method-description">
348
+ <p><a class="source-toggle" href="#"
349
+ onclick="toggleCode('M000028-source');return false;">[Source]</a></p>
350
+ <div class="method-source-code" id="M000028-source">
351
+ <pre>
352
+ <span class="ruby-comment cmt"># File lib/sip/sipper.rb, line 94</span>
353
+ <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">get_even_split</span>(<span class="ruby-identifier">min</span>, <span class="ruby-identifier">max</span>, <span class="ruby-identifier">index</span>, <span class="ruby-identifier">count</span>)
354
+ <span class="ruby-identifier">size</span> = ((<span class="ruby-identifier">max</span> <span class="ruby-operator">-</span> <span class="ruby-identifier">min</span> <span class="ruby-operator">+</span> <span class="ruby-value">1</span>).<span class="ruby-identifier">to_f</span> <span class="ruby-operator">/</span> <span class="ruby-identifier">count</span>.<span class="ruby-identifier">to_f</span>).<span class="ruby-identifier">ceil</span>
355
+ <span class="ruby-identifier">first</span> = (<span class="ruby-identifier">size</span> <span class="ruby-operator">*</span> <span class="ruby-identifier">index</span>) <span class="ruby-operator">+</span> <span class="ruby-identifier">min</span>
356
+ <span class="ruby-identifier">last</span> = (<span class="ruby-identifier">size</span> <span class="ruby-operator">*</span> (<span class="ruby-identifier">index</span><span class="ruby-operator">+</span><span class="ruby-value">1</span>)) <span class="ruby-operator">+</span> <span class="ruby-identifier">min</span> <span class="ruby-operator">-</span> <span class="ruby-value">1</span>
357
+ [<span class="ruby-identifier">first</span>, [<span class="ruby-identifier">max</span>, <span class="ruby-identifier">last</span>].<span class="ruby-identifier">min</span>]
358
+ <span class="ruby-keyword kw">end</span>
359
+ </pre>
360
+ </div>
361
+ </div>
362
+ </div>
363
+
364
+ <div id="method-M000022" class="method-detail">
365
+ <a name="M000022"></a>
366
+
367
+ <div class="method-heading">
368
+ <a href="#M000022" class="method-signature">
369
+ <span class="method-name">log</span><span class="method-args">(msg)</span>
370
+ </a>
371
+ </div>
372
+
373
+ <div class="method-description">
374
+ <p><a class="source-toggle" href="#"
375
+ onclick="toggleCode('M000022-source');return false;">[Source]</a></p>
376
+ <div class="method-source-code" id="M000022-source">
377
+ <pre>
378
+ <span class="ruby-comment cmt"># File lib/sip/sipper.rb, line 17</span>
379
+ <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">log</span>(<span class="ruby-identifier">msg</span>)
380
+ <span class="ruby-identifier">puts</span> <span class="ruby-node">&quot;#{Utils::hostname} #{Time.now.strftime '%Y-%m-%d %H:%M:%S'}: #{msg}&quot;</span> <span class="ruby-keyword kw">if</span> <span class="ruby-ivar">@config</span>[<span class="ruby-identifier">:debug</span>]
381
+ <span class="ruby-keyword kw">end</span>
382
+ </pre>
383
+ </div>
384
+ </div>
385
+ </div>
386
+
387
+ <div id="method-M000030" class="method-detail">
388
+ <a name="M000030"></a>
389
+
390
+ <div class="method-heading">
391
+ <a href="#M000030" class="method-signature">
392
+ <span class="method-name">run_hive_import</span><span class="method-args">(tableconf)</span>
393
+ </a>
394
+ </div>
395
+
396
+ <div class="method-description">
397
+ <p><a class="source-toggle" href="#"
398
+ onclick="toggleCode('M000030-source');return false;">[Source]</a></p>
399
+ <div class="method-source-code" id="M000030-source">
400
+ <pre>
401
+ <span class="ruby-comment cmt"># File lib/sip/sipper.rb, line 114</span>
402
+ <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">run_hive_import</span>(<span class="ruby-identifier">tableconf</span>)
403
+ <span class="ruby-constant">Hive</span>.<span class="ruby-identifier">run_import</span> <span class="ruby-keyword kw">self</span>, <span class="ruby-identifier">tableconf</span>
404
+ <span class="ruby-keyword kw">end</span>
405
+ </pre>
406
+ </div>
407
+ </div>
408
+ </div>
409
+
410
+ <div id="method-M000029" class="method-detail">
411
+ <a name="M000029"></a>
412
+
413
+ <div class="method-heading">
414
+ <a href="#M000029" class="method-signature">
415
+ <span class="method-name">run_scripts</span><span class="method-args">()</span>
416
+ </a>
417
+ </div>
418
+
419
+ <div class="method-description">
420
+ <p><a class="source-toggle" href="#"
421
+ onclick="toggleCode('M000029-source');return false;">[Source]</a></p>
422
+ <div class="method-source-code" id="M000029-source">
423
+ <pre>
424
+ <span class="ruby-comment cmt"># File lib/sip/sipper.rb, line 101</span>
425
+ <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">run_scripts</span>
426
+ <span class="ruby-identifier">pids</span> = {}
427
+ <span class="ruby-ivar">@scripts</span>.<span class="ruby-identifier">each</span> { <span class="ruby-operator">|</span><span class="ruby-identifier">script</span><span class="ruby-operator">|</span>
428
+ <span class="ruby-identifier">log</span> <span class="ruby-node">&quot;Running #{script}...&quot;</span>
429
+ <span class="ruby-identifier">pid</span> = <span class="ruby-identifier">fork</span> { <span class="ruby-constant">Kernel</span>.<span class="ruby-identifier">exit</span> <span class="ruby-identifier">system</span>(<span class="ruby-node">&quot;sh #{script}&quot;</span>) }
430
+ <span class="ruby-identifier">pids</span>[<span class="ruby-identifier">pid</span>] = <span class="ruby-identifier">script</span>
431
+ <span class="ruby-identifier">sleep</span> <span class="ruby-value">1</span>
432
+ }
433
+ <span class="ruby-constant">Process</span>.<span class="ruby-identifier">waitall</span>.<span class="ruby-identifier">map</span> { <span class="ruby-operator">|</span><span class="ruby-identifier">r</span><span class="ruby-operator">|</span> <span class="ruby-identifier">r</span>.<span class="ruby-identifier">last</span> }.<span class="ruby-identifier">each</span> { <span class="ruby-operator">|</span><span class="ruby-identifier">status</span><span class="ruby-operator">|</span>
434
+ <span class="ruby-identifier">raise</span> <span class="ruby-constant">ImportScriptExecutionError</span>, <span class="ruby-node">&quot;Error runing script '#{pids[status.pid]}'&quot;</span> <span class="ruby-keyword kw">if</span> <span class="ruby-identifier">status</span>.<span class="ruby-identifier">exitstatus</span> <span class="ruby-operator">!=</span> <span class="ruby-value">0</span>
435
+ }
436
+ <span class="ruby-keyword kw">end</span>
437
+ </pre>
438
+ </div>
439
+ </div>
440
+ </div>
441
+
442
+
443
+ </div>
444
+
445
+
446
+ </div>
447
+
448
+
449
+ <div id="validator-badges">
450
+ <p><small><a href="http://validator.w3.org/check/referer">[Validate]</a></small></p>
451
+ </div>
452
+
453
+ </body>
454
+ </html>