forklift_etl 1.0.8

Sign up to get free protection for your applications and to get access to all the features.
Files changed (121) hide show
  1. checksums.yaml +15 -0
  2. data/.gitignore +10 -0
  3. data/.rbenv-version +1 -0
  4. data/.travis.yml +10 -0
  5. data/Gemfile +10 -0
  6. data/Gemfile.lock +74 -0
  7. data/Rakefile +13 -0
  8. data/bin/forklift +61 -0
  9. data/doc/EmailSuffix.html +228 -0
  10. data/doc/Forklift.html +187 -0
  11. data/doc/Forklift/Base.html +167 -0
  12. data/doc/Forklift/Base/Connection.html +590 -0
  13. data/doc/Forklift/Base/Logger.html +453 -0
  14. data/doc/Forklift/Base/Mailer.html +399 -0
  15. data/doc/Forklift/Base/Mailer/ERBBinding.html +256 -0
  16. data/doc/Forklift/Base/Pid.html +489 -0
  17. data/doc/Forklift/Base/Utils.html +252 -0
  18. data/doc/Forklift/Connection.html +164 -0
  19. data/doc/Forklift/Connection/Elasticsearch.html +419 -0
  20. data/doc/Forklift/Connection/Mysql.html +939 -0
  21. data/doc/Forklift/Patterns.html +164 -0
  22. data/doc/Forklift/Patterns/Elasticsearch.html +169 -0
  23. data/doc/Forklift/Patterns/Mysql.html +402 -0
  24. data/doc/Forklift/Plan.html +704 -0
  25. data/doc/Gemfile.html +132 -0
  26. data/doc/Object.html +326 -0
  27. data/doc/Rakefile.html +138 -0
  28. data/doc/SpecClient.html +291 -0
  29. data/doc/SpecPlan.html +253 -0
  30. data/doc/SpecSeeds.html +303 -0
  31. data/doc/created.rid +35 -0
  32. data/doc/example/Gemfile.html +129 -0
  33. data/doc/images/add.png +0 -0
  34. data/doc/images/brick.png +0 -0
  35. data/doc/images/brick_link.png +0 -0
  36. data/doc/images/bug.png +0 -0
  37. data/doc/images/bullet_black.png +0 -0
  38. data/doc/images/bullet_toggle_minus.png +0 -0
  39. data/doc/images/bullet_toggle_plus.png +0 -0
  40. data/doc/images/date.png +0 -0
  41. data/doc/images/delete.png +0 -0
  42. data/doc/images/find.png +0 -0
  43. data/doc/images/loadingAnimation.gif +0 -0
  44. data/doc/images/macFFBgHack.png +0 -0
  45. data/doc/images/package.png +0 -0
  46. data/doc/images/page_green.png +0 -0
  47. data/doc/images/page_white_text.png +0 -0
  48. data/doc/images/page_white_width.png +0 -0
  49. data/doc/images/plugin.png +0 -0
  50. data/doc/images/ruby.png +0 -0
  51. data/doc/images/tag_blue.png +0 -0
  52. data/doc/images/tag_green.png +0 -0
  53. data/doc/images/transparent.png +0 -0
  54. data/doc/images/wrench.png +0 -0
  55. data/doc/images/wrench_orange.png +0 -0
  56. data/doc/images/zoom.png +0 -0
  57. data/doc/index.html +122 -0
  58. data/doc/js/darkfish.js +155 -0
  59. data/doc/js/jquery.js +18 -0
  60. data/doc/js/navigation.js +142 -0
  61. data/doc/js/search.js +94 -0
  62. data/doc/js/search_index.js +1 -0
  63. data/doc/js/searcher.js +228 -0
  64. data/doc/rdoc.css +543 -0
  65. data/doc/table_of_contents.html +309 -0
  66. data/example/Gemfile +3 -0
  67. data/example/Gemfile.lock +55 -0
  68. data/example/config/connections/elasticsearch/source.yml +1 -0
  69. data/example/config/connections/mysql/destination.yml +6 -0
  70. data/example/config/connections/mysql/source.yml +6 -0
  71. data/example/config/email.yml +18 -0
  72. data/example/plan.rb +87 -0
  73. data/example/template/email.erb +6 -0
  74. data/example/transformations/cleanup.sql +1 -0
  75. data/example/transformations/combined_name.sql +7 -0
  76. data/example/transformations/email_suffix.rb +20 -0
  77. data/forklift.jpg +0 -0
  78. data/forklift_etl.gemspec +28 -0
  79. data/lib/forklift/base/connection.rb +72 -0
  80. data/lib/forklift/base/logger.rb +49 -0
  81. data/lib/forklift/base/mailer.rb +83 -0
  82. data/lib/forklift/base/pid.rb +55 -0
  83. data/lib/forklift/base/utils.rb +23 -0
  84. data/lib/forklift/forklift.rb +19 -0
  85. data/lib/forklift/patterns/elasticsearch_patterns.rb +7 -0
  86. data/lib/forklift/patterns/mysql_patterns.rb +87 -0
  87. data/lib/forklift/plan.rb +138 -0
  88. data/lib/forklift/transports/elasticsearch.rb +75 -0
  89. data/lib/forklift/transports/mysql.rb +241 -0
  90. data/lib/forklift/version.rb +3 -0
  91. data/readme.md +410 -0
  92. data/spec/config/connections/elasticsearch/forklift_test.yml +1 -0
  93. data/spec/config/connections/mysql/forklift_test_destination.yml +6 -0
  94. data/spec/config/connections/mysql/forklift_test_source_a.yml +6 -0
  95. data/spec/config/connections/mysql/forklift_test_source_b.yml +6 -0
  96. data/spec/config/connections/mysql/forklift_test_working.yml +6 -0
  97. data/spec/config/email.yml +4 -0
  98. data/spec/integration/basic_spec.rb +29 -0
  99. data/spec/integration/elasticsearch_patterns_spec.rb +5 -0
  100. data/spec/integration/elasticsearch_spec.rb +95 -0
  101. data/spec/integration/multi_transport_spec.rb +112 -0
  102. data/spec/integration/mysql_patterns_spec.rb +76 -0
  103. data/spec/integration/mysql_spec.rb +138 -0
  104. data/spec/spec_helper.rb +30 -0
  105. data/spec/support/dumps/elasticsearch/forklift_test.json +7 -0
  106. data/spec/support/dumps/mysql/forklift_test_source_a.sql +79 -0
  107. data/spec/support/dumps/mysql/forklift_test_source_b.sql +23 -0
  108. data/spec/support/spec_client.rb +30 -0
  109. data/spec/support/spec_plan.rb +15 -0
  110. data/spec/support/spec_seeds.rb +69 -0
  111. data/spec/template/spec_email_template.erb +4 -0
  112. data/spec/unit/connection/mysql_spec.rb +102 -0
  113. data/spec/unit/misc/email_spec.rb +37 -0
  114. data/spec/unit/misc/pid_spec.rb +25 -0
  115. data/spec/unit/misc/step_spec.rb +53 -0
  116. data/template/destination.yml +6 -0
  117. data/template/email.erb +1 -0
  118. data/template/email.yml +18 -0
  119. data/template/plan.rb +10 -0
  120. data/template/source.yml +6 -0
  121. metadata +289 -0
data/doc/table_of_contents.html ADDED
@@ -0,0 +1,309 @@
1
+ <!DOCTYPE html>
2
+
3
+ <html>
4
+ <head>
5
+ <meta content="text/html; charset=UTF-8" http-equiv="Content-Type">
6
+
7
+ <title>Table of Contents - RDoc Documentation</title>
8
+
9
+ <link type="text/css" media="screen" href="./rdoc.css" rel="stylesheet">
10
+
11
+ <script type="text/javascript">
12
+ var rdoc_rel_prefix = "./";
13
+ </script>
14
+
15
+ <script type="text/javascript" charset="utf-8" src="./js/jquery.js"></script>
16
+ <script type="text/javascript" charset="utf-8" src="./js/navigation.js"></script>
17
+ <script type="text/javascript" charset="utf-8" src="./js/search_index.js"></script>
18
+ <script type="text/javascript" charset="utf-8" src="./js/search.js"></script>
19
+ <script type="text/javascript" charset="utf-8" src="./js/searcher.js"></script>
20
+ <script type="text/javascript" charset="utf-8" src="./js/darkfish.js"></script>
21
+
22
+
23
+ <body class="indexpage">
24
+ <h1>Table of Contents - RDoc Documentation</h1>
25
+
26
+ <h2>Pages</h2>
27
+ <ul>
28
+ <li class="file">
29
+ <a href="Gemfile.html">Gemfile</a>
30
+ </li>
31
+ <li class="file">
32
+ <a href="Rakefile.html">Rakefile</a>
33
+ </li>
34
+ <li class="file">
35
+ <a href="example/Gemfile.html">Gemfile</a>
36
+ </li>
37
+ <li class="file">
38
+ <a href="spec/pid/pidfile.html">pidfile</a>
39
+ </li>
40
+
41
+ </ul>
42
+
43
+ <h2 id="classes">Classes/Modules</h2>
44
+ <ul>
45
+ <li class="module">
46
+ <a href="Forklift.html">Forklift</a>
47
+ </li>
48
+ <li class="module">
49
+ <a href="Forklift/Base.html">Forklift::Base</a>
50
+ </li>
51
+ <li class="class">
52
+ <a href="Forklift/Base/Connection.html">Forklift::Base::Connection</a>
53
+ </li>
54
+ <li class="class">
55
+ <a href="Forklift/Base/Logger.html">Forklift::Base::Logger</a>
56
+ </li>
57
+ <li class="class">
58
+ <a href="Forklift/Base/Mailer.html">Forklift::Base::Mailer</a>
59
+ </li>
60
+ <li class="class">
61
+ <a href="Forklift/Base/Mailer/ERBBinding.html">Forklift::Base::Mailer::ERBBinding</a>
62
+ </li>
63
+ <li class="class">
64
+ <a href="Forklift/Base/Pid.html">Forklift::Base::Pid</a>
65
+ </li>
66
+ <li class="class">
67
+ <a href="Forklift/Base/Utils.html">Forklift::Base::Utils</a>
68
+ </li>
69
+ <li class="module">
70
+ <a href="Forklift/Connection.html">Forklift::Connection</a>
71
+ </li>
72
+ <li class="class">
73
+ <a href="Forklift/Connection/Elasticsearch.html">Forklift::Connection::Elasticsearch</a>
74
+ </li>
75
+ <li class="class">
76
+ <a href="Forklift/Connection/Mysql.html">Forklift::Connection::Mysql</a>
77
+ </li>
78
+ <li class="module">
79
+ <a href="Forklift/Patterns.html">Forklift::Patterns</a>
80
+ </li>
81
+ <li class="class">
82
+ <a href="Forklift/Patterns/Elasticsearch.html">Forklift::Patterns::Elasticsearch</a>
83
+ </li>
84
+ <li class="class">
85
+ <a href="Forklift/Patterns/Mysql.html">Forklift::Patterns::Mysql</a>
86
+ </li>
87
+ <li class="class">
88
+ <a href="Forklift/Plan.html">Forklift::Plan</a>
89
+ </li>
90
+ <li class="class">
91
+ <a href="EmailSuffix.html">EmailSuffix</a>
92
+ </li>
93
+ <li class="class">
94
+ <a href="Object.html">Object</a>
95
+ </li>
96
+ <li class="class">
97
+ <a href="SpecClient.html">SpecClient</a>
98
+ </li>
99
+ <li class="class">
100
+ <a href="SpecPlan.html">SpecPlan</a>
101
+ </li>
102
+ <li class="class">
103
+ <a href="SpecSeeds.html">SpecSeeds</a>
104
+ </li>
105
+
106
+ </ul>
107
+
108
+ <h2 id="methods">Methods</h2>
109
+ <ul>
110
+
111
+ <li class="method"><a href="Forklift/Patterns/Mysql.html#method-c-can_incremental_pipe-3F">::can_incremental_pipe? &mdash; Forklift::Patterns::Mysql</a>
112
+
113
+ <li class="method"><a href="SpecPlan.html#method-c-config">::config &mdash; SpecPlan</a>
114
+
115
+ <li class="method"><a href="SpecClient.html#method-c-elasticsearch">::elasticsearch &mdash; SpecClient</a>
116
+
117
+ <li class="method"><a href="Forklift/Patterns/Mysql.html#method-c-incremental_pipe">::incremental_pipe &mdash; Forklift::Patterns::Mysql</a>
118
+
119
+ <li class="method"><a href="SpecClient.html#method-c-load_config">::load_config &mdash; SpecClient</a>
120
+
121
+ <li class="method"><a href="SpecClient.html#method-c-mysql">::mysql &mdash; SpecClient</a>
122
+
123
+ <li class="method"><a href="Forklift/Patterns/Mysql.html#method-c-mysql_optimistic_import">::mysql_optimistic_import &mdash; Forklift::Patterns::Mysql</a>
124
+
125
+ <li class="method"><a href="Forklift/Base/Mailer/ERBBinding.html#method-c-new">::new &mdash; Forklift::Base::Mailer::ERBBinding</a>
126
+
127
+ <li class="method"><a href="Forklift/Base/Connection.html#method-c-new">::new &mdash; Forklift::Base::Connection</a>
128
+
129
+ <li class="method"><a href="Forklift/Plan.html#method-c-new">::new &mdash; Forklift::Plan</a>
130
+
131
+ <li class="method"><a href="SpecPlan.html#method-c-new">::new &mdash; SpecPlan</a>
132
+
133
+ <li class="method"><a href="Forklift/Base/Mailer.html#method-c-new">::new &mdash; Forklift::Base::Mailer</a>
134
+
135
+ <li class="method"><a href="Forklift/Base/Logger.html#method-c-new">::new &mdash; Forklift::Base::Logger</a>
136
+
137
+ <li class="method"><a href="Forklift/Connection/Elasticsearch.html#method-c-new">::new &mdash; Forklift::Connection::Elasticsearch</a>
138
+
139
+ <li class="method"><a href="Forklift/Connection/Mysql.html#method-c-new">::new &mdash; Forklift::Connection::Mysql</a>
140
+
141
+ <li class="method"><a href="Forklift/Base/Pid.html#method-c-new">::new &mdash; Forklift::Base::Pid</a>
142
+
143
+ <li class="method"><a href="Forklift/Patterns/Mysql.html#method-c-optimistic_pipe">::optimistic_pipe &mdash; Forklift::Patterns::Mysql</a>
144
+
145
+ <li class="method"><a href="Forklift/Patterns/Mysql.html#method-c-pipe">::pipe &mdash; Forklift::Patterns::Mysql</a>
146
+
147
+ <li class="method"><a href="SpecSeeds.html#method-c-setup_elasticsearch">::setup_elasticsearch &mdash; SpecSeeds</a>
148
+
149
+ <li class="method"><a href="SpecSeeds.html#method-c-setup_mysql">::setup_mysql &mdash; SpecSeeds</a>
150
+
151
+ <li class="method"><a href="Forklift/Plan.html#method-i-activate_steps">#activate_steps &mdash; Forklift::Plan</a>
152
+
153
+ <li class="method"><a href="Forklift/Plan.html#method-i-argv">#argv &mdash; Forklift::Plan</a>
154
+
155
+ <li class="method"><a href="Forklift/Base/Utils.html#method-i-class_name_from_file">#class_name_from_file &mdash; Forklift::Base::Utils</a>
156
+
157
+ <li class="method"><a href="Forklift/Base/Connection.html#method-i-client">#client &mdash; Forklift::Base::Connection</a>
158
+
159
+ <li class="method"><a href="Forklift/Connection/Mysql.html#method-i-columns">#columns &mdash; Forklift::Connection::Mysql</a>
160
+
161
+ <li class="method"><a href="Forklift/Base/Connection.html#method-i-config">#config &mdash; Forklift::Base::Connection</a>
162
+
163
+ <li class="method"><a href="Forklift/Base/Mailer.html#method-i-config">#config &mdash; Forklift::Base::Mailer</a>
164
+
165
+ <li class="method"><a href="Forklift/Connection/Elasticsearch.html#method-i-config">#config &mdash; Forklift::Connection::Elasticsearch</a>
166
+
167
+ <li class="method"><a href="Forklift/Connection/Mysql.html#method-i-config">#config &mdash; Forklift::Connection::Mysql</a>
168
+
169
+ <li class="method"><a href="Forklift/Plan.html#method-i-config">#config &mdash; Forklift::Plan</a>
170
+
171
+ <li class="method"><a href="Forklift/Base/Connection.html#method-i-connect">#connect &mdash; Forklift::Base::Connection</a>
172
+
173
+ <li class="method"><a href="Forklift/Plan.html#method-i-connect-21">#connect! &mdash; Forklift::Plan</a>
174
+
175
+ <li class="method"><a href="Forklift/Plan.html#method-i-connections">#connections &mdash; Forklift::Plan</a>
176
+
177
+ <li class="method"><a href="Forklift/Connection/Mysql.html#method-i-count">#count &mdash; Forklift::Connection::Mysql</a>
178
+
179
+ <li class="method"><a href="Forklift/Connection/Mysql.html#method-i-current_database">#current_database &mdash; Forklift::Connection::Mysql</a>
180
+
181
+ <li class="method"><a href="Forklift/Base/Logger.html#method-i-debug">#debug &mdash; Forklift::Base::Logger</a>
182
+
183
+ <li class="method"><a href="Forklift/Connection/Mysql.html#method-i-default_matcher">#default_matcher &mdash; Forklift::Connection::Mysql</a>
184
+
185
+ <li class="method"><a href="Forklift/Base/Pid.html#method-i-delete-21">#delete! &mdash; Forklift::Base::Pid</a>
186
+
187
+ <li class="method"><a href="Forklift/Connection/Elasticsearch.html#method-i-delete_index">#delete_index &mdash; Forklift::Connection::Elasticsearch</a>
188
+
189
+ <li class="method"><a href="Forklift/Base/Connection.html#method-i-disconnect">#disconnect &mdash; Forklift::Base::Connection</a>
190
+
191
+ <li class="method"><a href="Forklift/Plan.html#method-i-do-21">#do! &mdash; Forklift::Plan</a>
192
+
193
+ <li class="method"><a href="EmailSuffix.html#method-i-do-21">#do! &mdash; EmailSuffix</a>
194
+
195
+ <li class="method"><a href="Forklift/Plan.html#method-i-do_step-21">#do_step! &mdash; Forklift::Plan</a>
196
+
197
+ <li class="method"><a href="Forklift/Connection/Mysql.html#method-i-drop-21">#drop! &mdash; Forklift::Connection::Mysql</a>
198
+
199
+ <li class="method"><a href="Forklift/Connection/Mysql.html#method-i-dump">#dump &mdash; Forklift::Connection::Mysql</a>
200
+
201
+ <li class="method"><a href="Forklift/Base/Logger.html#method-i-emphatically">#emphatically &mdash; Forklift::Base::Logger</a>
202
+
203
+ <li class="method"><a href="Forklift/Base/Pid.html#method-i-ensure_pid_dir">#ensure_pid_dir &mdash; Forklift::Base::Pid</a>
204
+
205
+ <li class="method"><a href="Forklift/Base/Connection.html#method-i-exec">#exec &mdash; Forklift::Base::Connection</a>
206
+
207
+ <li class="method"><a href="Forklift/Base/Connection.html#method-i-exec-21">#exec! &mdash; Forklift::Base::Connection</a>
208
+
209
+ <li class="method"><a href="Forklift/Base/Connection.html#method-i-exec_ruby">#exec_ruby &mdash; Forklift::Base::Connection</a>
210
+
211
+ <li class="method"><a href="Forklift/Base/Connection.html#method-i-exec_script">#exec_script &mdash; Forklift::Base::Connection</a>
212
+
213
+ <li class="method"><a href="Forklift/Connection/Mysql.html#method-i-exec_script">#exec_script &mdash; Forklift::Connection::Mysql</a>
214
+
215
+ <li class="method"><a href="Forklift/Base/Logger.html#method-i-fatal">#fatal &mdash; Forklift::Base::Logger</a>
216
+
217
+ <li class="method"><a href="Forklift/Connection/Elasticsearch.html#method-i-forklift">#forklift &mdash; Forklift::Connection::Elasticsearch</a>
218
+
219
+ <li class="method"><a href="Forklift/Base/Logger.html#method-i-forklift">#forklift &mdash; Forklift::Base::Logger</a>
220
+
221
+ <li class="method"><a href="Forklift/Connection/Mysql.html#method-i-forklift">#forklift &mdash; Forklift::Connection::Mysql</a>
222
+
223
+ <li class="method"><a href="Forklift/Base/Pid.html#method-i-forklift">#forklift &mdash; Forklift::Base::Pid</a>
224
+
225
+ <li class="method"><a href="Forklift/Base/Mailer.html#method-i-forklift">#forklift &mdash; Forklift::Base::Mailer</a>
226
+
227
+ <li class="method"><a href="Object.html#method-i-generate">#generate &mdash; Object</a>
228
+
229
+ <li class="method"><a href="Forklift/Base/Mailer/ERBBinding.html#method-i-get_binding">#get_binding &mdash; Forklift::Base::Mailer::ERBBinding</a>
230
+
231
+ <li class="method"><a href="Forklift/Connection/Mysql.html#method-i-lazy_table_create">#lazy_table_create &mdash; Forklift::Connection::Mysql</a>
232
+
233
+ <li class="method"><a href="Forklift/Base/Utils.html#method-i-load_yml">#load_yml &mdash; Forklift::Base::Utils</a>
234
+
235
+ <li class="method"><a href="Forklift/Base/Logger.html#method-i-log">#log &mdash; Forklift::Base::Logger</a>
236
+
237
+ <li class="method"><a href="Forklift/Plan.html#method-i-logger">#logger &mdash; Forklift::Plan</a>
238
+
239
+ <li class="method"><a href="Forklift/Base/Logger.html#method-i-logger">#logger &mdash; Forklift::Base::Logger</a>
240
+
241
+ <li class="method"><a href="Forklift/Plan.html#method-i-mailer">#mailer &mdash; Forklift::Plan</a>
242
+
243
+ <li class="method"><a href="Forklift/Connection/Mysql.html#method-i-max_timestamp">#max_timestamp &mdash; Forklift::Connection::Mysql</a>
244
+
245
+ <li class="method"><a href="Forklift/Base/Mailer.html#method-i-message_defaults">#message_defaults &mdash; Forklift::Base::Mailer</a>
246
+
247
+ <li class="method"><a href="Forklift/Base/Logger.html#method-i-messages">#messages &mdash; Forklift::Base::Logger</a>
248
+
249
+ <li class="method"><a href="Forklift/Plan.html#method-i-pid">#pid &mdash; Forklift::Plan</a>
250
+
251
+ <li class="method"><a href="Forklift/Base/Pid.html#method-i-pid_dir">#pid_dir &mdash; Forklift::Base::Pid</a>
252
+
253
+ <li class="method"><a href="Forklift/Base/Pid.html#method-i-pidfile">#pidfile &mdash; Forklift::Base::Pid</a>
254
+
255
+ <li class="method"><a href="Forklift/Base/Connection.html#method-i-pipe">#pipe &mdash; Forklift::Base::Connection</a>
256
+
257
+ <li class="method"><a href="Forklift/Connection/Mysql.html#method-i-q">#q &mdash; Forklift::Connection::Mysql</a>
258
+
259
+ <li class="method"><a href="Forklift/Connection/Mysql.html#method-i-read">#read &mdash; Forklift::Connection::Mysql</a>
260
+
261
+ <li class="method"><a href="Forklift/Connection/Elasticsearch.html#method-i-read">#read &mdash; Forklift::Connection::Elasticsearch</a>
262
+
263
+ <li class="method"><a href="Forklift/Base/Connection.html#method-i-read">#read &mdash; Forklift::Base::Connection</a>
264
+
265
+ <li class="method"><a href="Forklift/Connection/Mysql.html#method-i-read_since">#read_since &mdash; Forklift::Connection::Mysql</a>
266
+
267
+ <li class="method"><a href="Forklift/Base/Pid.html#method-i-recall">#recall &mdash; Forklift::Base::Pid</a>
268
+
269
+ <li class="method"><a href="Object.html#method-i-run_plan">#run_plan &mdash; Object</a>
270
+
271
+ <li class="method"><a href="Forklift/Base/Pid.html#method-i-safe_to_run-3F">#safe_to_run? &mdash; Forklift::Base::Pid</a>
272
+
273
+ <li class="method"><a href="Forklift/Base/Mailer.html#method-i-send">#send &mdash; Forklift::Base::Mailer</a>
274
+
275
+ <li class="method"><a href="Forklift/Base/Mailer.html#method-i-send_template">#send_template &mdash; Forklift::Base::Mailer</a>
276
+
277
+ <li class="method"><a href="Forklift/Connection/Mysql.html#method-i-sql_type">#sql_type &mdash; Forklift::Connection::Mysql</a>
278
+
279
+ <li class="method"><a href="Forklift/Plan.html#method-i-step">#step &mdash; Forklift::Plan</a>
280
+
281
+ <li class="method"><a href="Forklift/Plan.html#method-i-steps">#steps &mdash; Forklift::Plan</a>
282
+
283
+ <li class="method"><a href="Forklift/Base/Pid.html#method-i-store-21">#store! &mdash; Forklift::Base::Pid</a>
284
+
285
+ <li class="method"><a href="Forklift/Connection/Mysql.html#method-i-tables">#tables &mdash; Forklift::Connection::Mysql</a>
286
+
287
+ <li class="method"><a href="Object.html#method-i-template">#template &mdash; Object</a>
288
+
289
+ <li class="method"><a href="Forklift/Connection/Mysql.html#method-i-truncate">#truncate &mdash; Forklift::Connection::Mysql</a>
290
+
291
+ <li class="method"><a href="Forklift/Connection/Mysql.html#method-i-truncate-21">#truncate! &mdash; Forklift::Connection::Mysql</a>
292
+
293
+ <li class="method"><a href="Forklift/Plan.html#method-i-utils">#utils &mdash; Forklift::Plan</a>
294
+
295
+ <li class="method"><a href="Forklift/Connection/Elasticsearch.html#method-i-write">#write &mdash; Forklift::Connection::Elasticsearch</a>
296
+
297
+ <li class="method"><a href="Forklift/Base/Connection.html#method-i-write">#write &mdash; Forklift::Base::Connection</a>
298
+
299
+ <li class="method"><a href="Forklift/Connection/Mysql.html#method-i-write">#write &mdash; Forklift::Connection::Mysql</a>
300
+
301
+ </ul>
302
+
303
+
304
+ <footer id="validator-badges">
305
+ <p><a href="http://validator.w3.org/check/referer">[Validate]</a>
306
+ <p>Generated by <a href="https://github.com/rdoc/rdoc">RDoc</a> 3.12.2.
307
+ <p>Generated with the <a href="http://deveiate.org/projects/Darkfish-Rdoc/">Darkfish Rdoc Generator</a> 3.
308
+ </footer>
309
+
data/example/Gemfile ADDED
@@ -0,0 +1,3 @@
1
+ source 'https://rubygems.org'
2
+
3
+ gem 'forklift_etl', :path => '../'
data/example/Gemfile.lock ADDED
@@ -0,0 +1,55 @@
1
+ PATH
2
+ remote: ../
3
+ specs:
4
+ forklift (1.0.3)
5
+ activesupport
6
+ elasticsearch
7
+ lumberjack
8
+ mysql2
9
+ pony
10
+
11
+ GEM
12
+ remote: https://rubygems.org/
13
+ specs:
14
+ activesupport (4.0.3)
15
+ i18n (~> 0.6, >= 0.6.4)
16
+ minitest (~> 4.2)
17
+ multi_json (~> 1.3)
18
+ thread_safe (~> 0.1)
19
+ tzinfo (~> 0.3.37)
20
+ atomic (1.1.15)
21
+ elasticsearch (1.0.1)
22
+ elasticsearch-api (= 1.0.1)
23
+ elasticsearch-transport (= 1.0.1)
24
+ elasticsearch-api (1.0.1)
25
+ multi_json
26
+ elasticsearch-transport (1.0.1)
27
+ faraday
28
+ multi_json
29
+ faraday (0.9.0)
30
+ multipart-post (>= 1.2, < 3)
31
+ i18n (0.6.9)
32
+ lumberjack (1.0.5)
33
+ mail (2.5.4)
34
+ mime-types (~> 1.16)
35
+ treetop (~> 1.4.8)
36
+ mime-types (1.25.1)
37
+ minitest (4.7.5)
38
+ multi_json (1.9.2)
39
+ multipart-post (2.0.0)
40
+ mysql2 (0.3.15)
41
+ polyglot (0.3.4)
42
+ pony (1.8)
43
+ mail (>= 2.0)
44
+ thread_safe (0.2.0)
45
+ atomic (>= 1.1.7, < 2)
46
+ treetop (1.4.15)
47
+ polyglot
48
+ polyglot (>= 0.3.1)
49
+ tzinfo (0.3.39)
50
+
51
+ PLATFORMS
52
+ ruby
53
+
54
+ DEPENDENCIES
55
+ forklift!
data/example/config/connections/elasticsearch/source.yml ADDED
@@ -0,0 +1 @@
1
+ :host: http://localhost:9200
data/example/config/connections/mysql/destination.yml ADDED
@@ -0,0 +1,6 @@
1
+ :encoding: utf8
2
+ :database: destination
3
+ :username: root
4
+ :password:
5
+ :host: 127.0.0.1
6
+ :port: 3306
data/example/config/connections/mysql/source.yml ADDED
@@ -0,0 +1,6 @@
1
+ :encoding: utf8
2
+ :database: source
3
+ :username: root
4
+ :password:
5
+ :host: 127.0.0.1
6
+ :port: 3306
data/example/config/email.yml ADDED
@@ -0,0 +1,18 @@
1
+ # Configuration is passed to Pony (https://github.com/benprew/pony)
2
+
3
+ # ==> SMTP
4
+ # If testing locally, mailcatcher (https://github.com/sj26/mailcatcher) is a helpful gem
5
+ via: smtp
6
+ via_options:
7
+ address: localhost
8
+ port: 1025
9
+ # user_name: user
10
+ # password: password
11
+ # authentication: :plain # :plain, :login, :cram_md5, no auth by default
12
+ # domain: "localhost.localdomain" # the HELO domain provided by the client to the server
13
+
14
+ # ==> Sendmail
15
+ # via: sendmail
16
+ # via_options:
17
+ # location: /usr/sbin/sendmail
18
+ # arguments: '-t -i'
data/example/plan.rb ADDED
@@ -0,0 +1,87 @@
1
+ # plan = Forklift::Plan.new
2
+ # Or, you can pass configs
3
+ plan = Forklift::Plan.new ({
4
+ # :logger => {:debug => true}
5
+ })
6
+
7
+ plan.do! {
8
+ # do! is a wrapper around common setup methods (pidfile locking, setting up the logger, etc)
9
+ # you don't need to use do! if you want finer control
10
+
11
+ # cleanup from a previous run
12
+ plan.step('Cleanup'){
13
+ destination = plan.connections[:mysql][:destination]
14
+ destination.exec("./transformations/cleanup.sql");
15
+ end
16
+
17
+ # mySQL -> mySQL
18
+ plan.step('Mysql Import'){
19
+ source = plan.connections[:mysql][:source]
20
+ destination = plan.connections[:mysql][:destination]
21
+ source.tables.each do |table|
22
+ Forklift::Patterns::Mysql.optimistic_pipe(source, table, destination, table)
23
+ # will attempt to do an incremental pipe, will fall back to a full table copy
24
+ # by default, incremental updates happen off of the `created_at` column, but you can modify this with "matcher"
25
+ end
26
+ }
27
+
28
+ # Elasticsearch -> mySQL
29
+ plan.step('Elasticsearch Import'){
30
+ source = plan.connections[:elasticsearch][:source]
31
+ destination = plan.connections[:mysql][:destination]
32
+ table = 'es_import'
33
+ index = 'aaa'
34
+ query = { :query => { :match_all => {} } } # pagination will happen automatically
35
+ destination.truncate!(table) if destination.tables.include? table
36
+ source.read(index, query) {|data| destination.write(data, table) }
37
+ }
38
+
39
+ # mySQL -> Elasticsearch
40
+ plan.step('Elasticsearch Load'){
41
+ source = plan.connections[:mysql][:source]
42
+ destination = plan.connections[:elasticsearch][:source]
43
+ table = 'users'
44
+ index = 'users'
45
+ query = "select * from users" # pagination will happen automatically
46
+ source.read(query) {|data| destination.write(data, table, true, 'user') }
47
+ }
48
+
49
+ # ... and you can write your own connections [LINK GOES HERE]
50
+
51
+ # Do some SQL transformations
52
+ plan.step('Transformations'){
53
+ # SQL transformations are done exactly as they are written
54
+ destination = plan.connections[:mysql][:destination]
55
+ destination.exec!("./transformations/combined_name.sql")
56
+
57
+ # Do some Ruby transformations
58
+ # Ruby transformations expect `do!(connection, forklift)` to be defined
59
+ destination = plan.connections[:mysql][:destination]
60
+ destination.exec!("./transformations/email_suffix.rb")
61
+ }
62
+
63
+ # mySQL Dump the destination
64
+ plan.step('Mysql Dump'){
65
+ destination = plan.connections[:mysql][:destination]
66
+ destination.dump('/tmp/destination.sql.gz')
67
+ }
68
+
69
+ # email the logs and a summary
70
+ plan.step('Email'){
71
+ destination = plan.connections[:mysql][:destination]
72
+
73
+ email_args = {
74
+ :to => "YOU@FAKE.com",
75
+ :from => "Forklift",
76
+ :subject => "Forklift has moved your database @ #{Time.new}",
77
+ }
78
+
79
+ email_variables = {
80
+ :total_users_count => destination.read('select count(1) as "count" from users')[0][:count],
81
+ :new_users_count => destination.read('select count(1) as "count" from users where date(created_at) = date(NOW())')[0][:count],
82
+ }
83
+
84
+ email_template = "./template/email.erb"
85
+ plan.mailer.send_template(email_args, email_template, email_variables, plan.logger.messages) unless ENV['EMAIL'] == 'false'
86
+ }
87
+ }