forklift_etl 1.0.8 → 1.0.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +8 -8
- data/forklift.jpg +0 -0
- data/lib/forklift/transports/mysql.rb +3 -2
- data/lib/forklift/version.rb +1 -1
- metadata +1 -58
- data/doc/EmailSuffix.html +0 -228
- data/doc/Forklift/Base/Connection.html +0 -590
- data/doc/Forklift/Base/Logger.html +0 -453
- data/doc/Forklift/Base/Mailer/ERBBinding.html +0 -256
- data/doc/Forklift/Base/Mailer.html +0 -399
- data/doc/Forklift/Base/Pid.html +0 -489
- data/doc/Forklift/Base/Utils.html +0 -252
- data/doc/Forklift/Base.html +0 -167
- data/doc/Forklift/Connection/Elasticsearch.html +0 -419
- data/doc/Forklift/Connection/Mysql.html +0 -939
- data/doc/Forklift/Connection.html +0 -164
- data/doc/Forklift/Patterns/Elasticsearch.html +0 -169
- data/doc/Forklift/Patterns/Mysql.html +0 -402
- data/doc/Forklift/Patterns.html +0 -164
- data/doc/Forklift/Plan.html +0 -704
- data/doc/Forklift.html +0 -187
- data/doc/Gemfile.html +0 -132
- data/doc/Object.html +0 -326
- data/doc/Rakefile.html +0 -138
- data/doc/SpecClient.html +0 -291
- data/doc/SpecPlan.html +0 -253
- data/doc/SpecSeeds.html +0 -303
- data/doc/created.rid +0 -35
- data/doc/example/Gemfile.html +0 -129
- data/doc/images/add.png +0 -0
- data/doc/images/brick.png +0 -0
- data/doc/images/brick_link.png +0 -0
- data/doc/images/bug.png +0 -0
- data/doc/images/bullet_black.png +0 -0
- data/doc/images/bullet_toggle_minus.png +0 -0
- data/doc/images/bullet_toggle_plus.png +0 -0
- data/doc/images/date.png +0 -0
- data/doc/images/delete.png +0 -0
- data/doc/images/find.png +0 -0
- data/doc/images/loadingAnimation.gif +0 -0
- data/doc/images/macFFBgHack.png +0 -0
- data/doc/images/package.png +0 -0
- data/doc/images/page_green.png +0 -0
- data/doc/images/page_white_text.png +0 -0
- data/doc/images/page_white_width.png +0 -0
- data/doc/images/plugin.png +0 -0
- data/doc/images/ruby.png +0 -0
- data/doc/images/tag_blue.png +0 -0
- data/doc/images/tag_green.png +0 -0
- data/doc/images/transparent.png +0 -0
- data/doc/images/wrench.png +0 -0
- data/doc/images/wrench_orange.png +0 -0
- data/doc/images/zoom.png +0 -0
- data/doc/index.html +0 -122
- data/doc/js/darkfish.js +0 -155
- data/doc/js/jquery.js +0 -18
- data/doc/js/navigation.js +0 -142
- data/doc/js/search.js +0 -94
- data/doc/js/search_index.js +0 -1
- data/doc/js/searcher.js +0 -228
- data/doc/rdoc.css +0 -543
- data/doc/table_of_contents.html +0 -309
|
@@ -1,402 +0,0 @@
|
|
|
1
|
-
<!DOCTYPE html>
|
|
2
|
-
|
|
3
|
-
<html>
|
|
4
|
-
<head>
|
|
5
|
-
<meta content="text/html; charset=UTF-8" http-equiv="Content-Type">
|
|
6
|
-
|
|
7
|
-
<title>class Forklift::Patterns::Mysql - RDoc Documentation</title>
|
|
8
|
-
|
|
9
|
-
<link type="text/css" media="screen" href="../../rdoc.css" rel="stylesheet">
|
|
10
|
-
|
|
11
|
-
<script type="text/javascript">
|
|
12
|
-
var rdoc_rel_prefix = "../../";
|
|
13
|
-
</script>
|
|
14
|
-
|
|
15
|
-
<script type="text/javascript" charset="utf-8" src="../../js/jquery.js"></script>
|
|
16
|
-
<script type="text/javascript" charset="utf-8" src="../../js/navigation.js"></script>
|
|
17
|
-
<script type="text/javascript" charset="utf-8" src="../../js/search_index.js"></script>
|
|
18
|
-
<script type="text/javascript" charset="utf-8" src="../../js/search.js"></script>
|
|
19
|
-
<script type="text/javascript" charset="utf-8" src="../../js/searcher.js"></script>
|
|
20
|
-
<script type="text/javascript" charset="utf-8" src="../../js/darkfish.js"></script>
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
<body id="top" class="class">
|
|
24
|
-
<nav id="metadata">
|
|
25
|
-
<nav id="home-section" class="section">
|
|
26
|
-
<h3 class="section-header">
|
|
27
|
-
<a href="../../index.html">Home</a>
|
|
28
|
-
<a href="../../table_of_contents.html#classes">Classes</a>
|
|
29
|
-
<a href="../../table_of_contents.html#methods">Methods</a>
|
|
30
|
-
</h3>
|
|
31
|
-
</nav>
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
<nav id="search-section" class="section project-section" class="initially-hidden">
|
|
35
|
-
<form action="#" method="get" accept-charset="utf-8">
|
|
36
|
-
<h3 class="section-header">
|
|
37
|
-
<input type="text" name="search" placeholder="Search" id="search-field"
|
|
38
|
-
title="Type to search, Up and Down to navigate, Enter to load">
|
|
39
|
-
</h3>
|
|
40
|
-
</form>
|
|
41
|
-
|
|
42
|
-
<ul id="search-results" class="initially-hidden"></ul>
|
|
43
|
-
</nav>
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
<div id="file-metadata">
|
|
47
|
-
<nav id="file-list-section" class="section">
|
|
48
|
-
<h3 class="section-header">Defined In</h3>
|
|
49
|
-
<ul>
|
|
50
|
-
<li>lib/forklift/patterns/mysql_patterns.rb
|
|
51
|
-
</ul>
|
|
52
|
-
</nav>
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
</div>
|
|
56
|
-
|
|
57
|
-
<div id="class-metadata">
|
|
58
|
-
|
|
59
|
-
<nav id="parent-class-section" class="section">
|
|
60
|
-
<h3 class="section-header">Parent</h3>
|
|
61
|
-
|
|
62
|
-
<p class="link"><a href="../../Object.html">Object</a>
|
|
63
|
-
|
|
64
|
-
</nav>
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
<!-- Method Quickref -->
|
|
68
|
-
<nav id="method-list-section" class="section">
|
|
69
|
-
<h3 class="section-header">Methods</h3>
|
|
70
|
-
|
|
71
|
-
<ul class="link-list">
|
|
72
|
-
|
|
73
|
-
<li><a href="#method-c-can_incremental_pipe-3F">::can_incremental_pipe?</a>
|
|
74
|
-
|
|
75
|
-
<li><a href="#method-c-incremental_pipe">::incremental_pipe</a>
|
|
76
|
-
|
|
77
|
-
<li><a href="#method-c-mysql_optimistic_import">::mysql_optimistic_import</a>
|
|
78
|
-
|
|
79
|
-
<li><a href="#method-c-optimistic_pipe">::optimistic_pipe</a>
|
|
80
|
-
|
|
81
|
-
<li><a href="#method-c-pipe">::pipe</a>
|
|
82
|
-
|
|
83
|
-
</ul>
|
|
84
|
-
</nav>
|
|
85
|
-
|
|
86
|
-
</div>
|
|
87
|
-
|
|
88
|
-
<div id="project-metadata">
|
|
89
|
-
<nav id="fileindex-section" class="section project-section">
|
|
90
|
-
<h3 class="section-header">Pages</h3>
|
|
91
|
-
|
|
92
|
-
<ul>
|
|
93
|
-
|
|
94
|
-
<li class="file"><a href="../../Gemfile.html">Gemfile</a>
|
|
95
|
-
|
|
96
|
-
<li class="file"><a href="../../Rakefile.html">Rakefile</a>
|
|
97
|
-
|
|
98
|
-
<li class="file"><a href="../../example/Gemfile.html">Gemfile</a>
|
|
99
|
-
|
|
100
|
-
<li class="file"><a href="../../spec/pid/pidfile.html">pidfile</a>
|
|
101
|
-
|
|
102
|
-
</ul>
|
|
103
|
-
</nav>
|
|
104
|
-
|
|
105
|
-
<nav id="classindex-section" class="section project-section">
|
|
106
|
-
<h3 class="section-header">Class and Module Index</h3>
|
|
107
|
-
|
|
108
|
-
<ul class="link-list">
|
|
109
|
-
|
|
110
|
-
<li><a href="../../Forklift.html">Forklift</a>
|
|
111
|
-
|
|
112
|
-
<li><a href="../../Forklift/Base.html">Forklift::Base</a>
|
|
113
|
-
|
|
114
|
-
<li><a href="../../Forklift/Base/Connection.html">Forklift::Base::Connection</a>
|
|
115
|
-
|
|
116
|
-
<li><a href="../../Forklift/Base/Logger.html">Forklift::Base::Logger</a>
|
|
117
|
-
|
|
118
|
-
<li><a href="../../Forklift/Base/Mailer.html">Forklift::Base::Mailer</a>
|
|
119
|
-
|
|
120
|
-
<li><a href="../../Forklift/Base/Mailer/ERBBinding.html">Forklift::Base::Mailer::ERBBinding</a>
|
|
121
|
-
|
|
122
|
-
<li><a href="../../Forklift/Base/Pid.html">Forklift::Base::Pid</a>
|
|
123
|
-
|
|
124
|
-
<li><a href="../../Forklift/Base/Utils.html">Forklift::Base::Utils</a>
|
|
125
|
-
|
|
126
|
-
<li><a href="../../Forklift/Connection.html">Forklift::Connection</a>
|
|
127
|
-
|
|
128
|
-
<li><a href="../../Forklift/Connection/Elasticsearch.html">Forklift::Connection::Elasticsearch</a>
|
|
129
|
-
|
|
130
|
-
<li><a href="../../Forklift/Connection/Mysql.html">Forklift::Connection::Mysql</a>
|
|
131
|
-
|
|
132
|
-
<li><a href="../../Forklift/Patterns.html">Forklift::Patterns</a>
|
|
133
|
-
|
|
134
|
-
<li><a href="../../Forklift/Patterns/Elasticsearch.html">Forklift::Patterns::Elasticsearch</a>
|
|
135
|
-
|
|
136
|
-
<li><a href="../../Forklift/Patterns/Mysql.html">Forklift::Patterns::Mysql</a>
|
|
137
|
-
|
|
138
|
-
<li><a href="../../Forklift/Plan.html">Forklift::Plan</a>
|
|
139
|
-
|
|
140
|
-
<li><a href="../../EmailSuffix.html">EmailSuffix</a>
|
|
141
|
-
|
|
142
|
-
<li><a href="../../Object.html">Object</a>
|
|
143
|
-
|
|
144
|
-
<li><a href="../../SpecClient.html">SpecClient</a>
|
|
145
|
-
|
|
146
|
-
<li><a href="../../SpecPlan.html">SpecPlan</a>
|
|
147
|
-
|
|
148
|
-
<li><a href="../../SpecSeeds.html">SpecSeeds</a>
|
|
149
|
-
|
|
150
|
-
</ul>
|
|
151
|
-
</nav>
|
|
152
|
-
|
|
153
|
-
</div>
|
|
154
|
-
</nav>
|
|
155
|
-
|
|
156
|
-
<div id="documentation">
|
|
157
|
-
<h1 class="class">class Forklift::Patterns::Mysql</h1>
|
|
158
|
-
|
|
159
|
-
<div id="description" class="description">
|
|
160
|
-
|
|
161
|
-
</div><!-- description -->
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
<section id="5Buntitled-5D" class="documentation-section">
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
<!-- Methods -->
|
|
176
|
-
|
|
177
|
-
<section id="public-class-5Buntitled-5D-method-details" class="method-section section">
|
|
178
|
-
<h3 class="section-header">Public Class Methods</h3>
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
<div id="method-c-can_incremental_pipe-3F" class="method-detail ">
|
|
182
|
-
|
|
183
|
-
<div class="method-heading">
|
|
184
|
-
<span class="method-name">can_incremental_pipe?</span><span
|
|
185
|
-
class="method-args">(conn, table, matcher=conn.default_matcher)</span>
|
|
186
|
-
<span class="method-click-advice">click to toggle source</span>
|
|
187
|
-
</div>
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
<div class="method-description">
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
<div class="method-source-code" id="can_incremental_pipe-3F-source">
|
|
197
|
-
<pre><span class="ruby-comment"># File lib/forklift/patterns/mysql_patterns.rb, line 66</span>
|
|
198
|
-
<span class="ruby-keyword">def</span> <span class="ruby-keyword">self</span>.<span class="ruby-identifier">can_incremental_pipe?</span>(<span class="ruby-identifier">conn</span>, <span class="ruby-identifier">table</span>, <span class="ruby-identifier">matcher</span>=<span class="ruby-identifier">conn</span>.<span class="ruby-identifier">default_matcher</span>)
|
|
199
|
-
<span class="ruby-identifier">conn</span>.<span class="ruby-identifier">columns</span>(<span class="ruby-identifier">table</span>, <span class="ruby-identifier">conn</span>.<span class="ruby-identifier">current_database</span>).<span class="ruby-identifier">include?</span>(<span class="ruby-identifier">matcher</span>)
|
|
200
|
-
<span class="ruby-keyword">end</span></pre>
|
|
201
|
-
</div><!-- can_incremental_pipe-3F-source -->
|
|
202
|
-
|
|
203
|
-
</div>
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
</div><!-- can_incremental_pipe-3F-method -->
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
<div id="method-c-incremental_pipe" class="method-detail ">
|
|
212
|
-
|
|
213
|
-
<div class="method-heading">
|
|
214
|
-
<span class="method-name">incremental_pipe</span><span
|
|
215
|
-
class="method-args">(source, from_table, destination, to_table, matcher=source.default_matcher, primary_key='id')</span>
|
|
216
|
-
<span class="method-click-advice">click to toggle source</span>
|
|
217
|
-
</div>
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
<div class="method-description">
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
<div class="method-source-code" id="incremental_pipe-source">
|
|
227
|
-
<pre><span class="ruby-comment"># File lib/forklift/patterns/mysql_patterns.rb, line 17</span>
|
|
228
|
-
<span class="ruby-keyword">def</span> <span class="ruby-keyword">self</span>.<span class="ruby-identifier">incremental_pipe</span>(<span class="ruby-identifier">source</span>, <span class="ruby-identifier">from_table</span>, <span class="ruby-identifier">destination</span>, <span class="ruby-identifier">to_table</span>, <span class="ruby-identifier">matcher</span>=<span class="ruby-identifier">source</span>.<span class="ruby-identifier">default_matcher</span>, <span class="ruby-identifier">primary_key</span>=<span class="ruby-string">'id'</span>)
|
|
229
|
-
<span class="ruby-identifier">start</span> = <span class="ruby-constant">Time</span>.<span class="ruby-identifier">new</span>.<span class="ruby-identifier">to_i</span>
|
|
230
|
-
<span class="ruby-identifier">from_db</span> = <span class="ruby-identifier">source</span>.<span class="ruby-identifier">current_database</span>
|
|
231
|
-
<span class="ruby-identifier">to_db</span> = <span class="ruby-identifier">destination</span>.<span class="ruby-identifier">current_database</span>
|
|
232
|
-
<span class="ruby-identifier">source</span>.<span class="ruby-identifier">forklift</span>.<span class="ruby-identifier">logger</span>.<span class="ruby-identifier">log</span>(<span class="ruby-node">"mysql incremental_pipe: `#{from_db}`.`#{from_table}` => `#{to_db}`.`#{to_table}`"</span>)
|
|
233
|
-
<span class="ruby-identifier">source</span>.<span class="ruby-identifier">q</span>(<span class="ruby-node">"create table if not exists `#{to_db}`.`#{to_table}` like `#{from_db}`.`#{from_table}`"</span>)
|
|
234
|
-
|
|
235
|
-
<span class="ruby-comment"># Count the number of rows in to_table</span>
|
|
236
|
-
<span class="ruby-identifier">original_count</span> = <span class="ruby-identifier">source</span>.<span class="ruby-identifier">count</span>(<span class="ruby-identifier">to_table</span>, <span class="ruby-identifier">to_db</span>)
|
|
237
|
-
|
|
238
|
-
<span class="ruby-comment"># Find the latest/max/newest timestamp from the final table</span>
|
|
239
|
-
<span class="ruby-comment"># in order to determine the last copied row.</span>
|
|
240
|
-
<span class="ruby-identifier">latest_timestamp</span> = <span class="ruby-identifier">source</span>.<span class="ruby-identifier">max_timestamp</span>(<span class="ruby-identifier">to_table</span>, <span class="ruby-identifier">matcher</span>, <span class="ruby-identifier">to_db</span>)
|
|
241
|
-
|
|
242
|
-
<span class="ruby-comment"># If to_table has existing rows, ensure none of them are "stale."</span>
|
|
243
|
-
<span class="ruby-comment"># A stale row in to_table means a previously copied row was</span>
|
|
244
|
-
<span class="ruby-comment"># updated in from_table, so let's delete it from the to_table</span>
|
|
245
|
-
<span class="ruby-comment"># so we can get a fresh copy of that row.</span>
|
|
246
|
-
<span class="ruby-keyword">if</span> <span class="ruby-identifier">original_count</span> <span class="ruby-operator">></span> <span class="ruby-value">0</span>
|
|
247
|
-
<span class="ruby-comment"># Get the ids of rows in from_table that are newer than the newest row in to_table.</span>
|
|
248
|
-
<span class="ruby-comment"># Some of these rows could either be a) stale or b) new.</span>
|
|
249
|
-
<span class="ruby-identifier">source</span>.<span class="ruby-identifier">read</span>(<span class="ruby-node">"select `#{primary_key}` from `#{from_db}`.`#{from_table}` where `#{matcher}` > \"#{latest_timestamp}\" order by `#{matcher}`"</span>) <span class="ruby-keyword">do</span> <span class="ruby-operator">|</span><span class="ruby-identifier">stale_rows</span><span class="ruby-operator">|</span>
|
|
250
|
-
<span class="ruby-keyword">if</span> <span class="ruby-identifier">stale_rows</span>.<span class="ruby-identifier">length</span> <span class="ruby-operator">></span> <span class="ruby-value">0</span>
|
|
251
|
-
<span class="ruby-comment"># Delete these ids from to_table.</span>
|
|
252
|
-
<span class="ruby-comment"># If the ids are stale, then they'll be deleted. If they're new, they won't exist, and nothing will happen.</span>
|
|
253
|
-
<span class="ruby-identifier">stale_ids</span> = <span class="ruby-identifier">stale_rows</span>.<span class="ruby-identifier">map</span> { <span class="ruby-operator">|</span><span class="ruby-identifier">row</span><span class="ruby-operator">|</span> <span class="ruby-identifier">row</span>[<span class="ruby-identifier">primary_key</span>.<span class="ruby-identifier">to_sym</span>] }.<span class="ruby-identifier">join</span>(<span class="ruby-string">','</span>)
|
|
254
|
-
<span class="ruby-identifier">source</span>.<span class="ruby-identifier">q</span>(<span class="ruby-node">"delete from `#{to_db}`.`#{to_table}` where `#{primary_key}` in (#{stale_ids})"</span>)
|
|
255
|
-
<span class="ruby-identifier">source</span>.<span class="ruby-identifier">forklift</span>.<span class="ruby-identifier">logger</span>.<span class="ruby-identifier">log</span>(<span class="ruby-node">" ^ deleted up to #{stale_rows.length} stale rows from `#{to_db}`.`#{to_table}`"</span>)
|
|
256
|
-
<span class="ruby-keyword">end</span>
|
|
257
|
-
<span class="ruby-keyword">end</span>
|
|
258
|
-
<span class="ruby-keyword">end</span>
|
|
259
|
-
|
|
260
|
-
<span class="ruby-comment"># Do the insert into to_table</span>
|
|
261
|
-
<span class="ruby-identifier">destination</span>.<span class="ruby-identifier">q</span>(<span class="ruby-node">"insert into `#{to_db}`.`#{to_table}` select * from `#{from_db}`.`#{from_table}` where `#{matcher}` > \"#{latest_timestamp}\" order by `#{matcher}`"</span>)
|
|
262
|
-
<span class="ruby-identifier">delta</span> = <span class="ruby-constant">Time</span>.<span class="ruby-identifier">new</span>.<span class="ruby-identifier">to_i</span> <span class="ruby-operator">-</span> <span class="ruby-identifier">start</span>
|
|
263
|
-
<span class="ruby-identifier">new_count</span> = <span class="ruby-identifier">destination</span>.<span class="ruby-identifier">count</span>(<span class="ruby-identifier">to_table</span>, <span class="ruby-identifier">to_db</span>) <span class="ruby-operator">-</span> <span class="ruby-identifier">original_count</span>
|
|
264
|
-
<span class="ruby-identifier">source</span>.<span class="ruby-identifier">forklift</span>.<span class="ruby-identifier">logger</span>.<span class="ruby-identifier">log</span>(<span class="ruby-node">" ^ created #{new_count} new rows in #{delta}s"</span>)
|
|
265
|
-
<span class="ruby-keyword">end</span></pre>
|
|
266
|
-
</div><!-- incremental_pipe-source -->
|
|
267
|
-
|
|
268
|
-
</div>
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
</div><!-- incremental_pipe-method -->
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
<div id="method-c-mysql_optimistic_import" class="method-detail ">
|
|
277
|
-
|
|
278
|
-
<div class="method-heading">
|
|
279
|
-
<span class="method-name">mysql_optimistic_import</span><span
|
|
280
|
-
class="method-args">(source, destination)</span>
|
|
281
|
-
<span class="method-click-advice">click to toggle source</span>
|
|
282
|
-
</div>
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
<div class="method-description">
|
|
286
|
-
|
|
287
|
-
<p>When you are copying data to and from mysql An implamentation of “pipe”
|
|
288
|
-
for remote databases</p>
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
<div class="method-source-code" id="mysql_optimistic_import-source">
|
|
293
|
-
<pre><span class="ruby-comment"># File lib/forklift/patterns/mysql_patterns.rb, line 72</span>
|
|
294
|
-
<span class="ruby-keyword">def</span> <span class="ruby-keyword">self</span>.<span class="ruby-identifier">mysql_optimistic_import</span>(<span class="ruby-identifier">source</span>, <span class="ruby-identifier">destination</span>)
|
|
295
|
-
<span class="ruby-comment">#TODO: allow passing in of matcher and primary_key</span>
|
|
296
|
-
<span class="ruby-identifier">source</span>.<span class="ruby-identifier">tables</span>.<span class="ruby-identifier">each</span> <span class="ruby-keyword">do</span> <span class="ruby-operator">|</span><span class="ruby-identifier">table</span><span class="ruby-operator">|</span>
|
|
297
|
-
<span class="ruby-keyword">if</span>( <span class="ruby-identifier">source</span>.<span class="ruby-identifier">columns</span>(<span class="ruby-identifier">table</span>).<span class="ruby-identifier">include?</span>(<span class="ruby-identifier">source</span>.<span class="ruby-identifier">default_matcher</span>) <span class="ruby-operator">&&</span> <span class="ruby-identifier">destination</span>.<span class="ruby-identifier">tables</span>.<span class="ruby-identifier">include?</span>(<span class="ruby-identifier">table</span>) )
|
|
298
|
-
<span class="ruby-identifier">since</span> = <span class="ruby-identifier">destination</span>.<span class="ruby-identifier">max_timestamp</span>(<span class="ruby-identifier">table</span>)
|
|
299
|
-
<span class="ruby-identifier">source</span>.<span class="ruby-identifier">read_since</span>(<span class="ruby-identifier">table</span>, <span class="ruby-identifier">since</span>){ <span class="ruby-operator">|</span><span class="ruby-identifier">data</span><span class="ruby-operator">|</span> <span class="ruby-identifier">destination</span>.<span class="ruby-identifier">write</span>(<span class="ruby-identifier">data</span>, <span class="ruby-identifier">table</span>) }
|
|
300
|
-
<span class="ruby-keyword">else</span>
|
|
301
|
-
<span class="ruby-identifier">destination</span>.<span class="ruby-identifier">truncate</span> <span class="ruby-identifier">table</span>
|
|
302
|
-
<span class="ruby-identifier">source</span>.<span class="ruby-identifier">read</span>(<span class="ruby-node">"select * from #{table}"</span>){ <span class="ruby-operator">|</span><span class="ruby-identifier">data</span><span class="ruby-operator">|</span> <span class="ruby-identifier">destination</span>.<span class="ruby-identifier">write</span>(<span class="ruby-identifier">data</span>, <span class="ruby-identifier">table</span>) }
|
|
303
|
-
<span class="ruby-keyword">end</span>
|
|
304
|
-
<span class="ruby-keyword">end</span>
|
|
305
|
-
<span class="ruby-keyword">end</span></pre>
|
|
306
|
-
</div><!-- mysql_optimistic_import-source -->
|
|
307
|
-
|
|
308
|
-
</div>
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
</div><!-- mysql_optimistic_import-method -->
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
<div id="method-c-optimistic_pipe" class="method-detail ">
|
|
317
|
-
|
|
318
|
-
<div class="method-heading">
|
|
319
|
-
<span class="method-name">optimistic_pipe</span><span
|
|
320
|
-
class="method-args">(source, from_table, destination, to_table, matcher=source.default_matcher, primary_key='id')</span>
|
|
321
|
-
<span class="method-click-advice">click to toggle source</span>
|
|
322
|
-
</div>
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
<div class="method-description">
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
<div class="method-source-code" id="optimistic_pipe-source">
|
|
332
|
-
<pre><span class="ruby-comment"># File lib/forklift/patterns/mysql_patterns.rb, line 56</span>
|
|
333
|
-
<span class="ruby-keyword">def</span> <span class="ruby-keyword">self</span>.<span class="ruby-identifier">optimistic_pipe</span>(<span class="ruby-identifier">source</span>, <span class="ruby-identifier">from_table</span>, <span class="ruby-identifier">destination</span>, <span class="ruby-identifier">to_table</span>, <span class="ruby-identifier">matcher</span>=<span class="ruby-identifier">source</span>.<span class="ruby-identifier">default_matcher</span>, <span class="ruby-identifier">primary_key</span>=<span class="ruby-string">'id'</span>)
|
|
334
|
-
<span class="ruby-identifier">from_db</span> = <span class="ruby-identifier">source</span>.<span class="ruby-identifier">current_database</span>
|
|
335
|
-
<span class="ruby-identifier">to_db</span> = <span class="ruby-identifier">destination</span>.<span class="ruby-identifier">current_database</span>
|
|
336
|
-
<span class="ruby-keyword">if</span> <span class="ruby-keyword">self</span>.<span class="ruby-identifier">can_incremental_pipe?</span>(<span class="ruby-identifier">from_db</span>, <span class="ruby-identifier">from_table</span>)
|
|
337
|
-
<span class="ruby-identifier">incremental_pipe</span>(<span class="ruby-identifier">from_db</span>, <span class="ruby-identifier">from_table</span>, <span class="ruby-identifier">to_db</span>, <span class="ruby-identifier">to_table</span>, <span class="ruby-identifier">matcher</span>, <span class="ruby-identifier">primary_key</span>)
|
|
338
|
-
<span class="ruby-keyword">else</span>
|
|
339
|
-
<span class="ruby-identifier">pipe</span>(<span class="ruby-identifier">from_db</span>, <span class="ruby-identifier">from_table</span>, <span class="ruby-identifier">to_db</span>, <span class="ruby-identifier">to_table</span>)
|
|
340
|
-
<span class="ruby-keyword">end</span>
|
|
341
|
-
<span class="ruby-keyword">end</span></pre>
|
|
342
|
-
</div><!-- optimistic_pipe-source -->
|
|
343
|
-
|
|
344
|
-
</div>
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
</div><!-- optimistic_pipe-method -->
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
<div id="method-c-pipe" class="method-detail ">
|
|
353
|
-
|
|
354
|
-
<div class="method-heading">
|
|
355
|
-
<span class="method-name">pipe</span><span
|
|
356
|
-
class="method-args">(source, from_table, destination, to_table)</span>
|
|
357
|
-
<span class="method-click-advice">click to toggle source</span>
|
|
358
|
-
</div>
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
<div class="method-description">
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
<div class="method-source-code" id="pipe-source">
|
|
368
|
-
<pre><span class="ruby-comment"># File lib/forklift/patterns/mysql_patterns.rb, line 5</span>
|
|
369
|
-
<span class="ruby-keyword">def</span> <span class="ruby-keyword">self</span>.<span class="ruby-identifier">pipe</span>(<span class="ruby-identifier">source</span>, <span class="ruby-identifier">from_table</span>, <span class="ruby-identifier">destination</span>, <span class="ruby-identifier">to_table</span>)
|
|
370
|
-
<span class="ruby-identifier">start</span> = <span class="ruby-constant">Time</span>.<span class="ruby-identifier">new</span>.<span class="ruby-identifier">to_i</span>
|
|
371
|
-
<span class="ruby-identifier">from_db</span> = <span class="ruby-identifier">source</span>.<span class="ruby-identifier">current_database</span>
|
|
372
|
-
<span class="ruby-identifier">to_db</span> = <span class="ruby-identifier">destination</span>.<span class="ruby-identifier">current_database</span>
|
|
373
|
-
<span class="ruby-identifier">source</span>.<span class="ruby-identifier">forklift</span>.<span class="ruby-identifier">logger</span>.<span class="ruby-identifier">log</span>(<span class="ruby-node">"mysql pipe: `#{from_db}`.`#{from_table}` => `#{to_db}`.`#{to_table}`"</span>)
|
|
374
|
-
<span class="ruby-identifier">source</span>.<span class="ruby-identifier">q</span>(<span class="ruby-node">"drop table if exists `#{to_db}`.`#{to_table}`"</span>)
|
|
375
|
-
<span class="ruby-identifier">source</span>.<span class="ruby-identifier">q</span>(<span class="ruby-node">"create table `#{to_db}`.`#{to_table}` like `#{from_db}`.`#{from_table}`"</span>)
|
|
376
|
-
<span class="ruby-identifier">source</span>.<span class="ruby-identifier">q</span>(<span class="ruby-node">"insert into `#{to_db}`.`#{to_table}` select * from `#{from_db}`.`#{from_table}`"</span>)
|
|
377
|
-
<span class="ruby-identifier">delta</span> = <span class="ruby-constant">Time</span>.<span class="ruby-identifier">new</span>.<span class="ruby-identifier">to_i</span> <span class="ruby-operator">-</span> <span class="ruby-identifier">start</span>
|
|
378
|
-
<span class="ruby-identifier">source</span>.<span class="ruby-identifier">forklift</span>.<span class="ruby-identifier">logger</span>.<span class="ruby-identifier">log</span>(<span class="ruby-node">" ^ moved #{destination.count(to_table, to_db)} rows in #{delta}s"</span>)
|
|
379
|
-
<span class="ruby-keyword">end</span></pre>
|
|
380
|
-
</div><!-- pipe-source -->
|
|
381
|
-
|
|
382
|
-
</div>
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
</div><!-- pipe-method -->
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
</section><!-- public-class-method-details -->
|
|
391
|
-
|
|
392
|
-
</section><!-- 5Buntitled-5D -->
|
|
393
|
-
|
|
394
|
-
</div><!-- documentation -->
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
<footer id="validator-badges">
|
|
398
|
-
<p><a href="http://validator.w3.org/check/referer">[Validate]</a>
|
|
399
|
-
<p>Generated by <a href="https://github.com/rdoc/rdoc">RDoc</a> 3.12.2.
|
|
400
|
-
<p>Generated with the <a href="http://deveiate.org/projects/Darkfish-Rdoc/">Darkfish Rdoc Generator</a> 3.
|
|
401
|
-
</footer>
|
|
402
|
-
|
data/doc/Forklift/Patterns.html
DELETED
|
@@ -1,164 +0,0 @@
|
|
|
1
|
-
<!DOCTYPE html>
|
|
2
|
-
|
|
3
|
-
<html>
|
|
4
|
-
<head>
|
|
5
|
-
<meta content="text/html; charset=UTF-8" http-equiv="Content-Type">
|
|
6
|
-
|
|
7
|
-
<title>module Forklift::Patterns - RDoc Documentation</title>
|
|
8
|
-
|
|
9
|
-
<link type="text/css" media="screen" href="../rdoc.css" rel="stylesheet">
|
|
10
|
-
|
|
11
|
-
<script type="text/javascript">
|
|
12
|
-
var rdoc_rel_prefix = "../";
|
|
13
|
-
</script>
|
|
14
|
-
|
|
15
|
-
<script type="text/javascript" charset="utf-8" src="../js/jquery.js"></script>
|
|
16
|
-
<script type="text/javascript" charset="utf-8" src="../js/navigation.js"></script>
|
|
17
|
-
<script type="text/javascript" charset="utf-8" src="../js/search_index.js"></script>
|
|
18
|
-
<script type="text/javascript" charset="utf-8" src="../js/search.js"></script>
|
|
19
|
-
<script type="text/javascript" charset="utf-8" src="../js/searcher.js"></script>
|
|
20
|
-
<script type="text/javascript" charset="utf-8" src="../js/darkfish.js"></script>
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
<body id="top" class="module">
|
|
24
|
-
<nav id="metadata">
|
|
25
|
-
<nav id="home-section" class="section">
|
|
26
|
-
<h3 class="section-header">
|
|
27
|
-
<a href="../index.html">Home</a>
|
|
28
|
-
<a href="../table_of_contents.html#classes">Classes</a>
|
|
29
|
-
<a href="../table_of_contents.html#methods">Methods</a>
|
|
30
|
-
</h3>
|
|
31
|
-
</nav>
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
<nav id="search-section" class="section project-section" class="initially-hidden">
|
|
35
|
-
<form action="#" method="get" accept-charset="utf-8">
|
|
36
|
-
<h3 class="section-header">
|
|
37
|
-
<input type="text" name="search" placeholder="Search" id="search-field"
|
|
38
|
-
title="Type to search, Up and Down to navigate, Enter to load">
|
|
39
|
-
</h3>
|
|
40
|
-
</form>
|
|
41
|
-
|
|
42
|
-
<ul id="search-results" class="initially-hidden"></ul>
|
|
43
|
-
</nav>
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
<div id="file-metadata">
|
|
47
|
-
<nav id="file-list-section" class="section">
|
|
48
|
-
<h3 class="section-header">Defined In</h3>
|
|
49
|
-
<ul>
|
|
50
|
-
<li>lib/forklift/patterns/elasticsearch_patterns.rb
|
|
51
|
-
<li>lib/forklift/patterns/mysql_patterns.rb
|
|
52
|
-
</ul>
|
|
53
|
-
</nav>
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
</div>
|
|
57
|
-
|
|
58
|
-
<div id="class-metadata">
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
</div>
|
|
64
|
-
|
|
65
|
-
<div id="project-metadata">
|
|
66
|
-
<nav id="fileindex-section" class="section project-section">
|
|
67
|
-
<h3 class="section-header">Pages</h3>
|
|
68
|
-
|
|
69
|
-
<ul>
|
|
70
|
-
|
|
71
|
-
<li class="file"><a href="../Gemfile.html">Gemfile</a>
|
|
72
|
-
|
|
73
|
-
<li class="file"><a href="../Rakefile.html">Rakefile</a>
|
|
74
|
-
|
|
75
|
-
<li class="file"><a href="../example/Gemfile.html">Gemfile</a>
|
|
76
|
-
|
|
77
|
-
<li class="file"><a href="../spec/pid/pidfile.html">pidfile</a>
|
|
78
|
-
|
|
79
|
-
</ul>
|
|
80
|
-
</nav>
|
|
81
|
-
|
|
82
|
-
<nav id="classindex-section" class="section project-section">
|
|
83
|
-
<h3 class="section-header">Class and Module Index</h3>
|
|
84
|
-
|
|
85
|
-
<ul class="link-list">
|
|
86
|
-
|
|
87
|
-
<li><a href="../Forklift.html">Forklift</a>
|
|
88
|
-
|
|
89
|
-
<li><a href="../Forklift/Base.html">Forklift::Base</a>
|
|
90
|
-
|
|
91
|
-
<li><a href="../Forklift/Base/Connection.html">Forklift::Base::Connection</a>
|
|
92
|
-
|
|
93
|
-
<li><a href="../Forklift/Base/Logger.html">Forklift::Base::Logger</a>
|
|
94
|
-
|
|
95
|
-
<li><a href="../Forklift/Base/Mailer.html">Forklift::Base::Mailer</a>
|
|
96
|
-
|
|
97
|
-
<li><a href="../Forklift/Base/Mailer/ERBBinding.html">Forklift::Base::Mailer::ERBBinding</a>
|
|
98
|
-
|
|
99
|
-
<li><a href="../Forklift/Base/Pid.html">Forklift::Base::Pid</a>
|
|
100
|
-
|
|
101
|
-
<li><a href="../Forklift/Base/Utils.html">Forklift::Base::Utils</a>
|
|
102
|
-
|
|
103
|
-
<li><a href="../Forklift/Connection.html">Forklift::Connection</a>
|
|
104
|
-
|
|
105
|
-
<li><a href="../Forklift/Connection/Elasticsearch.html">Forklift::Connection::Elasticsearch</a>
|
|
106
|
-
|
|
107
|
-
<li><a href="../Forklift/Connection/Mysql.html">Forklift::Connection::Mysql</a>
|
|
108
|
-
|
|
109
|
-
<li><a href="../Forklift/Patterns.html">Forklift::Patterns</a>
|
|
110
|
-
|
|
111
|
-
<li><a href="../Forklift/Patterns/Elasticsearch.html">Forklift::Patterns::Elasticsearch</a>
|
|
112
|
-
|
|
113
|
-
<li><a href="../Forklift/Patterns/Mysql.html">Forklift::Patterns::Mysql</a>
|
|
114
|
-
|
|
115
|
-
<li><a href="../Forklift/Plan.html">Forklift::Plan</a>
|
|
116
|
-
|
|
117
|
-
<li><a href="../EmailSuffix.html">EmailSuffix</a>
|
|
118
|
-
|
|
119
|
-
<li><a href="../Object.html">Object</a>
|
|
120
|
-
|
|
121
|
-
<li><a href="../SpecClient.html">SpecClient</a>
|
|
122
|
-
|
|
123
|
-
<li><a href="../SpecPlan.html">SpecPlan</a>
|
|
124
|
-
|
|
125
|
-
<li><a href="../SpecSeeds.html">SpecSeeds</a>
|
|
126
|
-
|
|
127
|
-
</ul>
|
|
128
|
-
</nav>
|
|
129
|
-
|
|
130
|
-
</div>
|
|
131
|
-
</nav>
|
|
132
|
-
|
|
133
|
-
<div id="documentation">
|
|
134
|
-
<h1 class="module">module Forklift::Patterns</h1>
|
|
135
|
-
|
|
136
|
-
<div id="description" class="description">
|
|
137
|
-
|
|
138
|
-
</div><!-- description -->
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
<section id="5Buntitled-5D" class="documentation-section">
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
<!-- Methods -->
|
|
153
|
-
|
|
154
|
-
</section><!-- 5Buntitled-5D -->
|
|
155
|
-
|
|
156
|
-
</div><!-- documentation -->
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
<footer id="validator-badges">
|
|
160
|
-
<p><a href="http://validator.w3.org/check/referer">[Validate]</a>
|
|
161
|
-
<p>Generated by <a href="https://github.com/rdoc/rdoc">RDoc</a> 3.12.2.
|
|
162
|
-
<p>Generated with the <a href="http://deveiate.org/projects/Darkfish-Rdoc/">Darkfish Rdoc Generator</a> 3.
|
|
163
|
-
</footer>
|
|
164
|
-
|