forklift_etl 1.1.12 → 1.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.ruby-version +1 -0
- data/Gemfile.lock +5 -5
- data/lib/forklift/base/connection.rb +8 -8
- data/lib/forklift/plan.rb +8 -0
- data/lib/forklift/transports/csv.rb +3 -0
- data/lib/forklift/transports/elasticsearch.rb +8 -1
- data/lib/forklift/transports/mysql.rb +8 -2
- data/lib/forklift/version.rb +1 -1
- data/readme.md +13 -2
- data/spec/integration/basic_spec.rb +2 -0
- data/spec/integration/elasticsearch_spec.rb +5 -0
- data/spec/integration/multi_transport_spec.rb +5 -0
- data/spec/integration/mysql_patterns_spec.rb +4 -0
- data/spec/integration/mysql_spec.rb +10 -0
- data/spec/integration/transformations_spec.rb +54 -0
- data/spec/template/spec_user_transformation.rb +8 -0
- data/spec/template/spec_user_transformation.sql +2 -0
- data/spec/unit/connection/mysql_spec.rb +7 -0
- data/spec/unit/misc/email_spec.rb +1 -0
- data/spec/unit/misc/error_spec.rb +2 -0
- data/spec/unit/misc/pid_spec.rb +2 -0
- data/spec/unit/misc/step_spec.rb +3 -0
- metadata +38 -43
- data/.rbenv-version +0 -1
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 72c15001e2a6f2164be9fbe2ef78da4d5e131d6c
|
4
|
+
data.tar.gz: 5dfd8551cee1c2a19af8ffe49c9a73bb9b1b5bdf
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: a0763e36c35340ee5e55e78fcef2aa54533d6e4aa18fa9ac16b64835b7fa58f32195482a78f76e4e8a2dc3e380e8513cdd48c364b9bd65b20180af1b55e2ca7c
|
7
|
+
data.tar.gz: ae1472bc5ce7a6ebf0be6b6e896bc22211818bc6db9e405d085c68ae570ecef2fa1eee757ff85163af9771b928cecd4aa69fce84a865c449ddfe28c35ce3b2df
|
data/.ruby-version
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
2.1.5
|
data/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
forklift_etl (1.
|
4
|
+
forklift_etl (1.2.0)
|
5
5
|
activesupport (~> 4.0, >= 4.0.0)
|
6
6
|
elasticsearch (~> 1.0, >= 1.0.0)
|
7
7
|
lumberjack (~> 1.0, >= 1.0.0)
|
@@ -18,7 +18,7 @@ GEM
|
|
18
18
|
thread_safe (~> 0.3, >= 0.3.4)
|
19
19
|
tzinfo (~> 1.1)
|
20
20
|
addressable (2.3.6)
|
21
|
-
awesome_print (1.6.
|
21
|
+
awesome_print (1.6.1)
|
22
22
|
diff-lcs (1.2.5)
|
23
23
|
elasticsearch (1.0.6)
|
24
24
|
elasticsearch-api (= 1.0.6)
|
@@ -31,17 +31,17 @@ GEM
|
|
31
31
|
email_spec (1.6.0)
|
32
32
|
launchy (~> 2.1)
|
33
33
|
mail (~> 2.2)
|
34
|
-
faraday (0.9.
|
34
|
+
faraday (0.9.1)
|
35
35
|
multipart-post (>= 1.2, < 3)
|
36
36
|
i18n (0.7.0)
|
37
|
-
json (1.8.
|
37
|
+
json (1.8.2)
|
38
38
|
launchy (2.4.3)
|
39
39
|
addressable (~> 2.3)
|
40
40
|
lumberjack (1.0.9)
|
41
41
|
mail (2.6.3)
|
42
42
|
mime-types (>= 1.16, < 3)
|
43
43
|
mime-types (2.4.3)
|
44
|
-
minitest (5.5.
|
44
|
+
minitest (5.5.1)
|
45
45
|
multi_json (1.10.1)
|
46
46
|
multipart-post (2.0.0)
|
47
47
|
mysql2 (0.3.17)
|
@@ -38,32 +38,32 @@ module Forklift
|
|
38
38
|
raise 'not implemented'
|
39
39
|
end
|
40
40
|
|
41
|
-
def exec(path)
|
41
|
+
def exec(path, *args)
|
42
42
|
begin
|
43
|
-
exec!(path)
|
43
|
+
exec!(path, &args)
|
44
44
|
rescue Exception => e
|
45
45
|
forklift.logger.log(e)
|
46
46
|
end
|
47
47
|
end
|
48
48
|
|
49
|
-
def exec!(path)
|
49
|
+
def exec!(path, *args)
|
50
50
|
forklift.logger.log "Running script: #{path}"
|
51
51
|
extension = path.split(".").last
|
52
52
|
if(extension == "rb" || extension == "ruby")
|
53
|
-
exec_ruby(path)
|
53
|
+
exec_ruby(path, *args)
|
54
54
|
else
|
55
|
-
exec_script(path)
|
55
|
+
exec_script(path, *args)
|
56
56
|
end
|
57
57
|
end
|
58
58
|
|
59
|
-
def exec_ruby(path)
|
59
|
+
def exec_ruby(path, *args)
|
60
60
|
klass = forklift.utils.class_name_from_file(path)
|
61
61
|
require path
|
62
62
|
model = eval("#{klass}.new")
|
63
|
-
model.do!(self, forklift)
|
63
|
+
model.do!(self, forklift, *args)
|
64
64
|
end
|
65
65
|
|
66
|
-
def exec_script(path)
|
66
|
+
def exec_script(path, *args)
|
67
67
|
raise 'not implemented'
|
68
68
|
end
|
69
69
|
|
data/lib/forklift/plan.rb
CHANGED
@@ -33,6 +33,7 @@ module Forklift
|
|
33
33
|
begin
|
34
34
|
loader = "Forklift::Connection::#{type.camelcase}.new(db_config, self)"
|
35
35
|
connection = eval(loader)
|
36
|
+
connection.connect
|
36
37
|
connections[type.to_sym][name.to_sym] = connection
|
37
38
|
logger.debug "loaded a #{type.camelcase} connection from #{f}"
|
38
39
|
rescue Exception => e
|
@@ -42,6 +43,13 @@ module Forklift
|
|
42
43
|
end
|
43
44
|
end
|
44
45
|
|
46
|
+
def disconnect!
|
47
|
+
connections.each do |k, collection|
|
48
|
+
collection.each do |k, connection|
|
49
|
+
connection.disconnect
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
45
53
|
|
46
54
|
def default_error_handler
|
47
55
|
return lambda {|name, e| raise e }
|
@@ -7,7 +7,6 @@ module Forklift
|
|
7
7
|
def initialize(config, forklift)
|
8
8
|
@config = config
|
9
9
|
@forklift = forklift
|
10
|
-
@client = ::Elasticsearch::Client.new(config)
|
11
10
|
end
|
12
11
|
|
13
12
|
def config
|
@@ -18,6 +17,14 @@ module Forklift
|
|
18
17
|
@forklift
|
19
18
|
end
|
20
19
|
|
20
|
+
def connect
|
21
|
+
@client = ::Elasticsearch::Client.new(config)
|
22
|
+
end
|
23
|
+
|
24
|
+
def disconnect
|
25
|
+
@client = nil
|
26
|
+
end
|
27
|
+
|
21
28
|
def read(index, query, looping=true, from=0, size=forklift.config[:batch_size])
|
22
29
|
offset = 0
|
23
30
|
loop_count = 0
|
@@ -8,8 +8,14 @@ module Forklift
|
|
8
8
|
def initialize(config, forklift)
|
9
9
|
@config = config
|
10
10
|
@forklift = forklift
|
11
|
-
|
12
|
-
|
11
|
+
end
|
12
|
+
|
13
|
+
def connect
|
14
|
+
@client = Mysql2::Client.new(@config)
|
15
|
+
end
|
16
|
+
|
17
|
+
def disconnect
|
18
|
+
@client.close
|
13
19
|
end
|
14
20
|
|
15
21
|
def config
|
data/lib/forklift/version.rb
CHANGED
data/readme.md
CHANGED
@@ -413,20 +413,31 @@ Forklift allows you to create both Ruby transformations and script transformatio
|
|
413
413
|
- It is up to the transport to define `exec_script`, and not all transports will support it. Mysql can run `.sql` files, but there is not an equivalent for elasticsearch.
|
414
414
|
- `.exec` runs and logs exceptions, while `.exec!` will raise on an error. For example, `destination.exec("./transformations/cleanup.rb")` will run cleanup.rb on the destination database.
|
415
415
|
- Script files are run as-is, but ruby transformations must define a `do!` method in their class and are passed `def do!(connection, forklift)`
|
416
|
+
- args is optional, and can be passed in from your plan
|
416
417
|
|
417
418
|
```ruby
|
418
419
|
# Example transformation to count users
|
419
420
|
# count_users.rb
|
420
421
|
|
421
422
|
class CountUsers
|
422
|
-
def do!(connection, forklift)
|
423
|
+
def do!(connection, forklift, args)
|
423
424
|
forklift.logger.log "counting users"
|
424
425
|
count = connection.count('users')
|
425
|
-
forklift.logger.log "found #{count} users"
|
426
|
+
forklift.logger.log "[#{args.name}] found #{count} users"
|
426
427
|
end
|
427
428
|
end
|
428
429
|
```
|
429
430
|
|
431
|
+
```ruby
|
432
|
+
# in your plan.rb
|
433
|
+
plan = Forklift::Plan.new
|
434
|
+
plan.do! do
|
435
|
+
destination = plan.connections[:mysql][:destination]
|
436
|
+
destination.exec!("./transformations/combined_name.sql", {name: 'user counter'})
|
437
|
+
|
438
|
+
end
|
439
|
+
```
|
440
|
+
|
430
441
|
## Options & Notes
|
431
442
|
- Thanks to [@rahilsondhi](https://github.com/rahilsondhi) from [InternMatch](http://www.internmatch.com/) for all his help
|
432
443
|
- email_options is a hash consumed by the [Pony mail gem](https://github.com/benprew/pony)
|
@@ -10,6 +10,7 @@ describe 'basics' do
|
|
10
10
|
tables << row.values[0]
|
11
11
|
end
|
12
12
|
expect(tables.count).to eql 3
|
13
|
+
client.close
|
13
14
|
|
14
15
|
client = SpecClient.mysql('forklift_test_source_b')
|
15
16
|
tables = []
|
@@ -17,6 +18,7 @@ describe 'basics' do
|
|
17
18
|
tables << row.values[0]
|
18
19
|
end
|
19
20
|
expect(tables.count).to eql 1
|
21
|
+
client.close
|
20
22
|
end
|
21
23
|
|
22
24
|
it 'seeded the elasticsearch db' do
|
@@ -17,6 +17,7 @@ describe 'elasticsearch' do
|
|
17
17
|
@rows = (@rows + data)
|
18
18
|
}
|
19
19
|
}
|
20
|
+
plan.disconnect!
|
20
21
|
|
21
22
|
expect(@rows.length).to eql 5
|
22
23
|
end
|
@@ -32,6 +33,7 @@ describe 'elasticsearch' do
|
|
32
33
|
@rows = (@rows + data)
|
33
34
|
}
|
34
35
|
}
|
36
|
+
plan.disconnect!
|
35
37
|
|
36
38
|
expect(@rows.length).to eql 3
|
37
39
|
end
|
@@ -46,6 +48,7 @@ describe 'elasticsearch' do
|
|
46
48
|
destination = plan.connections[:elasticsearch][:forklift_test]
|
47
49
|
destination.write(data, index)
|
48
50
|
}
|
51
|
+
plan.disconnect!
|
49
52
|
|
50
53
|
destination = SpecClient.elasticsearch('forklift_test')
|
51
54
|
count = destination.count({ index: index })["count"]
|
@@ -63,6 +66,7 @@ describe 'elasticsearch' do
|
|
63
66
|
destination = plan.connections[:elasticsearch][:forklift_test]
|
64
67
|
destination.write(data, index, true)
|
65
68
|
}
|
69
|
+
plan.disconnect!
|
66
70
|
|
67
71
|
destination = SpecClient.elasticsearch('forklift_test')
|
68
72
|
count = destination.count({ index: index })["count"]
|
@@ -91,5 +95,6 @@ describe 'elasticsearch' do
|
|
91
95
|
destination.delete_index(index)
|
92
96
|
expect { client.search({ index: index }) }.to raise_error(/IndexMissingException/)
|
93
97
|
}
|
98
|
+
plan.disconnect!
|
94
99
|
end
|
95
100
|
end
|
@@ -18,6 +18,7 @@ describe 'multiple trasport types' do
|
|
18
18
|
destination = plan.connections[:mysql][:forklift_test_destination]
|
19
19
|
source.read(index, query) {|data| destination.write(data, table) }
|
20
20
|
}
|
21
|
+
plan.disconnect!
|
21
22
|
|
22
23
|
destination = SpecClient.mysql('forklift_test_destination')
|
23
24
|
rows = destination.query("select count(1) as 'count' from es_import").first["count"]
|
@@ -34,6 +35,7 @@ describe 'multiple trasport types' do
|
|
34
35
|
destination = plan.connections[:mysql][:forklift_test_destination]
|
35
36
|
source.read(index, query, false, 0, 3) {|data| destination.write(data, table) }
|
36
37
|
}
|
38
|
+
plan.disconnect!
|
37
39
|
|
38
40
|
destination = SpecClient.mysql('forklift_test_destination')
|
39
41
|
rows = destination.query("select count(1) as 'count' from es_import").first["count"]
|
@@ -61,6 +63,7 @@ describe 'multiple trasport types' do
|
|
61
63
|
destination.write(clean_data, table)
|
62
64
|
}
|
63
65
|
}
|
66
|
+
plan.disconnect!
|
64
67
|
|
65
68
|
destination = SpecClient.mysql('forklift_test_destination')
|
66
69
|
max = destination.query("select max(viewed_at) as 'max' from es_import").first["max"]
|
@@ -85,6 +88,7 @@ describe 'multiple trasport types' do
|
|
85
88
|
destination = plan.connections[:elasticsearch][:forklift_test]
|
86
89
|
source.read("select * from #{table}") {|data| destination.write(data, index) }
|
87
90
|
}
|
91
|
+
plan.disconnect!
|
88
92
|
|
89
93
|
destination = SpecClient.elasticsearch('forklift_test')
|
90
94
|
count = destination.count({ index: index })["count"]
|
@@ -102,6 +106,7 @@ describe 'multiple trasport types' do
|
|
102
106
|
destination.write(data, index)
|
103
107
|
}
|
104
108
|
}
|
109
|
+
plan.disconnect!
|
105
110
|
|
106
111
|
destination = SpecClient.elasticsearch('forklift_test')
|
107
112
|
count = destination.count({ index: index })["count"]
|
@@ -21,6 +21,7 @@ describe 'mysql patterns' do
|
|
21
21
|
|
22
22
|
expect(destination.tables.length).to eql 3
|
23
23
|
}
|
24
|
+
plan.disconnect!
|
24
25
|
end
|
25
26
|
|
26
27
|
it "can do an incramental data pipe with only updated data" do
|
@@ -42,6 +43,7 @@ describe 'mysql patterns' do
|
|
42
43
|
expect(destination.count('users')).to eql 5
|
43
44
|
expect(destination.read('select first_name from users where id = 1')[0][:first_name]).to eql 'EvanAgain'
|
44
45
|
}
|
46
|
+
plan.disconnect!
|
45
47
|
end
|
46
48
|
|
47
49
|
it "(optimistic_pipe) can determine if it should do an incramental or full pipe" do
|
@@ -52,6 +54,7 @@ describe 'mysql patterns' do
|
|
52
54
|
expect(Forklift::Patterns::Mysql.can_incremental_pipe?(source, 'sales', source, 'sales')).to eql false
|
53
55
|
expect(Forklift::Patterns::Mysql.can_incremental_pipe?(source, 'products', source, 'products')).to eql true
|
54
56
|
}
|
57
|
+
plan.disconnect!
|
55
58
|
end
|
56
59
|
|
57
60
|
it "can run the mysql_optimistic_import pattern" do
|
@@ -72,6 +75,7 @@ describe 'mysql patterns' do
|
|
72
75
|
expect(destination.count('users')).to eql 5
|
73
76
|
expect(destination.read('select first_name from users where id = 1')[0][:first_name]).to eql 'EvanAgain'
|
74
77
|
}
|
78
|
+
plan.disconnect!
|
75
79
|
end
|
76
80
|
|
77
81
|
it "can write the high_water_mark"
|
@@ -16,6 +16,7 @@ describe 'mysql' do
|
|
16
16
|
@rows = (@rows + data)
|
17
17
|
}
|
18
18
|
}
|
19
|
+
plan.disconnect!
|
19
20
|
|
20
21
|
expect(@rows.length).to eql 5
|
21
22
|
end
|
@@ -30,6 +31,7 @@ describe 'mysql' do
|
|
30
31
|
@rows = (@rows + data)
|
31
32
|
}
|
32
33
|
}
|
34
|
+
plan.disconnect!
|
33
35
|
|
34
36
|
expect(@rows.length).to eql 3
|
35
37
|
end
|
@@ -45,6 +47,7 @@ describe 'mysql' do
|
|
45
47
|
destination = plan.connections[:mysql][:forklift_test_source_a]
|
46
48
|
destination.write(data, table)
|
47
49
|
}
|
50
|
+
plan.disconnect!
|
48
51
|
|
49
52
|
destination = SpecClient.mysql('forklift_test_source_a')
|
50
53
|
count = destination.query('select count(1) as "count" from users').first['count']
|
@@ -61,6 +64,7 @@ describe 'mysql' do
|
|
61
64
|
destination = plan.connections[:mysql][:forklift_test_source_a]
|
62
65
|
destination.write(data, table)
|
63
66
|
}
|
67
|
+
plan.disconnect!
|
64
68
|
|
65
69
|
destination = SpecClient.mysql('forklift_test_source_a')
|
66
70
|
count = destination.query('select count(1) as "count" from users').first['count']
|
@@ -88,6 +92,7 @@ describe 'mysql' do
|
|
88
92
|
destination = plan.connections[:mysql][:forklift_test_source_a]
|
89
93
|
destination.write(data, table)
|
90
94
|
}
|
95
|
+
plan.disconnect!
|
91
96
|
|
92
97
|
destination = SpecClient.mysql('forklift_test_source_a')
|
93
98
|
cols = []
|
@@ -117,6 +122,7 @@ describe 'mysql' do
|
|
117
122
|
destination = plan.connections[:mysql][:forklift_test_source_a]
|
118
123
|
destination.write(data, table)
|
119
124
|
}
|
125
|
+
plan.disconnect!
|
120
126
|
|
121
127
|
destination = SpecClient.mysql('forklift_test_source_a')
|
122
128
|
cols = []
|
@@ -151,6 +157,7 @@ describe 'mysql' do
|
|
151
157
|
destination = plan.connections[:mysql][:forklift_test_source_a]
|
152
158
|
destination.write(data, table)
|
153
159
|
}
|
160
|
+
plan.disconnect!
|
154
161
|
|
155
162
|
count = raw.query("SHOW COLUMNS FROM #{table}").count
|
156
163
|
expect(count).to eql 7
|
@@ -168,6 +175,7 @@ describe 'mysql' do
|
|
168
175
|
destination = plan.connections[:mysql][:forklift_test_source_a]
|
169
176
|
destination.write(data, table)
|
170
177
|
}
|
178
|
+
plan.disconnect!
|
171
179
|
|
172
180
|
destination = SpecClient.mysql('forklift_test_source_a')
|
173
181
|
cols = []
|
@@ -200,6 +208,7 @@ describe 'mysql' do
|
|
200
208
|
destination = plan.connections[:mysql][:forklift_test_source_a]
|
201
209
|
destination.write(data, table)
|
202
210
|
}
|
211
|
+
plan.disconnect!
|
203
212
|
|
204
213
|
destination = SpecClient.mysql('forklift_test_source_a')
|
205
214
|
cols = []
|
@@ -226,6 +235,7 @@ describe 'mysql' do
|
|
226
235
|
destination = plan.connections[:mysql][:forklift_test_source_a]
|
227
236
|
destination.write(data, table)
|
228
237
|
}
|
238
|
+
plan.disconnect!
|
229
239
|
|
230
240
|
destination = SpecClient.mysql('forklift_test_source_a')
|
231
241
|
cols = []
|
@@ -0,0 +1,54 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe 'transformations' do
|
4
|
+
|
5
|
+
before(:each) do
|
6
|
+
SpecSeeds.setup_mysql
|
7
|
+
end
|
8
|
+
|
9
|
+
it "can run a native transformation" do
|
10
|
+
plan = SpecPlan.new
|
11
|
+
@rows = []
|
12
|
+
|
13
|
+
raw = SpecClient.mysql('forklift_test_destination')
|
14
|
+
|
15
|
+
plan.do! {
|
16
|
+
source = plan.connections[:mysql][:forklift_test_source_a]
|
17
|
+
destination = plan.connections[:mysql][:forklift_test_destination]
|
18
|
+
source.read('select * from `users`') {|data| destination.write(data, 'users') }
|
19
|
+
|
20
|
+
expect( destination.columns("users").include?("full_name") ).to eql false
|
21
|
+
|
22
|
+
transformation_file = "#{File.dirname(__FILE__)}/../template/spec_user_transformation.sql"
|
23
|
+
destination.exec!(transformation_file)
|
24
|
+
|
25
|
+
expect( destination.columns("users").include?("full_name") ).to eql true
|
26
|
+
}
|
27
|
+
plan.disconnect!
|
28
|
+
end
|
29
|
+
|
30
|
+
it "can run a ruby transformation" do
|
31
|
+
plan = SpecPlan.new
|
32
|
+
@rows = []
|
33
|
+
|
34
|
+
raw = SpecClient.mysql('forklift_test_destination')
|
35
|
+
|
36
|
+
plan.do! {
|
37
|
+
source = plan.connections[:mysql][:forklift_test_source_a]
|
38
|
+
destination = plan.connections[:mysql][:forklift_test_destination]
|
39
|
+
source.read('select * from `users`') {|data| destination.write(data, 'users') }
|
40
|
+
|
41
|
+
expect( destination.columns("users").include?("full_name") ).to eql false
|
42
|
+
|
43
|
+
transformation_file = "#{File.dirname(__FILE__)}/../template/spec_user_transformation.rb"
|
44
|
+
destination.exec!(transformation_file, {prefix: 'my_prefix' })
|
45
|
+
|
46
|
+
expect( destination.columns("users").include?("full_name") ).to eql true
|
47
|
+
|
48
|
+
data = destination.read('select * from `users` where email="evan@example.com"')
|
49
|
+
expect( data.first[:full_name] ).to eql 'my_prefix Evan T'
|
50
|
+
}
|
51
|
+
plan.disconnect!
|
52
|
+
end
|
53
|
+
|
54
|
+
end
|
@@ -0,0 +1,8 @@
|
|
1
|
+
class SpecUserTransformation
|
2
|
+
|
3
|
+
def do!(connection, forklift, args)
|
4
|
+
connection.q("ALTER TABLE `users` ADD `full_name` VARCHAR(255) NULL DEFAULT NULL AFTER `updated_at`;")
|
5
|
+
connection.q("UPDATE `users` SET full_name = CONCAT('#{args[:prefix]}', ' ', first_name, ' ', last_name);")
|
6
|
+
end
|
7
|
+
|
8
|
+
end
|
@@ -16,6 +16,7 @@ describe Forklift::Connection::Mysql do
|
|
16
16
|
expect(source.tables).to include 'products'
|
17
17
|
expect(source.tables).to include 'sales'
|
18
18
|
}
|
19
|
+
plan.disconnect!
|
19
20
|
end
|
20
21
|
|
21
22
|
it "can delte a table" do
|
@@ -27,6 +28,7 @@ describe Forklift::Connection::Mysql do
|
|
27
28
|
source.drop! table
|
28
29
|
expect(source.tables).to_not include 'users'
|
29
30
|
}
|
31
|
+
plan.disconnect!
|
30
32
|
end
|
31
33
|
|
32
34
|
it "can count the rows in a table" do
|
@@ -36,6 +38,7 @@ describe Forklift::Connection::Mysql do
|
|
36
38
|
source = plan.connections[:mysql][:forklift_test_source_a]
|
37
39
|
expect(source.count(table)).to eql 5
|
38
40
|
}
|
41
|
+
plan.disconnect!
|
39
42
|
end
|
40
43
|
|
41
44
|
it "can truncate a table (both with and without !)" do
|
@@ -48,6 +51,7 @@ describe Forklift::Connection::Mysql do
|
|
48
51
|
expect(source.count(table)).to eql 0
|
49
52
|
expect { source.truncate(table) }.to_not raise_error
|
50
53
|
}
|
54
|
+
plan.disconnect!
|
51
55
|
end
|
52
56
|
|
53
57
|
it 'trunacte! will raise if the table does not exist' do
|
@@ -57,6 +61,7 @@ describe Forklift::Connection::Mysql do
|
|
57
61
|
source = plan.connections[:mysql][:forklift_test_source_a]
|
58
62
|
expect { source.truncate!(table) }.to raise_error(/Table 'forklift_test_source_a.other_table' doesn't exist/)
|
59
63
|
}
|
64
|
+
plan.disconnect!
|
60
65
|
end
|
61
66
|
|
62
67
|
it "can get the columns of a table" do
|
@@ -69,6 +74,7 @@ describe Forklift::Connection::Mysql do
|
|
69
74
|
expect(source.columns(table)).to include 'product_id'
|
70
75
|
expect(source.columns(table)).to include 'timestamp'
|
71
76
|
}
|
77
|
+
plan.disconnect!
|
72
78
|
end
|
73
79
|
|
74
80
|
it "can create a mysqldump" do
|
@@ -78,6 +84,7 @@ describe Forklift::Connection::Mysql do
|
|
78
84
|
source = plan.connections[:mysql][:forklift_test_source_a]
|
79
85
|
source.dump(dump)
|
80
86
|
}
|
87
|
+
plan.disconnect!
|
81
88
|
|
82
89
|
expect(File.exists?(dump)).to eql true
|
83
90
|
contents = Zlib::GzipReader.new(StringIO.new(File.read(dump))).read
|
@@ -21,6 +21,7 @@ describe 'misc forklift core' do
|
|
21
21
|
email_template = "#{File.dirname(__FILE__)}/../../template/spec_email_template.erb"
|
22
22
|
@email = plan.mailer.send_template(email_args, email_template, email_variables).first
|
23
23
|
}
|
24
|
+
plan.disconnect!
|
24
25
|
|
25
26
|
expect(@email).to deliver_to("YOU@FAKE.com")
|
26
27
|
expect(@email).to have_subject(/Forklift has moved your database/)
|
@@ -11,6 +11,7 @@ describe 'misc forklift core' do
|
|
11
11
|
}
|
12
12
|
}.to raise_error 'BREAK'
|
13
13
|
plan.pid.delete!
|
14
|
+
plan.disconnect!
|
14
15
|
end
|
15
16
|
|
16
17
|
it 'can make error handlers' do
|
@@ -24,6 +25,7 @@ describe 'misc forklift core' do
|
|
24
25
|
plan.do! {
|
25
26
|
plan.step("step_a", error_handler){ raise 'BREAK' }
|
26
27
|
}
|
28
|
+
plan.disconnect!
|
27
29
|
|
28
30
|
expect(name).to eql :step_a
|
29
31
|
expect(ex.to_s).to eql 'BREAK'
|
data/spec/unit/misc/pid_spec.rb
CHANGED
@@ -11,6 +11,7 @@ describe 'misc forklift core' do
|
|
11
11
|
expect(File.exists?(pid)).to eql true
|
12
12
|
expect(File.read(pid).to_i).to eql Process.pid
|
13
13
|
}
|
14
|
+
plan.disconnect!
|
14
15
|
expect(File.exists?(pid)).to eql false
|
15
16
|
end
|
16
17
|
|
@@ -19,6 +20,7 @@ describe 'misc forklift core' do
|
|
19
20
|
plan.pid.store!
|
20
21
|
expect { plan.do! }.to raise_error SystemExit
|
21
22
|
plan.pid.delete!
|
23
|
+
plan.disconnect!
|
22
24
|
end
|
23
25
|
end
|
24
26
|
|
data/spec/unit/misc/step_spec.rb
CHANGED
@@ -20,6 +20,7 @@ describe 'misc forklift core' do
|
|
20
20
|
plan.step("b"){ steps_run << 'b' }
|
21
21
|
plan.step("c"){ steps_run << 'c' }
|
22
22
|
}
|
23
|
+
plan.disconnect!
|
23
24
|
expect(steps_run).to include 'a'
|
24
25
|
expect(steps_run).to include 'b'
|
25
26
|
expect(steps_run).to include 'c'
|
@@ -34,6 +35,7 @@ describe 'misc forklift core' do
|
|
34
35
|
plan.step("b"){ steps_run << 'b' }
|
35
36
|
plan.step("c"){ steps_run << 'c' }
|
36
37
|
}
|
38
|
+
plan.disconnect!
|
37
39
|
expect(steps_run).to include 'a'
|
38
40
|
expect(steps_run).to_not include 'b'
|
39
41
|
expect(steps_run).to include 'c'
|
@@ -46,6 +48,7 @@ describe 'misc forklift core' do
|
|
46
48
|
plan.do! {
|
47
49
|
plan.step("a"){ raise 'never should get here' }
|
48
50
|
}
|
51
|
+
plan.disconnect!
|
49
52
|
}.to raise_error SystemExit
|
50
53
|
end
|
51
54
|
end
|
metadata
CHANGED
@@ -1,124 +1,113 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: forklift_etl
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
5
|
-
prerelease:
|
4
|
+
version: 1.2.0
|
6
5
|
platform: ruby
|
7
6
|
authors:
|
8
7
|
- Evan Tahler
|
9
8
|
autorequire:
|
10
9
|
bindir: bin
|
11
10
|
cert_chain: []
|
12
|
-
date:
|
11
|
+
date: 2015-01-16 00:00:00.000000000 Z
|
13
12
|
dependencies:
|
14
13
|
- !ruby/object:Gem::Dependency
|
15
14
|
name: activesupport
|
16
15
|
requirement: !ruby/object:Gem::Requirement
|
17
|
-
none: false
|
18
16
|
requirements:
|
19
|
-
- - ~>
|
17
|
+
- - "~>"
|
20
18
|
- !ruby/object:Gem::Version
|
21
19
|
version: '4.0'
|
22
|
-
- -
|
20
|
+
- - ">="
|
23
21
|
- !ruby/object:Gem::Version
|
24
22
|
version: 4.0.0
|
25
23
|
type: :runtime
|
26
24
|
prerelease: false
|
27
25
|
version_requirements: !ruby/object:Gem::Requirement
|
28
|
-
none: false
|
29
26
|
requirements:
|
30
|
-
- - ~>
|
27
|
+
- - "~>"
|
31
28
|
- !ruby/object:Gem::Version
|
32
29
|
version: '4.0'
|
33
|
-
- -
|
30
|
+
- - ">="
|
34
31
|
- !ruby/object:Gem::Version
|
35
32
|
version: 4.0.0
|
36
33
|
- !ruby/object:Gem::Dependency
|
37
34
|
name: mysql2
|
38
35
|
requirement: !ruby/object:Gem::Requirement
|
39
|
-
none: false
|
40
36
|
requirements:
|
41
|
-
- - ~>
|
37
|
+
- - "~>"
|
42
38
|
- !ruby/object:Gem::Version
|
43
39
|
version: '0.0'
|
44
|
-
- -
|
40
|
+
- - ">="
|
45
41
|
- !ruby/object:Gem::Version
|
46
42
|
version: 0.0.1
|
47
43
|
type: :runtime
|
48
44
|
prerelease: false
|
49
45
|
version_requirements: !ruby/object:Gem::Requirement
|
50
|
-
none: false
|
51
46
|
requirements:
|
52
|
-
- - ~>
|
47
|
+
- - "~>"
|
53
48
|
- !ruby/object:Gem::Version
|
54
49
|
version: '0.0'
|
55
|
-
- -
|
50
|
+
- - ">="
|
56
51
|
- !ruby/object:Gem::Version
|
57
52
|
version: 0.0.1
|
58
53
|
- !ruby/object:Gem::Dependency
|
59
54
|
name: elasticsearch
|
60
55
|
requirement: !ruby/object:Gem::Requirement
|
61
|
-
none: false
|
62
56
|
requirements:
|
63
|
-
- - ~>
|
57
|
+
- - "~>"
|
64
58
|
- !ruby/object:Gem::Version
|
65
59
|
version: '1.0'
|
66
|
-
- -
|
60
|
+
- - ">="
|
67
61
|
- !ruby/object:Gem::Version
|
68
62
|
version: 1.0.0
|
69
63
|
type: :runtime
|
70
64
|
prerelease: false
|
71
65
|
version_requirements: !ruby/object:Gem::Requirement
|
72
|
-
none: false
|
73
66
|
requirements:
|
74
|
-
- - ~>
|
67
|
+
- - "~>"
|
75
68
|
- !ruby/object:Gem::Version
|
76
69
|
version: '1.0'
|
77
|
-
- -
|
70
|
+
- - ">="
|
78
71
|
- !ruby/object:Gem::Version
|
79
72
|
version: 1.0.0
|
80
73
|
- !ruby/object:Gem::Dependency
|
81
74
|
name: pony
|
82
75
|
requirement: !ruby/object:Gem::Requirement
|
83
|
-
none: false
|
84
76
|
requirements:
|
85
|
-
- - ~>
|
77
|
+
- - "~>"
|
86
78
|
- !ruby/object:Gem::Version
|
87
79
|
version: '1.0'
|
88
|
-
- -
|
80
|
+
- - ">="
|
89
81
|
- !ruby/object:Gem::Version
|
90
82
|
version: 1.0.0
|
91
83
|
type: :runtime
|
92
84
|
prerelease: false
|
93
85
|
version_requirements: !ruby/object:Gem::Requirement
|
94
|
-
none: false
|
95
86
|
requirements:
|
96
|
-
- - ~>
|
87
|
+
- - "~>"
|
97
88
|
- !ruby/object:Gem::Version
|
98
89
|
version: '1.0'
|
99
|
-
- -
|
90
|
+
- - ">="
|
100
91
|
- !ruby/object:Gem::Version
|
101
92
|
version: 1.0.0
|
102
93
|
- !ruby/object:Gem::Dependency
|
103
94
|
name: lumberjack
|
104
95
|
requirement: !ruby/object:Gem::Requirement
|
105
|
-
none: false
|
106
96
|
requirements:
|
107
|
-
- - ~>
|
97
|
+
- - "~>"
|
108
98
|
- !ruby/object:Gem::Version
|
109
99
|
version: '1.0'
|
110
|
-
- -
|
100
|
+
- - ">="
|
111
101
|
- !ruby/object:Gem::Version
|
112
102
|
version: 1.0.0
|
113
103
|
type: :runtime
|
114
104
|
prerelease: false
|
115
105
|
version_requirements: !ruby/object:Gem::Requirement
|
116
|
-
none: false
|
117
106
|
requirements:
|
118
|
-
- - ~>
|
107
|
+
- - "~>"
|
119
108
|
- !ruby/object:Gem::Version
|
120
109
|
version: '1.0'
|
121
|
-
- -
|
110
|
+
- - ">="
|
122
111
|
- !ruby/object:Gem::Version
|
123
112
|
version: 1.0.0
|
124
113
|
description: A collection of ETL tools and patterns for mysql and elasticsearch.
|
@@ -129,9 +118,9 @@ executables:
|
|
129
118
|
extensions: []
|
130
119
|
extra_rdoc_files: []
|
131
120
|
files:
|
132
|
-
- .gitignore
|
133
|
-
- .
|
134
|
-
- .travis.yml
|
121
|
+
- ".gitignore"
|
122
|
+
- ".ruby-version"
|
123
|
+
- ".travis.yml"
|
135
124
|
- Gemfile
|
136
125
|
- Gemfile.lock
|
137
126
|
- LICENSE.txt
|
@@ -180,6 +169,7 @@ files:
|
|
180
169
|
- spec/integration/multi_transport_spec.rb
|
181
170
|
- spec/integration/mysql_patterns_spec.rb
|
182
171
|
- spec/integration/mysql_spec.rb
|
172
|
+
- spec/integration/transformations_spec.rb
|
183
173
|
- spec/spec_helper.rb
|
184
174
|
- spec/support/dumps/csv/source.csv
|
185
175
|
- spec/support/dumps/elasticsearch/forklift_test.json
|
@@ -189,6 +179,8 @@ files:
|
|
189
179
|
- spec/support/spec_plan.rb
|
190
180
|
- spec/support/spec_seeds.rb
|
191
181
|
- spec/template/spec_email_template.erb
|
182
|
+
- spec/template/spec_user_transformation.rb
|
183
|
+
- spec/template/spec_user_transformation.sql
|
192
184
|
- spec/unit/connection/mysql_spec.rb
|
193
185
|
- spec/unit/misc/email_spec.rb
|
194
186
|
- spec/unit/misc/error_spec.rb
|
@@ -202,28 +194,27 @@ files:
|
|
202
194
|
homepage: https://github.com/taskrabbit/forklift
|
203
195
|
licenses:
|
204
196
|
- MIT
|
197
|
+
metadata: {}
|
205
198
|
post_install_message:
|
206
199
|
rdoc_options: []
|
207
200
|
require_paths:
|
208
201
|
- lib
|
209
202
|
required_ruby_version: !ruby/object:Gem::Requirement
|
210
|
-
none: false
|
211
203
|
requirements:
|
212
|
-
- -
|
204
|
+
- - ">="
|
213
205
|
- !ruby/object:Gem::Version
|
214
206
|
version: '0'
|
215
207
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
216
|
-
none: false
|
217
208
|
requirements:
|
218
|
-
- -
|
209
|
+
- - ">="
|
219
210
|
- !ruby/object:Gem::Version
|
220
211
|
version: '0'
|
221
212
|
requirements: []
|
222
213
|
rubyforge_project: forklift_etl
|
223
|
-
rubygems_version:
|
214
|
+
rubygems_version: 2.2.2
|
224
215
|
signing_key:
|
225
|
-
specification_version:
|
226
|
-
summary:
|
216
|
+
specification_version: 4
|
217
|
+
summary: 'Forklift: Moving big databases around. A ruby ETL tool.'
|
227
218
|
test_files:
|
228
219
|
- spec/config/connections/csv/forklift_test_destination.yml
|
229
220
|
- spec/config/connections/csv/forklift_test_source.yml
|
@@ -240,6 +231,7 @@ test_files:
|
|
240
231
|
- spec/integration/multi_transport_spec.rb
|
241
232
|
- spec/integration/mysql_patterns_spec.rb
|
242
233
|
- spec/integration/mysql_spec.rb
|
234
|
+
- spec/integration/transformations_spec.rb
|
243
235
|
- spec/spec_helper.rb
|
244
236
|
- spec/support/dumps/csv/source.csv
|
245
237
|
- spec/support/dumps/elasticsearch/forklift_test.json
|
@@ -249,8 +241,11 @@ test_files:
|
|
249
241
|
- spec/support/spec_plan.rb
|
250
242
|
- spec/support/spec_seeds.rb
|
251
243
|
- spec/template/spec_email_template.erb
|
244
|
+
- spec/template/spec_user_transformation.rb
|
245
|
+
- spec/template/spec_user_transformation.sql
|
252
246
|
- spec/unit/connection/mysql_spec.rb
|
253
247
|
- spec/unit/misc/email_spec.rb
|
254
248
|
- spec/unit/misc/error_spec.rb
|
255
249
|
- spec/unit/misc/pid_spec.rb
|
256
250
|
- spec/unit/misc/step_spec.rb
|
251
|
+
has_rdoc:
|
data/.rbenv-version
DELETED
@@ -1 +0,0 @@
|
|
1
|
-
1.9.3-p194
|