forklift_etl 1.2.1 → 1.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile.lock +3 -0
- data/lib/forklift/transports/elasticsearch.rb +1 -1
- data/lib/forklift/transports/mysql.rb +18 -14
- data/lib/forklift/version.rb +1 -1
- data/readme.md +8 -0
- data/spec/integration/elasticsearch_spec.rb +7 -7
- data/spec/integration/mysql_spec.rb +31 -31
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 1718155b7b286739b163fe0c386488035dad0d0d
|
4
|
+
data.tar.gz: 00e9c7b12f1f1c46da88c2ea6051ca3fd2c38b85
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a254bafa76f5bf176fcd4ec75233732961e35522afd12f04c4e994f125641bc7ed554137fca541761e630e334c119bb7d89ec3ef1f4a476e06a6540692db9970
|
7
|
+
data.tar.gz: 9b4d36cfa659ac78511b235c3ccd14dea7fe817de65f4c7c6507fc14350ec85bb21b6a1778c7a440c178145ab6bb0404e0d842b4f8cc78de6052626337a26f7c
|
data/Gemfile.lock
CHANGED
@@ -99,7 +99,7 @@ module Forklift
|
|
99
99
|
insert_q << safe_values(columns, d)
|
100
100
|
insert_q << ","
|
101
101
|
end
|
102
|
-
|
102
|
+
|
103
103
|
if delete_keys.length > 0
|
104
104
|
delete_q << "(#{delete_keys.join(',')})"
|
105
105
|
q(delete_q)
|
@@ -129,22 +129,26 @@ module Forklift
|
|
129
129
|
keys = {}
|
130
130
|
data.each do |item|
|
131
131
|
item.each do |k,v|
|
132
|
-
keys[k] = sql_type(v) if (
|
132
|
+
keys[k.to_s] = sql_type(v) if (keys[k.to_s].nil? || keys[k.to_s] == sql_type(nil))
|
133
133
|
end
|
134
134
|
end
|
135
|
+
keys[primary_key] = 'bigint(20)' unless keys.has_key?(primary_key)
|
135
136
|
|
136
|
-
|
137
|
-
|
137
|
+
col_defn = keys.map do |col, type|
|
138
|
+
if col == primary_key
|
139
|
+
"`#{col}` #{type} NOT NULL AUTO_INCREMENT"
|
140
|
+
else
|
141
|
+
"`#{col}` #{type} DEFAULT NULL"
|
142
|
+
end
|
138
143
|
end
|
144
|
+
col_defn << "PRIMARY KEY (`#{primary_key}`)"
|
145
|
+
col_defn << "KEY `#{matcher}` (`#{matcher}`)" if keys.include?(matcher.to_sym)
|
139
146
|
|
140
|
-
command =
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
command << " PRIMARY KEY (`#{primary_key}`) "
|
146
|
-
command << " , KEY `#{matcher}` (`#{matcher}`) " if keys.include?(matcher.to_sym)
|
147
|
-
command << " ) "
|
147
|
+
command = <<-EOS
|
148
|
+
CREATE TABLE `#{database}`.`#{table}` (
|
149
|
+
#{col_defn.join(', ')}
|
150
|
+
)
|
151
|
+
EOS
|
148
152
|
|
149
153
|
q(command)
|
150
154
|
forklift.logger.log "lazy-created table `#{database}`.`#{table}`"
|
@@ -238,7 +242,7 @@ module Forklift
|
|
238
242
|
cmd << " | gzip > #{file}"
|
239
243
|
forklift.logger.log "Dumping #{config['database']} to #{file}"
|
240
244
|
forklift.logger.debug cmd
|
241
|
-
|
245
|
+
|
242
246
|
stdin, stdout, stderr = Open3.popen3(cmd)
|
243
247
|
stdout = stdout.readlines
|
244
248
|
stderr = stderr.readlines
|
@@ -285,7 +289,7 @@ module Forklift
|
|
285
289
|
end
|
286
290
|
|
287
291
|
end
|
288
|
-
end
|
292
|
+
end
|
289
293
|
end
|
290
294
|
|
291
295
|
def safe_columns(cols)
|
data/lib/forklift/version.rb
CHANGED
data/readme.md
CHANGED
@@ -1,4 +1,12 @@
|
|
1
1
|
# Forklift ETL
|
2
|
+
|
3
|
+
---
|
4
|
+
|
5
|
+
## THIS TOOL IS NO LONGER SUPORTED.
|
6
|
+
It has been succeded by [Empujar](https://github.com/taskrabbit/empujar)
|
7
|
+
|
8
|
+
---
|
9
|
+
|
2
10
|
Moving heavy databases around. [](http://badge.fury.io/rb/forklift_etl)
|
3
11
|
[](http://travis-ci.org/taskrabbit/forklift)
|
4
12
|
|
@@ -29,7 +29,7 @@ describe 'elasticsearch' do
|
|
29
29
|
@rows = []
|
30
30
|
plan.do! {
|
31
31
|
source = plan.connections[:elasticsearch][:forklift_test]
|
32
|
-
source.read(index, query, false, 0, 3) {|data|
|
32
|
+
source.read(index, query, false, 0, 3) {|data|
|
33
33
|
@rows = (@rows + data)
|
34
34
|
}
|
35
35
|
}
|
@@ -56,7 +56,7 @@ describe 'elasticsearch' do
|
|
56
56
|
expect(count).to eql 6
|
57
57
|
end
|
58
58
|
|
59
|
-
it "can overwrite existing data, probided a primary key" do
|
59
|
+
it "can overwrite existing data, probided a primary key" do
|
60
60
|
index = 'forklift_test'
|
61
61
|
plan = SpecPlan.new
|
62
62
|
data = [
|
@@ -80,7 +80,7 @@ describe 'elasticsearch' do
|
|
80
80
|
expect(obj["viewed_at"]).to eql 99
|
81
81
|
end
|
82
82
|
|
83
|
-
it "can delete an index" do
|
83
|
+
it "can delete an index" do
|
84
84
|
index = 'other_test_index'
|
85
85
|
plan = SpecPlan.new
|
86
86
|
client = SpecClient.elasticsearch('forklift_test')
|
@@ -89,12 +89,12 @@ describe 'elasticsearch' do
|
|
89
89
|
]
|
90
90
|
plan.do! {
|
91
91
|
destination = plan.connections[:elasticsearch][:forklift_test]
|
92
|
-
expect { client.search({ index: index }) }.to raise_error(/IndexMissingException/)
|
92
|
+
expect { client.search({ index: index }) }.to raise_error(/index_not_found_exception|IndexMissingException/)
|
93
93
|
destination.write(data, index, true)
|
94
94
|
expect { client.search({ index: index }) }.to_not raise_error
|
95
95
|
destination.delete_index(index)
|
96
|
-
expect { client.search({ index: index }) }.to raise_error(/IndexMissingException/)
|
97
|
-
}
|
96
|
+
expect { client.search({ index: index }) }.to raise_error(/index_not_found_exception|IndexMissingException/)
|
97
|
+
}
|
98
98
|
plan.disconnect!
|
99
99
|
end
|
100
|
-
end
|
100
|
+
end
|
@@ -1,6 +1,6 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
|
3
|
-
describe 'mysql' do
|
3
|
+
describe 'mysql' do
|
4
4
|
|
5
5
|
before(:each) do
|
6
6
|
SpecSeeds.setup_mysql
|
@@ -12,7 +12,7 @@ describe 'mysql' do
|
|
12
12
|
@rows = []
|
13
13
|
plan.do! {
|
14
14
|
source = plan.connections[:mysql][:forklift_test_source_a]
|
15
|
-
source.read(query) {|data|
|
15
|
+
source.read(query) {|data|
|
16
16
|
@rows = (@rows + data)
|
17
17
|
}
|
18
18
|
}
|
@@ -20,14 +20,14 @@ describe 'mysql' do
|
|
20
20
|
|
21
21
|
expect(@rows.length).to eql 5
|
22
22
|
end
|
23
|
-
|
23
|
+
|
24
24
|
it "can read data (filtered)" do
|
25
25
|
query = 'select * from `users`'
|
26
26
|
plan = SpecPlan.new
|
27
27
|
@rows = []
|
28
28
|
plan.do! {
|
29
29
|
source = plan.connections[:mysql][:forklift_test_source_a]
|
30
|
-
source.read(query, source.current_database, false, 3, 0) {|data|
|
30
|
+
source.read(query, source.current_database, false, 3, 0) {|data|
|
31
31
|
@rows = (@rows + data)
|
32
32
|
}
|
33
33
|
}
|
@@ -54,7 +54,7 @@ describe 'mysql' do
|
|
54
54
|
expect(count).to eql 7
|
55
55
|
end
|
56
56
|
|
57
|
-
it "can update existing data" do
|
57
|
+
it "can update existing data" do
|
58
58
|
table = "users"
|
59
59
|
data = [
|
60
60
|
{id: 1, email: 'evan@example.com', first_name: 'New Name', last_name: 'T', created_at: Time.new.to_s(:db), updated_at: Time.new.to_s(:db)}
|
@@ -79,8 +79,8 @@ describe 'mysql' do
|
|
79
79
|
destination = SpecClient.mysql('forklift_test_source_a')
|
80
80
|
destination.query('drop table if exists `new_table`')
|
81
81
|
end
|
82
|
-
|
83
|
-
it "can lazy-create a table with primary keys provided" do
|
82
|
+
|
83
|
+
it "can lazy-create a table with primary keys provided" do
|
84
84
|
data = [
|
85
85
|
{id: 1, thing: 'stuff a', updated_at: Time.new},
|
86
86
|
{id: 2, thing: 'stuff b', updated_at: Time.new},
|
@@ -99,17 +99,17 @@ describe 'mysql' do
|
|
99
99
|
destination.query("describe #{table}").each do |row|
|
100
100
|
cols << row["Field"]
|
101
101
|
case row["Field"]
|
102
|
-
when "id"
|
102
|
+
when "id"
|
103
103
|
expect(row["Type"]).to eql "bigint(20)"
|
104
|
-
when "thing"
|
104
|
+
when "thing"
|
105
105
|
expect(row["Type"]).to eql "text"
|
106
|
-
when "updated_at"
|
106
|
+
when "updated_at"
|
107
107
|
expect(row["Type"]).to eql "datetime"
|
108
108
|
end
|
109
109
|
end
|
110
110
|
expect(cols).to eql ['id', 'thing', 'updated_at']
|
111
111
|
end
|
112
|
-
|
112
|
+
|
113
113
|
it "can lazy-create a table without primary keys provided" do
|
114
114
|
data = [
|
115
115
|
{thing: 'stuff a', number: 1.123, updated_at: Time.new},
|
@@ -129,17 +129,17 @@ describe 'mysql' do
|
|
129
129
|
destination.query("describe #{table}").each do |row|
|
130
130
|
cols << row["Field"]
|
131
131
|
case row["Field"]
|
132
|
-
when "id"
|
132
|
+
when "id"
|
133
133
|
expect(row["Type"]).to eql "bigint(20)"
|
134
|
-
when "thing"
|
134
|
+
when "thing"
|
135
135
|
expect(row["Type"]).to eql "text"
|
136
|
-
when "number"
|
136
|
+
when "number"
|
137
137
|
expect(row["Type"]).to eql "float"
|
138
|
-
when "updated_at"
|
138
|
+
when "updated_at"
|
139
139
|
expect(row["Type"]).to eql "datetime"
|
140
140
|
end
|
141
141
|
end
|
142
|
-
expect(cols).to
|
142
|
+
expect(cols).to include('id', 'thing', 'number', 'updated_at')
|
143
143
|
end
|
144
144
|
|
145
145
|
it "can add columns to exiting tables when new keys are provided" do
|
@@ -163,7 +163,7 @@ describe 'mysql' do
|
|
163
163
|
expect(count).to eql 7
|
164
164
|
end
|
165
165
|
|
166
|
-
it "can will seek further for null-ish values" do
|
166
|
+
it "can will seek further for null-ish values" do
|
167
167
|
data = [
|
168
168
|
{id: 1, thing: 'stuff a', number: nil, updated_at: Time.new},
|
169
169
|
{id: 2, thing: 'stuff b', number: nil, updated_at: Time.new},
|
@@ -182,25 +182,25 @@ describe 'mysql' do
|
|
182
182
|
destination.query("describe #{table}").each do |row|
|
183
183
|
cols << row["Field"]
|
184
184
|
case row["Field"]
|
185
|
-
when "id"
|
185
|
+
when "id"
|
186
186
|
expect(row["Type"]).to eql "bigint(20)"
|
187
|
-
when "thing"
|
187
|
+
when "thing"
|
188
188
|
expect(row["Type"]).to eql "text"
|
189
|
-
when "number"
|
189
|
+
when "number"
|
190
190
|
expect(row["Type"]).to eql "bigint(20)"
|
191
|
-
when "updated_at"
|
191
|
+
when "updated_at"
|
192
192
|
expect(row["Type"]).to eql "datetime"
|
193
193
|
end
|
194
194
|
end
|
195
|
-
expect(cols).to
|
195
|
+
expect(cols).to include('id', 'thing', 'updated_at', 'number')
|
196
196
|
end
|
197
197
|
|
198
|
-
it "null rows will be text, and can be updated on subsequent writes" do
|
198
|
+
it "null rows will be text, and can be updated on subsequent writes" do
|
199
199
|
data = [
|
200
200
|
{id: 1, number: nil, updated_at: Time.new},
|
201
201
|
{id: 2, number: nil, updated_at: Time.new},
|
202
202
|
]
|
203
|
-
|
203
|
+
|
204
204
|
table = "new_table"
|
205
205
|
|
206
206
|
plan = SpecPlan.new
|
@@ -216,15 +216,15 @@ describe 'mysql' do
|
|
216
216
|
destination.query("describe #{table}").each do |row|
|
217
217
|
cols << row["Field"]
|
218
218
|
case row["Field"]
|
219
|
-
when "id"
|
219
|
+
when "id"
|
220
220
|
expect(row["Type"]).to eql "bigint(20)"
|
221
|
-
when "number"
|
221
|
+
when "number"
|
222
222
|
expect(row["Type"]).to eql "varchar(0)"
|
223
|
-
when "updated_at"
|
223
|
+
when "updated_at"
|
224
224
|
expect(row["Type"]).to eql "datetime"
|
225
225
|
end
|
226
226
|
end
|
227
|
-
expect(cols).to
|
227
|
+
expect(cols).to include('id', 'updated_at', 'number')
|
228
228
|
|
229
229
|
data = [
|
230
230
|
{id: 3, number: 123, updated_at: Time.new},
|
@@ -243,13 +243,13 @@ describe 'mysql' do
|
|
243
243
|
destination.query("describe #{table}").each do |row|
|
244
244
|
cols << row["Field"]
|
245
245
|
case row["Field"]
|
246
|
-
when "number"
|
246
|
+
when "number"
|
247
247
|
expect(row["Type"]).to eql "bigint(20)"
|
248
248
|
end
|
249
249
|
end
|
250
|
-
expect(cols).to
|
250
|
+
expect(cols).to include('id', 'updated_at', 'number')
|
251
251
|
end
|
252
252
|
|
253
253
|
end
|
254
254
|
|
255
|
-
end
|
255
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: forklift_etl
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.2.
|
4
|
+
version: 1.2.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Evan Tahler
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2016-03-11 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activesupport
|