forklift_etl 1.2.1 → 1.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 94c0245d54361f22228d060adfc79dddc9dcf39b
4
- data.tar.gz: 85ff71007431776dc19480d24567611a6a03a222
3
+ metadata.gz: 1718155b7b286739b163fe0c386488035dad0d0d
4
+ data.tar.gz: 00e9c7b12f1f1c46da88c2ea6051ca3fd2c38b85
5
5
  SHA512:
6
- metadata.gz: 7285a14abf86b2b0f509ab3d4aa2197509ccd462132d5a2d81d0fd6ddba5373ff61c850de4d49cfdd072fe55c329efdf917c489f70fa07184d6c03f4f0b5ee74
7
- data.tar.gz: 67e20f3b594f3d85fcee6cfe74e8051512ad2b2e2e4ef9847a050f00bcb340ea083b2c11e5ae3cc1e31635a1624be43c50cbe37b062eb90130826ce05c71bcfb
6
+ metadata.gz: a254bafa76f5bf176fcd4ec75233732961e35522afd12f04c4e994f125641bc7ed554137fca541761e630e334c119bb7d89ec3ef1f4a476e06a6540692db9970
7
+ data.tar.gz: 9b4d36cfa659ac78511b235c3ccd14dea7fe817de65f4c7c6507fc14350ec85bb21b6a1778c7a440c178145ab6bb0404e0d842b4f8cc78de6052626337a26f7c
@@ -73,3 +73,6 @@ DEPENDENCIES
73
73
  forklift_etl!
74
74
  rake
75
75
  rspec
76
+
77
+ BUNDLED WITH
78
+ 1.11.2
@@ -17,7 +17,7 @@ module Forklift
17
17
  @forklift
18
18
  end
19
19
 
20
- def connect
20
+ def connect
21
21
  @client = ::Elasticsearch::Client.new(config)
22
22
  end
23
23
 
@@ -99,7 +99,7 @@ module Forklift
99
99
  insert_q << safe_values(columns, d)
100
100
  insert_q << ","
101
101
  end
102
-
102
+
103
103
  if delete_keys.length > 0
104
104
  delete_q << "(#{delete_keys.join(',')})"
105
105
  q(delete_q)
@@ -129,22 +129,26 @@ module Forklift
129
129
  keys = {}
130
130
  data.each do |item|
131
131
  item.each do |k,v|
132
- keys[k] = sql_type(v) if ( keys[k].nil? && v.class != NilClass )
132
+ keys[k.to_s] = sql_type(v) if (keys[k.to_s].nil? || keys[k.to_s] == sql_type(nil))
133
133
  end
134
134
  end
135
+ keys[primary_key] = 'bigint(20)' unless keys.has_key?(primary_key)
135
136
 
136
- data.first.each do |k,v|
137
- keys[k] = sql_type(v) if ( keys[k].nil? )
137
+ col_defn = keys.map do |col, type|
138
+ if col == primary_key
139
+ "`#{col}` #{type} NOT NULL AUTO_INCREMENT"
140
+ else
141
+ "`#{col}` #{type} DEFAULT NULL"
142
+ end
138
143
  end
144
+ col_defn << "PRIMARY KEY (`#{primary_key}`)"
145
+ col_defn << "KEY `#{matcher}` (`#{matcher}`)" if keys.include?(matcher.to_sym)
139
146
 
140
- command = "CREATE TABLE `#{database}`.`#{table}` ( "
141
- command << " `#{primary_key}` bigint(20) NOT NULL AUTO_INCREMENT, " if ( data.first[primary_key.to_sym].nil? )
142
- keys.each do |col, type|
143
- command << " `#{col}` #{type} DEFAULT NULL, "
144
- end
145
- command << " PRIMARY KEY (`#{primary_key}`) "
146
- command << " , KEY `#{matcher}` (`#{matcher}`) " if keys.include?(matcher.to_sym)
147
- command << " ) "
147
+ command = <<-EOS
148
+ CREATE TABLE `#{database}`.`#{table}` (
149
+ #{col_defn.join(', ')}
150
+ )
151
+ EOS
148
152
 
149
153
  q(command)
150
154
  forklift.logger.log "lazy-created table `#{database}`.`#{table}`"
@@ -238,7 +242,7 @@ module Forklift
238
242
  cmd << " | gzip > #{file}"
239
243
  forklift.logger.log "Dumping #{config['database']} to #{file}"
240
244
  forklift.logger.debug cmd
241
-
245
+
242
246
  stdin, stdout, stderr = Open3.popen3(cmd)
243
247
  stdout = stdout.readlines
244
248
  stderr = stderr.readlines
@@ -285,7 +289,7 @@ module Forklift
285
289
  end
286
290
 
287
291
  end
288
- end
292
+ end
289
293
  end
290
294
 
291
295
  def safe_columns(cols)
@@ -1,3 +1,3 @@
1
1
  module Forklift
2
- VERSION = "1.2.1"
2
+ VERSION = "1.2.2"
3
3
  end
data/readme.md CHANGED
@@ -1,4 +1,12 @@
1
1
  # Forklift ETL
2
+
3
+ ---
4
+
5
+ ## THIS TOOL IS NO LONGER SUPORTED.
6
+ It has been succeded by [Empujar](https://github.com/taskrabbit/empujar)
7
+
8
+ ---
9
+
2
10
  Moving heavy databases around. [![Gem Version](https://badge.fury.io/rb/forklift_etl.svg)](http://badge.fury.io/rb/forklift_etl)
3
11
  [![Build Status](https://secure.travis-ci.org/taskrabbit/forklift.png?branch=master)](http://travis-ci.org/taskrabbit/forklift)
4
12
 
@@ -29,7 +29,7 @@ describe 'elasticsearch' do
29
29
  @rows = []
30
30
  plan.do! {
31
31
  source = plan.connections[:elasticsearch][:forklift_test]
32
- source.read(index, query, false, 0, 3) {|data|
32
+ source.read(index, query, false, 0, 3) {|data|
33
33
  @rows = (@rows + data)
34
34
  }
35
35
  }
@@ -56,7 +56,7 @@ describe 'elasticsearch' do
56
56
  expect(count).to eql 6
57
57
  end
58
58
 
59
- it "can overwrite existing data, probided a primary key" do
59
+ it "can overwrite existing data, probided a primary key" do
60
60
  index = 'forklift_test'
61
61
  plan = SpecPlan.new
62
62
  data = [
@@ -80,7 +80,7 @@ describe 'elasticsearch' do
80
80
  expect(obj["viewed_at"]).to eql 99
81
81
  end
82
82
 
83
- it "can delete an index" do
83
+ it "can delete an index" do
84
84
  index = 'other_test_index'
85
85
  plan = SpecPlan.new
86
86
  client = SpecClient.elasticsearch('forklift_test')
@@ -89,12 +89,12 @@ describe 'elasticsearch' do
89
89
  ]
90
90
  plan.do! {
91
91
  destination = plan.connections[:elasticsearch][:forklift_test]
92
- expect { client.search({ index: index }) }.to raise_error(/IndexMissingException/)
92
+ expect { client.search({ index: index }) }.to raise_error(/index_not_found_exception|IndexMissingException/)
93
93
  destination.write(data, index, true)
94
94
  expect { client.search({ index: index }) }.to_not raise_error
95
95
  destination.delete_index(index)
96
- expect { client.search({ index: index }) }.to raise_error(/IndexMissingException/)
97
- }
96
+ expect { client.search({ index: index }) }.to raise_error(/index_not_found_exception|IndexMissingException/)
97
+ }
98
98
  plan.disconnect!
99
99
  end
100
- end
100
+ end
@@ -1,6 +1,6 @@
1
1
  require 'spec_helper'
2
2
 
3
- describe 'mysql' do
3
+ describe 'mysql' do
4
4
 
5
5
  before(:each) do
6
6
  SpecSeeds.setup_mysql
@@ -12,7 +12,7 @@ describe 'mysql' do
12
12
  @rows = []
13
13
  plan.do! {
14
14
  source = plan.connections[:mysql][:forklift_test_source_a]
15
- source.read(query) {|data|
15
+ source.read(query) {|data|
16
16
  @rows = (@rows + data)
17
17
  }
18
18
  }
@@ -20,14 +20,14 @@ describe 'mysql' do
20
20
 
21
21
  expect(@rows.length).to eql 5
22
22
  end
23
-
23
+
24
24
  it "can read data (filtered)" do
25
25
  query = 'select * from `users`'
26
26
  plan = SpecPlan.new
27
27
  @rows = []
28
28
  plan.do! {
29
29
  source = plan.connections[:mysql][:forklift_test_source_a]
30
- source.read(query, source.current_database, false, 3, 0) {|data|
30
+ source.read(query, source.current_database, false, 3, 0) {|data|
31
31
  @rows = (@rows + data)
32
32
  }
33
33
  }
@@ -54,7 +54,7 @@ describe 'mysql' do
54
54
  expect(count).to eql 7
55
55
  end
56
56
 
57
- it "can update existing data" do
57
+ it "can update existing data" do
58
58
  table = "users"
59
59
  data = [
60
60
  {id: 1, email: 'evan@example.com', first_name: 'New Name', last_name: 'T', created_at: Time.new.to_s(:db), updated_at: Time.new.to_s(:db)}
@@ -79,8 +79,8 @@ describe 'mysql' do
79
79
  destination = SpecClient.mysql('forklift_test_source_a')
80
80
  destination.query('drop table if exists `new_table`')
81
81
  end
82
-
83
- it "can lazy-create a table with primary keys provided" do
82
+
83
+ it "can lazy-create a table with primary keys provided" do
84
84
  data = [
85
85
  {id: 1, thing: 'stuff a', updated_at: Time.new},
86
86
  {id: 2, thing: 'stuff b', updated_at: Time.new},
@@ -99,17 +99,17 @@ describe 'mysql' do
99
99
  destination.query("describe #{table}").each do |row|
100
100
  cols << row["Field"]
101
101
  case row["Field"]
102
- when "id"
102
+ when "id"
103
103
  expect(row["Type"]).to eql "bigint(20)"
104
- when "thing"
104
+ when "thing"
105
105
  expect(row["Type"]).to eql "text"
106
- when "updated_at"
106
+ when "updated_at"
107
107
  expect(row["Type"]).to eql "datetime"
108
108
  end
109
109
  end
110
110
  expect(cols).to eql ['id', 'thing', 'updated_at']
111
111
  end
112
-
112
+
113
113
  it "can lazy-create a table without primary keys provided" do
114
114
  data = [
115
115
  {thing: 'stuff a', number: 1.123, updated_at: Time.new},
@@ -129,17 +129,17 @@ describe 'mysql' do
129
129
  destination.query("describe #{table}").each do |row|
130
130
  cols << row["Field"]
131
131
  case row["Field"]
132
- when "id"
132
+ when "id"
133
133
  expect(row["Type"]).to eql "bigint(20)"
134
- when "thing"
134
+ when "thing"
135
135
  expect(row["Type"]).to eql "text"
136
- when "number"
136
+ when "number"
137
137
  expect(row["Type"]).to eql "float"
138
- when "updated_at"
138
+ when "updated_at"
139
139
  expect(row["Type"]).to eql "datetime"
140
140
  end
141
141
  end
142
- expect(cols).to eql ['id', 'thing', 'number', 'updated_at']
142
+ expect(cols).to include('id', 'thing', 'number', 'updated_at')
143
143
  end
144
144
 
145
145
  it "can add columns to exiting tables when new keys are provided" do
@@ -163,7 +163,7 @@ describe 'mysql' do
163
163
  expect(count).to eql 7
164
164
  end
165
165
 
166
- it "can will seek further for null-ish values" do
166
+ it "can will seek further for null-ish values" do
167
167
  data = [
168
168
  {id: 1, thing: 'stuff a', number: nil, updated_at: Time.new},
169
169
  {id: 2, thing: 'stuff b', number: nil, updated_at: Time.new},
@@ -182,25 +182,25 @@ describe 'mysql' do
182
182
  destination.query("describe #{table}").each do |row|
183
183
  cols << row["Field"]
184
184
  case row["Field"]
185
- when "id"
185
+ when "id"
186
186
  expect(row["Type"]).to eql "bigint(20)"
187
- when "thing"
187
+ when "thing"
188
188
  expect(row["Type"]).to eql "text"
189
- when "number"
189
+ when "number"
190
190
  expect(row["Type"]).to eql "bigint(20)"
191
- when "updated_at"
191
+ when "updated_at"
192
192
  expect(row["Type"]).to eql "datetime"
193
193
  end
194
194
  end
195
- expect(cols).to eql ['id', 'thing', 'updated_at', 'number']
195
+ expect(cols).to include('id', 'thing', 'updated_at', 'number')
196
196
  end
197
197
 
198
- it "null rows will be text, and can be updated on subsequent writes" do
198
+ it "null rows will be text, and can be updated on subsequent writes" do
199
199
  data = [
200
200
  {id: 1, number: nil, updated_at: Time.new},
201
201
  {id: 2, number: nil, updated_at: Time.new},
202
202
  ]
203
-
203
+
204
204
  table = "new_table"
205
205
 
206
206
  plan = SpecPlan.new
@@ -216,15 +216,15 @@ describe 'mysql' do
216
216
  destination.query("describe #{table}").each do |row|
217
217
  cols << row["Field"]
218
218
  case row["Field"]
219
- when "id"
219
+ when "id"
220
220
  expect(row["Type"]).to eql "bigint(20)"
221
- when "number"
221
+ when "number"
222
222
  expect(row["Type"]).to eql "varchar(0)"
223
- when "updated_at"
223
+ when "updated_at"
224
224
  expect(row["Type"]).to eql "datetime"
225
225
  end
226
226
  end
227
- expect(cols).to eql ['id', 'updated_at', 'number']
227
+ expect(cols).to include('id', 'updated_at', 'number')
228
228
 
229
229
  data = [
230
230
  {id: 3, number: 123, updated_at: Time.new},
@@ -243,13 +243,13 @@ describe 'mysql' do
243
243
  destination.query("describe #{table}").each do |row|
244
244
  cols << row["Field"]
245
245
  case row["Field"]
246
- when "number"
246
+ when "number"
247
247
  expect(row["Type"]).to eql "bigint(20)"
248
248
  end
249
249
  end
250
- expect(cols).to eql ['id', 'updated_at', 'number']
250
+ expect(cols).to include('id', 'updated_at', 'number')
251
251
  end
252
252
 
253
253
  end
254
254
 
255
- end
255
+ end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: forklift_etl
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.2.1
4
+ version: 1.2.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Evan Tahler
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-01-16 00:00:00.000000000 Z
11
+ date: 2016-03-11 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: activesupport