forklift_etl 1.1.2 → 1.1.3

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    NjcxMzMwMTNkNzgxNDcyYWJmZWMyYWRiNWZlMGJkMDA3YzJlMGRjNQ==
+    NmIzZGJlZDAwZDhlNjE3ZjIxMzdkNGQ2ZDQ5NDk3ZTUzMzkxM2Y5MQ==
   data.tar.gz: !binary |-
-    ODQ5OTg3NGIwNDAyMTRiZDY2MzdmYWNlZjAzMTkwYTQ5MjI2ZmQ1ZA==
+    NzE3NzZjOGNiOWZiMDYxNDk4ZGIyZjRlNjdmOTk4NjU5MDYzNWZiZA==
 SHA512:
   metadata.gz: !binary |-
-    NmM4ZDRkMzNlYTdiOGFmNGZhZTJiZTI4NDVjNGVjOTFmYzc2ZGE5NTk3NDU5
-    ZGI0NGRlM2FiYjliYmQzZWQzYmM0N2U0NTU5NGFkNTJiZmJmZGY2NjZlYmMx
-    MDgzMDQ2Y2Q0NGUxYjg1ZmI5NDVjOGU5ZDY1MTc1YWI0MTgzZGY=
+    YTIxMmJiMWE5Nzg3ZTdiNDQyZGZjNDBhNzllNTg5ZTAyZDk0NTNmMGU1NzAy
+    ZjI4ZDY2NGM2ODkwNmViZGJjYTdmNzM0ZWFkMmM2MDFhNDUwY2ViMDJmODVk
+    Y2NmYmM5YmYwMDg1ZGM5NTNmMTk4MGNhYzg4ZWM2ZTA1M2IxZDY=
   data.tar.gz: !binary |-
-    ZjMzMDgwNDM1NmQwYjI0Njg4YjU4NDYwNWU0Y2RlZTE3YzgzNjI3ZWJkOWE1
-    MGE5YjU5MjVjODA4NmY5ZTA4OWRkZDdiZTI0Y2E0YzBiYjNkM2FlNjQ5YjMz
-    NDY2OTAwMWFlYTJmMWE4OGE0MzBmN2MzZjgwYzY0ZDMxODdhNTM=
+    YTI1YWYxNzM4MDdlOWY1NTUyNzA4NTIwOWEwZWQwMjBkOGM0MGFkNzdiMGI4
+    YmI1YjhiNDY3NjkzYmNhYTdlYzg2ZWY4OGMyODUxYzk0ODU1NTM2OWQxY2Uw
+    NzE4MjNiZGY0YTNhMWU3OTg4ZDRhY2NhMTNhNzBjM2NhMTcyNDc=

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    forklift_etl (1.1.0)
+    forklift_etl (1.1.3)
       activesupport (~> 4.0, >= 4.0.0)
       elasticsearch (~> 1.0, >= 1.0.0)
       lumberjack (~> 1.0, >= 1.0.0)
@@ -20,12 +20,12 @@ GEM
     addressable (2.3.6)
     awesome_print (1.2.0)
     diff-lcs (1.2.5)
-    elasticsearch (1.0.1)
-      elasticsearch-api (= 1.0.1)
-      elasticsearch-transport (= 1.0.1)
-    elasticsearch-api (1.0.1)
+    elasticsearch (1.0.2)
+      elasticsearch-api (= 1.0.2)
+      elasticsearch-transport (= 1.0.2)
+    elasticsearch-api (1.0.2)
       multi_json
-    elasticsearch-transport (1.0.1)
+    elasticsearch-transport (1.0.2)
       faraday
       multi_json
     email_spec (1.5.0)
@@ -37,17 +37,17 @@ GEM
     json (1.8.1)
     launchy (2.4.2)
       addressable (~> 2.3)
-    lumberjack (1.0.5)
+    lumberjack (1.0.7)
     mail (2.5.4)
       mime-types (~> 1.16)
       treetop (~> 1.4.8)
     mime-types (1.25.1)
-    minitest (5.3.4)
-    multi_json (1.10.0)
+    minitest (5.3.5)
+    multi_json (1.10.1)
     multipart-post (2.0.0)
     mysql2 (0.3.16)
     polyglot (0.3.4)
-    pony (1.8)
+    pony (1.9)
       mail (>= 2.0)
     rake (10.2.2)
     rspec (2.14.1)
@@ -58,11 +58,11 @@ GEM
     rspec-expectations (2.14.5)
       diff-lcs (>= 1.1.3, < 2.0)
     rspec-mocks (2.14.6)
-    thread_safe (0.3.3)
+    thread_safe (0.3.4)
     treetop (1.4.15)
       polyglot
       polyglot (>= 0.3.1)
-    tzinfo (1.1.0)
+    tzinfo (1.2.1)
       thread_safe (~> 0.1)
 
 PLATFORMS

data/lib/forklift/patterns/mysql_patterns.rb

@@ -57,7 +57,11 @@ module Forklift
         from_db = source.current_database 
         to_db = destination.current_database 
         if self.can_incremental_pipe?(source, from_table, destination, to_table, matcher)
-          incremental_pipe(source, from_table, destination, to_table, matcher, primary_key)
+          begin
+            incremental_pipe(source, from_table, destination, to_table, matcher, primary_key)
+          rescue
+            pipe(source, from_table, destination, to_table)
+          end
         else
           pipe(source, from_table, destination, to_table)
         end

data/lib/forklift/transports/mysql.rb

@@ -82,10 +82,14 @@ module Forklift
         keys = {}
         data.each do |item|
           item.each do |k,v|
-            keys[k] = sql_type(v) if ( keys[k].nil? )
+            keys[k] = sql_type(v) if ( keys[k].nil? && v.class != NilClass )
           end
         end
 
+        data.first do |k,v|
+          keys[k] = sql_type(v) if ( keys[k].nil? )
+        end
+
         command = "CREATE TABLE `#{database}`.`#{table}` ( "
         command << " `#{primary_key}` int(11) NOT NULL AUTO_INCREMENT, " if ( data.first[primary_key.to_sym].nil? )
         keys.each do |col, type|

data/lib/forklift/version.rb

@@ -1,3 +1,3 @@
 module Forklift
-  VERSION = "1.1.2"
+  VERSION = "1.1.3"
 end

data/spec/integration/mysql_spec.rb

@@ -133,6 +133,37 @@ describe 'mysql' do
       expect(cols).to eql ['id', 'thing', 'updated_at']
     end
 
+    it "can will seek further for null-ish values" do 
+      data = [
+        {id: 1, thing: 'stuff a', number: nil, updated_at: Time.new},
+        {id: 2, thing: 'stuff b', number: nil, updated_at: Time.new},
+        {id: 3, thing: 'stuff c', number: 100, updated_at: Time.new},
+      ]
+      table = "new_table"
+      plan = SpecPlan.new
+      plan.do! {
+        destination = plan.connections[:mysql][:forklift_test_source_a]
+        destination.write(data, table)
+      }
+
+      destination = SpecClient.mysql('forklift_test_source_a')
+      cols = []
+      destination.query("describe #{table}").each do |row|
+        cols << row["Field"]
+        case row["Field"]
+        when "id" 
+          expect(row["Type"]).to eql "bigint(20)"
+        when "thing" 
+          expect(row["Type"]).to eql "text"
+        when "number" 
+          expect(row["Type"]).to eql "bigint(20)"
+        when "updated_at" 
+          expect(row["Type"]).to eql "datetime"
+        end
+      end
+      expect(cols).to eql ['id', 'thing', 'updated_at', 'number']
+    end
+
   end
 
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: forklift_etl
 version: !ruby/object:Gem::Version
-  version: 1.1.2
+  version: 1.1.3
 platform: ruby
 authors:
 - Evan Tahler
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-06-23 00:00:00.000000000 Z
+date: 2014-06-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport