forklift_etl 1.1.3 → 1.1.4

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    NmIzZGJlZDAwZDhlNjE3ZjIxMzdkNGQ2ZDQ5NDk3ZTUzMzkxM2Y5MQ==
+    MTFiNTAwY2RlOGQxOTUzY2Y3N2RiNTg3Njg2ZDJiZDE3NTY3MGNmZg==
   data.tar.gz: !binary |-
-    NzE3NzZjOGNiOWZiMDYxNDk4ZGIyZjRlNjdmOTk4NjU5MDYzNWZiZA==
+    ZmQ2NTQwZjEyZGFhMzY5MmRjMzQyNzVkZTIxYTM4YzNmM2FmZTFiOA==
 SHA512:
   metadata.gz: !binary |-
-    YTIxMmJiMWE5Nzg3ZTdiNDQyZGZjNDBhNzllNTg5ZTAyZDk0NTNmMGU1NzAy
-    ZjI4ZDY2NGM2ODkwNmViZGJjYTdmNzM0ZWFkMmM2MDFhNDUwY2ViMDJmODVk
-    Y2NmYmM5YmYwMDg1ZGM5NTNmMTk4MGNhYzg4ZWM2ZTA1M2IxZDY=
+    YTU5YzAyZDQ5NmI3YTVkMGZmZmY3Yjc0MTc0NjBhZDQzODkwMDI0NDhjMDdi
+    YWM1ODkwNGI4NWVkZjA2MTlhOTkzNmE0MDY2MTFjMDk0ZDJhNWYyMTI3MzM3
+    MmZjMjVkNTM5YTdmYzY1NTU3ZDkxZDE5MGIzN2M3ZjNmYWU4OWY=
   data.tar.gz: !binary |-
-    YTI1YWYxNzM4MDdlOWY1NTUyNzA4NTIwOWEwZWQwMjBkOGM0MGFkNzdiMGI4
-    YmI1YjhiNDY3NjkzYmNhYTdlYzg2ZWY4OGMyODUxYzk0ODU1NTM2OWQxY2Uw
-    NzE4MjNiZGY0YTNhMWU3OTg4ZDRhY2NhMTNhNzBjM2NhMTcyNDc=
+    MTI5YzU5YWQ4MjEyNmFhZDE1NTMyYjlkOTE2ZTUzODNhMzhmODQ5OTQyNGNm
+    OGJiZjJkZTgyNTc2ZGI0M2QyNzA5NDQxMzg5MjY4MWRiZTFlZTgzMTk0Nzhj
+    YzM4NmQxNzg5OGYxMGE5Y2JlYzVmYjg5NWI5MGQ0YzdkZTdhM2U=

data/lib/forklift/transports/mysql.rb

@@ -60,7 +60,7 @@ module Forklift
         data.map{|l| l.symbolize_keys! }
 
         if tables.include? table
-          # all good, cary on
+          ensure_row_types(data, table, database)
         elsif(lazy == true && data.length > 0)
           lazy_table_create(table, data, database, primary_key)
         end
@@ -72,7 +72,8 @@ module Forklift
             if(to_update == true && !d[primary_key.to_sym].nil?)
               q("DELETE FROM `#{database}`.`#{table}` WHERE `#{primary_key}` = #{d[primary_key.to_sym]}")
             end
-            q("INSERT INTO `#{database}`.`#{table}` (#{safe_columns(d.keys)}) VALUES (#{safe_values(d.values)});")
+            insert_q = "INSERT INTO `#{database}`.`#{table}` (#{safe_columns(d.keys)}) VALUES (#{safe_values(d.values)});"
+            q(insert_q)
           end
           forklift.logger.log "wrote #{data.length} rows to `#{database}`.`#{table}`"
         end
@@ -86,12 +87,12 @@ module Forklift
           end
         end
 
-        data.first do |k,v|
+        data.first.each do |k,v|
           keys[k] = sql_type(v) if ( keys[k].nil? )
         end
 
         command = "CREATE TABLE `#{database}`.`#{table}` ( "
-        command << " `#{primary_key}` int(11) NOT NULL AUTO_INCREMENT, " if ( data.first[primary_key.to_sym].nil? )
+        command << " `#{primary_key}` bigint(20) NOT NULL AUTO_INCREMENT, " if ( data.first[primary_key.to_sym].nil? )
         keys.each do |col, type|
           command << " `#{col}` #{type} DEFAULT NULL, "
         end
@@ -113,7 +114,7 @@ module Forklift
         return "tinyint(1)"   if v.class == TrueClass
         return "tinyint(1)"   if v.class == FalseClass
         return "text"         if v.class == String
-        return "text"         if v.class == NilClass
+        return "varchar(0)"   if v.class == NilClass
         return "text"         # catchall
       end
 
@@ -214,6 +215,29 @@ module Forklift
 
       private
 
+      def ensure_row_types(data, table, database=current_database)
+        read("describe `#{database}`.`#{table}`").each do |row|
+          if row[:Type] == 'varchar(0)'
+
+            value = nil
+            data.each do |r|
+              if ( !r[row[:Field].to_sym].nil? )
+                value = r[row[:Field].to_sym]
+                break
+              end
+            end
+
+            if !value.nil?
+              sql_type = sql_type(value)
+              alter_sql = "ALTER TABLE `#{database}`.`#{table}` CHANGE `#{row[:Field]}` `#{row[:Field]}` #{sql_type};"
+              puts alter_sql
+              q(alter_sql)
+            end
+
+          end
+        end 
+      end
+
       def safe_columns(cols)
         a = []
         cols.each do |c|

data/lib/forklift/version.rb

@@ -1,3 +1,3 @@
 module Forklift
-  VERSION = "1.1.3"
+  VERSION = "1.1.4"
 end

data/spec/integration/mysql_spec.rb

@@ -123,7 +123,7 @@ describe 'mysql' do
         cols << row["Field"]
         case row["Field"]
         when "id" 
-          expect(row["Type"]).to eql "int(11)"
+          expect(row["Type"]).to eql "bigint(20)"
         when "thing" 
           expect(row["Type"]).to eql "text"
         when "updated_at" 
@@ -164,6 +164,59 @@ describe 'mysql' do
       expect(cols).to eql ['id', 'thing', 'updated_at', 'number']
     end
 
+    it "null rows will be text, and can be updated on subsequent writes" do 
+      data = [
+        {id: 1, number: nil, updated_at: Time.new},
+        {id: 2, number: nil, updated_at: Time.new},
+      ]
+      
+      table = "new_table"
+
+      plan = SpecPlan.new
+      plan.do! {
+        destination = plan.connections[:mysql][:forklift_test_source_a]
+        destination.write(data, table)
+      }
+
+      destination = SpecClient.mysql('forklift_test_source_a')
+      cols = []
+
+      destination.query("describe #{table}").each do |row|
+        cols << row["Field"]
+        case row["Field"]
+        when "id" 
+          expect(row["Type"]).to eql "bigint(20)"
+        when "number" 
+          expect(row["Type"]).to eql "varchar(0)"
+        when "updated_at" 
+          expect(row["Type"]).to eql "datetime"
+        end
+      end
+      expect(cols).to eql ['id', 'updated_at', 'number']
+
+      data = [
+        {id: 3, number: 123, updated_at: Time.new},
+      ]
+
+      plan = SpecPlan.new
+      plan.do! {
+        destination = plan.connections[:mysql][:forklift_test_source_a]
+        destination.write(data, table)
+      }
+
+      destination = SpecClient.mysql('forklift_test_source_a')
+      cols = []
+
+      destination.query("describe #{table}").each do |row|
+        cols << row["Field"]
+        case row["Field"]
+        when "number" 
+          expect(row["Type"]).to eql "bigint(20)"
+        end
+      end
+      expect(cols).to eql ['id', 'updated_at', 'number']
+    end
+
   end
 
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: forklift_etl
 version: !ruby/object:Gem::Version
-  version: 1.1.3
+  version: 1.1.4
 platform: ruby
 authors:
 - Evan Tahler