forklift_etl 1.1.9 → 1.1.10

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    forklift_etl (1.1.9)
+    forklift_etl (1.1.10)
       activesupport (~> 4.0, >= 4.0.0)
       elasticsearch (~> 1.0, >= 1.0.0)
       lumberjack (~> 1.0, >= 1.0.0)

data/lib/forklift/patterns/mysql_patterns.rb

@@ -107,21 +107,27 @@ module Forklift
       # The high water method will stub a row in all tables with a `default_matcher` column prentending to have a record from `time`
       # This enabled partial forklift funs which will only extract data "later than X"
       # TODO: assumes all columns have a default NULL setting
-      def self.write_high_water_mark(db, time, matcher=source.default_matcher)
+      def self.write_high_water_mark(db, time, matcher=db.default_matcher)
         db.tables.each do |table|
-          columns, types = db.columns(table, nil, true)
+          columns, types = db.columns(table, db.current_database, true)
           if columns.include?(matcher)
             row = {}
             i = 0
             while( i < columns.length )
               if(columns[i] == matcher)
-                row[column[i]] << time.to_s(:db)
-              elsif( types[i] =~ /text/ || types[i] =~ /varchar/ )
-                row[column[i]] << "~~stub~~" 
+                row[columns[i]] = time.to_s(:db)
+              elsif( types[i] =~ /text/ )
+                row[columns[i]] = "~~stub~~" 
+              elsif( types[i] =~ /varchar/  )
+                row[columns[i]] = "~~stub~~".to_sym
               elsif( types[i] =~ /float/ || types[i] =~ /int/ )
-                row[column[i]] << 0
+                row[columns[i]] = 0
+              elsif( types[i] =~ /datetime/ || types[i] =~ /timetsamp/ )
+                row[columns[i]] = time.to_s(:db)
+              elsif( types[i] =~ /date/ )
+                row[columns[i]] = time.to_s(:db).split(" ").first
               else
-                row[column[i]] << "NULL"
+                row[columns[i]] = "NULL"
               end
               i = i + 1
             end

data/lib/forklift/plan.rb

@@ -138,6 +138,7 @@ module Forklift
       return {
         project_root: Dir.pwd,
         batch_size: 1000,
+        char_bytecode_max: 65535, # the utf8 char limit
         logger: {
           stdout: true,
           debug:  false,

data/lib/forklift/transports/mysql.rb

@@ -99,7 +99,22 @@ module Forklift
             q(delete_q)
           end
           insert_q = insert_q[0...-1]
-          q(insert_q)
+
+          begin
+            q(insert_q)
+          rescue Mysql2::Error => ex
+            # UTF8 Safety.  Open a PR if you don't want UTF8 data...
+            # https://github.com/taskrabbit/demoji
+            raise ex unless ex.message.match /Incorrect string value:/
+            safer_insert_q = ""
+            for i in (0...insert_q.length)
+              char = insert_q[i]
+              char = '???' if char.ord > forklift.config[:char_bytecode_max]
+              safer_insert_q << char
+            end
+            q(safer_insert_q)
+          end
+
           forklift.logger.log "wrote #{data.length} rows to `#{database}`.`#{table}`"
         end
       end

data/lib/forklift/version.rb

@@ -1,3 +1,3 @@
 module Forklift
-  VERSION = "1.1.9"
+  VERSION = "1.1.10"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: forklift_etl
 version: !ruby/object:Gem::Version
-  version: 1.1.9
+  version: 1.1.10
   prerelease: 
 platform: ruby
 authors: