forklift_etl 1.1.8 → 1.1.9

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    forklift_etl (1.1.8)
+    forklift_etl (1.1.9)
       activesupport (~> 4.0, >= 4.0.0)
       elasticsearch (~> 1.0, >= 1.0.0)
       lumberjack (~> 1.0, >= 1.0.0)

data/lib/forklift/patterns/mysql_patterns.rb

@@ -2,14 +2,18 @@ module Forklift
   module Patterns
     class Mysql
 
-      def self.pipe(source, from_table, destination, to_table)
+      def self.pipe(source, from_table, destination, to_table, tmp_table="_forklift_tmp")
         start = Time.new.to_i
         from_db = source.current_database 
         to_db = destination.current_database 
         source.forklift.logger.log("mysql pipe: `#{from_db}`.`#{from_table}` => `#{to_db}`.`#{to_table}`")
+
+        source.q("drop table if exists `#{to_db}`.`#{tmp_table}`")
+        source.q("create table `#{to_db}`.`#{tmp_table}` like `#{from_db}`.`#{from_table}`")
+        source.q("insert into `#{to_db}`.`#{tmp_table}` select * from `#{from_db}`.`#{from_table}`")
         source.q("drop table if exists `#{to_db}`.`#{to_table}`")
-        source.q("create table `#{to_db}`.`#{to_table}` like `#{from_db}`.`#{from_table}`")
-        source.q("insert into `#{to_db}`.`#{to_table}` select * from `#{from_db}`.`#{from_table}`")
+        source.q("rename table `#{to_db}`.`#{tmp_table}` to `#{to_db}`.`#{to_table}`")
+
         delta = Time.new.to_i - start
         source.forklift.logger.log("  ^ moved #{destination.count(to_table, to_db)} rows in #{delta}s")
       end
@@ -59,7 +63,9 @@ module Forklift
         if self.can_incremental_pipe?(source, from_table, destination, to_table, matcher)
           begin
             incremental_pipe(source, from_table, destination, to_table, matcher, primary_key)
-          rescue
+          rescue Exception => e
+            source.forklift.logger.log("! incremental_pipe failure on #{from_table} => #{to_table}: #{e} ")
+            source.forklift.logger.log("! falling back to pipe...")
             pipe(source, from_table, destination, to_table)
           end
         else
@@ -98,6 +104,32 @@ module Forklift
         end
       end
 
+      # The high water method will stub a row in all tables with a `default_matcher` column prentending to have a record from `time`
+      # This enabled partial forklift funs which will only extract data "later than X"
+      # TODO: assumes all columns have a default NULL setting
+      def self.write_high_water_mark(db, time, matcher=source.default_matcher)
+        db.tables.each do |table|
+          columns, types = db.columns(table, nil, true)
+          if columns.include?(matcher)
+            row = {}
+            i = 0
+            while( i < columns.length )
+              if(columns[i] == matcher)
+                row[column[i]] << time.to_s(:db)
+              elsif( types[i] =~ /text/ || types[i] =~ /varchar/ )
+                row[column[i]] << "~~stub~~" 
+              elsif( types[i] =~ /float/ || types[i] =~ /int/ )
+                row[column[i]] << 0
+              else
+                row[column[i]] << "NULL"
+              end
+              i = i + 1
+            end
+            db.write([row], table)
+          end
+        end
+      end
+
     end
   end
 end

data/lib/forklift/plan.rb

@@ -137,6 +137,7 @@ module Forklift
     def default_config
       return {
         project_root: Dir.pwd,
+        batch_size: 1000,
         logger: {
           stdout: true,
           debug:  false,

data/lib/forklift/transports/csv.rb

@@ -18,7 +18,7 @@ module Forklift
         @forklift
       end
 
-      def read(size=1000)
+      def read(size=forklift.config[:batch_size])
         data = []
         CSV.foreach(config[:file], headers: true, converters: :all) do |row|
           data << row.to_hash.symbolize_keys

data/lib/forklift/transports/elasticsearch.rb

@@ -18,7 +18,7 @@ module Forklift
         @forklift
       end
 
-      def read(index, query, looping=true, from=0, size=1000)
+      def read(index, query, looping=true, from=0, size=forklift.config[:batch_size])
         offset = 0
         loop_count = 0
 

data/lib/forklift/transports/mysql.rb

@@ -28,7 +28,11 @@ module Forklift
         q("DROP table `#{database}`.`#{table}`");
       end
 
-      def read(query, database=current_database, looping=true, limit=1000, offset=0)
+      def rename(table, new_table, database=current_database, new_database=current_database)
+        q("RENAME TABLE `#{database}`.`#{table}` TO `#{new_database}`.`#{new_table}`")
+      end
+
+      def read(query, database=current_database, looping=true, limit=forklift.config[:batch_size], offset=0)
         loop_count = 0
         # TODO: Detect limit/offset already present in query
         q("USE `#{database}`")
@@ -77,14 +81,25 @@ module Forklift
                 end
               end
             end
+          end
 
+          insert_q = "INSERT INTO `#{database}`.`#{table}` (#{safe_columns(columns)}) VALUES "
+          delete_q = "DELETE FROM `#{database}`.`#{table}` WHERE `#{primary_key}` IN "
+          delete_keys = []
+          data.each do |d|
             if(to_update == true && !d[primary_key.to_sym].nil?)
-              q("DELETE FROM `#{database}`.`#{table}` WHERE `#{primary_key}` = #{d[primary_key.to_sym]}")
+              delete_keys << d[primary_key.to_sym]
             end
-
-            insert_q = "INSERT INTO `#{database}`.`#{table}` (#{safe_columns(d.keys)}) VALUES (#{safe_values(d.values)});"
-            q(insert_q)
+            insert_q << safe_values(columns, d)
+            insert_q << ","
+          end
+          
+          if delete_keys.length > 0
+            delete_q << "(#{delete_keys.join(',')})"
+            q(delete_q)
           end
+          insert_q = insert_q[0...-1]
+          q(insert_q)
           forklift.logger.log "wrote #{data.length} rows to `#{database}`.`#{table}`"
         end
       end
@@ -177,12 +192,15 @@ module Forklift
         end
       end
 
-      def columns(table, database=current_database)
+      def columns(table, database=current_database, return_types=false)
         cols = []
+        types = []
         read("describe `#{database}`.`#{table}`").each do |row|
-          cols << row[:Field]
+          cols  << row[:Field]
+          types << row[:Type]
         end
-        cols
+        return cols if return_types == false
+        return cols, types
       end
 
       def dump(file, options=[])
@@ -257,27 +275,31 @@ module Forklift
         return a.join(', ')
       end
 
-      def safe_values(values)
+      def safe_values(columns, d)
         a = []
-        values.each do |v|
+        sym_cols = columns.map { |s| s.to_sym }
+        sym_cols.each do |c|
           part = "NULL"
-          if( [::String, ::Symbol].include?(v.class) )
-            s = v.to_s
-            s.gsub!('\\') { '\\\\' }
-            s.gsub!('\"', '\/"')
-            s.gsub!('"', '\"')
-            part = "\"#{s}\""
-          elsif( [::Date, ::Time, ::DateTime].include?(v.class) )
-            s = v.to_s(:db)
-            part = "\"#{s}\""
-          elsif( [::Fixnum].include?(v.class) )
-            part = v
-          elsif( [::Float, ::BigDecimal].include?(v.class) )
-            part = v.to_f
+          v = d[c]
+          unless v.nil?
+            if( [::String, ::Symbol].include?(v.class) )
+              s = v.to_s
+              s.gsub!('\\') { '\\\\' }
+              s.gsub!('\"', '\/"')
+              s.gsub!('"', '\"')
+              part = "\"#{s}\""
+            elsif( [::Date, ::Time, ::DateTime].include?(v.class) )
+              s = v.to_s(:db)
+              part = "\"#{s}\""
+            elsif( [::Fixnum].include?(v.class) )
+              part = v
+            elsif( [::Float, ::BigDecimal].include?(v.class) )
+              part = v.to_f
+            end
           end
           a << part
         end
-        return a.join(', ')
+        return "( #{a.join(', ')} )"
       end
 
       #/private

data/lib/forklift/version.rb

@@ -1,3 +1,3 @@
 module Forklift
-  VERSION = "1.1.8"
+  VERSION = "1.1.9"
 end

data/readme.md

@@ -370,7 +370,8 @@ end
 - max_timestamp(table, matcher=default_matcher, database=current_database)
   - return the timestamp of the max(matcher) or 1970-01-01
 - truncate!(table, database=current_database)
-- columns(table, database=current_database)
+- columns(table, database=current_database, return_types=false)
+- rename(table, new_table, database, new_database)
 - dump(file)
   - mysqldump the database to `file` via gzip
 
@@ -384,6 +385,8 @@ end
 - mysql_optimistic_import(source, destination)
   - tries to do an incramental table copy, falls back to a full table copy
   - this differs from `pipe`, as all data is loaded into forklift, rather than relying on mysql transfer methods
+- write_high_water_mark(db, time, matcher)
+  - The high water method will stub a row in all tables with a `default_matcher` column prentending to have a record from `time`
 
 ### Elasticsearch
 

data/spec/integration/mysql_patterns_spec.rb

@@ -73,4 +73,6 @@ describe 'mysql patterns' do
       expect(destination.read('select first_name from users where id = 1')[0][:first_name]).to eql 'EvanAgain'
     }
   end
+
+  it "can write the high_water_mark"
 end

data/spec/integration/mysql_spec.rb

@@ -37,7 +37,8 @@ describe 'mysql' do
   it "can write new data" do
     table = "users"
     data = [
-      {email: 'other@example.com', first_name: 'other', last_name: 'n', created_at: Time.new.to_s(:db), updated_at: Time.new.to_s(:db)}
+      {email: 'other@example.com', first_name: 'other', last_name: 'n', created_at: Time.new.to_s(:db), updated_at: Time.new.to_s(:db)},
+      {email: 'else@example.com',  first_name: 'else', last_name: 'n', created_at: Time.new.to_s(:db), updated_at: Time.new.to_s(:db)}
     ]
     plan = SpecPlan.new
     plan.do! {
@@ -47,7 +48,7 @@ describe 'mysql' do
 
     destination = SpecClient.mysql('forklift_test_source_a')
     count = destination.query('select count(1) as "count" from users').first['count']
-    expect(count).to eql 6
+    expect(count).to eql 7
   end
 
   it "can update existing data" do 

data/spec/unit/connection/mysql_spec.rb

@@ -90,13 +90,15 @@ describe Forklift::Connection::Mysql do
     subject { described_class.new({}, {}) }
 
     it "escapes one trailing backslash" do
-      values = ["foo\\"]
-      expect(subject.send(:safe_values, values)).to eq("\"foo\\\\\"")
+      columns = ['col']
+      values = {:col => "foo\\"}
+      expect(subject.send(:safe_values, columns, values)).to eq("( \"foo\\\\\" )")
     end
 
     it "escapes two trailing backslashes" do
-      values = ["foo\\\\"]
-      expect(subject.send(:safe_values, values)).to eq("\"foo\\\\\\\\\"")
+      columns = ['col']
+      values = {:col => "foo\\\\" }
+      expect(subject.send(:safe_values, columns, values)).to eq("( \"foo\\\\\\\\\" )")
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: forklift_etl
 version: !ruby/object:Gem::Version
-  version: 1.1.8
+  version: 1.1.9
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-08-20 00:00:00.000000000 Z
+date: 2014-08-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport