forklift_etl 1.0.14 → 1.0.15

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    MjQ1MzQwOGNlOWMxNWExZTc2OTBjODAyNzhlMzQyNjU2N2EyNzE0ZQ==
+    ZmFkNWMxMTVjYTkxMDUyOWU4MTk1ZTljZDRhNzk1ODQwNGNjYmRjYw==
   data.tar.gz: !binary |-
-    Nzk0NDM0MWJlODQ0YmIwYjdkNDgyNWU3ZGU1ZWYyOWI0ZWM3YTg2Ng==
+    MzJjYjQzOTJiOGJiYzZjNjU2M2UwZWM4NDU1MGU5YjhjMTBlZGNlZQ==
 SHA512:
   metadata.gz: !binary |-
-    NDZjNzhhMDQ0NzkwOGFiNDUyMzIwYWFmNjdiMWUzNGRjOTdiMWQyMjEzMTBk
-    MmQ4OWEwYjhjZWMwZGE2ZTk5ZDA4YTBjOGIxMTAxOTYxYTQ1ODQzZWIyOTgx
-    Y2NiM2MzNWU5ZWQ4MzQ5NzEyOGYxMGJhZmY2YmZiMjkyMDQ5ZmY=
+    OWM3ZTMyNTAwM2Q1NDAwOWJlOWUyOTUyYTBjNjk1YjE5ZThkZDZhYzY1Y2Q2
+    MmY4ZGM4OTBiNmViMDhmM2Q5Yjg0ODRhNGQ3ZDA4NzZiMGY2ODcwNjM0ODA4
+    OTRkMjk1ZmZkNDQ4YmY4Y2FiYWIyYjcxZmI1NTlmOGQyNTU5ZTI=
   data.tar.gz: !binary |-
-    YWNmZjk4NWRjYzlhMTEyMWIzNTk1YWE2NGVlYWViYjA0NGNiNmIyMzJhOWY2
-    MmFhMDY2MjMwMWYzYzQyNjA2M2NmY2EwOTcyOTM0MTRlMzc2OTcxODk0M2I5
-    NDAyZmVjYmEzNjhiYzIzMWJhYzVjMjNjYmRiYTAwMjk1MjlhNzI=
+    YjAzMTA3MjYzMjFlZWQ2OTYxYjI4NmI2M2E5ODE0ODNiZWRhMDBlZThkZTc2
+    NDVkYjE3NjM3ODgyMjk5YWEyMmU1NGE4MmQ3M2M3N2EwNWM5NTM2M2E1MmFm
+    ZDdkYTcwYTE2MWU3YzkxZmEwZmE0NGEwOTAwMWU0NmQyYzhjZmI=

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    forklift_etl (1.0.14)
+    forklift_etl (1.0.15)
       activesupport (~> 4.0, >= 4.0.0)
       elasticsearch (~> 1.0, >= 1.0.0)
       lumberjack (~> 1.0, >= 1.0.0)

data/bin/forklift

@@ -4,9 +4,9 @@ require 'rubygems'
 require 'fileutils'
 
 begin
-  require 'forklift/forklift'
+  require 'forklift'
 rescue LoadError
-  require "#{File.expand_path(File.dirname(__FILE__))}/../lib/forklift/forklift.rb"
+  require "#{File.expand_path(File.dirname(__FILE__))}/../lib/forklift.rb"
 end
 
 def generate

data/lib/{forklift/forklift.rb → forklift.rb}

@@ -1,8 +1,8 @@
-require 'rubygems'
+require 'forklift/version'
 
 module Forklift
 
-  lib = File.expand_path(File.dirname(__FILE__))
+  lib = File.join(File.expand_path(File.dirname(__FILE__)), 'forklift')
 
   require "#{lib}/base/utils.rb"
   require "#{lib}/base/pid.rb"

data/lib/forklift/patterns/mysql_patterns.rb

@@ -56,27 +56,29 @@ module Forklift
       def self.optimistic_pipe(source, from_table, destination, to_table, matcher=source.default_matcher, primary_key='id')
         from_db = source.current_database 
         to_db = destination.current_database 
-        if self.can_incremental_pipe?(source, from_table)
+        if self.can_incremental_pipe?(source, from_table, destination, to_table, matcher)
           incremental_pipe(source, from_table, destination, to_table, matcher, primary_key)
         else
           pipe(source, from_table, destination, to_table)
         end
       end
 
-      def self.can_incremental_pipe?(conn, table, matcher=conn.default_matcher)
-        conn.columns(table, conn.current_database).include?(matcher)
+      def self.can_incremental_pipe?(source, from_table, destination, to_table, matcher=source.default_matcher)
+        a = source.columns(from_table, source.current_database).include?(matcher)
+        b = destination.columns(to_table, destination.current_database).include?(matcher)
+        return (a && b)
       end
 
       ## When you are copying data to and from mysql 
-      ## An implamentation of "pipe" for remote databases
-      def self.mysql_optimistic_import(source, destination)
-        #TODO: allow passing in of matcher and primary_key
+      ## An implementation of "pipe" for remote databases
+      def self.mysql_optimistic_import(source, destination, matcher=source.default_matcher)
         source.tables.each do |table|
-          if( source.columns(table).include?(source.default_matcher) && destination.tables.include?(table) )
+          if( source.columns(table).include?(matcher) && destination.tables.include?(table) && destination.columns(table).include?(matcher) )
             since = destination.max_timestamp(table)
             source.read_since(table, since){ |data| destination.write(data, table) }
           else
-            destination.truncate table
+            # destination.truncate table
+            destination.drop! table if destination.tables.include?(table)
             source.read("select * from #{table}"){ |data| destination.write(data, table) }
           end
         end

data/lib/forklift/version.rb

@@ -1,3 +1,3 @@
 module Forklift
-  VERSION = "1.0.14"
+  VERSION = "1.0.15"
 end

data/spec/integration/mysql_patterns_spec.rb

@@ -48,9 +48,9 @@ describe 'mysql patterns' do
     plan = SpecPlan.new
     plan.do! {
       source = plan.connections[:mysql][:forklift_test_source_a]
-      expect(Forklift::Patterns::Mysql.can_incremental_pipe?(source, 'users')).to eql true
-      expect(Forklift::Patterns::Mysql.can_incremental_pipe?(source, 'sales')).to eql false
-      expect(Forklift::Patterns::Mysql.can_incremental_pipe?(source, 'products')).to eql true
+      expect(Forklift::Patterns::Mysql.can_incremental_pipe?(source, 'users', source, 'users')).to eql true
+      expect(Forklift::Patterns::Mysql.can_incremental_pipe?(source, 'sales', source, 'sales')).to eql false
+      expect(Forklift::Patterns::Mysql.can_incremental_pipe?(source, 'products', source, 'products')).to eql true
     }
   end
 

data/spec/spec_helper.rb

@@ -9,7 +9,7 @@ $LOAD_PATH.unshift(File.dirname(__FILE__))
 $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
 APP_DIR ||= File.expand_path('../../', __FILE__)
 
-require 'forklift/forklift'
+require 'forklift'
 require 'awesome_print'
 require 'rspec'
 require 'fileutils'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: forklift_etl
 version: !ruby/object:Gem::Version
-  version: 1.0.14
+  version: 1.0.15
 platform: ruby
 authors:
 - Evan Tahler
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-04-17 00:00:00.000000000 Z
+date: 2014-04-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -139,12 +139,12 @@ files:
 - example/transformations/email_suffix.rb
 - forklift.jpg
 - forklift_etl.gemspec
+- lib/forklift.rb
 - lib/forklift/base/connection.rb
 - lib/forklift/base/logger.rb
 - lib/forklift/base/mailer.rb
 - lib/forklift/base/pid.rb
 - lib/forklift/base/utils.rb
-- lib/forklift/forklift.rb
 - lib/forklift/patterns/elasticsearch_patterns.rb
 - lib/forklift/patterns/mysql_patterns.rb
 - lib/forklift/plan.rb