forklift_etl 1.0.14 → 1.0.15

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,15 +1,15 @@
1
1
  ---
2
2
  !binary "U0hBMQ==":
3
3
  metadata.gz: !binary |-
4
- MjQ1MzQwOGNlOWMxNWExZTc2OTBjODAyNzhlMzQyNjU2N2EyNzE0ZQ==
4
+ ZmFkNWMxMTVjYTkxMDUyOWU4MTk1ZTljZDRhNzk1ODQwNGNjYmRjYw==
5
5
  data.tar.gz: !binary |-
6
- Nzk0NDM0MWJlODQ0YmIwYjdkNDgyNWU3ZGU1ZWYyOWI0ZWM3YTg2Ng==
6
+ MzJjYjQzOTJiOGJiYzZjNjU2M2UwZWM4NDU1MGU5YjhjMTBlZGNlZQ==
7
7
  SHA512:
8
8
  metadata.gz: !binary |-
9
- NDZjNzhhMDQ0NzkwOGFiNDUyMzIwYWFmNjdiMWUzNGRjOTdiMWQyMjEzMTBk
10
- MmQ4OWEwYjhjZWMwZGE2ZTk5ZDA4YTBjOGIxMTAxOTYxYTQ1ODQzZWIyOTgx
11
- Y2NiM2MzNWU5ZWQ4MzQ5NzEyOGYxMGJhZmY2YmZiMjkyMDQ5ZmY=
9
+ OWM3ZTMyNTAwM2Q1NDAwOWJlOWUyOTUyYTBjNjk1YjE5ZThkZDZhYzY1Y2Q2
10
+ MmY4ZGM4OTBiNmViMDhmM2Q5Yjg0ODRhNGQ3ZDA4NzZiMGY2ODcwNjM0ODA4
11
+ OTRkMjk1ZmZkNDQ4YmY4Y2FiYWIyYjcxZmI1NTlmOGQyNTU5ZTI=
12
12
  data.tar.gz: !binary |-
13
- YWNmZjk4NWRjYzlhMTEyMWIzNTk1YWE2NGVlYWViYjA0NGNiNmIyMzJhOWY2
14
- MmFhMDY2MjMwMWYzYzQyNjA2M2NmY2EwOTcyOTM0MTRlMzc2OTcxODk0M2I5
15
- NDAyZmVjYmEzNjhiYzIzMWJhYzVjMjNjYmRiYTAwMjk1MjlhNzI=
13
+ YjAzMTA3MjYzMjFlZWQ2OTYxYjI4NmI2M2E5ODE0ODNiZWRhMDBlZThkZTc2
14
+ NDVkYjE3NjM3ODgyMjk5YWEyMmU1NGE4MmQ3M2M3N2EwNWM5NTM2M2E1MmFm
15
+ ZDdkYTcwYTE2MWU3YzkxZmEwZmE0NGEwOTAwMWU0NmQyYzhjZmI=
data/Gemfile.lock CHANGED
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- forklift_etl (1.0.14)
4
+ forklift_etl (1.0.15)
5
5
  activesupport (~> 4.0, >= 4.0.0)
6
6
  elasticsearch (~> 1.0, >= 1.0.0)
7
7
  lumberjack (~> 1.0, >= 1.0.0)
data/bin/forklift CHANGED
@@ -4,9 +4,9 @@ require 'rubygems'
4
4
  require 'fileutils'
5
5
 
6
6
  begin
7
- require 'forklift/forklift'
7
+ require 'forklift'
8
8
  rescue LoadError
9
- require "#{File.expand_path(File.dirname(__FILE__))}/../lib/forklift/forklift.rb"
9
+ require "#{File.expand_path(File.dirname(__FILE__))}/../lib/forklift.rb"
10
10
  end
11
11
 
12
12
  def generate
@@ -1,8 +1,8 @@
1
- require 'rubygems'
1
+ require 'forklift/version'
2
2
 
3
3
  module Forklift
4
4
 
5
- lib = File.expand_path(File.dirname(__FILE__))
5
+ lib = File.join(File.expand_path(File.dirname(__FILE__)), 'forklift')
6
6
 
7
7
  require "#{lib}/base/utils.rb"
8
8
  require "#{lib}/base/pid.rb"
@@ -56,27 +56,29 @@ module Forklift
56
56
  def self.optimistic_pipe(source, from_table, destination, to_table, matcher=source.default_matcher, primary_key='id')
57
57
  from_db = source.current_database
58
58
  to_db = destination.current_database
59
- if self.can_incremental_pipe?(source, from_table)
59
+ if self.can_incremental_pipe?(source, from_table, destination, to_table, matcher)
60
60
  incremental_pipe(source, from_table, destination, to_table, matcher, primary_key)
61
61
  else
62
62
  pipe(source, from_table, destination, to_table)
63
63
  end
64
64
  end
65
65
 
66
- def self.can_incremental_pipe?(conn, table, matcher=conn.default_matcher)
67
- conn.columns(table, conn.current_database).include?(matcher)
66
+ def self.can_incremental_pipe?(source, from_table, destination, to_table, matcher=source.default_matcher)
67
+ a = source.columns(from_table, source.current_database).include?(matcher)
68
+ b = destination.columns(to_table, destination.current_database).include?(matcher)
69
+ return (a && b)
68
70
  end
69
71
 
70
72
  ## When you are copying data to and from mysql
71
- ## An implamentation of "pipe" for remote databases
72
- def self.mysql_optimistic_import(source, destination)
73
- #TODO: allow passing in of matcher and primary_key
73
+ ## An implementation of "pipe" for remote databases
74
+ def self.mysql_optimistic_import(source, destination, matcher=source.default_matcher)
74
75
  source.tables.each do |table|
75
- if( source.columns(table).include?(source.default_matcher) && destination.tables.include?(table) )
76
+ if( source.columns(table).include?(matcher) && destination.tables.include?(table) && destination.columns(table).include?(matcher) )
76
77
  since = destination.max_timestamp(table)
77
78
  source.read_since(table, since){ |data| destination.write(data, table) }
78
79
  else
79
- destination.truncate table
80
+ # destination.truncate table
81
+ destination.drop! table if destination.tables.include?(table)
80
82
  source.read("select * from #{table}"){ |data| destination.write(data, table) }
81
83
  end
82
84
  end
@@ -1,3 +1,3 @@
1
1
  module Forklift
2
- VERSION = "1.0.14"
2
+ VERSION = "1.0.15"
3
3
  end
@@ -48,9 +48,9 @@ describe 'mysql patterns' do
48
48
  plan = SpecPlan.new
49
49
  plan.do! {
50
50
  source = plan.connections[:mysql][:forklift_test_source_a]
51
- expect(Forklift::Patterns::Mysql.can_incremental_pipe?(source, 'users')).to eql true
52
- expect(Forklift::Patterns::Mysql.can_incremental_pipe?(source, 'sales')).to eql false
53
- expect(Forklift::Patterns::Mysql.can_incremental_pipe?(source, 'products')).to eql true
51
+ expect(Forklift::Patterns::Mysql.can_incremental_pipe?(source, 'users', source, 'users')).to eql true
52
+ expect(Forklift::Patterns::Mysql.can_incremental_pipe?(source, 'sales', source, 'sales')).to eql false
53
+ expect(Forklift::Patterns::Mysql.can_incremental_pipe?(source, 'products', source, 'products')).to eql true
54
54
  }
55
55
  end
56
56
 
data/spec/spec_helper.rb CHANGED
@@ -9,7 +9,7 @@ $LOAD_PATH.unshift(File.dirname(__FILE__))
9
9
  $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
10
10
  APP_DIR ||= File.expand_path('../../', __FILE__)
11
11
 
12
- require 'forklift/forklift'
12
+ require 'forklift'
13
13
  require 'awesome_print'
14
14
  require 'rspec'
15
15
  require 'fileutils'
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: forklift_etl
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.14
4
+ version: 1.0.15
5
5
  platform: ruby
6
6
  authors:
7
7
  - Evan Tahler
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2014-04-17 00:00:00.000000000 Z
11
+ date: 2014-04-23 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: activesupport
@@ -139,12 +139,12 @@ files:
139
139
  - example/transformations/email_suffix.rb
140
140
  - forklift.jpg
141
141
  - forklift_etl.gemspec
142
+ - lib/forklift.rb
142
143
  - lib/forklift/base/connection.rb
143
144
  - lib/forklift/base/logger.rb
144
145
  - lib/forklift/base/mailer.rb
145
146
  - lib/forklift/base/pid.rb
146
147
  - lib/forklift/base/utils.rb
147
- - lib/forklift/forklift.rb
148
148
  - lib/forklift/patterns/elasticsearch_patterns.rb
149
149
  - lib/forklift/patterns/mysql_patterns.rb
150
150
  - lib/forklift/plan.rb