forklift_etl 1.0.14 → 1.0.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +8 -8
- data/Gemfile.lock +1 -1
- data/bin/forklift +2 -2
- data/lib/{forklift/forklift.rb → forklift.rb} +2 -2
- data/lib/forklift/patterns/mysql_patterns.rb +10 -8
- data/lib/forklift/version.rb +1 -1
- data/spec/integration/mysql_patterns_spec.rb +3 -3
- data/spec/spec_helper.rb +1 -1
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
---
|
2
2
|
!binary "U0hBMQ==":
|
3
3
|
metadata.gz: !binary |-
|
4
|
-
|
4
|
+
ZmFkNWMxMTVjYTkxMDUyOWU4MTk1ZTljZDRhNzk1ODQwNGNjYmRjYw==
|
5
5
|
data.tar.gz: !binary |-
|
6
|
-
|
6
|
+
MzJjYjQzOTJiOGJiYzZjNjU2M2UwZWM4NDU1MGU5YjhjMTBlZGNlZQ==
|
7
7
|
SHA512:
|
8
8
|
metadata.gz: !binary |-
|
9
|
-
|
10
|
-
|
11
|
-
|
9
|
+
OWM3ZTMyNTAwM2Q1NDAwOWJlOWUyOTUyYTBjNjk1YjE5ZThkZDZhYzY1Y2Q2
|
10
|
+
MmY4ZGM4OTBiNmViMDhmM2Q5Yjg0ODRhNGQ3ZDA4NzZiMGY2ODcwNjM0ODA4
|
11
|
+
OTRkMjk1ZmZkNDQ4YmY4Y2FiYWIyYjcxZmI1NTlmOGQyNTU5ZTI=
|
12
12
|
data.tar.gz: !binary |-
|
13
|
-
|
14
|
-
|
15
|
-
|
13
|
+
YjAzMTA3MjYzMjFlZWQ2OTYxYjI4NmI2M2E5ODE0ODNiZWRhMDBlZThkZTc2
|
14
|
+
NDVkYjE3NjM3ODgyMjk5YWEyMmU1NGE4MmQ3M2M3N2EwNWM5NTM2M2E1MmFm
|
15
|
+
ZDdkYTcwYTE2MWU3YzkxZmEwZmE0NGEwOTAwMWU0NmQyYzhjZmI=
|
data/Gemfile.lock
CHANGED
data/bin/forklift
CHANGED
@@ -4,9 +4,9 @@ require 'rubygems'
|
|
4
4
|
require 'fileutils'
|
5
5
|
|
6
6
|
begin
|
7
|
-
require 'forklift
|
7
|
+
require 'forklift'
|
8
8
|
rescue LoadError
|
9
|
-
require "#{File.expand_path(File.dirname(__FILE__))}/../lib/forklift
|
9
|
+
require "#{File.expand_path(File.dirname(__FILE__))}/../lib/forklift.rb"
|
10
10
|
end
|
11
11
|
|
12
12
|
def generate
|
@@ -56,27 +56,29 @@ module Forklift
|
|
56
56
|
def self.optimistic_pipe(source, from_table, destination, to_table, matcher=source.default_matcher, primary_key='id')
|
57
57
|
from_db = source.current_database
|
58
58
|
to_db = destination.current_database
|
59
|
-
if self.can_incremental_pipe?(source, from_table)
|
59
|
+
if self.can_incremental_pipe?(source, from_table, destination, to_table, matcher)
|
60
60
|
incremental_pipe(source, from_table, destination, to_table, matcher, primary_key)
|
61
61
|
else
|
62
62
|
pipe(source, from_table, destination, to_table)
|
63
63
|
end
|
64
64
|
end
|
65
65
|
|
66
|
-
def self.can_incremental_pipe?(
|
67
|
-
|
66
|
+
def self.can_incremental_pipe?(source, from_table, destination, to_table, matcher=source.default_matcher)
|
67
|
+
a = source.columns(from_table, source.current_database).include?(matcher)
|
68
|
+
b = destination.columns(to_table, destination.current_database).include?(matcher)
|
69
|
+
return (a && b)
|
68
70
|
end
|
69
71
|
|
70
72
|
## When you are copying data to and from mysql
|
71
|
-
## An
|
72
|
-
def self.mysql_optimistic_import(source, destination)
|
73
|
-
#TODO: allow passing in of matcher and primary_key
|
73
|
+
## An implementation of "pipe" for remote databases
|
74
|
+
def self.mysql_optimistic_import(source, destination, matcher=source.default_matcher)
|
74
75
|
source.tables.each do |table|
|
75
|
-
if( source.columns(table).include?(
|
76
|
+
if( source.columns(table).include?(matcher) && destination.tables.include?(table) && destination.columns(table).include?(matcher) )
|
76
77
|
since = destination.max_timestamp(table)
|
77
78
|
source.read_since(table, since){ |data| destination.write(data, table) }
|
78
79
|
else
|
79
|
-
destination.truncate table
|
80
|
+
# destination.truncate table
|
81
|
+
destination.drop! table if destination.tables.include?(table)
|
80
82
|
source.read("select * from #{table}"){ |data| destination.write(data, table) }
|
81
83
|
end
|
82
84
|
end
|
data/lib/forklift/version.rb
CHANGED
@@ -48,9 +48,9 @@ describe 'mysql patterns' do
|
|
48
48
|
plan = SpecPlan.new
|
49
49
|
plan.do! {
|
50
50
|
source = plan.connections[:mysql][:forklift_test_source_a]
|
51
|
-
expect(Forklift::Patterns::Mysql.can_incremental_pipe?(source, 'users')).to eql true
|
52
|
-
expect(Forklift::Patterns::Mysql.can_incremental_pipe?(source, 'sales')).to eql false
|
53
|
-
expect(Forklift::Patterns::Mysql.can_incremental_pipe?(source, 'products')).to eql true
|
51
|
+
expect(Forklift::Patterns::Mysql.can_incremental_pipe?(source, 'users', source, 'users')).to eql true
|
52
|
+
expect(Forklift::Patterns::Mysql.can_incremental_pipe?(source, 'sales', source, 'sales')).to eql false
|
53
|
+
expect(Forklift::Patterns::Mysql.can_incremental_pipe?(source, 'products', source, 'products')).to eql true
|
54
54
|
}
|
55
55
|
end
|
56
56
|
|
data/spec/spec_helper.rb
CHANGED
@@ -9,7 +9,7 @@ $LOAD_PATH.unshift(File.dirname(__FILE__))
|
|
9
9
|
$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
|
10
10
|
APP_DIR ||= File.expand_path('../../', __FILE__)
|
11
11
|
|
12
|
-
require 'forklift
|
12
|
+
require 'forklift'
|
13
13
|
require 'awesome_print'
|
14
14
|
require 'rspec'
|
15
15
|
require 'fileutils'
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: forklift_etl
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.
|
4
|
+
version: 1.0.15
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Evan Tahler
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-04-
|
11
|
+
date: 2014-04-23 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activesupport
|
@@ -139,12 +139,12 @@ files:
|
|
139
139
|
- example/transformations/email_suffix.rb
|
140
140
|
- forklift.jpg
|
141
141
|
- forklift_etl.gemspec
|
142
|
+
- lib/forklift.rb
|
142
143
|
- lib/forklift/base/connection.rb
|
143
144
|
- lib/forklift/base/logger.rb
|
144
145
|
- lib/forklift/base/mailer.rb
|
145
146
|
- lib/forklift/base/pid.rb
|
146
147
|
- lib/forklift/base/utils.rb
|
147
|
-
- lib/forklift/forklift.rb
|
148
148
|
- lib/forklift/patterns/elasticsearch_patterns.rb
|
149
149
|
- lib/forklift/patterns/mysql_patterns.rb
|
150
150
|
- lib/forklift/plan.rb
|