forklift_etl 1.1.12 → 1.2.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 72c15001e2a6f2164be9fbe2ef78da4d5e131d6c
+  data.tar.gz: 5dfd8551cee1c2a19af8ffe49c9a73bb9b1b5bdf
+SHA512:
+  metadata.gz: a0763e36c35340ee5e55e78fcef2aa54533d6e4aa18fa9ac16b64835b7fa58f32195482a78f76e4e8a2dc3e380e8513cdd48c364b9bd65b20180af1b55e2ca7c
+  data.tar.gz: ae1472bc5ce7a6ebf0be6b6e896bc22211818bc6db9e405d085c68ae570ecef2fa1eee757ff85163af9771b928cecd4aa69fce84a865c449ddfe28c35ce3b2df

data/.ruby-version

@@ -0,0 +1 @@
+2.1.5

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    forklift_etl (1.1.12)
+    forklift_etl (1.2.0)
       activesupport (~> 4.0, >= 4.0.0)
       elasticsearch (~> 1.0, >= 1.0.0)
       lumberjack (~> 1.0, >= 1.0.0)
@@ -18,7 +18,7 @@ GEM
       thread_safe (~> 0.3, >= 0.3.4)
       tzinfo (~> 1.1)
     addressable (2.3.6)
-    awesome_print (1.6.0)
+    awesome_print (1.6.1)
     diff-lcs (1.2.5)
     elasticsearch (1.0.6)
       elasticsearch-api (= 1.0.6)
@@ -31,17 +31,17 @@ GEM
     email_spec (1.6.0)
       launchy (~> 2.1)
       mail (~> 2.2)
-    faraday (0.9.0)
+    faraday (0.9.1)
       multipart-post (>= 1.2, < 3)
     i18n (0.7.0)
-    json (1.8.1)
+    json (1.8.2)
     launchy (2.4.3)
       addressable (~> 2.3)
     lumberjack (1.0.9)
     mail (2.6.3)
       mime-types (>= 1.16, < 3)
     mime-types (2.4.3)
-    minitest (5.5.0)
+    minitest (5.5.1)
     multi_json (1.10.1)
     multipart-post (2.0.0)
     mysql2 (0.3.17)

data/lib/forklift/base/connection.rb

@@ -38,32 +38,32 @@ module Forklift
         raise 'not implemented'
       end
 
-      def exec(path)
+      def exec(path, *args)
         begin
-          exec!(path)
+          exec!(path, &args)
         rescue Exception => e
           forklift.logger.log(e)
         end
       end
 
-      def exec!(path)
+      def exec!(path, *args)
         forklift.logger.log "Running script: #{path}"
         extension = path.split(".").last
         if(extension == "rb" || extension == "ruby")
-          exec_ruby(path)
+          exec_ruby(path, *args)
         else
-          exec_script(path)
+          exec_script(path, *args)
         end
       end
 
-      def exec_ruby(path)
+      def exec_ruby(path, *args)
         klass = forklift.utils.class_name_from_file(path)
         require path
         model = eval("#{klass}.new")
-        model.do!(self, forklift)
+        model.do!(self, forklift, *args)
       end
 
-      def exec_script(path)
+      def exec_script(path, *args)
         raise 'not implemented'
       end
 

data/lib/forklift/plan.rb

@@ -33,6 +33,7 @@ module Forklift
         begin
           loader = "Forklift::Connection::#{type.camelcase}.new(db_config, self)"
           connection = eval(loader)
+          connection.connect
           connections[type.to_sym][name.to_sym] = connection
           logger.debug "loaded a #{type.camelcase} connection from #{f}"
         rescue Exception => e
@@ -42,6 +43,13 @@ module Forklift
       end
     end
 
+    def disconnect!
+      connections.each do |k, collection|
+        collection.each do |k, connection|
+          connection.disconnect
+        end
+      end
+    end
 
     def default_error_handler
       return lambda {|name, e| raise e }

data/lib/forklift/transports/csv.rb

@@ -18,6 +18,9 @@ module Forklift
         @forklift
       end
 
+      def connect; end
+      def disconnect; end
+
       def read(size=forklift.config[:batch_size])
         data = []
         CSV.foreach(config[:file], headers: true, converters: :all) do |row|

data/lib/forklift/transports/elasticsearch.rb

@@ -7,7 +7,6 @@ module Forklift
       def initialize(config, forklift)
         @config = config
         @forklift = forklift
-        @client = ::Elasticsearch::Client.new(config)
       end
 
       def config
@@ -18,6 +17,14 @@ module Forklift
         @forklift
       end
 
+      def connect 
+        @client = ::Elasticsearch::Client.new(config)
+      end
+
+      def disconnect
+        @client = nil
+      end
+
       def read(index, query, looping=true, from=0, size=forklift.config[:batch_size])
         offset = 0
         loop_count = 0

data/lib/forklift/transports/mysql.rb

@@ -8,8 +8,14 @@ module Forklift
       def initialize(config, forklift)
         @config = config
         @forklift = forklift
-        @client = Mysql2::Client.new(config)
-        # q("USE #{config['database']}")
+      end
+
+      def connect
+        @client = Mysql2::Client.new(@config)
+      end
+
+      def disconnect
+        @client.close
       end
 
       def config

data/lib/forklift/version.rb

@@ -1,3 +1,3 @@
 module Forklift
-  VERSION = "1.1.12"
+  VERSION = "1.2.0"
 end

data/readme.md

@@ -413,20 +413,31 @@ Forklift allows you to create both Ruby transformations and script transformatio
 - It is up to the transport to define `exec_script`, and not all transports will support it.  Mysql can run `.sql` files, but there is not an equivalent for elasticsearch.
 - `.exec` runs and logs exceptions, while `.exec!` will raise on an error.  For example, `destination.exec("./transformations/cleanup.rb")` will run cleanup.rb on the destination database.
 - Script files are run as-is, but ruby transformations must define a `do!` method in their class and are passed `def do!(connection, forklift)`
+- args is optional, and can be passed in from your plan
 
 ```ruby
 # Example transformation to count users
 # count_users.rb
 
 class CountUsers
-  def do!(connection, forklift)
+  def do!(connection, forklift, args)
     forklift.logger.log "counting users"
     count = connection.count('users')
-    forklift.logger.log "found #{count} users"
+    forklift.logger.log "[#{args.name}] found #{count} users"
   end
 end
 ```
 
+```ruby
+# in your plan.rb
+plan = Forklift::Plan.new
+plan.do! do
+  destination = plan.connections[:mysql][:destination]
+  destination.exec!("./transformations/combined_name.sql", {name: 'user counter'})
+
+  end
+```
+
 ## Options & Notes
 - Thanks to [@rahilsondhi](https://github.com/rahilsondhi) from [InternMatch](http://www.internmatch.com/) for all his help
 - email_options is a hash consumed by the [Pony mail gem](https://github.com/benprew/pony)

data/spec/integration/basic_spec.rb

@@ -10,6 +10,7 @@ describe 'basics' do
         tables << row.values[0]
       end
       expect(tables.count).to eql 3
+      client.close
 
       client = SpecClient.mysql('forklift_test_source_b')
       tables = []
@@ -17,6 +18,7 @@ describe 'basics' do
         tables << row.values[0]
       end
       expect(tables.count).to eql 1
+      client.close
     end
 
     it 'seeded the elasticsearch db' do

data/spec/integration/elasticsearch_spec.rb

@@ -17,6 +17,7 @@ describe 'elasticsearch' do
         @rows = (@rows + data)
       }
     }
+    plan.disconnect!
 
     expect(@rows.length).to eql 5
   end
@@ -32,6 +33,7 @@ describe 'elasticsearch' do
         @rows = (@rows + data)
       }
     }
+    plan.disconnect!
 
     expect(@rows.length).to eql 3
   end
@@ -46,6 +48,7 @@ describe 'elasticsearch' do
       destination = plan.connections[:elasticsearch][:forklift_test]
       destination.write(data, index)
     }
+    plan.disconnect!
 
     destination = SpecClient.elasticsearch('forklift_test')
     count = destination.count({ index: index })["count"]
@@ -63,6 +66,7 @@ describe 'elasticsearch' do
       destination = plan.connections[:elasticsearch][:forklift_test]
       destination.write(data, index, true)
     }
+    plan.disconnect!
 
     destination = SpecClient.elasticsearch('forklift_test')
     count = destination.count({ index: index })["count"]
@@ -91,5 +95,6 @@ describe 'elasticsearch' do
       destination.delete_index(index)
       expect { client.search({ index: index }) }.to raise_error(/IndexMissingException/)
     }    
+    plan.disconnect!
   end
 end

data/spec/integration/multi_transport_spec.rb

@@ -18,6 +18,7 @@ describe 'multiple trasport types' do
         destination = plan.connections[:mysql][:forklift_test_destination]
         source.read(index, query) {|data| destination.write(data, table) }
       }
+      plan.disconnect!
 
       destination = SpecClient.mysql('forklift_test_destination')
       rows = destination.query("select count(1) as 'count' from es_import").first["count"]
@@ -34,6 +35,7 @@ describe 'multiple trasport types' do
         destination = plan.connections[:mysql][:forklift_test_destination]
         source.read(index, query, false, 0, 3) {|data| destination.write(data, table) }
       }
+      plan.disconnect!
 
       destination = SpecClient.mysql('forklift_test_destination')
       rows = destination.query("select count(1) as 'count' from es_import").first["count"]
@@ -61,6 +63,7 @@ describe 'multiple trasport types' do
           destination.write(clean_data, table) 
         }
       }
+      plan.disconnect!
 
       destination = SpecClient.mysql('forklift_test_destination')
       max = destination.query("select max(viewed_at) as 'max' from es_import").first["max"]
@@ -85,6 +88,7 @@ describe 'multiple trasport types' do
         destination = plan.connections[:elasticsearch][:forklift_test]
         source.read("select * from #{table}") {|data| destination.write(data, index) }
       }
+      plan.disconnect!
 
       destination = SpecClient.elasticsearch('forklift_test')
       count = destination.count({ index: index })["count"]
@@ -102,6 +106,7 @@ describe 'multiple trasport types' do
           destination.write(data, index) 
         }
       }
+      plan.disconnect!
 
       destination = SpecClient.elasticsearch('forklift_test')
       count = destination.count({ index: index })["count"]

data/spec/integration/mysql_patterns_spec.rb

@@ -21,6 +21,7 @@ describe 'mysql patterns' do
 
       expect(destination.tables.length).to eql 3
     }
+    plan.disconnect!
   end
 
   it "can do an incramental data pipe with only updated data" do
@@ -42,6 +43,7 @@ describe 'mysql patterns' do
       expect(destination.count('users')).to eql 5
       expect(destination.read('select first_name from users where id = 1')[0][:first_name]).to eql 'EvanAgain'
     }
+    plan.disconnect!
   end
 
   it "(optimistic_pipe) can determine if it should do an incramental or full pipe" do
@@ -52,6 +54,7 @@ describe 'mysql patterns' do
       expect(Forklift::Patterns::Mysql.can_incremental_pipe?(source, 'sales', source, 'sales')).to eql false
       expect(Forklift::Patterns::Mysql.can_incremental_pipe?(source, 'products', source, 'products')).to eql true
     }
+    plan.disconnect!
   end
 
   it "can run the mysql_optimistic_import pattern" do
@@ -72,6 +75,7 @@ describe 'mysql patterns' do
       expect(destination.count('users')).to eql 5
       expect(destination.read('select first_name from users where id = 1')[0][:first_name]).to eql 'EvanAgain'
     }
+    plan.disconnect!
   end
 
   it "can write the high_water_mark"

data/spec/integration/mysql_spec.rb

@@ -16,6 +16,7 @@ describe 'mysql' do
         @rows = (@rows + data)
       }
     }
+    plan.disconnect!
 
     expect(@rows.length).to eql 5
   end
@@ -30,6 +31,7 @@ describe 'mysql' do
         @rows = (@rows + data)
       }
     }
+    plan.disconnect!
 
     expect(@rows.length).to eql 3
   end
@@ -45,6 +47,7 @@ describe 'mysql' do
       destination = plan.connections[:mysql][:forklift_test_source_a]
       destination.write(data, table)
     }
+    plan.disconnect!
 
     destination = SpecClient.mysql('forklift_test_source_a')
     count = destination.query('select count(1) as "count" from users').first['count']
@@ -61,6 +64,7 @@ describe 'mysql' do
       destination = plan.connections[:mysql][:forklift_test_source_a]
       destination.write(data, table)
     }
+    plan.disconnect!
 
     destination = SpecClient.mysql('forklift_test_source_a')
     count = destination.query('select count(1) as "count" from users').first['count']
@@ -88,6 +92,7 @@ describe 'mysql' do
         destination = plan.connections[:mysql][:forklift_test_source_a]
         destination.write(data, table)
       }
+      plan.disconnect!
 
       destination = SpecClient.mysql('forklift_test_source_a')
       cols = []
@@ -117,6 +122,7 @@ describe 'mysql' do
         destination = plan.connections[:mysql][:forklift_test_source_a]
         destination.write(data, table)
       }
+      plan.disconnect!
 
       destination = SpecClient.mysql('forklift_test_source_a')
       cols = []
@@ -151,6 +157,7 @@ describe 'mysql' do
         destination = plan.connections[:mysql][:forklift_test_source_a]
         destination.write(data, table)
       }
+      plan.disconnect!
 
       count = raw.query("SHOW COLUMNS FROM #{table}").count
       expect(count).to eql 7
@@ -168,6 +175,7 @@ describe 'mysql' do
         destination = plan.connections[:mysql][:forklift_test_source_a]
         destination.write(data, table)
       }
+      plan.disconnect!
 
       destination = SpecClient.mysql('forklift_test_source_a')
       cols = []
@@ -200,6 +208,7 @@ describe 'mysql' do
         destination = plan.connections[:mysql][:forklift_test_source_a]
         destination.write(data, table)
       }
+      plan.disconnect!
 
       destination = SpecClient.mysql('forklift_test_source_a')
       cols = []
@@ -226,6 +235,7 @@ describe 'mysql' do
         destination = plan.connections[:mysql][:forklift_test_source_a]
         destination.write(data, table)
       }
+      plan.disconnect!
 
       destination = SpecClient.mysql('forklift_test_source_a')
       cols = []

data/spec/integration/transformations_spec.rb

@@ -0,0 +1,54 @@
+require 'spec_helper'
+
+describe 'transformations' do  
+
+  before(:each) do
+    SpecSeeds.setup_mysql
+  end
+
+  it "can run a native transformation" do
+    plan = SpecPlan.new
+    @rows = []
+
+    raw = SpecClient.mysql('forklift_test_destination')
+
+    plan.do! {
+      source      = plan.connections[:mysql][:forklift_test_source_a]
+      destination = plan.connections[:mysql][:forklift_test_destination]
+      source.read('select * from `users`') {|data| destination.write(data, 'users') }
+
+      expect( destination.columns("users").include?("full_name") ).to eql false
+
+      transformation_file = "#{File.dirname(__FILE__)}/../template/spec_user_transformation.sql"
+      destination.exec!(transformation_file)
+
+      expect( destination.columns("users").include?("full_name") ).to eql true
+    }
+    plan.disconnect!
+  end
+
+  it "can run a ruby transformation" do
+    plan = SpecPlan.new
+    @rows = []
+
+    raw = SpecClient.mysql('forklift_test_destination')
+
+    plan.do! {
+      source      = plan.connections[:mysql][:forklift_test_source_a]
+      destination = plan.connections[:mysql][:forklift_test_destination]
+      source.read('select * from `users`') {|data| destination.write(data, 'users') }
+
+      expect( destination.columns("users").include?("full_name") ).to eql false
+
+      transformation_file = "#{File.dirname(__FILE__)}/../template/spec_user_transformation.rb"
+      destination.exec!(transformation_file, {prefix: 'my_prefix' })
+
+      expect( destination.columns("users").include?("full_name") ).to eql true
+
+      data = destination.read('select * from `users` where email="evan@example.com"')
+      expect( data.first[:full_name] ).to eql 'my_prefix Evan T'
+    }
+    plan.disconnect!
+  end
+
+end

data/spec/template/spec_user_transformation.rb

@@ -0,0 +1,8 @@
+class SpecUserTransformation
+
+  def do!(connection, forklift, args)
+    connection.q("ALTER TABLE `users` ADD `full_name` VARCHAR(255)  NULL  DEFAULT NULL  AFTER `updated_at`;")
+    connection.q("UPDATE `users` SET full_name = CONCAT('#{args[:prefix]}', ' ', first_name, ' ', last_name);")
+  end
+
+end

data/spec/template/spec_user_transformation.sql

@@ -0,0 +1,2 @@
+ALTER TABLE `users` ADD `full_name` VARCHAR(255)  NULL  DEFAULT NULL  AFTER `updated_at`;
+UPDATE `users` SET full_name = CONCAT(first_name, ' ', last_name);

data/spec/unit/connection/mysql_spec.rb

@@ -16,6 +16,7 @@ describe Forklift::Connection::Mysql do
         expect(source.tables).to include 'products'
         expect(source.tables).to include 'sales'
       }
+      plan.disconnect!
     end
 
     it "can delte a table" do 
@@ -27,6 +28,7 @@ describe Forklift::Connection::Mysql do
         source.drop! table
         expect(source.tables).to_not include 'users'
       }
+      plan.disconnect!
     end
 
     it "can count the rows in a table" do
@@ -36,6 +38,7 @@ describe Forklift::Connection::Mysql do
         source = plan.connections[:mysql][:forklift_test_source_a]
         expect(source.count(table)).to eql 5
       }
+      plan.disconnect!
     end
 
     it "can truncate a table (both with and without !)" do 
@@ -48,6 +51,7 @@ describe Forklift::Connection::Mysql do
         expect(source.count(table)).to eql 0
         expect { source.truncate(table) }.to_not raise_error
       }
+      plan.disconnect!
     end
 
     it 'trunacte! will raise if the table does not exist' do
@@ -57,6 +61,7 @@ describe Forklift::Connection::Mysql do
         source = plan.connections[:mysql][:forklift_test_source_a]
         expect { source.truncate!(table) }.to raise_error(/Table 'forklift_test_source_a.other_table' doesn't exist/)
       }
+      plan.disconnect!
     end
 
     it "can get the columns of a table" do 
@@ -69,6 +74,7 @@ describe Forklift::Connection::Mysql do
         expect(source.columns(table)).to include 'product_id' 
         expect(source.columns(table)).to include 'timestamp' 
       }
+      plan.disconnect!
     end
 
     it "can create a mysqldump" do
@@ -78,6 +84,7 @@ describe Forklift::Connection::Mysql do
         source = plan.connections[:mysql][:forklift_test_source_a]
         source.dump(dump)
       }
+      plan.disconnect!
 
       expect(File.exists?(dump)).to eql true
       contents = Zlib::GzipReader.new(StringIO.new(File.read(dump))).read  

data/spec/unit/misc/email_spec.rb

@@ -21,6 +21,7 @@ describe 'misc forklift core' do
         email_template = "#{File.dirname(__FILE__)}/../../template/spec_email_template.erb"
         @email = plan.mailer.send_template(email_args, email_template, email_variables).first
       }
+      plan.disconnect!
       
       expect(@email).to deliver_to("YOU@FAKE.com")
       expect(@email).to have_subject(/Forklift has moved your database/)

data/spec/unit/misc/error_spec.rb

@@ -11,6 +11,7 @@ describe 'misc forklift core' do
         }
       }.to raise_error 'BREAK'
       plan.pid.delete!
+      plan.disconnect!
     end
 
     it 'can make error handlers' do
@@ -24,6 +25,7 @@ describe 'misc forklift core' do
       plan.do! {
         plan.step("step_a", error_handler){ raise 'BREAK' }
       }
+      plan.disconnect!
 
       expect(name).to    eql :step_a
       expect(ex.to_s).to eql 'BREAK'

data/spec/unit/misc/pid_spec.rb

@@ -11,6 +11,7 @@ describe 'misc forklift core' do
         expect(File.exists?(pid)).to eql true
         expect(File.read(pid).to_i).to eql Process.pid
       }
+      plan.disconnect!
       expect(File.exists?(pid)).to eql false
     end
 
@@ -19,6 +20,7 @@ describe 'misc forklift core' do
       plan.pid.store!
       expect { plan.do! }.to raise_error SystemExit
       plan.pid.delete!
+      plan.disconnect!
     end 
   end
 

data/spec/unit/misc/step_spec.rb

@@ -20,6 +20,7 @@ describe 'misc forklift core' do
         plan.step("b"){ steps_run << 'b' }
         plan.step("c"){ steps_run << 'c' }
       }
+      plan.disconnect!
       expect(steps_run).to include 'a'
       expect(steps_run).to include 'b'
       expect(steps_run).to include 'c'
@@ -34,6 +35,7 @@ describe 'misc forklift core' do
         plan.step("b"){ steps_run << 'b' }
         plan.step("c"){ steps_run << 'c' }
       }
+      plan.disconnect!
       expect(steps_run).to include 'a'
       expect(steps_run).to_not include 'b'
       expect(steps_run).to include 'c'
@@ -46,6 +48,7 @@ describe 'misc forklift core' do
         plan.do! {
           plan.step("a"){ raise 'never should get here' }
         }
+        plan.disconnect!
       }.to raise_error SystemExit
     end
   end

metadata

@@ -1,124 +1,113 @@
 --- !ruby/object:Gem::Specification
 name: forklift_etl
 version: !ruby/object:Gem::Version
-  version: 1.1.12
-  prerelease: 
+  version: 1.2.0
 platform: ruby
 authors:
 - Evan Tahler
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-12-31 00:00:00.000000000 Z
+date: 2015-01-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '4.0'
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 4.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '4.0'
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 4.0.0
 - !ruby/object:Gem::Dependency
   name: mysql2
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.0'
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.0.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.0'
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.0.1
 - !ruby/object:Gem::Dependency
   name: elasticsearch
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.0.0
 - !ruby/object:Gem::Dependency
   name: pony
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.0.0
 - !ruby/object:Gem::Dependency
   name: lumberjack
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.0.0
 description: A collection of ETL tools and patterns for mysql and elasticsearch.
@@ -129,9 +118,9 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
-- .rbenv-version
-- .travis.yml
+- ".gitignore"
+- ".ruby-version"
+- ".travis.yml"
 - Gemfile
 - Gemfile.lock
 - LICENSE.txt
@@ -180,6 +169,7 @@ files:
 - spec/integration/multi_transport_spec.rb
 - spec/integration/mysql_patterns_spec.rb
 - spec/integration/mysql_spec.rb
+- spec/integration/transformations_spec.rb
 - spec/spec_helper.rb
 - spec/support/dumps/csv/source.csv
 - spec/support/dumps/elasticsearch/forklift_test.json
@@ -189,6 +179,8 @@ files:
 - spec/support/spec_plan.rb
 - spec/support/spec_seeds.rb
 - spec/template/spec_email_template.erb
+- spec/template/spec_user_transformation.rb
+- spec/template/spec_user_transformation.sql
 - spec/unit/connection/mysql_spec.rb
 - spec/unit/misc/email_spec.rb
 - spec/unit/misc/error_spec.rb
@@ -202,28 +194,27 @@ files:
 homepage: https://github.com/taskrabbit/forklift
 licenses:
 - MIT
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: forklift_etl
-rubygems_version: 1.8.23
+rubygems_version: 2.2.2
 signing_key: 
-specification_version: 3
-summary: ! 'Forklift: Moving big databases around. A ruby ETL tool.'
+specification_version: 4
+summary: 'Forklift: Moving big databases around. A ruby ETL tool.'
 test_files:
 - spec/config/connections/csv/forklift_test_destination.yml
 - spec/config/connections/csv/forklift_test_source.yml
@@ -240,6 +231,7 @@ test_files:
 - spec/integration/multi_transport_spec.rb
 - spec/integration/mysql_patterns_spec.rb
 - spec/integration/mysql_spec.rb
+- spec/integration/transformations_spec.rb
 - spec/spec_helper.rb
 - spec/support/dumps/csv/source.csv
 - spec/support/dumps/elasticsearch/forklift_test.json
@@ -249,8 +241,11 @@ test_files:
 - spec/support/spec_plan.rb
 - spec/support/spec_seeds.rb
 - spec/template/spec_email_template.erb
+- spec/template/spec_user_transformation.rb
+- spec/template/spec_user_transformation.sql
 - spec/unit/connection/mysql_spec.rb
 - spec/unit/misc/email_spec.rb
 - spec/unit/misc/error_spec.rb
 - spec/unit/misc/pid_spec.rb
 - spec/unit/misc/step_spec.rb
+has_rdoc: 

data/.rbenv-version

@@ -1 +0,0 @@
-1.9.3-p194