forklift_etl 1.2.1 → 1.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 94c0245d54361f22228d060adfc79dddc9dcf39b
-  data.tar.gz: 85ff71007431776dc19480d24567611a6a03a222
+  metadata.gz: 1718155b7b286739b163fe0c386488035dad0d0d
+  data.tar.gz: 00e9c7b12f1f1c46da88c2ea6051ca3fd2c38b85
 SHA512:
-  metadata.gz: 7285a14abf86b2b0f509ab3d4aa2197509ccd462132d5a2d81d0fd6ddba5373ff61c850de4d49cfdd072fe55c329efdf917c489f70fa07184d6c03f4f0b5ee74
-  data.tar.gz: 67e20f3b594f3d85fcee6cfe74e8051512ad2b2e2e4ef9847a050f00bcb340ea083b2c11e5ae3cc1e31635a1624be43c50cbe37b062eb90130826ce05c71bcfb
+  metadata.gz: a254bafa76f5bf176fcd4ec75233732961e35522afd12f04c4e994f125641bc7ed554137fca541761e630e334c119bb7d89ec3ef1f4a476e06a6540692db9970
+  data.tar.gz: 9b4d36cfa659ac78511b235c3ccd14dea7fe817de65f4c7c6507fc14350ec85bb21b6a1778c7a440c178145ab6bb0404e0d842b4f8cc78de6052626337a26f7c

data/Gemfile.lock

@@ -73,3 +73,6 @@ DEPENDENCIES
   forklift_etl!
   rake
   rspec
+
+BUNDLED WITH
+   1.11.2

data/lib/forklift/transports/elasticsearch.rb

@@ -17,7 +17,7 @@ module Forklift
         @forklift
       end
 
-      def connect 
+      def connect
         @client = ::Elasticsearch::Client.new(config)
       end
 

data/lib/forklift/transports/mysql.rb

@@ -99,7 +99,7 @@ module Forklift
             insert_q << safe_values(columns, d)
             insert_q << ","
           end
-          
+
           if delete_keys.length > 0
             delete_q << "(#{delete_keys.join(',')})"
             q(delete_q)
@@ -129,22 +129,26 @@ module Forklift
         keys = {}
         data.each do |item|
           item.each do |k,v|
-            keys[k] = sql_type(v) if ( keys[k].nil? && v.class != NilClass )
+            keys[k.to_s] = sql_type(v) if (keys[k.to_s].nil? || keys[k.to_s] == sql_type(nil))
           end
         end
+        keys[primary_key] = 'bigint(20)' unless keys.has_key?(primary_key)
 
-        data.first.each do |k,v|
-          keys[k] = sql_type(v) if ( keys[k].nil? )
+        col_defn = keys.map do |col, type|
+          if col == primary_key
+            "`#{col}` #{type} NOT NULL AUTO_INCREMENT"
+          else
+            "`#{col}` #{type} DEFAULT NULL"
+          end
         end
+        col_defn << "PRIMARY KEY (`#{primary_key}`)"
+        col_defn << "KEY `#{matcher}` (`#{matcher}`)" if keys.include?(matcher.to_sym)
 
-        command = "CREATE TABLE `#{database}`.`#{table}` ( "
-        command << " `#{primary_key}` bigint(20) NOT NULL AUTO_INCREMENT, " if ( data.first[primary_key.to_sym].nil? )
-        keys.each do |col, type|
-          command << " `#{col}` #{type} DEFAULT NULL, "
-        end
-        command << " PRIMARY KEY (`#{primary_key}`) "
-        command << " , KEY `#{matcher}` (`#{matcher}`) " if keys.include?(matcher.to_sym)
-        command << " ) "
+        command = <<-EOS
+        CREATE TABLE `#{database}`.`#{table}` (
+          #{col_defn.join(', ')}
+        )
+        EOS
 
         q(command)
         forklift.logger.log "lazy-created table `#{database}`.`#{table}`"
@@ -238,7 +242,7 @@ module Forklift
         cmd << " | gzip > #{file}"
         forklift.logger.log "Dumping #{config['database']} to #{file}"
         forklift.logger.debug cmd
-        
+
         stdin, stdout, stderr = Open3.popen3(cmd)
         stdout = stdout.readlines
         stderr = stderr.readlines
@@ -285,7 +289,7 @@ module Forklift
             end
 
           end
-        end 
+        end
       end
 
       def safe_columns(cols)

data/lib/forklift/version.rb

@@ -1,3 +1,3 @@
 module Forklift
-  VERSION = "1.2.1"
+  VERSION = "1.2.2"
 end

data/readme.md

@@ -1,4 +1,12 @@
 # Forklift ETL
+
+---
+
+## THIS TOOL IS NO LONGER SUPORTED.
+It has been succeded by [Empujar](https://github.com/taskrabbit/empujar)
+
+---
+
 Moving heavy databases around. [![Gem Version](https://badge.fury.io/rb/forklift_etl.svg)](http://badge.fury.io/rb/forklift_etl)
 [![Build Status](https://secure.travis-ci.org/taskrabbit/forklift.png?branch=master)](http://travis-ci.org/taskrabbit/forklift)
 

data/spec/integration/elasticsearch_spec.rb

@@ -29,7 +29,7 @@ describe 'elasticsearch' do
     @rows = []
     plan.do! {
       source = plan.connections[:elasticsearch][:forklift_test]
-      source.read(index, query, false, 0, 3) {|data| 
+      source.read(index, query, false, 0, 3) {|data|
         @rows = (@rows + data)
       }
     }
@@ -56,7 +56,7 @@ describe 'elasticsearch' do
     expect(count).to eql 6
   end
 
-  it "can overwrite existing data, probided a primary key" do 
+  it "can overwrite existing data, probided a primary key" do
     index = 'forklift_test'
     plan = SpecPlan.new
     data = [
@@ -80,7 +80,7 @@ describe 'elasticsearch' do
     expect(obj["viewed_at"]).to eql 99
   end
 
-  it "can delete an index" do 
+  it "can delete an index" do
     index = 'other_test_index'
     plan = SpecPlan.new
     client = SpecClient.elasticsearch('forklift_test')
@@ -89,12 +89,12 @@ describe 'elasticsearch' do
     ]
     plan.do! {
       destination = plan.connections[:elasticsearch][:forklift_test]
-      expect { client.search({ index: index }) }.to raise_error(/IndexMissingException/)
+      expect { client.search({ index: index }) }.to raise_error(/index_not_found_exception|IndexMissingException/)
       destination.write(data, index, true)
       expect { client.search({ index: index }) }.to_not raise_error
       destination.delete_index(index)
-      expect { client.search({ index: index }) }.to raise_error(/IndexMissingException/)
-    }    
+      expect { client.search({ index: index }) }.to raise_error(/index_not_found_exception|IndexMissingException/)
+    }
     plan.disconnect!
   end
-end
+end

data/spec/integration/mysql_spec.rb

@@ -1,6 +1,6 @@
 require 'spec_helper'
 
-describe 'mysql' do  
+describe 'mysql' do
 
   before(:each) do
     SpecSeeds.setup_mysql
@@ -12,7 +12,7 @@ describe 'mysql' do
     @rows = []
     plan.do! {
       source = plan.connections[:mysql][:forklift_test_source_a]
-      source.read(query) {|data| 
+      source.read(query) {|data|
         @rows = (@rows + data)
       }
     }
@@ -20,14 +20,14 @@ describe 'mysql' do
 
     expect(@rows.length).to eql 5
   end
-  
+
   it "can read data (filtered)" do
     query = 'select * from `users`'
     plan = SpecPlan.new
     @rows = []
     plan.do! {
       source = plan.connections[:mysql][:forklift_test_source_a]
-      source.read(query, source.current_database, false, 3, 0) {|data| 
+      source.read(query, source.current_database, false, 3, 0) {|data|
         @rows = (@rows + data)
       }
     }
@@ -54,7 +54,7 @@ describe 'mysql' do
     expect(count).to eql 7
   end
 
-  it "can update existing data" do 
+  it "can update existing data" do
     table = "users"
     data = [
       {id: 1, email: 'evan@example.com', first_name: 'New Name', last_name: 'T', created_at: Time.new.to_s(:db), updated_at: Time.new.to_s(:db)}
@@ -79,8 +79,8 @@ describe 'mysql' do
       destination = SpecClient.mysql('forklift_test_source_a')
       destination.query('drop table if exists `new_table`')
     end
-    
-    it "can lazy-create a table with primary keys provided" do 
+
+    it "can lazy-create a table with primary keys provided" do
       data = [
         {id: 1, thing: 'stuff a', updated_at: Time.new},
         {id: 2, thing: 'stuff b', updated_at: Time.new},
@@ -99,17 +99,17 @@ describe 'mysql' do
       destination.query("describe #{table}").each do |row|
         cols << row["Field"]
         case row["Field"]
-        when "id" 
+        when "id"
           expect(row["Type"]).to eql "bigint(20)"
-        when "thing" 
+        when "thing"
           expect(row["Type"]).to eql "text"
-        when "updated_at" 
+        when "updated_at"
           expect(row["Type"]).to eql "datetime"
         end
       end
       expect(cols).to eql ['id', 'thing', 'updated_at']
     end
-    
+
     it "can lazy-create a table without primary keys provided" do
       data = [
         {thing: 'stuff a', number: 1.123, updated_at: Time.new},
@@ -129,17 +129,17 @@ describe 'mysql' do
       destination.query("describe #{table}").each do |row|
         cols << row["Field"]
         case row["Field"]
-        when "id" 
+        when "id"
           expect(row["Type"]).to eql "bigint(20)"
-        when "thing" 
+        when "thing"
           expect(row["Type"]).to eql "text"
-        when "number" 
+        when "number"
           expect(row["Type"]).to eql "float"
-        when "updated_at" 
+        when "updated_at"
           expect(row["Type"]).to eql "datetime"
         end
       end
-      expect(cols).to eql ['id', 'thing', 'number', 'updated_at']
+      expect(cols).to include('id', 'thing', 'number', 'updated_at')
     end
 
     it "can add columns to exiting tables when new keys are provided" do
@@ -163,7 +163,7 @@ describe 'mysql' do
       expect(count).to eql 7
     end
 
-    it "can will seek further for null-ish values" do 
+    it "can will seek further for null-ish values" do
       data = [
         {id: 1, thing: 'stuff a', number: nil, updated_at: Time.new},
         {id: 2, thing: 'stuff b', number: nil, updated_at: Time.new},
@@ -182,25 +182,25 @@ describe 'mysql' do
       destination.query("describe #{table}").each do |row|
         cols << row["Field"]
         case row["Field"]
-        when "id" 
+        when "id"
           expect(row["Type"]).to eql "bigint(20)"
-        when "thing" 
+        when "thing"
           expect(row["Type"]).to eql "text"
-        when "number" 
+        when "number"
           expect(row["Type"]).to eql "bigint(20)"
-        when "updated_at" 
+        when "updated_at"
           expect(row["Type"]).to eql "datetime"
         end
       end
-      expect(cols).to eql ['id', 'thing', 'updated_at', 'number']
+      expect(cols).to include('id', 'thing', 'updated_at', 'number')
     end
 
-    it "null rows will be text, and can be updated on subsequent writes" do 
+    it "null rows will be text, and can be updated on subsequent writes" do
       data = [
         {id: 1, number: nil, updated_at: Time.new},
         {id: 2, number: nil, updated_at: Time.new},
       ]
-      
+
       table = "new_table"
 
       plan = SpecPlan.new
@@ -216,15 +216,15 @@ describe 'mysql' do
       destination.query("describe #{table}").each do |row|
         cols << row["Field"]
         case row["Field"]
-        when "id" 
+        when "id"
           expect(row["Type"]).to eql "bigint(20)"
-        when "number" 
+        when "number"
           expect(row["Type"]).to eql "varchar(0)"
-        when "updated_at" 
+        when "updated_at"
           expect(row["Type"]).to eql "datetime"
         end
       end
-      expect(cols).to eql ['id', 'updated_at', 'number']
+      expect(cols).to include('id', 'updated_at', 'number')
 
       data = [
         {id: 3, number: 123, updated_at: Time.new},
@@ -243,13 +243,13 @@ describe 'mysql' do
       destination.query("describe #{table}").each do |row|
         cols << row["Field"]
         case row["Field"]
-        when "number" 
+        when "number"
           expect(row["Type"]).to eql "bigint(20)"
         end
       end
-      expect(cols).to eql ['id', 'updated_at', 'number']
+      expect(cols).to include('id', 'updated_at', 'number')
     end
 
   end
 
-end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: forklift_etl
 version: !ruby/object:Gem::Version
-  version: 1.2.1
+  version: 1.2.2
 platform: ruby
 authors:
 - Evan Tahler
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-01-16 00:00:00.000000000 Z
+date: 2016-03-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport