forklift_etl 1.1.5 → 1.1.6

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    YWJmZWViNzNiMTljYzRmMDJjZmUyNDhlYWJhODU5ZTM5MmFkNmMwYg==
+    MThmMGU3YjE2YWU5YTNhMmM1NWU4ZjNlNGNjODk0Njk2ZGI2ZmZiMQ==
   data.tar.gz: !binary |-
-    NDU1OTJlN2Q2MWU4ZDEyYTJhNzRiYWI0YmIzZjA5NzQ1NjVhODVmMg==
+    YTA0N2JkYTQ2YTJiOWJjZDNjOTlkNTY2OGU0OTVlOGIwZDg2MWY4ZQ==
 SHA512:
   metadata.gz: !binary |-
-    OWRiM2QzNzVkYjE2OGM1Y2VmMmQxNWQyZGEyYmMyM2Q1YmUyYTI3NmZkZjI1
-    MzExMzA4M2EwY2IwMWIyNTI1NzI0YTMzZDU4MjE4ZWEwYzY2Y2U4YzgwYmEz
-    NjM3MDI3YjVjMjE2OWI0ZmQwY2FmMzI2NjQzZTBmYmNkZjFmNzY=
+    NzNlODE5ZDNmMzc3NTVhYjBhY2NkN2Q4NDBmMWQ3NmU4YzRlYzc4MzYzM2Fm
+    Nzk1NTA4YmRkOTkyZTViMGU0MzBlMjZjMzZhOGU3ZmUxMWUzNzI4YTAyZjMx
+    YzhmZWVmMmIwMjZjYTA4ODAzYjQ4NTBjOTI5Y2Y2NzY1YzVhM2Y=
   data.tar.gz: !binary |-
-    ODJlMGI3ODUwZWQ2M2VmYzljODMyNmQzZThjZjUzMDY2NWVhZDcxNGQwYWIy
-    MDFkNDZiMjg2MmRiNTIyOThkNDQxYWM4MjNjOGEyZDVjODA4N2YxMDA5NGY2
-    ZmYzZmU4YzljYjA5YjJkYWUxNDcwMGM0NDIyNjA3MDVlZWFkYjE=
+    YzcwZTg5YjZiOGI1ZjYxNzQ1ZjIwMmVkM2JmMDI0YTU1ZTQwYTQxYzRiMGE1
+    OWZhNTNhOTM4N2NhMGM5MGEyNDc0NzZkMTk2MjU3ODhhNDI1OTk5YWI4ODAx
+    MmMyMTdhZGE2YmIwN2FkZDQ0MjllYzlhNDAyNzFiMGJjMzk5NzA=

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    forklift_etl (1.1.5)
+    forklift_etl (1.1.6)
       activesupport (~> 4.0, >= 4.0.0)
       elasticsearch (~> 1.0, >= 1.0.0)
       lumberjack (~> 1.0, >= 1.0.0)
@@ -11,7 +11,7 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (4.1.2)
+    activesupport (4.1.4)
       i18n (~> 0.6, >= 0.6.9)
       json (~> 1.7, >= 1.7.7)
       minitest (~> 5.1)
@@ -28,40 +28,39 @@ GEM
     elasticsearch-transport (1.0.4)
       faraday
       multi_json
-    email_spec (1.5.0)
+    email_spec (1.6.0)
       launchy (~> 2.1)
       mail (~> 2.2)
     faraday (0.9.0)
       multipart-post (>= 1.2, < 3)
-    i18n (0.6.9)
+    i18n (0.6.11)
     json (1.8.1)
     launchy (2.4.2)
       addressable (~> 2.3)
     lumberjack (1.0.9)
-    mail (2.5.4)
-      mime-types (~> 1.16)
-      treetop (~> 1.4.8)
-    mime-types (1.25.1)
-    minitest (5.3.5)
+    mail (2.6.1)
+      mime-types (>= 1.16, < 3)
+    mime-types (2.3)
+    minitest (5.4.0)
     multi_json (1.10.1)
     multipart-post (2.0.0)
     mysql2 (0.3.16)
-    polyglot (0.3.4)
-    pony (1.9)
+    pony (1.10)
       mail (>= 2.0)
-    rake (10.2.2)
-    rspec (2.14.1)
-      rspec-core (~> 2.14.0)
-      rspec-expectations (~> 2.14.0)
-      rspec-mocks (~> 2.14.0)
-    rspec-core (2.14.8)
-    rspec-expectations (2.14.5)
-      diff-lcs (>= 1.1.3, < 2.0)
-    rspec-mocks (2.14.6)
+    rake (10.3.2)
+    rspec (3.0.0)
+      rspec-core (~> 3.0.0)
+      rspec-expectations (~> 3.0.0)
+      rspec-mocks (~> 3.0.0)
+    rspec-core (3.0.2)
+      rspec-support (~> 3.0.0)
+    rspec-expectations (3.0.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.0.0)
+    rspec-mocks (3.0.2)
+      rspec-support (~> 3.0.0)
+    rspec-support (3.0.2)
     thread_safe (0.3.4)
-    treetop (1.4.15)
-      polyglot
-      polyglot (>= 0.3.1)
     tzinfo (1.2.1)
       thread_safe (~> 0.1)
 

data/lib/forklift/transports/elasticsearch.rb

@@ -28,6 +28,7 @@ module Forklift
           prepared_query[:from] = from + offset
           prepared_query[:size] = size
 
+          forklift.logger.debug "    ELASTICSEARCH: #{query}"
           results = client.search( { index: index, body: prepared_query } )
           results["hits"]["hits"].each do |hit|
             data << hit["_source"]
@@ -57,12 +58,15 @@ module Forklift
             type:   type,
           }
           object[:id] = d[primary_key] if ( !d[primary_key].nil? && update == true )
+
+          forklift.logger.debug "    ELASTICSEARCH (store): #{object}"
           client.index object
         end
         client.indices.refresh({ index: index })
       end
 
       def delete_index(index)
+        forklift.logger.debug "    ELASTICSEARCH (delete index): #{index}"
         client.indices.delete({ index: index }) if client.indices.exists({ index: index })
       end
 

data/lib/forklift/version.rb

@@ -1,3 +1,3 @@
 module Forklift
-  VERSION = "1.1.5"
+  VERSION = "1.1.6"
 end

data/spec/support/dumps/mysql/forklift_test_source_a.sql

@@ -6,6 +6,7 @@ DROP TABLE IF EXISTS `products`;
 CREATE TABLE `products` (
   `id` int(11) unsigned NOT NULL AUTO_INCREMENT,
   `name` varchar(255) NOT NULL DEFAULT '',
+  `description` text NOT NULL,
   `inventory` int(11) DEFAULT NULL,
   `created_at` datetime NOT NULL,
   `updated_at` datetime NOT NULL,
@@ -14,13 +15,13 @@ CREATE TABLE `products` (
 
 LOCK TABLES `products` WRITE;
 
-INSERT INTO `products` (`id`, `name`, `inventory`, `created_at`, `updated_at`)
+INSERT INTO `products` (`id`, `name`, `description`, `inventory`, `created_at`, `updated_at`)
 VALUES
-	(1,'car',10,'2014-04-03 11:45:51','2014-04-03 11:45:51'),
-	(2,'boat',3,'2014-04-03 11:45:52','2014-04-03 11:45:52'),
-	(3,'bus',5,'2014-04-03 11:45:54','2014-04-03 11:45:54'),
-	(4,'motorcycle',23,'2014-04-03 11:45:56','2014-04-03 11:45:56'),
-	(5,'hang_glider',2,'2014-04-03 11:46:19','2014-04-03 11:46:19');
+	(1,'car','a car',10,'2014-04-03 11:45:51','2014-04-03 11:45:51'),
+	(2,'boat','a boat',3,'2014-04-03 11:45:52','2014-04-03 11:45:52'),
+	(3,'bus','a bus',5,'2014-04-03 11:45:54','2014-04-03 11:45:54'),
+	(4,'motorcycle','a motorcycle',23,'2014-04-03 11:45:56','2014-04-03 11:45:56'),
+	(5,'hang_glider','awesome',2,'2014-04-03 11:46:19','2014-04-03 11:46:19');
 
 UNLOCK TABLES;
 

data/spec/unit/connection/mysql_spec.rb

@@ -91,12 +91,12 @@ describe Forklift::Connection::Mysql do
 
     it "escapes one trailing backslash" do
       values = ["foo\\"]
-      subject.send(:safe_values, values).should == "\"foo\\\\\""
+      expect(subject.send(:safe_values, values)).to eq("\"foo\\\\\"")
     end
 
     it "escapes two trailing backslashes" do
       values = ["foo\\\\"]
-      subject.send(:safe_values, values).should == "\"foo\\\\\\\\\""
+      expect(subject.send(:safe_values, values)).to eq("\"foo\\\\\\\\\"")
     end
   end
 end

data/spec/unit/misc/email_spec.rb

@@ -22,15 +22,15 @@ describe 'misc forklift core' do
         @email = plan.mailer.send_template(email_args, email_template, email_variables).first
       }
       
-      @email.should deliver_to("YOU@FAKE.com")
-      @email.should have_subject(/Forklift has moved your database/)
-      @email.should have_body_text(/Your forklift email/) # base 
-      @email.should have_body_text(/Total Users: 10/) # template
-      @email.should have_body_text(/New Users: 5/) # template
+      expect(@email).to deliver_to("YOU@FAKE.com")
+      expect(@email).to have_subject(/Forklift has moved your database/)
+      expect(@email).to have_body_text(/Your forklift email/) # base
+      expect(@email).to have_body_text(/Total Users: 10/) # template
+      expect(@email).to have_body_text(/New Users: 5/) # template
     end
 
     it "can send mail with an attachment" do 
-      pending("how to test email attachments?")
+      skip("how to test email attachments?")
     end
   end
 

data/spec/unit/misc/step_spec.rb

@@ -13,7 +13,7 @@ describe 'misc forklift core' do
 
     it "will run all steps with no extra ARGV" do
       plan = SpecPlan.new
-      plan.stub(:argv){ ['/path/to/plan'] }
+      allow(plan).to receive(:argv){ ['/path/to/plan'] }
       steps_run = []
       plan.do! {
         plan.step("a"){ steps_run << 'a' }
@@ -27,7 +27,7 @@ describe 'misc forklift core' do
 
     it "will only run steps named within ARGV" do
       plan = SpecPlan.new
-      plan.stub(:argv){ ['/path/to/plan', 'a','c'] }
+      allow(plan).to receive(:argv){ ['/path/to/plan', 'a','c'] }
       steps_run = []
       plan.do! {
         plan.step("a"){ steps_run << 'a' }
@@ -41,7 +41,7 @@ describe 'misc forklift core' do
 
     it "won't run on a badly defined step" do
       plan = SpecPlan.new
-      plan.stub(:argv){ ['/path/to/plan', 'missing_step'] }      
+      allow(plan).to receive(:argv){ ['/path/to/plan', 'missing_step'] }
       expect{
         plan.do! {
           plan.step("a"){ raise 'never should get here' }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: forklift_etl
 version: !ruby/object:Gem::Version
-  version: 1.1.5
+  version: 1.1.6
 platform: ruby
 authors:
 - Evan Tahler
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-06-28 00:00:00.000000000 Z
+date: 2014-07-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport