forklift_etl 1.0.19 → 1.1.0

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    OTg3Mzg3YTI0MDg0YTMyMDZiNWZjYWNiNTllNmYwZWZjMGYzZTQxOA==
+    YjE2NGE4ZGQ0N2M3YWI2OTE4MDdkMGIzYTdhZTUzOWRmNzM0YzU3MQ==
   data.tar.gz: !binary |-
-    YzI1NWNkZWE0MzRlMTQ2NDQ5ZDc0MGYxMTUyYWQ1ZWRmMGRkYmEzYQ==
+    N2EzZTBmNDFhZjEyMTQ3MzZkNmZkMDU3NmU3NGY1MzUyYmY2YmU1MQ==
 SHA512:
   metadata.gz: !binary |-
-    MzI0NWNhOWFiNjViODEzNDQzOWRjNTFhZmUwMDdhYWIyMzIxNzE2MWM5ZDdm
-    ZTJlMTBjMjY1MjY4NDUzNjg5YjAxNGI4ODQ3MDYzMjNjOWU2NjFjN2Y1NzFi
-    NDBjYmUzODA0YmNlNmY4OTk3YWIzMmNhNzcwYmM3ZjUwYzFkM2Q=
+    NTQyNzc0OGI0YzMyOGQyNzE4YmQ3MDBlMDEwMDk2ZjQ4ODFlMTI1MmU2M2Mx
+    MzJjZDg2NzA1Y2Y1NDVmOWU0NTAzNDhkNmRhMGFjNGY2ZmE0MGYyYzJmNzBh
+    MGU5NmE3NmYyMTE2MzRjMTc2MzIwZjYwZTE3MWViOTE5ZmRlMDc=
   data.tar.gz: !binary |-
-    MjM1NWZkZWI2MDJhYmNmZmQ2ZGJhZDk3Y2M3YzNmODBhYjhjY2JiNzU3MTgy
-    ZThmM2Y5OWExNjZjYjU0YWNiYWZhZDdhYWE3YjE1MDgzODhkM2MxODk4ZWI0
-    OTA4ZGEzOWI4ZDdmMTdlNjA4OGIzZDQ2YmVkYzY1NzFjOWM5ZjY=
+    N2EwNjZiM2JlNzQzOTFlNGUzM2I5YWEwM2Y0ZGRlMjUyZTZhZTlkZWQ3MTJh
+    NzA3YTE5ZDgzMTkxMzFjZjZiMWYyYzMzZTEzZjVlMDg0NTlhYWUyNDBlODJm
+    Njk2ZGJmOTIwOTgyOWFkNTRkMTFkYjZhMjFkZDMzODZhOGJhOWQ=

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    forklift_etl (1.0.19)
+    forklift_etl (1.1.0)
       activesupport (~> 4.0, >= 4.0.0)
       elasticsearch (~> 1.0, >= 1.0.0)
       lumberjack (~> 1.0, >= 1.0.0)

data/lib/forklift/plan.rb

@@ -37,17 +37,25 @@ module Forklift
           logger.debug "loaded a #{type.camelcase} connection from #{f}"
         rescue Exception => e
           logger.fatal "cannot create a class type of #{loader} from #{f} | #{e}"
-          raise e
+          # raise e ## Don't raise here, but let a step fail so the error_handler can report
         end
       end
     end
 
+
+    def default_error_handler
+      return lambda {|name, e| raise e }
+    end
+
     def step(*args, &block)
-      name = args[0].to_sym
+      name          = args[0].to_sym
+      error_handler = default_error_handler
+      error_handler = args[1] unless args[1].nil?
       self.steps[name] = {
-        :ran => false,
-        :to_run => false,
-        :block => block
+        :ran           => false,
+        :to_run        => false,
+        :block         => block,
+        :error_handler => error_handler,
       }
     end
 
@@ -63,8 +71,12 @@ module Forklift
           self.logger.log "skipping step `#{name}`"
         else
           self.logger.log "*** step: #{name} ***"
-          step[:block].call
-          step[:ran] = true
+          begin
+            step[:block].call
+            step[:ran] = true
+          rescue Exception => e
+            step[:error_handler].call(name, e)
+          end
         end
       end
     end

data/lib/forklift/version.rb

@@ -1,3 +1,3 @@
 module Forklift
-  VERSION = "1.0.19"
+  VERSION = "1.1.0"
 end

data/readme.md

@@ -265,6 +265,29 @@ end
 
 When you use steps, you can run your whole plan, or just part if it with command line arguments.  For example, `forklift plan.rb "Elasticsearch Import"` would just run that single portion of the plan.  Note that any parts of your plan not within a step will be run each time. 
 
+### Error Handling
+
+By default, exceptions within your plan will raise and crash your application.  However, you can pass an optional `error_handler` lambda to your step about how to handle the error.  the `error_handler` will be passed (`step_name`,`exception`).  If you don't re-raise within your error handler, your plan will continue to excecute.  For example:
+
+```ruby
+
+error_handler = lambda { |name, exception|
+  if exception.class =~ /connection/
+    # I can't connect, I should halt
+    raise e
+  elsif exception.class =~ /SoftError/
+    # this type of error is OK
+  else
+    raise e
+  end
+}
+
+plan.step('a_complex_step', error_handler){
+  # ...
+}
+
+```
+
 ## Transports
 
 Transports are how you interact with your data.  Every transport defines `read` and `write` methods which handle arrays of data objects (and the helper methods required).  

data/spec/unit/misc/error_spec.rb

@@ -0,0 +1,33 @@
+require 'spec_helper'
+
+describe 'misc forklift core' do  
+  describe 'error handling' do
+
+    it "un-caught errors will raise" do
+      plan = SpecPlan.new
+      expect{
+        plan.do! {
+          plan.step("step_a"){ raise 'BREAK' }
+        }
+      }.to raise_error 'BREAK'
+      plan.pid.delete!
+    end
+
+    it 'can make error handlers' do
+      plan = SpecPlan.new
+      name = ''
+      ex   = ''
+      error_handler = lambda{ |n, e| 
+        ex   = e
+        name = n
+      }
+      plan.do! {
+        plan.step("step_a", error_handler){ raise 'BREAK' }
+      }
+
+      expect(name).to    eql :step_a
+      expect(ex.to_s).to eql 'BREAK'
+    end
+
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: forklift_etl
 version: !ruby/object:Gem::Version
-  version: 1.0.19
+  version: 1.1.0
 platform: ruby
 authors:
 - Evan Tahler
@@ -179,6 +179,7 @@ files:
 - spec/template/spec_email_template.erb
 - spec/unit/connection/mysql_spec.rb
 - spec/unit/misc/email_spec.rb
+- spec/unit/misc/error_spec.rb
 - spec/unit/misc/pid_spec.rb
 - spec/unit/misc/step_spec.rb
 - template/destination.yml
@@ -237,6 +238,7 @@ test_files:
 - spec/template/spec_email_template.erb
 - spec/unit/connection/mysql_spec.rb
 - spec/unit/misc/email_spec.rb
+- spec/unit/misc/error_spec.rb
 - spec/unit/misc/pid_spec.rb
 - spec/unit/misc/step_spec.rb
 has_rdoc: