retl 0.0.4 → 0.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1262b0170c7b1fdbd343d510a6e8bab6edb3bc01
-  data.tar.gz: 4eeafb6b53d22901171185786c297b3b18fbfcb6
+  metadata.gz: 5fb3e97322814aed123d196693eacd2eb5560719
+  data.tar.gz: 778407acae9f684ddaa8b746d0c8dd751ed2e9be
 SHA512:
-  metadata.gz: ff134b73c4e49380cb63b6a2251d6d2a1823c8ff4c8c87fbae64860d0e96bfb4b740b3670a2858db00e1aac84ffdc7cb8251aeb93750e4544a563c5b4c2d0233
-  data.tar.gz: f02e88e10931ac4ff35e33223f539bb408e55faf2e9d5d7212fbeffd871a410f10fdc3f8df131b1b7be71f1370fea807839420f09e2cb4703b3b26e84bc429f1
+  metadata.gz: 8b0182706dbf3a4350d53cb967b576075a9fbf85dd1dd1efb778bc0bd4d648b15bddea9904ece0a973bf15896d60ed38ae4833f09b1c5ffc986f48650415fa8d
+  data.tar.gz: 29a71295ac8f7025fde450c1f4f079e100fe082c2891d20fe34950f28f588c12affe2a0e1e8bcc50a3bd43ca20e80ad6ea97f9839fccf8535514b6c43a93fe84

data/CHANGELOG.md

@@ -1,5 +1,16 @@
 # Change Log
 
+### 0.0.5
+
+- New Feature: Step descriptions allow steps to be described while being
+  built. This will allow the description of the step to appear in the 
+  error message if an error occurs.
+- Bug Fix: transform was mutating source data. Instead we just want it to 
+  mutate a duplicate and pass that along.
+- New Feature: adds a configuration block to alter the behavior of rETL
+- New Feature: error handling. Errors can simple be raised (for development)
+  or be captured so they can be evaluated later (for production)
+
 ### 0.0.4
 
 - Bug Fix: fixes memoized fork results

data/README.md

@@ -232,8 +232,6 @@ end
 result = my_path.transform([6])
 result.to_a
 #=> [3, 9, 15]
-
-expect(result.to_a).to eq([3, 9, 15])
 ```
 
 #### Path Reuse
@@ -355,6 +353,76 @@ result.to_a
 #=> [{years_since_birth: 7, age_classification: "child"}]
 ```
 
+### Error Handling
+
+Sometimes errors happen while a step is being performed. By default, errors are
+wrapped in a `StepExecutionError` and the transformation will abort (this
+can be changed with configuration, however). The `StepExecutionError` will
+provide context to aid in debugging the error.
+
+Pay attention to the `desc` step in the path below. `desc` simply describes the
+next step. Not only does this provide good documentation, but the description
+will also be available in the `StepExecutionError` to make debugging easier.
+
+```
+my_path = Retl::Path.new do 
+  desc "calculates the number 5"
+  calc(:five) { 5 }
+
+  desc "Check for sane ages"
+  transform do |row|
+    raise StandardError, "bad age" if row[:age] > 100 
+  end
+end
+
+data = [
+  {age: 33},
+  {age: 3},
+  {age: 1000}
+]
+
+begin
+  result = my_path.transform(data).to_a
+rescue Retl::StepExecutionError => e
+  e.message
+  #=> "bad age (at step: Check for sane ages))"
+
+  e.step_description
+  #=> "Check for sane ages"
+
+  e.input_data
+  #=> { :age => 1000 }
+
+  e.current_data
+  #=> { :age => 1000, :five => 5 }
+end
+```
+
+rETL can be globally configured to continue transforming data despite an error
+like so:
+
+```
+Retl.configure do |config|
+  config.raise_errors = false
+end
+```
+
+With this configuration change, the transformation would continue to process the
+data and errors would be accessible via the `#errors` method on the
+transformation.
+
+```
+# same path and data as above
+
+result = my_path.transform(data)
+
+result.to_a
+#=> [ { age: 3, five: 5 } ]   # the error data isn't in the result
+
+result.errors.to_a
+#=> [ StepExecutionError ]    # the error information is still available, see
+                              # above for use.
+```
 
 ## Roadmap
 
@@ -372,7 +440,6 @@ universal for any type of data transformation requirement in Ruby.
 
 ### Next Steps
 
-- Error handling
 - Tracing and logging
 - Extract patterns
 - Load patterns

data/lib/retl.rb

@@ -1,7 +1,18 @@
 require "retl/version"
 
 require "retl/path"
+require "retl/configuration"
 
 module Retl
+  @configuration = Configuration.new
+
+  class << self
+    attr_reader :configuration
+
+    def configure(&block)
+      block.call(@configuration)
+    end
+  end
+
   # Your code goes here...
 end

data/lib/retl/configuration.rb

@@ -0,0 +1,7 @@
+class Configuration
+  attr_accessor :raise_errors
+
+  def initialize
+    self.raise_errors = true
+  end
+end

data/lib/retl/errors/step_execution_error.rb

@@ -0,0 +1,17 @@
+module Retl
+  class StepExecutionError < StandardError
+    attr_reader :input_data, :current_data, :step, :cause
+
+    def initialize(input_data: nil, current_data: nil, step: nil, cause: $!)
+      @input_data, @current_data = input_data, current_data
+      @step, @cause = step, cause
+
+      super("#{cause} (at step: #{step_description}))")
+      set_backtrace(cause.backtrace)
+    end
+
+    def step_description
+      @step.description
+    end
+  end
+end

data/lib/retl/handlers/explode_handler.rb

@@ -2,7 +2,7 @@ require_relative "step_handler"
 
 module Retl
   class ExplodeHandler < StepHandler
-    def push_in(data, context)
+    def call(data, context)
       context.execute_step(step, data).each do |result|
         push_out result
       end

data/lib/retl/handlers/filter_handler.rb

@@ -2,7 +2,7 @@ require_relative "step_handler"
 
 module Retl
   class FilterHandler < StepHandler
-    def push_in(data, context)
+    def call(data, context)
       keep = context.execute_step(step, data)
       push_out data if keep
     end

data/lib/retl/handlers/fork_handler.rb

@@ -7,7 +7,7 @@ module Retl
       @fork = fork
     end
 
-    def push_in(data, context)
+    def call(data, context)
       context._events.trigger(:fork_data, fork_name: @fork, data: data.dup)
       push_out data
     end

data/lib/retl/handlers/handler.rb

@@ -1,7 +1,10 @@
 module Retl
   class Handler
+    attr_accessor :description
+
     def initialize
-      @output = []
+      @output      = []
+      @description = "unknown"
     end
 
     def output
@@ -9,6 +12,10 @@ module Retl
     end
 
     def push_in(data, context)
+      call(data, context)
+    end
+
+    def call(data, context)
       raise NotImplementedError, "Handlers much implement the #push_in(data, context) method"
     end
 
@@ -18,4 +25,4 @@ module Retl
       @output.push data
     end
   end
-end
+end

data/lib/retl/handlers/inspect_handler.rb

@@ -2,7 +2,7 @@ require_relative "step_handler"
 
 module Retl
   class InspectHandler < StepHandler
-    def push_in(data, context)
+    def call(data, context)
       context.execute_step(step, data.dup)
       push_out data
     end

data/lib/retl/handlers/path_handler.rb

@@ -9,7 +9,7 @@ module Retl
       @context = Context.new(@path, dependencies)
     end
 
-    def push_in(data, context)
+    def call(data, context)
       @context.execute_step(@path, data).each do |result|
         push_out(result)
       end

data/lib/retl/handlers/step_handler.rb

@@ -9,7 +9,7 @@ module Retl
       @step = step
     end
 
-    def push_in(data, context)
+    def call(data, context)
       push_out context.execute_step(step, data)
     end
   end

data/lib/retl/handlers/transform_handler.rb

@@ -2,9 +2,10 @@ require_relative "step_handler"
 
 module Retl
   class TransformHandler < StepHandler
-    def push_in(data, context)
-      context.execute_step(step, data)
-      push_out data
+    def call(data, context)
+      dup = data.dup
+      context.execute_step(step, dup)
+      push_out dup
     end
   end
 end

data/lib/retl/next_description.rb

@@ -0,0 +1,22 @@
+module Retl
+  class NextDescription
+    def initialize(default="unknown")
+      @default = default
+      reset
+    end
+
+    def reset
+      @next_description = @default
+    end
+
+    def describe_next(description)
+      @next_description = description
+    end
+
+    def take
+      description = @next_description
+      reset
+      description
+    end
+  end
+end

data/lib/retl/path.rb

@@ -5,6 +5,7 @@ require "retl/handlers/handler"
 require "retl/handlers/step_handler"
 require "retl/handlers/explode_handler"
 require "retl/handlers/fork_handler"
+require "retl/errors/step_execution_error"
 
 module Retl
   # A Path is a blueprint for transforming data
@@ -106,10 +107,18 @@ module Retl
     # @param context [Context] the execution context for the transformation
     #
     # @return [Array<Hash>] the transformed data
-    def call(data, context=Context.new(self))
-      @steps.reduce([data]) do |queue, handler|
+    def call(input, context=Context.new(self))
+      @steps.reduce([input]) do |queue, handler|
         queue.each do |data|
-          handler.push_in(data, context)
+          begin
+            handler.push_in(data, context)
+          rescue Exception => e
+            raise StepExecutionError.new(
+              input_data: input, 
+              current_data: data, 
+              step: handler
+            )
+          end
         end
         handler.output
       end

data/lib/retl/path_builder.rb

@@ -4,18 +4,24 @@ require "retl/handlers/filter_handler"
 require "retl/handlers/inspect_handler"
 require "retl/handlers/explode_handler"
 require "retl/handlers/path_handler"
+require "retl/next_description"
 
 
 module Retl
   class PathBuilder
     def initialize(path, &block)
       @path = path
+      @next_descripion = NextDescription.new
       instance_eval(&block)
     end
 
     def step(step=nil, handler: StepHandler, &block)
       step ||= block
-      @path.add_step step, handler: handler
+
+      handler = handler.new(step)
+      handler.description = @next_descripion.take
+      
+      @path.add_handler handler
     end
     alias_method :replace, :step
 
@@ -63,5 +69,9 @@ module Retl
     def path(path, dependencies={}, &block)
       @path.add_handler PathHandler.new(path, dependencies, &block)
     end
+
+    def desc(step_description)
+      @next_descripion.describe_next(step_description)
+    end
   end
 end

data/lib/retl/transformation.rb

@@ -1,15 +1,17 @@
 require "retl/context"
 require "retl/fork_data_collector"
+require "retl/errors/step_execution_error"
 
 module Retl
   class Transformation
     include Enumerable
-    
+
     def initialize(enumerable, path, options={})
       @enumerable, @path, @options = enumerable, path, options
       @context   = Context.new(@path, @options)
       @fork_data = ForkDataCollector.new(@context)
       @forks     = {}
+      @errors    = []
     end
 
     def each(&block)
@@ -52,13 +54,25 @@ module Retl
       end
     end
 
+    def errors
+      @errors.each
+    end
+
     private
 
     def build_each_result(&block)
       @each ||= @enumerable.reduce([]) do |result, data|
-        @path.call(data, @context).each do |data|
-          yield data if block_given?
-          result << data
+        begin
+          @path.call(data, @context).each do |data|
+            yield data if block_given?
+            result << data
+          end
+        rescue StepExecutionError => e
+          if Retl.configuration.raise_errors
+            raise e
+          else
+            @errors << e
+          end
         end
         result
       end

data/lib/retl/version.rb

@@ -1,3 +1,3 @@
 module Retl
-  VERSION = "0.0.4"
+  VERSION = "0.0.5"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: retl
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.5
 platform: ruby
 authors:
 - David Biehl
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2015-11-22 00:00:00.000000000 Z
+date: 2015-11-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -71,7 +71,9 @@ files:
 - bin/console
 - bin/setup
 - lib/retl.rb
+- lib/retl/configuration.rb
 - lib/retl/context.rb
+- lib/retl/errors/step_execution_error.rb
 - lib/retl/event_router.rb
 - lib/retl/fork_data_collector.rb
 - lib/retl/handlers/explode_handler.rb
@@ -82,6 +84,7 @@ files:
 - lib/retl/handlers/path_handler.rb
 - lib/retl/handlers/step_handler.rb
 - lib/retl/handlers/transform_handler.rb
+- lib/retl/next_description.rb
 - lib/retl/path.rb
 - lib/retl/path_builder.rb
 - lib/retl/transformation.rb