retl 0.0.5 → 0.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5fb3e97322814aed123d196693eacd2eb5560719
-  data.tar.gz: 778407acae9f684ddaa8b746d0c8dd751ed2e9be
+  metadata.gz: b72439339d6a06a2bef173718fbc40d953ee6cae
+  data.tar.gz: 6a35ec42b81631654b2e0a4ced399628e279185d
 SHA512:
-  metadata.gz: 8b0182706dbf3a4350d53cb967b576075a9fbf85dd1dd1efb778bc0bd4d648b15bddea9904ece0a973bf15896d60ed38ae4833f09b1c5ffc986f48650415fa8d
-  data.tar.gz: 29a71295ac8f7025fde450c1f4f079e100fe082c2891d20fe34950f28f588c12affe2a0e1e8bcc50a3bd43ca20e80ad6ea97f9839fccf8535514b6c43a93fe84
+  metadata.gz: 7b6b0431f65fa58a2d9a0f85779d3b284d81f51a9528dee74cf90ca334380f4a76ad92d88ec35e9fa3047552d8f6eb33dc4844f7608e745c03d2f43208d5c0ba
+  data.tar.gz: baef9bc311e17363a1aafeff0ad913184e2f6121b4241c11e0b82af13a762fa6e227a5e2681039b5187d587dad15d7456b61a12cc43c020625dd5c9f10e09089

data/CHANGELOG.md

@@ -1,5 +1,14 @@
 # Change Log
 
+### 0.0.6
+
+- Improvement: No longer attempts to memoize transformation results. If you
+  want to re-iterate over the results, store them with `#to_a`
+- New Feature: adds support for horizontal threading between the source
+  and the transformation and the transformation and the load. 
+- Bug Fix: Defines single methods for dependencies on a Context instead
+  of insteance methods for the whole class
+
 ### 0.0.5
 
 - New Feature: Step descriptions allow steps to be described while being

data/README.md

@@ -424,6 +424,17 @@ result.errors.to_a
                               # above for use.
 ```
 
+### Multi-Thread Support
+
+Since extracting and loading can be I/O intense operations, it makes sense
+to run them in their own threads. This can be done with the `Path#transform!`
+method. It works just like `Path#transform` only it will separate extration,
+transformation and loading into separate threads.
+
+```
+path.transform!(data)
+```
+
 ## Roadmap
 
 Currently the rETL gem's strengths are transforming data and code reuse. However

data/lib/retl/configuration.rb

@@ -2,6 +2,10 @@ class Configuration
   attr_accessor :raise_errors
 
   def initialize
+    reset!
+  end
+
+  def reset!
     self.raise_errors = true
   end
 end

data/lib/retl/context.rb

@@ -8,7 +8,7 @@ module Retl
           raise ArgumentError, "This transformation depends on `name`"
         end
 
-        self.class.send(:define_method, name) do 
+        define_singleton_method(name) do 
           (dependency && dependency.call(options)) || options[name]
         end 
       end

data/lib/retl/default_execution.rb

@@ -0,0 +1,45 @@
+module Retl
+  class DefaultExecution
+    def initialize(enumerable, path, context, errors)
+      @enumerable, @path, @context, @errors = enumerable, path, context, errors
+      @executed = false
+    end
+
+    def each(&block)
+      @executed = true
+      @enumerable.each do |data|
+        execute(data, &block)
+      end
+    end
+
+    def execute(input)
+      @path.call(input, @context).each do |data|
+        yield data if block_given?
+      end
+    rescue StepExecutionError => e
+      if Retl.configuration.raise_errors
+        raise e
+      else
+        @errors << e
+      end
+    end
+
+    def load_into(*destinations)
+      destinations = Array(destinations)
+
+      each do |data|
+        destinations.each do |destination|
+          destination << data
+        end
+      end
+
+      destinations.each do |destination|
+        destination.close if destination.respond_to?(:close)
+      end
+    end
+
+    def executed?
+      @executed
+    end
+  end
+end

data/lib/retl/path.rb

@@ -139,6 +139,9 @@ module Retl
     # @return [void]
     def add_fork(name, &block)
       fork = Path.new(&block)
+      @dependencies.each do |name, dependency|
+        fork.add_dependency name, dependency
+      end
       add_handler ForkHandler.new(name)
       @forks[name] = fork 
     end
@@ -183,5 +186,11 @@ module Retl
     def transform(enumerable, options={})
       Transformation.new(enumerable, self, options)
     end
+
+    def transform!(enumerable, options={})
+      transform(enumerable, options).tap do |transformation|
+        transformation.execution_strategy = ThreadedExecution
+      end
+    end
   end
 end

data/lib/retl/threaded_execution.rb

@@ -0,0 +1,43 @@
+module Retl
+  class ThreadedExecution < DefaultExecution
+    def each(&block)
+      @executed = true
+      queue = Queue.new
+
+      producer = Thread.new do 
+        @enumerable.each { |item| queue.push item }
+        queue.push :eoq
+      end
+
+      while((data = queue.pop) != :eoq)
+        execute(data, &block)
+      end
+
+      producer.join
+    end
+
+    def load_into(*destinations)
+      destinations = Array(destinations)
+      queue = Queue.new
+
+      producer = Thread.new do 
+        each do |data|
+          queue.push data
+        end
+        queue.push :eoq
+      end
+
+      while((data = queue.pop) != :eoq)
+        destinations.each do |destination|
+          destination << data
+        end
+      end
+
+      producer.join
+
+      destinations.each do |destination|
+        destination.close if destination.respond_to?(:close)
+      end
+    end
+  end
+end

data/lib/retl/transformation.rb

@@ -1,89 +1,56 @@
 require "retl/context"
 require "retl/fork_data_collector"
 require "retl/errors/step_execution_error"
+require_relative "default_execution"
+require_relative "threaded_execution"
 
 module Retl
   class Transformation
     include Enumerable
 
+    attr_writer :execution_strategy
+
     def initialize(enumerable, path, options={})
       @enumerable, @path, @options = enumerable, path, options
       @context   = Context.new(@path, @options)
       @fork_data = ForkDataCollector.new(@context)
       @forks     = {}
       @errors    = []
+      self.execution_strategy = DefaultExecution
+    end
+
+    def execution_strategy=(strategy)
+      @execution_strategy = strategy.new(@enumerable, @path, @context, @errors)
     end
 
     def each(&block)
-      if @each
-        @each.each(&block)
-      else
-        build_each_result(&block)
-      end
+      @execution_strategy.each(&block)
     end
 
     def each_slice(size, &block)
-      @each_slice ||= {}
-      if @each_slice[size]
-        @each_slice[size].each(&block)
-      else
-        build_each_slice_result(size, &block)
+      @enumerable.each_slice(size).map do |slice|
+        Transformation.new(slice, @path, @options).tap do |transformed_slice|
+          yield transformed_slice if block_given?
+        end
       end
     end
 
     def forks(name)
       unless @forks[name]
-        build_each_result
+        each unless @execution_strategy.executed?
         @forks[name] = @path.forks(name).transform(@fork_data.take(name), @options)
+        @forks[name].execution_strategy = @execution_strategy.class
       end
 
       @forks[name]
     end
 
     def load_into(*destinations)
-      destinations = Array(destinations)
-
-      each do |data|
-        destinations.each do |destination|
-          destination << data
-        end
-      end
-
-      destinations.each do |destination|
-        destination.close if destination.respond_to?(:close)
-      end
+      @execution_strategy.load_into(*destinations)
     end
 
     def errors
       @errors.each
     end
-
-    private
-
-    def build_each_result(&block)
-      @each ||= @enumerable.reduce([]) do |result, data|
-        begin
-          @path.call(data, @context).each do |data|
-            yield data if block_given?
-            result << data
-          end
-        rescue StepExecutionError => e
-          if Retl.configuration.raise_errors
-            raise e
-          else
-            @errors << e
-          end
-        end
-        result
-      end
-    end
-
-    def build_each_slice_result(size, &block)
-      @each_slice[size] ||= @enumerable.each_slice(size).reduce([]) do |result, slice|
-        transformed_slice = Transformation.new(slice, @path, @options)
-        yield transformed_slice if block_given?
-        result << transformed_slice
-      end
-    end
   end
 end

data/lib/retl/version.rb

@@ -1,3 +1,3 @@
 module Retl
-  VERSION = "0.0.5"
+  VERSION = "0.0.6"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: retl
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.0.6
 platform: ruby
 authors:
 - David Biehl
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2015-11-23 00:00:00.000000000 Z
+date: 2015-12-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -73,6 +73,7 @@ files:
 - lib/retl.rb
 - lib/retl/configuration.rb
 - lib/retl/context.rb
+- lib/retl/default_execution.rb
 - lib/retl/errors/step_execution_error.rb
 - lib/retl/event_router.rb
 - lib/retl/fork_data_collector.rb
@@ -87,6 +88,7 @@ files:
 - lib/retl/next_description.rb
 - lib/retl/path.rb
 - lib/retl/path_builder.rb
+- lib/retl/threaded_execution.rb
 - lib/retl/transformation.rb
 - lib/retl/version.rb
 - retl.gemspec