retl 0.0.3 → 0.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 65892cccb09b0ee2bde26342b072f3c405b4b735
-  data.tar.gz: b56d3d24285ce3c40aa5e8e98924fe2cf0a09341
+  metadata.gz: 1262b0170c7b1fdbd343d510a6e8bab6edb3bc01
+  data.tar.gz: 4eeafb6b53d22901171185786c297b3b18fbfcb6
 SHA512:
-  metadata.gz: 50953eae75e94eb9383320c56268c0a18e3d1289d234509e13a90eb4c8d43ae2686c6adf6f45f45aed880680b0dd389f2a532a22555ed3bb140aa79a9f8f27cc
-  data.tar.gz: 7cd0b87b33d5810820e978722f3c2e3b63f160a01b65ba4105e4f800135b43a2ec68d65eacf034d78542a8da9de6d3aca263a19df9dced3d600ef304952f9940
+  metadata.gz: ff134b73c4e49380cb63b6a2251d6d2a1823c8ff4c8c87fbae64860d0e96bfb4b740b3670a2858db00e1aac84ffdc7cb8251aeb93750e4544a563c5b4c2d0233
+  data.tar.gz: f02e88e10931ac4ff35e33223f539bb408e55faf2e9d5d7212fbeffd871a410f10fdc3f8df131b1b7be71f1370fea807839420f09e2cb4703b3b26e84bc429f1

data/CHANGELOG.md

@@ -0,0 +1,18 @@
+# Change Log
+
+### 0.0.4
+
+- Bug Fix: fixes memoized fork results
+- Bug Fix: fixes memoized transformation results
+- New Feature: added the `path` step to include other paths
+- New Feature: paths that include other paths can now resolve their dependencies
+  with a hash or a block
+- Improvement: Dependencies that don't define a block will raise an
+  ArgumentError if the dependency is not resolved during translation. The block
+  then becomes the default value of the dependency when it is not passed in.
+- Specs: Added specs are all of the README examples.
+
+
+### 0.0.3
+
+- Proof of concept phase; too much going on too fast.

data/README.md

@@ -44,8 +44,8 @@ returned, which is `Enumerable`.
 
 ```
 data = [
-  { full_name: "David"  , last_name: "Biehl" },
-  { full_name: "Indiana", last_name: "Jones" }
+  { first_name: "David"  , last_name: "Biehl" },
+  { first_name: "Indiana", last_name: "Jones" }
 ]
 
 result = my_path.transform(data)
@@ -70,15 +70,25 @@ common use is to use `replace` as the first step to convert incoming objects
 into hashes.
 
 ```
-users = Users.where(active: true)  # a bunch of ActiveRecord objects
+users = [
+  ["David", "Biehl", 33],
+  ["Indiana", "Jones", 50]
+]
 
 my_path = Retl::Path.new do
-  replace do |user|
-    user.to_hash
+  replace do |row|
+    { first_name: row[0], last_name: row[1], age: row[2] }
   end
 
   # perform other steps with the hash
 end
+
+result = my_path.transform(users)
+result.to_a
+#=> [
+#=>   {:first_name=>"David", :last_name=>"Biehl", :age=>33}, 
+#=>   {:first_name=>"Indiana", :last_name=>"Jones", :age=>50}
+#=> ]
 ```
 
 #### Filter & Reject
@@ -89,9 +99,9 @@ truthy. `select` is an alias for `filter`.
 
 ```
 data = [
-  {name: "David"  , age: 33}
-  {name: "Indiana", age: 50}
-  {name: "Sully"  , age: 7}
+  {name: "David"  , age: 33},
+  {name: "Indiana", age: 50},
+  {name: "Sully"  , age: 7},
   {name: "Boo"    , age: 3}
 ]
 
@@ -111,7 +121,8 @@ end
 
 result = my_path.transform(data)
 result.to_a
-#=> [ { name: "David", ... } ]
+#=> [{:name=>"David", :age=>33, :adult_or_child=>"adult"}]
+
 ```
 
 #### Calculate
@@ -162,9 +173,9 @@ are unaffected by any steps that take place after the fork is defined.
 
 ```
 data = [
-  { name: "David"  , age: 33 }
-  { name: "Indiana", age: 50 }
-  { name: "Sully"  , age: 7 }
+  { name: "David"  , age: 33 },
+  { name: "Indiana", age: 50 },
+  { name: "Sully"  , age: 7 },
   { name: "Boo"    , age: 3 }
 ]
 
@@ -208,7 +219,7 @@ The `explode` step adds additional data to the Path. The return value of the
 block should respond to `#each`, like an Array.
 
 ```
-my_path = Reth::Path.new do
+my_path = Retl::Path.new do
   explode do |number|
     number.times.map { |x| x + x + x }
   end
@@ -218,32 +229,39 @@ my_path = Reth::Path.new do
   end
 end
 
-my_path.transform(6).to_a
+result = my_path.transform([6])
+result.to_a
 #=> [3, 9, 15]
+
+expect(result.to_a).to eq([3, 9, 15])
 ```
 
 #### Path Reuse
 
-In rETL, Paths can be re-used with the `step` step. Common Paths can be defined
-to ensure that calculations yield consistent results throughout the entire ETL
-project. Consistent data and meanings will make the data warehouse easier to
-understand for data consumers.
+In rETL, paths can be re-used with the `path` step. Common Paths can be
+defined to ensure that calculations yield consistent results throughout the
+entire ETL project. Consistent data and meanings will make the data warehouse
+easier to understand for data consumers.
 
 ```
 AdultOrChild = Retl::Path.new do 
-  calculate(:adult_or_child) do 
+  calculate(:adult_or_child) do |row|
     row[:age] >= 18 ? "adult" : "child"
   end
 end
 
 my_path = Retl::Path.new do 
-  step AdultOrChild
+  path AdultOrChild
 
-  ### perform other steps
+  ### perform other steps, if necessary
 end
+
+result = my_path.transform([{age: 3}])
+result.to_a
+#=> [ { age: 3, adult_or_child: "child" } ]
 ```
 
-### Dependencies
+#### Dependencies
 
 Dependencies can be defined with the `depends_on(name)`. The value of the
 dependency is accessible inside of each step by its name. In this example, we'll
@@ -251,21 +269,26 @@ define an `age_lookup` dependency.
 
 ```
 my_path = Retl::Path.new do
-  depends_on(:age_lookup) do 
+  depends_on(:age_lookup) do   # hint: the block returns the default value
     { 
       "adult" => "Adults are 18 or older",
       "child" => "Children are younger than 18"
     }
   end
 
-  step AdultOrChild   # see previous example
+  path AdultOrChild   # see previous example
 
   calculate(:age_description) do |row|
     age_lookup[row[:adult_or_child]]
   end
 end
+
+result = my_path.transform([{age: 19}])
+result.to_a
 ```
 
+##### Dependency Injection
+
 Dependencies can also be injected when the transformation takes place. This is
 useful for testing by passing in mocks or stubs. Also, concrete results from
 other paths can be merged merged into a single path making data integration
@@ -273,11 +296,9 @@ possible.
 
 ```
 my_path = Retl::Path.new do
-  depends_on(:age_lookup) do |options|  # transformation options are passed into `depends_on`
-    options[:age_lookup] || (raise ArgumentError, "This Path depends on an age lookup hash")
-  end
-
-  step AdultOrChild
+  depends_on(:age_lookup)   # hint: without a block, the dependency must
+                            # be provided when #transform is called
+  path AdultOrChild
 
   calculate(:age_description) do |row|
     age_lookup[row[:adult_or_child]]
@@ -289,8 +310,51 @@ age_lookup_hash = {
   "child" => "Children are younger than 18"
 }
 
-my_path.transform(data, age_lookup: age_lookup_hash)
+result = my_path.transform([{age: 4}], age_lookup: age_lookup_hash)
+result.to_a
+#=> [ { age: 4, adult_or_child: "child", age_description: "Children are younger than 18" } ]
+```
+
+#### Path Reuse with Dependencies
+
+The `AdultOrChild` path above isn't very flexible. It depends on an `:age` key
+to be present in the hash. What if our data uses a different key, like
+`:years_since_birth`? rETL can make this more flexible by adding dependencies.
+
 ```
+FlexibleAdultOrChild = Retl::Path.new do 
+  depends_on(:from) 
+
+  depends_on(:to) do |options|
+    options[:to] || :adult_or_child   # use a default value
+  end
+
+  transform do |row|
+    row[to] = row[from] >= 18 ? "adult" : "child"
+  end
+end
+
+
+path_with_age = Retl::Path.new do 
+  path FlexibleAdultOrChild, from: :age
+end
+
+path_with_age_result = path_with_age.transform([{age: 33}])
+path_with_age_result.to_a
+#=> [{age: 33, adult_or_child: "adult"}]
+
+
+path_with_years_since_birth = Retl::Path.new do 
+  path FlexibleAdultOrChild do
+    { from: :years_since_birth, to: :age_classification }   # blocks work too
+  end
+end
+
+result = path_with_years_since_birth.transform([{years_since_birth: 7}])
+result.to_a
+#=> [{years_since_birth: 7, age_classification: "child"}]
+```
+
 
 ## Roadmap
 
@@ -306,6 +370,13 @@ path.transform(Enumerable)
 Enumerales in, Enumerales out. This makes the application of the gem pretty much
 universal for any type of data transformation requirement in Ruby.
 
+### Next Steps
+
+- Error handling
+- Tracing and logging
+- Extract patterns
+- Load patterns
+
 ## Installation
 
 Add this line to your application's Gemfile:

data/lib/retl/context.rb

@@ -4,7 +4,13 @@ module Retl
   class Context
     def initialize(path, options={})
       path.dependencies.each do |name, dependency|
-        self.class.send(:define_method, name) { dependency.call(options) }
+        if dependency.nil? && !options[name]
+          raise ArgumentError, "This transformation depends on `name`"
+        end
+
+        self.class.send(:define_method, name) do 
+          (dependency && dependency.call(options)) || options[name]
+        end 
       end
 
       @_events = EventRouter.new

data/lib/retl/handlers/path_handler.rb

@@ -0,0 +1,18 @@
+require_relative "handler"
+
+module Retl
+  class PathHandler < Handler
+    def initialize(path, dependencies={}, &block)
+      super()
+      @path = path
+      dependencies.merge!(block.call) if block
+      @context = Context.new(@path, dependencies)
+    end
+
+    def push_in(data, context)
+      @context.execute_step(@path, data).each do |result|
+        push_out(result)
+      end
+    end
+  end
+end

data/lib/retl/path_builder.rb

@@ -3,6 +3,7 @@ require "retl/handlers/transform_handler"
 require "retl/handlers/filter_handler"
 require "retl/handlers/inspect_handler"
 require "retl/handlers/explode_handler"
+require "retl/handlers/path_handler"
 
 
 module Retl
@@ -58,5 +59,9 @@ module Retl
       action ||= block
       step(action, handler: ExplodeHandler)
     end
+
+    def path(path, dependencies={}, &block)
+      @path.add_handler PathHandler.new(path, dependencies, &block)
+    end
   end
 end

data/lib/retl/transformation.rb

@@ -9,6 +9,7 @@ module Retl
       @enumerable, @path, @options = enumerable, path, options
       @context   = Context.new(@path, @options)
       @fork_data = ForkDataCollector.new(@context)
+      @forks     = {}
     end
 
     def each(&block)
@@ -29,8 +30,12 @@ module Retl
     end
 
     def forks(name)
-      build_each_result
-      @path.forks(name).transform(@fork_data.take(name), @options)
+      unless @forks[name]
+        build_each_result
+        @forks[name] = @path.forks(name).transform(@fork_data.take(name), @options)
+      end
+
+      @forks[name]
     end
 
     def load_into(*destinations)
@@ -55,6 +60,7 @@ module Retl
           yield data if block_given?
           result << data
         end
+        result
       end
     end
 

data/lib/retl/version.rb

@@ -1,3 +1,3 @@
 module Retl
-  VERSION = "0.0.3"
+  VERSION = "0.0.4"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: retl
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
 platform: ruby
 authors:
 - David Biehl
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2015-11-20 00:00:00.000000000 Z
+date: 2015-11-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -62,6 +62,7 @@ files:
 - ".gitignore"
 - ".rspec"
 - ".travis.yml"
+- CHANGELOG.md
 - CODE_OF_CONDUCT.md
 - Gemfile
 - LICENSE.txt
@@ -78,6 +79,7 @@ files:
 - lib/retl/handlers/fork_handler.rb
 - lib/retl/handlers/handler.rb
 - lib/retl/handlers/inspect_handler.rb
+- lib/retl/handlers/path_handler.rb
 - lib/retl/handlers/step_handler.rb
 - lib/retl/handlers/transform_handler.rb
 - lib/retl/path.rb