kiba 0.6.0 → 0.6.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: faa6cbb049d4b35cdd62c647c1510bc9d296cbb5
-  data.tar.gz: c6d522da427b0b2388771279b146fd48ac117605
+  metadata.gz: b46106e516df027f47b160c8a015ef86379c3994
+  data.tar.gz: da5ecf0dd661b1dc4960bf4bfc149df9be4c3a89
 SHA512:
-  metadata.gz: b26ef4488c4aa78c86f99fb001565260dacae1ef529f2c7b8c37533ef179e39c57dfc8c6e35fa1f31acf8e9e4ab6d22b418d90316317a6f4db898c3a93c22108
-  data.tar.gz: f4d7cc78c3ccdb04fc3b98310ac2f4f9163aa60112481b81f22119f4864c6485117087718ec3512559ac668f62d2f8bcc7b54275999303941d9373225851b620
+  metadata.gz: 04d98ac5c1b9d2bed003930cbf1f70230b9639758b09562f0e4cdb69a5fb566bb2fcae7f392617f3dae44d929d405eb915db0982ba7a5388c505ce1295c91f40
+  data.tar.gz: a561008267ef7b2b436095187592aaa811aaec788453d7f6edfb129950a9e9e058862ea22b812dd7caecffc10a257cbf4c069f6f7e524ef74541257ff637bb9b

data/.travis.yml

@@ -4,3 +4,4 @@ rvm:
   - 2.1
   - 2.0
   - jruby
+  - rbx-2.5.7

data/Changes.md

@@ -1,6 +1,12 @@
 Unreleased
 ----------
 
+0.6.1
+-----
+
+- Bugfix: make sure files generated by `pre_process` can safely be read from constructor of sources [#16]. Thanks @bcherrington for catching this.
+- Internal refactoring of processing engine (should not affect regular use)
+
 0.6.0
 -----
 
@@ -9,4 +15,4 @@ Unreleased
 0.5.0
 -----
 
-- Initial release
+- Initial release

data/README.md

@@ -6,6 +6,8 @@ Learn more on the [Kiba blog](http://thibautbarrere.com):
 
 * [Rubyists - are you doing ETL unknowningly?](http://thibautbarrere.com/2015/03/25/rubyists-are-you-doing-etl-unknowingly/)
 * [How to write solid data processing code](http://thibautbarrere.com/2015/04/05/how-to-write-solid-data-processing-code/)
+* [How to reformat CSV files with Kiba](http://thibautbarrere.com/2015/06/04/how-to-reformat-csv-files-with-kiba/) (in-depth, hands-on tutorial)
+* [How to explode multivalued attributes with Kiba ETL?](http://thibautbarrere.com/2015/06/25/how-to-explode-multivalued-attributes-with-kiba/)
 
 [![Gem Version](https://badge.fury.io/rb/kiba.svg)](http://badge.fury.io/rb/kiba)
 [![Build Status](https://travis-ci.org/thbar/kiba.svg?branch=master)](https://travis-ci.org/thbar/kiba) [![Code Climate](https://codeclimate.com/github/thbar/kiba/badges/gpa.svg)](https://codeclimate.com/github/thbar/kiba) [![Dependency Status](https://gemnasium.com/thbar/kiba.svg)](https://gemnasium.com/thbar/kiba)
@@ -254,7 +256,7 @@ Make sure to subscribe to my [Ruby ETL blog](http://thibautbarrere.com) where I'
 
 ## Supported Ruby versions
 
-Kiba currently supports Ruby 2.0+ and JRuby (with its default 1.9 syntax).
+Kiba currently supports Ruby 2.0+, JRuby (with its default 1.9 syntax) and Rubinius (see [test matrix](https://travis-ci.org/thbar/kiba)).
 
 ## History & Credits
 

data/lib/kiba/context.rb

@@ -1,12 +1,11 @@
 module Kiba
   class Context
     def initialize(control)
-      # TODO: forbid access to control from context? use cleanroom?
       @control = control
     end
 
     def pre_process(&block)
-      @control.pre_processes << block
+      @control.pre_processes << { block: block }
     end
 
     def source(klass, *initialization_params)
@@ -14,11 +13,7 @@ module Kiba
     end
 
     def transform(klass = nil, *initialization_params, &block)
-      if klass
-        @control.transforms << { klass: klass, args: initialization_params }
-      else
-        @control.transforms << block
-      end
+      @control.transforms << { klass: klass, args: initialization_params, block: block }
     end
 
     def destination(klass, *initialization_params)
@@ -26,7 +21,7 @@ module Kiba
     end
 
     def post_process(&block)
-      @control.post_processes << block
+      @control.post_processes << { block: block }
     end
   end
 end

data/lib/kiba/runner.rb

@@ -1,30 +1,38 @@
 module Kiba
   module Runner
+    # allow to handle a block form just like a regular transform
+    class AliasingProc < Proc
+      alias_method :process, :call
+    end
+
     def run(control)
-      # instantiate early so that error are raised before any processing occurs
-      pre_processes = to_instances(control.pre_processes, true, false)
-      sources = to_instances(control.sources)
-      destinations = to_instances(control.destinations)
-      transforms = to_instances(control.transforms, true)
-      post_processes = to_instances(control.post_processes, true, false)
+      # TODO: add a dry-run (not instantiating mode) to_instances call
+      # that will validate the job definition from a syntax pov before
+      # going any further. This could be shared with the parser.
+      run_pre_processes(control)
+      process_rows(
+        to_instances(control.sources),
+        to_instances(control.transforms, true),
+        to_instances(control.destinations)
+      )
+      # TODO: when I add post processes as class, I'll have to add a test to
+      # make sure instantiation occurs after the main processing is done (#16)
+      run_post_processes(control)
+    end
 
-      pre_processes.each(&:call)
-      process_rows(sources, transforms, destinations)
-      destinations.each(&:close)
-      post_processes.each(&:call)
+    def run_pre_processes(control)
+      to_instances(control.pre_processes, true, false).each(&:call)
+    end
+
+    def run_post_processes(control)
+      to_instances(control.post_processes, true, false).each(&:call)
     end
-    
+
     def process_rows(sources, transforms, destinations)
       sources.each do |source|
         source.each do |row|
           transforms.each do |transform|
-            # TODO: avoid the case completely by e.g. subclassing Proc
-            # and aliasing `process` to `call`. Benchmark needed first though.
-            if transform.is_a?(Proc)
-              row = transform.call(row)
-            else
-              row = transform.process(row)
-            end
+            row = transform.process(row)
             break unless row
           end
           next unless row
@@ -33,19 +41,29 @@ module Kiba
           end
         end
       end
+      destinations.each(&:close)
     end
 
     # not using keyword args because JRuby defaults to 1.9 syntax currently
     def to_instances(definitions, allow_block = false, allow_class = true)
-      definitions.map do |d|
-        case d
-        when Proc
-          fail 'Block form is not allowed here' unless allow_block
-          d
-        else
-          fail 'Class form is not allowed here' unless allow_class
-          d[:klass].new(*d[:args])
-        end
+      definitions.map do |definition|
+        to_instance(
+          *definition.values_at(:klass, :args, :block),
+          allow_block, allow_class
+        )
+      end
+    end
+
+    def to_instance(klass, args, block, allow_block, allow_class)
+      if klass
+        fail 'Class form is not allowed here' unless allow_class
+        klass.new(*args)
+      elsif block
+        fail 'Block form is not allowed here' unless allow_block
+        AliasingProc.new(&block)
+      else
+        # TODO: support block passing to a class form definition?
+        fail 'Class and block form cannot be used together at the moment'
       end
     end
   end

data/lib/kiba/version.rb

@@ -1,3 +1,3 @@
 module Kiba
-  VERSION = '0.6.0'
+  VERSION = '0.6.1'
 end

data/test/support/test_source_that_reads_at_instantiation_time.rb

@@ -0,0 +1,8 @@
+class SourceThatReadsAtInstantionTime
+  def initialize(file)
+    IO.read(file)
+  end
+
+  def each
+  end
+end

data/test/test_cli.rb

@@ -11,7 +11,7 @@ class TestCli < Kiba::Test
       Kiba::Cli.run([fixture('bogus.etl')])
     end
 
-    assert_match(/uninitialized constant (.*)UnknownThing/, exception.message)
+    assert_match(/uninitialized constant(.*)UnknownThing/, exception.message)
     assert_includes exception.backtrace.to_s, 'test/fixtures/bogus.etl:2:in'
   end
 end

data/test/test_integration.rb

@@ -3,6 +3,8 @@ require_relative 'helper'
 require_relative 'support/test_csv_source'
 require_relative 'support/test_csv_destination'
 require_relative 'support/test_rename_field_transform'
+require_relative 'support/test_enumerable_source'
+require_relative 'support/test_source_that_reads_at_instantiation_time'
 
 # End-to-end tests go here
 class TestIntegration < Kiba::Test
@@ -19,13 +21,17 @@ Patrick,McWire,M
 CSV
   end
 
+  def clean
+    remove_files(*Dir['test/tmp/*.csv'])
+  end
+
   def setup
-    remove_files(input_file, output_file)
+    clean
     IO.write(input_file, sample_csv_data)
   end
 
   def teardown
-    remove_files(input_file, output_file)
+    clean
   end
 
   def test_csv_to_csv
@@ -71,7 +77,7 @@ CSV
 
       # assign a first value at parsing time
       count = 0
-      
+
       pre_process do
         # then change it from there (run time)
         count += 100
@@ -93,4 +99,18 @@ CSV
 
     assert_equal 'Count is now 103', message
   end
+
+  def test_file_created_by_pre_process_can_be_read_by_source_at_instantiation_time
+    remove_files('test/tmp/eager.csv')
+
+    control = Kiba.parse do
+      pre_process do
+        IO.write('test/tmp/eager.csv', 'something')
+      end
+
+      source SourceThatReadsAtInstantionTime, 'test/tmp/eager.csv'
+    end
+
+    Kiba.run(control)
+  end
 end

data/test/test_parser.rb

@@ -20,7 +20,7 @@ class TestParser < Kiba::Test
       transform { |row| row }
     end
 
-    assert_instance_of Proc, control.transforms[0]
+    assert_instance_of Proc, control.transforms[0][:block]
   end
 
   def test_class_transform_definition
@@ -46,7 +46,7 @@ class TestParser < Kiba::Test
       post_process {}
     end
 
-    assert_instance_of Proc, control.post_processes[0]
+    assert_instance_of Proc, control.post_processes[0][:block]
   end
   
   def test_block_pre_process_definition
@@ -54,7 +54,7 @@ class TestParser < Kiba::Test
       pre_process {}
     end
 
-    assert_instance_of Proc, control.pre_processes[0]
+    assert_instance_of Proc, control.pre_processes[0][:block]
   end
 
   def test_source_as_string_parsing

data/test/test_runner.rb

@@ -1,5 +1,5 @@
 require_relative 'helper'
-
+require 'minitest/mock'
 require_relative 'support/test_enumerable_source'
 
 class TestRunner < Kiba::Test
@@ -22,14 +22,14 @@ class TestRunner < Kiba::Test
 
   def test_block_transform_processing
     # is there a better way to assert a block was called in minitest?
-    control.transforms << lambda { |r| @called = true; r }
+    control.transforms << { block: lambda { |r| @called = true; r } }
     Kiba.run(control)
     assert_equal true, @called
   end
 
   def test_dismissed_row_not_passed_to_next_transform
-    control.transforms << lambda { |_| nil }
-    control.transforms << lambda { |_| @called = true; nil }
+    control.transforms << { block: lambda { |_| nil } }
+    control.transforms << { block: lambda { |_| @called = true; nil } }
     Kiba.run(control)
     assert_nil @called
   end
@@ -37,14 +37,14 @@ class TestRunner < Kiba::Test
   def test_post_process_runs_once
     assert_equal 2, rows.size
     @called = 0
-    control.post_processes << lambda { @called += 1 }
+    control.post_processes << { block: lambda { @called += 1 } }
     Kiba.run(control)
     assert_equal 1, @called
   end
 
   def test_post_process_not_called_after_row_failure
-    control.transforms << lambda { |_| fail 'FAIL' }
-    control.post_processes << lambda { @called = true }
+    control.transforms << { block: lambda { |_| fail 'FAIL' } }
+    control.post_processes << { block: lambda { @called = true } }
     assert_raises(RuntimeError, 'FAIL') { Kiba.run(control) }
     assert_nil @called
   end
@@ -52,8 +52,24 @@ class TestRunner < Kiba::Test
   def test_pre_process_runs_once
     assert_equal 2, rows.size
     @called = 0
-    control.pre_processes << lambda { @called += 1 }
+    control.pre_processes << { block: lambda { @called += 1 } }
     Kiba.run(control)
     assert_equal 1, @called
   end
+
+  def test_pre_process_runs_before_source_is_instantiated
+    calls = []
+
+    mock_source_class = MiniTest::Mock.new
+    mock_source_class.expect(:new, TestEnumerableSource.new([1, 2, 3])) do
+      calls << :source_instantiated
+    end
+
+    control = Kiba::Control.new
+    control.pre_processes << { block: lambda { calls << :pre_processor_executed } }
+    control.sources << { klass: mock_source_class }
+    Kiba.run(control)
+
+    assert_equal [:pre_processor_executed, :source_instantiated], calls
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kiba
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.6.1
 platform: ruby
 authors:
 - Thibaut Barrère
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-05-14 00:00:00.000000000 Z
+date: 2015-06-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -82,6 +82,7 @@ files:
 - test/support/test_csv_source.rb
 - test/support/test_enumerable_source.rb
 - test/support/test_rename_field_transform.rb
+- test/support/test_source_that_reads_at_instantiation_time.rb
 - test/test_cli.rb
 - test/test_integration.rb
 - test/test_parser.rb
@@ -119,6 +120,7 @@ test_files:
 - test/support/test_csv_source.rb
 - test/support/test_enumerable_source.rb
 - test/support/test_rename_field_transform.rb
+- test/support/test_source_that_reads_at_instantiation_time.rb
 - test/test_cli.rb
 - test/test_integration.rb
 - test/test_parser.rb