kiba 1.0.0 → 3.5.0

data/lib/kiba/parser.rb

@@ -1,14 +1,9 @@
 module Kiba
   module Parser
-    def parse(source_as_string = nil, source_file = nil, &source_as_block)
-      control = Control.new
-      context = Context.new(control)
-      if source_as_string
-        # this somewhat weird construct allows to remove a nil source_file
-        context.instance_eval(*[source_as_string, source_file].compact)
-      else
-        context.instance_eval(&source_as_block)
-      end
+    def parse(&source_as_block)
+      control = Kiba::Control.new
+      context = Kiba::Context.new(control)
+      context.instance_eval(&source_as_block)
       control
     end
   end

data/lib/kiba/runner.rb

@@ -1,5 +1,7 @@
 module Kiba
   module Runner
+    extend self
+
     # allow to handle a block form just like a regular transform
     class AliasingProc < Proc
       alias_method :process, :call
@@ -13,8 +15,9 @@ module Kiba
       process_rows(
         to_instances(control.sources),
         to_instances(control.transforms, true),
-        to_instances(control.destinations)
+        destinations = to_instances(control.destinations)
       )
+      close_destinations(destinations)
       # TODO: when I add post processes as class, I'll have to add a test to
       # make sure instantiation occurs after the main processing is done (#16)
       run_post_processes(control)
@@ -28,6 +31,12 @@ module Kiba
       to_instances(control.post_processes, true, false).each(&:call)
     end
 
+    def close_destinations(destinations)
+      destinations
+      .find_all { |d| d.respond_to?(:close) }
+      .each(&:close)
+    end
+
     def process_rows(sources, transforms, destinations)
       sources.each do |source|
         source.each do |row|
@@ -41,7 +50,6 @@ module Kiba
           end
         end
       end
-      destinations.find_all { |d| d.respond_to?(:close) }.each(&:close)
     end
 
     # not using keyword args because JRuby defaults to 1.9 syntax currently
@@ -55,15 +63,16 @@ module Kiba
     end
 
     def to_instance(klass, args, block, allow_block, allow_class)
-      if klass
+      if klass && block
+        fail 'Class and block form cannot be used together at the moment'
+      elsif klass
         fail 'Class form is not allowed here' unless allow_class
         klass.new(*args)
       elsif block
         fail 'Block form is not allowed here' unless allow_block
         AliasingProc.new(&block)
       else
-        # TODO: support block passing to a class form definition?
-        fail 'Class and block form cannot be used together at the moment'
+        fail 'Nil parameters not allowed here'
       end
     end
   end

data/lib/kiba/streaming_runner.rb

@@ -0,0 +1,38 @@
+module Kiba
+  module StreamingRunner
+    include Runner
+    extend self
+    
+    def transform_stream(stream, t)
+      Enumerator.new do |y|
+        stream.each do |input_row|
+          returned_row = t.process(input_row) do |yielded_row|
+            y << yielded_row
+          end
+          y << returned_row if returned_row
+        end
+        if t.respond_to?(:close)
+          t.close do |close_row|
+            y << close_row
+          end
+        end
+      end
+    end
+    
+    def source_stream(sources)
+      Enumerator.new do |y|
+        sources.each do |source|
+          source.each { |r| y << r }
+        end
+      end
+    end
+
+    def process_rows(sources, transforms, destinations)
+      stream = source_stream(sources)
+      recurser = lambda { |s,t| transform_stream(s, t) }
+      transforms.inject(stream, &recurser).each do |r|
+        destinations.each { |d| d.write(r) }
+      end
+    end
+  end
+end

data/lib/kiba/version.rb

@@ -1,3 +1,3 @@
 module Kiba
-  VERSION = '1.0.0'
+  VERSION = '3.5.0'
 end

data/test/helper.rb

@@ -1,10 +1,13 @@
 require 'minitest/autorun'
 require 'minitest/pride'
+require 'minitest/focus'
 require 'kiba'
 
-class Kiba::Test < Minitest::Test
-  extend Minitest::Spec::DSL
+if ENV['CI'] == 'true'
+  puts "Running with MiniTest version #{MiniTest::VERSION}"
+end
 
+class Kiba::Test < Minitest::Test
   def remove_files(*files)
     files.each do |file|
       File.delete(file) if File.exist?(file)
@@ -14,4 +17,10 @@ class Kiba::Test < Minitest::Test
   def fixture(file)
     File.join(File.dirname(__FILE__), 'fixtures', file)
   end
+  
+  unless self.method_defined?(:assert_mock)
+    def assert_mock(mock)
+      mock.verify
+    end
+  end
 end

data/test/shared_runner_tests.rb

@@ -0,0 +1,228 @@
+require 'minitest/mock'
+require_relative 'support/test_enumerable_source'
+require_relative 'support/test_destination_returning_nil'
+
+module SharedRunnerTests
+  def rows
+    @rows ||= [
+      { identifier: 'first-row' },
+      { identifier: 'second-row' }
+    ]
+  end
+  
+  def control
+    @control ||= begin
+      control = Kiba::Control.new
+      # this will yield a single row for testing
+      control.sources << {
+        klass: TestEnumerableSource,
+        args: [rows]
+      }
+      control
+    end
+  end
+  
+  def test_block_transform_processing
+    # is there a better way to assert a block was called in minitest?
+    control.transforms << { block: lambda { |r| @called = true; r } }
+    kiba_run(control)
+    assert_equal true, @called
+  end
+
+  def test_dismissed_row_not_passed_to_next_transform
+    @called = nil
+    control.transforms << { block: lambda { |_| nil } }
+    control.transforms << { block: lambda { |_| @called = true; nil } }
+    kiba_run(control)
+    assert_nil @called
+  end
+
+  def test_post_process_runs_once
+    assert_equal 2, rows.size
+    @called = 0
+    control.post_processes << { block: lambda { @called += 1 } }
+    kiba_run(control)
+    assert_equal 1, @called
+  end
+
+  def test_post_process_not_called_after_row_failure
+    @called = nil
+    control.transforms << { block: lambda { |_| fail 'FAIL' } }
+    control.post_processes << { block: lambda { @called = true } }
+    assert_raises(RuntimeError, 'FAIL') { kiba_run(control) }
+    assert_nil @called
+  end
+  
+  def test_pre_process_runs_once
+    assert_equal 2, rows.size
+    @called = 0
+    control.pre_processes << { block: lambda { @called += 1 } }
+    kiba_run(control)
+    assert_equal 1, @called
+  end
+
+  def test_pre_process_runs_before_source_is_instantiated
+    calls = []
+
+    mock_source_class = MiniTest::Mock.new
+    mock_source_class.expect(:new, TestEnumerableSource.new([1, 2, 3])) do
+      calls << :source_instantiated
+    end
+
+    control = Kiba::Control.new
+    control.pre_processes << { block: lambda { calls << :pre_processor_executed } }
+    control.sources << { klass: mock_source_class }
+    kiba_run(control)
+
+    assert_equal [:pre_processor_executed, :source_instantiated], calls
+    assert_mock mock_source_class
+  end
+  
+  def test_no_error_raised_if_destination_close_not_implemented
+    # NOTE: this fake destination does not implement `close`
+    destination_instance = MiniTest::Mock.new
+
+    mock_destination_class = MiniTest::Mock.new
+    mock_destination_class.expect(:new, destination_instance)
+    
+    control = Kiba::Control.new
+    control.destinations << { klass: mock_destination_class }
+    kiba_run(control)
+    assert_mock mock_destination_class
+  end
+  
+  def test_destination_close_called_if_defined
+    destination_instance = MiniTest::Mock.new
+    destination_instance.expect(:close, nil)
+    mock_destination_class = MiniTest::Mock.new
+    mock_destination_class.expect(:new, destination_instance)
+
+    control = Kiba::Control.new
+    control.destinations << { klass: mock_destination_class }
+    kiba_run(control)
+    assert_mock destination_instance
+    assert_mock mock_destination_class
+  end
+  
+  def test_use_next_to_exit_early_from_block_transform
+    assert_equal 2, rows.size
+
+    # calling "return row" from a block is forbidden, but you can use "next" instead
+    b = lambda do |row|
+      if row.fetch(:identifier) == 'first-row'
+        # demonstrate how to remove a row from the pipeline via next
+        next
+      else
+        # demonstrate how you can reformat via next
+        next({new_identifier: row.fetch(:identifier)})
+      end
+      fail "This should not be called"
+    end
+    control.transforms << { block: b }
+
+    # keep track of the rows
+    @remaining_rows = []
+    checker = lambda { |row| @remaining_rows << row; row }
+    control.transforms << { block: checker }
+
+    kiba_run(control)
+    
+    # the first row should have been removed
+    # and the second row should have been reformatted
+    assert_equal [{new_identifier: 'second-row'}], @remaining_rows
+  end
+  
+  def test_destination_returning_nil_does_not_remove_row_from_pipeline
+    # safeguard to avoid modification on the support code
+    assert_nil TestDestinationReturningNil.new.write("FOOBAR")
+
+    destinations = []
+    control = Kiba.parse do
+      source TestEnumerableSource, [{key: 'value'}]
+      2.times do
+        destination TestDestinationReturningNil, on_init: lambda { |d| destinations << d }
+      end
+    end
+    kiba_run(control)
+    2.times do |i|
+      assert_equal [{key: 'value'}], destinations[i].instance_variable_get(:@written_rows)
+    end
+  end
+
+  def test_nil_transform_error_message
+    control = Kiba.parse do
+      transform
+    end
+    assert_raises(RuntimeError, 'Nil parameters not allowed here') { kiba_run(control) }
+  end
+
+  def test_ruby_3_source_kwargs
+    # NOTE: before Ruby 3 kwargs support, a Ruby warning would 
+    # be captured here with Ruby 2.7 & ensure we fail,
+    # and an error would be raised with Ruby 2.8.0-dev
+    # NOTE: only the first warning will be captured, though, but
+    # having 3 different tests is still better
+    storage = nil
+    assert_silent do
+      Kiba.run(Kiba.parse do
+        source TestKeywordArgumentsComponent,
+          mandatory: "first",
+          on_init: -> (values) { storage = values }
+      end)
+    end
+    assert_equal({
+      mandatory: "first",
+      optional: nil
+    }, storage)
+  end
+  
+  def test_ruby_3_transform_kwargs
+    storage = nil
+    assert_silent do
+      Kiba.run(Kiba.parse do
+        transform TestKeywordArgumentsComponent,
+          mandatory: "first",
+          on_init: -> (values) { storage = values }
+      end)
+    end
+    assert_equal({
+      mandatory: "first",
+      optional: nil
+    }, storage)
+  end
+  
+  def test_ruby_3_destination_kwargs
+    storage = nil
+    assert_silent do
+      Kiba.run(Kiba.parse do
+        destination TestKeywordArgumentsComponent,
+          mandatory: "first",
+          on_init: -> (values) { storage = values }
+      end)
+    end
+    assert_equal({
+      mandatory: "first",
+      optional: nil
+    }, storage)
+  end
+      
+  def test_positional_plus_keyword_arguments
+    storage = nil
+    assert_silent do
+      Kiba.run(Kiba.parse do
+        source TestMixedArgumentsComponent,
+          "some positional argument",
+          mandatory: "first",
+          on_init: -> (values) {
+            storage = values
+          }
+      end)
+    end
+    
+    assert_equal({
+      some_value: "some positional argument",
+      mandatory: "first",
+      optional: nil
+    }, storage)
+  end
+end

data/test/support/shared_tests.rb

@@ -0,0 +1,10 @@
+module SharedTests
+  def shared_tests_for(desc, &block)
+    @@shared_tests ||= {}
+    @@shared_tests[desc] = block
+  end
+
+  def shared_tests(desc, *args)
+    self.class_exec(*args, &@@shared_tests.fetch(desc))
+  end
+end

data/test/support/test_aggregate_transform.rb

@@ -0,0 +1,19 @@
+class AggregateTransform
+  def initialize(options)
+    @aggregate_size = options.fetch(:aggregate_size)
+  end
+  
+  def process(row)
+    @buffer ||= []
+    @buffer << row
+    if @buffer.size == @aggregate_size
+      yield @buffer
+      @buffer = []
+    end
+    nil
+  end
+  
+  def close
+    yield @buffer unless @buffer.empty?
+  end
+end

data/test/support/test_array_destination.rb

@@ -0,0 +1,9 @@
+class TestArrayDestination
+  def initialize(array)
+    @array = array
+  end
+  
+  def write(row)
+    @array << row
+  end
+end

data/test/support/test_close_yielding_transform.rb

@@ -0,0 +1,11 @@
+class CloseYieldingTransform
+  def initialize(options)
+    @yield_on_close = options.fetch(:yield_on_close)
+  end
+
+  def close
+    @yield_on_close.each do |item|
+      yield item
+    end
+  end
+end

data/test/support/test_destination_returning_nil.rb

@@ -0,0 +1,12 @@
+class TestDestinationReturningNil
+  def initialize(options = {})
+    on_init = options[:on_init]
+    # A little trick to allow outer references to this instance
+    on_init.call(self) if on_init
+  end
+  
+  def write(row)
+    (@written_rows ||= []) << row
+    nil
+  end
+end