kiba 2.0.0 → 4.0.0

data/lib/kiba/version.rb

@@ -1,3 +1,3 @@
 module Kiba
-  VERSION = '2.0.0'
+  VERSION = "4.0.0"
 end

data/test/helper.rb

@@ -1,11 +1,13 @@
-require 'minitest/autorun'
-require 'minitest/pride'
-require 'minitest/focus'
-require 'kiba'
+require "minitest/autorun"
+require "minitest/pride"
+require "minitest/focus"
+require "kiba"
 
-class Kiba::Test < Minitest::Test
-  extend Minitest::Spec::DSL
+if ENV["CI"] == "true"
+  puts "Running with MiniTest version #{MiniTest::VERSION}"
+end
 
+class Kiba::Test < Minitest::Test
   def remove_files(*files)
     files.each do |file|
       File.delete(file) if File.exist?(file)
@@ -13,6 +15,12 @@ class Kiba::Test < Minitest::Test
   end
 
   def fixture(file)
-    File.join(File.dirname(__FILE__), 'fixtures', file)
+    File.join(File.dirname(__FILE__), "fixtures", file)
+  end
+
+  unless method_defined?(:assert_mock)
+    def assert_mock(mock)
+      mock.verify
+    end
   end
 end

data/test/shared_runner_tests.rb

@@ -0,0 +1,227 @@
+require "minitest/mock"
+require_relative "support/test_enumerable_source"
+require_relative "support/test_destination_returning_nil"
+
+module SharedRunnerTests
+  def rows
+    @rows ||= [
+      {identifier: "first-row"},
+      {identifier: "second-row"}
+    ]
+  end
+
+  def control
+    @control ||= begin
+      control = Kiba::Control.new
+      # this will yield a single row for testing
+      control.sources << {
+        klass: TestEnumerableSource,
+        args: [rows]
+      }
+      control
+    end
+  end
+
+  def test_block_transform_processing
+    # is there a better way to assert a block was called in minitest?
+    control.transforms << {block: lambda { |r| (@called = true) && (return r) }}
+    kiba_run(control)
+    assert_equal true, @called
+  end
+
+  def test_dismissed_row_not_passed_to_next_transform
+    @called = nil
+    control.transforms << {block: lambda { |_| }}
+    control.transforms << {block: lambda { |_| (@called = true) && (return nil) }}
+    kiba_run(control)
+    assert_nil @called
+  end
+
+  def test_post_process_runs_once
+    assert_equal 2, rows.size
+    @called = 0
+    control.post_processes << {block: lambda { @called += 1 }}
+    kiba_run(control)
+    assert_equal 1, @called
+  end
+
+  def test_post_process_not_called_after_row_failure
+    @called = nil
+    control.transforms << {block: lambda { |_| fail "FAIL" }}
+    control.post_processes << {block: lambda { @called = true }}
+    assert_raises(RuntimeError, "FAIL") { kiba_run(control) }
+    assert_nil @called
+  end
+
+  def test_pre_process_runs_once
+    assert_equal 2, rows.size
+    @called = 0
+    control.pre_processes << {block: lambda { @called += 1 }}
+    kiba_run(control)
+    assert_equal 1, @called
+  end
+
+  def test_pre_process_runs_before_source_is_instantiated
+    calls = []
+
+    mock_source_class = MiniTest::Mock.new
+    mock_source_class.expect(:new, TestEnumerableSource.new([1, 2, 3])) do
+      calls << :source_instantiated
+    end
+
+    control = Kiba::Control.new
+    control.pre_processes << {block: lambda { calls << :pre_processor_executed }}
+    control.sources << {klass: mock_source_class}
+    kiba_run(control)
+
+    assert_equal [:pre_processor_executed, :source_instantiated], calls
+    assert_mock mock_source_class
+  end
+
+  def test_no_error_raised_if_destination_close_not_implemented
+    # NOTE: this fake destination does not implement `close`
+    destination_instance = MiniTest::Mock.new
+
+    mock_destination_class = MiniTest::Mock.new
+    mock_destination_class.expect(:new, destination_instance)
+
+    control = Kiba::Control.new
+    control.destinations << {klass: mock_destination_class}
+    kiba_run(control)
+    assert_mock mock_destination_class
+  end
+
+  def test_destination_close_called_if_defined
+    destination_instance = MiniTest::Mock.new
+    destination_instance.expect(:close, nil)
+    mock_destination_class = MiniTest::Mock.new
+    mock_destination_class.expect(:new, destination_instance)
+
+    control = Kiba::Control.new
+    control.destinations << {klass: mock_destination_class}
+    kiba_run(control)
+    assert_mock destination_instance
+    assert_mock mock_destination_class
+  end
+
+  def test_use_next_to_exit_early_from_block_transform
+    assert_equal 2, rows.size
+
+    # calling "return row" from a block is forbidden, but you can use "next" instead
+    b = lambda do |row|
+      if row.fetch(:identifier) == "first-row"
+        # demonstrate how to remove a row from the pipeline via next
+        next
+      else
+        # demonstrate how you can reformat via next
+        next({new_identifier: row.fetch(:identifier)})
+      end
+    end
+    control.transforms << {block: b}
+
+    # keep track of the rows
+    @remaining_rows = []
+    checker = lambda { |row| (@remaining_rows << row) && (return row) }
+    control.transforms << {block: checker}
+
+    kiba_run(control)
+
+    # the first row should have been removed
+    # and the second row should have been reformatted
+    assert_equal [{new_identifier: "second-row"}], @remaining_rows
+  end
+
+  def test_destination_returning_nil_does_not_remove_row_from_pipeline
+    # safeguard to avoid modification on the support code
+    assert_nil TestDestinationReturningNil.new.write("FOOBAR")
+
+    destinations = []
+    control = Kiba.parse do
+      source TestEnumerableSource, [{key: "value"}]
+      2.times do
+        destination TestDestinationReturningNil, on_init: lambda { |d| destinations << d }
+      end
+    end
+    kiba_run(control)
+    2.times do |i|
+      assert_equal [{key: "value"}], destinations[i].instance_variable_get(:@written_rows)
+    end
+  end
+
+  def test_nil_transform_error_message
+    control = Kiba.parse do
+      transform
+    end
+    assert_raises(RuntimeError, "Nil parameters not allowed here") { kiba_run(control) }
+  end
+
+  def test_ruby_3_source_kwargs
+    # NOTE: before Ruby 3 kwargs support, a Ruby warning would
+    # be captured here with Ruby 2.7 & ensure we fail,
+    # and an error would be raised with Ruby 2.8.0-dev
+    # NOTE: only the first warning will be captured, though, but
+    # having 3 different tests is still better
+    storage = nil
+    assert_silent do
+      Kiba.run(Kiba.parse do
+        source TestKeywordArgumentsComponent,
+          mandatory: "first",
+          on_init: ->(values) { storage = values }
+      end)
+    end
+    assert_equal({
+      mandatory: "first",
+      optional: nil
+    }, storage)
+  end
+
+  def test_ruby_3_transform_kwargs
+    storage = nil
+    assert_silent do
+      Kiba.run(Kiba.parse do
+        transform TestKeywordArgumentsComponent,
+          mandatory: "first",
+          on_init: ->(values) { storage = values }
+      end)
+    end
+    assert_equal({
+      mandatory: "first",
+      optional: nil
+    }, storage)
+  end
+
+  def test_ruby_3_destination_kwargs
+    storage = nil
+    assert_silent do
+      Kiba.run(Kiba.parse do
+        destination TestKeywordArgumentsComponent,
+          mandatory: "first",
+          on_init: ->(values) { storage = values }
+      end)
+    end
+    assert_equal({
+      mandatory: "first",
+      optional: nil
+    }, storage)
+  end
+
+  def test_positional_plus_keyword_arguments
+    storage = nil
+    assert_silent do
+      Kiba.run(Kiba.parse do
+        source TestMixedArgumentsComponent,
+          "some positional argument",
+          mandatory: "first",
+          on_init: ->(values) {
+            storage = values
+          }
+      end)
+    end
+
+    assert_equal({
+      some_value: "some positional argument",
+      mandatory: "first",
+      optional: nil
+    }, storage)
+  end
+end

data/test/support/shared_tests.rb

@@ -5,6 +5,6 @@ module SharedTests
   end
 
   def shared_tests(desc, *args)
-    self.class_exec(*args, &@@shared_tests.fetch(desc))
+    class_exec(*args, &@@shared_tests.fetch(desc))
   end
 end

data/test/support/test_aggregate_transform.rb

@@ -0,0 +1,19 @@
+class AggregateTransform
+  def initialize(options)
+    @aggregate_size = options.fetch(:aggregate_size)
+  end
+
+  def process(row)
+    @buffer ||= []
+    @buffer << row
+    if @buffer.size == @aggregate_size
+      yield @buffer
+      @buffer = []
+    end
+    nil
+  end
+
+  def close
+    yield @buffer unless @buffer.empty?
+  end
+end

data/test/support/test_array_destination.rb

@@ -2,8 +2,8 @@ class TestArrayDestination
   def initialize(array)
     @array = array
   end
-  
+
   def write(row)
     @array << row
   end
-end
+end

data/test/support/test_close_yielding_transform.rb

@@ -0,0 +1,11 @@
+class CloseYieldingTransform
+  def initialize(options)
+    @yield_on_close = options.fetch(:yield_on_close)
+  end
+
+  def close
+    @yield_on_close.each do |item|
+      yield item
+    end
+  end
+end

data/test/support/test_csv_destination.rb

@@ -1,9 +1,9 @@
-require 'csv'
+require "csv"
 
 # simple destination, not checking that each row has all the fields
 class TestCsvDestination
   def initialize(output_file)
-    @csv = CSV.open(output_file, 'w')
+    @csv = CSV.open(output_file, "w")
     @headers_written = false
   end
 

data/test/support/test_csv_source.rb

@@ -1,4 +1,4 @@
-require 'csv'
+require "csv"
 
 class TestCsvSource
   def initialize(input_file)

data/test/support/test_destination_returning_nil.rb

@@ -0,0 +1,12 @@
+class TestDestinationReturningNil
+  def initialize(options = {})
+    on_init = options[:on_init]
+    # A little trick to allow outer references to this instance
+    on_init&.call(self)
+  end
+
+  def write(row)
+    (@written_rows ||= []) << row
+    nil
+  end
+end

data/test/support/test_duplicate_row_transform.rb

@@ -6,4 +6,4 @@ class TestDuplicateRowTranform
     end
     nil
   end
-end
+end

data/test/support/test_keyword_arguments_component.rb

@@ -0,0 +1,14 @@
+# a mock component to test Ruby 3 keyword argument support
+class TestKeywordArgumentsComponent
+  def initialize(mandatory:, optional: nil, on_init: nil)
+    values = {
+      mandatory: mandatory,
+      optional: optional
+    }
+    on_init&.call(values)
+  end
+
+  def each
+    # no-op
+  end
+end

data/test/support/test_mixed_arguments_component.rb

@@ -0,0 +1,14 @@
+# a mock component to test Ruby 3 keyword argument support
+class TestMixedArgumentsComponent
+  def initialize(some_value, mandatory:, on_init:, optional: nil)
+    @values = {}
+    @values[:some_value] = some_value
+    @values[:mandatory] = mandatory
+    @values[:optional] = optional
+    on_init&.call(@values)
+  end
+
+  def each
+    # no-op
+  end
+end

data/test/support/test_non_closing_transform.rb

@@ -0,0 +1,5 @@
+class NonClosingTransform
+  def process(row)
+    row
+  end
+end

data/test/support/test_yielding_transform.rb

@@ -5,4 +5,4 @@ class TestYieldingTransform
     end
     {item: "classic-return-value"}
   end
-end
+end

data/test/test_integration.rb

@@ -1,28 +1,33 @@
-require_relative 'helper'
+require_relative "helper"
 
-require_relative 'support/test_csv_source'
-require_relative 'support/test_csv_destination'
-require_relative 'support/test_rename_field_transform'
-require_relative 'support/test_enumerable_source'
-require_relative 'support/test_source_that_reads_at_instantiation_time'
+require_relative "support/test_csv_source"
+require_relative "support/test_csv_destination"
+require_relative "support/test_rename_field_transform"
+require_relative "support/test_enumerable_source"
+require_relative "support/test_source_that_reads_at_instantiation_time"
 
 # End-to-end tests go here
 class TestIntegration < Kiba::Test
-  let(:output_file) { 'test/tmp/output.csv' }
-  let(:input_file) { 'test/tmp/input.csv' }
-
-  let(:sample_csv_data) do
-    <<CSV
-first_name,last_name,sex
-John,Doe,M
-Mary,Johnson,F
-Cindy,Backgammon,F
-Patrick,McWire,M
-CSV
+  def output_file
+    "test/tmp/output.csv"
+  end
+
+  def input_file
+    "test/tmp/input.csv"
+  end
+
+  def sample_csv_data
+    <<~CSV
+      first_name,last_name,sex
+      John,Doe,M
+      Mary,Johnson,F
+      Cindy,Backgammon,F
+      Patrick,McWire,M
+    CSV
   end
 
   def clean
-    remove_files(*Dir['test/tmp/*.csv'])
+    remove_files(*Dir["test/tmp/*.csv"])
   end
 
   def setup
@@ -37,36 +42,36 @@ CSV
   def test_csv_to_csv
     # parse the ETL script (this won't run it)
     control = Kiba.parse do
-      source TestCsvSource, 'test/tmp/input.csv'
+      source TestCsvSource, "test/tmp/input.csv"
 
       transform do |row|
         row[:sex] = case row[:sex]
-                    when 'M' then 'Male'
-                    when 'F' then 'Female'
-                    else 'Unknown'
+                    when "M" then "Male"
+                    when "F" then "Female"
+                    else "Unknown"
         end
         row # must be returned
       end
 
       # returning nil dismisses the row
       transform do |row|
-        row[:sex] == 'Female' ? row : nil
+        row[:sex] == "Female" ? row : nil
       end
 
       transform TestRenameFieldTransform, :sex, :sex_2015
 
-      destination TestCsvDestination, 'test/tmp/output.csv'
+      destination TestCsvDestination, "test/tmp/output.csv"
     end
 
     # run the parsed ETL script
     Kiba.run(control)
 
     # verify the output
-    assert_equal <<CSV, IO.read(output_file)
-first_name,last_name,sex_2015
-Mary,Johnson,Female
-Cindy,Backgammon,Female
-CSV
+    assert_equal <<~CSV, IO.read(output_file)
+      first_name,last_name,sex_2015
+      Mary,Johnson,Female
+      Cindy,Backgammon,Female
+    CSV
   end
 
   def test_variable_access
@@ -97,18 +102,18 @@ CSV
 
     Kiba.run(control)
 
-    assert_equal 'Count is now 103', message
+    assert_equal "Count is now 103", message
   end
 
   def test_file_created_by_pre_process_can_be_read_by_source_at_instantiation_time
-    remove_files('test/tmp/eager.csv')
+    remove_files("test/tmp/eager.csv")
 
     control = Kiba.parse do
       pre_process do
-        IO.write('test/tmp/eager.csv', 'something')
+        IO.write("test/tmp/eager.csv", "something")
       end
 
-      source SourceThatReadsAtInstantionTime, 'test/tmp/eager.csv'
+      source SourceThatReadsAtInstantionTime, "test/tmp/eager.csv"
     end
 
     Kiba.run(control)