kiba 1.0.0 → 3.5.0

data/test/support/test_duplicate_row_transform.rb

@@ -0,0 +1,9 @@
+class TestDuplicateRowTranform
+  def process(row)
+    2.times do
+      # NOTE: it's a good idea to carefully avoid data reuse between rows
+      yield({item: row.fetch(:item).dup})
+    end
+    nil
+  end
+end

data/test/support/test_keyword_arguments_component.rb

@@ -0,0 +1,14 @@
+# a mock component to test Ruby 3 keyword argument support
+class TestKeywordArgumentsComponent
+  def initialize(mandatory:, optional: nil, on_init: nil)
+    values = {
+      mandatory: mandatory,
+      optional: optional
+    }
+    on_init&.call(values)
+  end
+  
+  def each
+    # no-op
+  end
+end

data/test/support/test_mixed_arguments_component.rb

@@ -0,0 +1,14 @@
+# a mock component to test Ruby 3 keyword argument support
+class TestMixedArgumentsComponent
+  def initialize(some_value, mandatory:, optional: nil, on_init:)
+    @values = {}
+    @values[:some_value] = some_value
+    @values[:mandatory] = mandatory
+    @values[:optional] = optional
+    on_init&.call(@values)
+  end
+  
+  def each
+    # no-op
+  end
+end

data/test/support/test_non_closing_transform.rb

@@ -0,0 +1,5 @@
+class NonClosingTransform
+  def process(row)
+    row
+  end
+end

data/test/support/test_yielding_transform.rb

@@ -0,0 +1,8 @@
+class TestYieldingTransform
+  def process(row)
+    row.fetch(:tags).each do |value|
+      yield({item: value})
+    end
+    {item: "classic-return-value"}
+  end
+end

data/test/test_integration.rb

@@ -8,10 +8,10 @@ require_relative 'support/test_source_that_reads_at_instantiation_time'
 
 # End-to-end tests go here
 class TestIntegration < Kiba::Test
-  let(:output_file) { 'test/tmp/output.csv' }
-  let(:input_file) { 'test/tmp/input.csv' }
+  def output_file; 'test/tmp/output.csv'; end
+  def input_file; 'test/tmp/input.csv'; end
 
-  let(:sample_csv_data) do
+  def sample_csv_data
     <<CSV
 first_name,last_name,sex
 John,Doe,M

data/test/test_parser.rb

@@ -14,6 +14,17 @@ class TestParser < Kiba::Test
     assert_equal DummyClass, control.sources[0][:klass]
     assert_equal %w(has args), control.sources[0][:args]
   end
+  
+  # NOTE: useful for anything not using the CLI (e.g. sidekiq)
+  def test_block_parsing_with_reference_to_outside_variable
+    some_variable = Object.new
+    
+    control = Kiba.parse do
+      source DummyClass, some_variable
+    end
+    
+    assert_equal [some_variable], control.sources[0][:args]
+  end
 
   def test_block_transform_definition
     control = Kiba.parse do
@@ -57,36 +68,30 @@ class TestParser < Kiba::Test
     assert_instance_of Proc, control.pre_processes[0][:block]
   end
 
-  def test_source_as_string_parsing
-    control = Kiba.parse <<RUBY
-      source DummyClass, 'from', 'file'
-RUBY
-
-    assert_equal 1, control.sources.size
-    assert_equal DummyClass, control.sources[0][:klass]
-    assert_equal %w(from file), control.sources[0][:args]
+  def test_config
+    control = Kiba.parse do
+      extend Kiba::DSLExtensions::Config
+      
+      config :context, key: "value", other_key: "other_value"
+    end
+    
+    assert_equal({ context: {
+      key: "value",
+      other_key: "other_value"
+    }}, control.config)
   end
+  
+  def test_config_override
+    control = Kiba.parse do
+      extend Kiba::DSLExtensions::Config
 
-  def test_source_as_file_doing_require
-    IO.write 'test/tmp/etl-common.rb', <<RUBY
-      def common_source_declaration
-        source DummyClass, 'from', 'common'
-      end
-RUBY
-    IO.write 'test/tmp/etl-main.rb', <<RUBY
-      require './test/tmp/etl-common.rb'
-
-      source DummyClass, 'from', 'main'
-      common_source_declaration
-RUBY
-    control = Kiba.parse IO.read('test/tmp/etl-main.rb')
-
-    assert_equal 2, control.sources.size
-
-    assert_equal %w(from main), control.sources[0][:args]
-    assert_equal %w(from common), control.sources[1][:args]
-
-  ensure
-    remove_files('test/tmp/etl-common.rb', 'test/tmp/etl-main.rb')
+      config :context, key: "value", other_key: "other_value"
+      config :context, key: "new_value"
+    end
+    
+    assert_equal({ context: {
+      key: "new_value",
+      other_key: "other_value"
+    }}, control.config)
   end
 end

data/test/test_run.rb

@@ -0,0 +1,12 @@
+require_relative 'helper'
+require 'minitest/mock'
+
+class TestRun < Kiba::Test
+  def test_ensure_kiba_defaults_to_streaming_runner
+    cb = -> (job) { "Streaming runner called" }
+    Kiba::StreamingRunner.stub(:run, cb) do
+      job = Kiba::Control.new
+      assert_equal "Streaming runner called", Kiba.run(job)
+    end
+  end
+end

data/test/test_runner.rb

@@ -1,87 +1,11 @@
 require_relative 'helper'
-require 'minitest/mock'
-require_relative 'support/test_enumerable_source'
+require_relative 'shared_runner_tests'
 
 class TestRunner < Kiba::Test
-  let(:rows) do
-    [
-      { field: 'value' },
-      { field: 'other-value' }
-    ]
+  def kiba_run(job)
+    job.config[:kiba] = {runner: Kiba::Runner}
+    Kiba.run(job)
   end
-  
-  let(:control) do
-    control = Kiba::Control.new
-    # this will yield a single row for testing
-    control.sources << {
-      klass: TestEnumerableSource,
-      args: [rows]
-    }
-    control
-  end
-
-  def test_block_transform_processing
-    # is there a better way to assert a block was called in minitest?
-    control.transforms << { block: lambda { |r| @called = true; r } }
-    Kiba.run(control)
-    assert_equal true, @called
-  end
-
-  def test_dismissed_row_not_passed_to_next_transform
-    control.transforms << { block: lambda { |_| nil } }
-    control.transforms << { block: lambda { |_| @called = true; nil } }
-    Kiba.run(control)
-    assert_nil @called
-  end
-
-  def test_post_process_runs_once
-    assert_equal 2, rows.size
-    @called = 0
-    control.post_processes << { block: lambda { @called += 1 } }
-    Kiba.run(control)
-    assert_equal 1, @called
-  end
-
-  def test_post_process_not_called_after_row_failure
-    control.transforms << { block: lambda { |_| fail 'FAIL' } }
-    control.post_processes << { block: lambda { @called = true } }
-    assert_raises(RuntimeError, 'FAIL') { Kiba.run(control) }
-    assert_nil @called
-  end
-  
-  def test_pre_process_runs_once
-    assert_equal 2, rows.size
-    @called = 0
-    control.pre_processes << { block: lambda { @called += 1 } }
-    Kiba.run(control)
-    assert_equal 1, @called
-  end
-
-  def test_pre_process_runs_before_source_is_instantiated
-    calls = []
-
-    mock_source_class = MiniTest::Mock.new
-    mock_source_class.expect(:new, TestEnumerableSource.new([1, 2, 3])) do
-      calls << :source_instantiated
-    end
 
-    control = Kiba::Control.new
-    control.pre_processes << { block: lambda { calls << :pre_processor_executed } }
-    control.sources << { klass: mock_source_class }
-    Kiba.run(control)
-
-    assert_equal [:pre_processor_executed, :source_instantiated], calls
-  end
-  
-  def test_no_error_raised_if_destination_close_not_implemented
-    # NOTE: this fake destination does not implement `close`
-    destination_instance = MiniTest::Mock.new
-
-    mock_destination_class = MiniTest::Mock.new
-    mock_destination_class.expect(:new, destination_instance)
-    
-    control = Kiba::Control.new
-    control.destinations << { klass: mock_destination_class }
-    Kiba.run(control)
-  end
+  include SharedRunnerTests
 end

data/test/test_streaming_runner.rb

@@ -0,0 +1,70 @@
+require_relative 'helper'
+require_relative 'support/test_enumerable_source'
+require_relative 'support/test_array_destination'
+require_relative 'support/test_yielding_transform'
+require_relative 'support/test_duplicate_row_transform'
+require_relative 'support/test_close_yielding_transform'
+require_relative 'support/test_non_closing_transform'
+require_relative 'shared_runner_tests'
+require_relative 'support/test_keyword_arguments_component'
+require_relative 'support/test_mixed_arguments_component'
+
+class TestStreamingRunner < Kiba::Test
+  def kiba_run(job)
+    job.config[:kiba] = {runner: Kiba::StreamingRunner}
+    Kiba.run(job)
+  end
+
+  include SharedRunnerTests
+  
+  def test_yielding_class_transform
+    input_row = {tags: ["one", "two", "three"]}
+    destination_array = []
+    
+    job = Kiba.parse do
+      # provide a single row as the input
+      source TestEnumerableSource, [input_row]
+
+      # explode tags in one row each
+      transform TestYieldingTransform
+
+      # generate two rows out of each exploded tags row
+      transform TestDuplicateRowTranform
+
+      destination TestArrayDestination, destination_array
+    end
+    
+    kiba_run(job)
+  
+    assert_equal [
+      {item: 'one'},
+      {item: 'one'},
+
+      {item: 'two'},
+      {item: 'two'},
+
+      {item: 'three'},
+      {item: 'three'},
+
+      {item: 'classic-return-value'},
+      {item: 'classic-return-value'}
+    ], destination_array
+  end
+  
+  def test_transform_yielding_from_close
+    destination_array = []
+    job = Kiba.parse do
+      transform CloseYieldingTransform, yield_on_close: [1, 2]
+      destination TestArrayDestination, destination_array
+    end
+    Kiba.run(job)
+    assert_equal [1, 2], destination_array
+  end
+
+  def test_transform_with_no_close_must_not_raise
+    job = Kiba.parse do
+      transform NonClosingTransform
+    end
+    Kiba.run(job)
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kiba
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 3.5.0
 platform: ruby
 authors:
 - Thibaut Barrère
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-12-01 00:00:00.000000000 Z
+date: 2020-09-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -26,6 +26,20 @@ dependencies:
         version: '0'
 - !ruby/object:Gem::Dependency
   name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.9'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.9'
+- !ruby/object:Gem::Dependency
+  name: awesome_print
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -39,7 +53,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: awesome_print
+  name: minitest-focus
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -60,40 +74,58 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/FUNDING.yml"
 - ".gitignore"
 - ".travis.yml"
+- COMM-LICENSE.md
 - Changes.md
 - Gemfile
+- ISSUE_TEMPLATE.md
+- LICENSE
+- Pro-Changes.md
 - README.md
 - Rakefile
 - appveyor.yml
 - bin/kiba
 - kiba.gemspec
 - lib/kiba.rb
-- lib/kiba/cli.rb
 - lib/kiba/context.rb
 - lib/kiba/control.rb
+- lib/kiba/dsl_extensions/config.rb
 - lib/kiba/parser.rb
 - lib/kiba/runner.rb
+- lib/kiba/streaming_runner.rb
 - lib/kiba/version.rb
-- test/fixtures/bogus.etl
-- test/fixtures/valid.etl
 - test/helper.rb
+- test/shared_runner_tests.rb
+- test/support/shared_tests.rb
+- test/support/test_aggregate_transform.rb
+- test/support/test_array_destination.rb
+- test/support/test_close_yielding_transform.rb
 - test/support/test_csv_destination.rb
 - test/support/test_csv_source.rb
+- test/support/test_destination_returning_nil.rb
+- test/support/test_duplicate_row_transform.rb
 - test/support/test_enumerable_source.rb
+- test/support/test_keyword_arguments_component.rb
+- test/support/test_mixed_arguments_component.rb
+- test/support/test_non_closing_transform.rb
 - test/support/test_rename_field_transform.rb
 - test/support/test_source_that_reads_at_instantiation_time.rb
-- test/test_cli.rb
+- test/support/test_yielding_transform.rb
 - test/test_integration.rb
 - test/test_parser.rb
+- test/test_run.rb
 - test/test_runner.rb
+- test/test_streaming_runner.rb
 - test/tmp/.gitkeep
 homepage: http://thbar.github.io/kiba/
 licenses:
 - LGPL-3.0
-metadata: {}
-post_install_message: 
+metadata:
+  source_code_uri: https://github.com/thbar/kiba
+  documentation_uri: https://github.com/thbar/kiba/wiki
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -108,22 +140,31 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.4.8
-signing_key: 
+rubygems_version: 3.1.2
+signing_key:
 specification_version: 4
 summary: Lightweight ETL for Ruby
 test_files:
-- test/fixtures/bogus.etl
-- test/fixtures/valid.etl
 - test/helper.rb
+- test/shared_runner_tests.rb
+- test/support/shared_tests.rb
+- test/support/test_aggregate_transform.rb
+- test/support/test_array_destination.rb
+- test/support/test_close_yielding_transform.rb
 - test/support/test_csv_destination.rb
 - test/support/test_csv_source.rb
+- test/support/test_destination_returning_nil.rb
+- test/support/test_duplicate_row_transform.rb
 - test/support/test_enumerable_source.rb
+- test/support/test_keyword_arguments_component.rb
+- test/support/test_mixed_arguments_component.rb
+- test/support/test_non_closing_transform.rb
 - test/support/test_rename_field_transform.rb
 - test/support/test_source_that_reads_at_instantiation_time.rb
-- test/test_cli.rb
+- test/support/test_yielding_transform.rb
 - test/test_integration.rb
 - test/test_parser.rb
+- test/test_run.rb
 - test/test_runner.rb
+- test/test_streaming_runner.rb
 - test/tmp/.gitkeep