kiba 2.0.0.rc1 → 3.6.0

data/test/support/test_aggregate_transform.rb

@@ -0,0 +1,19 @@
+class AggregateTransform
+  def initialize(options)
+    @aggregate_size = options.fetch(:aggregate_size)
+  end
+  
+  def process(row)
+    @buffer ||= []
+    @buffer << row
+    if @buffer.size == @aggregate_size
+      yield @buffer
+      @buffer = []
+    end
+    nil
+  end
+  
+  def close
+    yield @buffer unless @buffer.empty?
+  end
+end

data/test/support/test_close_yielding_transform.rb

@@ -0,0 +1,11 @@
+class CloseYieldingTransform
+  def initialize(options)
+    @yield_on_close = options.fetch(:yield_on_close)
+  end
+
+  def close
+    @yield_on_close.each do |item|
+      yield item
+    end
+  end
+end

data/test/support/test_destination_returning_nil.rb

@@ -0,0 +1,12 @@
+class TestDestinationReturningNil
+  def initialize(options = {})
+    on_init = options[:on_init]
+    # A little trick to allow outer references to this instance
+    on_init.call(self) if on_init
+  end
+  
+  def write(row)
+    (@written_rows ||= []) << row
+    nil
+  end
+end

data/test/support/test_duplicate_row_transform.rb

@@ -0,0 +1,9 @@
+class TestDuplicateRowTranform
+  def process(row)
+    2.times do
+      # NOTE: it's a good idea to carefully avoid data reuse between rows
+      yield({item: row.fetch(:item).dup})
+    end
+    nil
+  end
+end

data/test/support/test_keyword_arguments_component.rb

@@ -0,0 +1,14 @@
+# a mock component to test Ruby 3 keyword argument support
+class TestKeywordArgumentsComponent
+  def initialize(mandatory:, optional: nil, on_init: nil)
+    values = {
+      mandatory: mandatory,
+      optional: optional
+    }
+    on_init&.call(values)
+  end
+  
+  def each
+    # no-op
+  end
+end

data/test/support/test_mixed_arguments_component.rb

@@ -0,0 +1,14 @@
+# a mock component to test Ruby 3 keyword argument support
+class TestMixedArgumentsComponent
+  def initialize(some_value, mandatory:, optional: nil, on_init:)
+    @values = {}
+    @values[:some_value] = some_value
+    @values[:mandatory] = mandatory
+    @values[:optional] = optional
+    on_init&.call(@values)
+  end
+  
+  def each
+    # no-op
+  end
+end

data/test/support/test_non_closing_transform.rb

@@ -0,0 +1,5 @@
+class NonClosingTransform
+  def process(row)
+    row
+  end
+end

data/test/test_integration.rb

@@ -8,10 +8,10 @@ require_relative 'support/test_source_that_reads_at_instantiation_time'
 
 # End-to-end tests go here
 class TestIntegration < Kiba::Test
-  let(:output_file) { 'test/tmp/output.csv' }
-  let(:input_file) { 'test/tmp/input.csv' }
+  def output_file; 'test/tmp/output.csv'; end
+  def input_file; 'test/tmp/input.csv'; end
 
-  let(:sample_csv_data) do
+  def sample_csv_data
     <<CSV
 first_name,last_name,sex
 John,Doe,M

data/test/test_parser.rb

@@ -68,39 +68,6 @@ class TestParser < Kiba::Test
     assert_instance_of Proc, control.pre_processes[0][:block]
   end
 
-  def test_source_as_string_parsing
-    control = Kiba.parse <<RUBY
-      source DummyClass, 'from', 'file'
-RUBY
-
-    assert_equal 1, control.sources.size
-    assert_equal DummyClass, control.sources[0][:klass]
-    assert_equal %w(from file), control.sources[0][:args]
-  end
-
-  def test_source_as_file_doing_require
-    IO.write 'test/tmp/etl-common.rb', <<RUBY
-      def common_source_declaration
-        source DummyClass, 'from', 'common'
-      end
-RUBY
-    IO.write 'test/tmp/etl-main.rb', <<RUBY
-      require './test/tmp/etl-common.rb'
-
-      source DummyClass, 'from', 'main'
-      common_source_declaration
-RUBY
-    control = Kiba.parse IO.read('test/tmp/etl-main.rb')
-
-    assert_equal 2, control.sources.size
-
-    assert_equal %w(from main), control.sources[0][:args]
-    assert_equal %w(from common), control.sources[1][:args]
-
-  ensure
-    remove_files('test/tmp/etl-common.rb', 'test/tmp/etl-main.rb')
-  end
-  
   def test_config
     control = Kiba.parse do
       extend Kiba::DSLExtensions::Config

data/test/test_run.rb

@@ -0,0 +1,38 @@
+require_relative 'helper'
+require 'minitest/mock'
+require_relative 'support/test_enumerable_source'
+require_relative 'support/test_array_destination'
+
+class TestRun < Kiba::Test
+  def test_ensure_kiba_defaults_to_streaming_runner
+    cb = -> (job) { "Streaming runner called" }
+    Kiba::StreamingRunner.stub(:run, cb) do
+      job = Kiba::Control.new
+      assert_equal "Streaming runner called", Kiba.run(job)
+    end
+  end
+
+  def test_run_allows_block_arg
+    rows = []
+    Kiba.run do
+      source TestEnumerableSource, (1..10)
+      destination TestArrayDestination, rows
+    end
+    assert_equal (1..10).to_a, rows
+  end
+
+  def test_forbids_no_arg
+    assert_raises ArgumentError do
+      Kiba.run
+    end
+  end
+
+  def test_forbids_multiple_args
+    assert_raises ArgumentError do
+      job = Kiba.parse { }
+      Kiba.run(job) do
+        #
+      end
+    end
+  end
+end

data/test/test_runner.rb

@@ -1,6 +1,11 @@
 require_relative 'helper'
-require_relative 'common/runner'
+require_relative 'shared_runner_tests'
 
 class TestRunner < Kiba::Test
+  def kiba_run(job)
+    job.config[:kiba] = {runner: Kiba::Runner}
+    Kiba.run(job)
+  end
+
   include SharedRunnerTests
 end

data/test/test_streaming_runner.rb

@@ -2,9 +2,19 @@ require_relative 'helper'
 require_relative 'support/test_enumerable_source'
 require_relative 'support/test_array_destination'
 require_relative 'support/test_yielding_transform'
-require_relative 'common/runner'
+require_relative 'support/test_duplicate_row_transform'
+require_relative 'support/test_close_yielding_transform'
+require_relative 'support/test_non_closing_transform'
+require_relative 'shared_runner_tests'
+require_relative 'support/test_keyword_arguments_component'
+require_relative 'support/test_mixed_arguments_component'
 
 class TestStreamingRunner < Kiba::Test
+  def kiba_run(job)
+    job.config[:kiba] = {runner: Kiba::StreamingRunner}
+    Kiba.run(job)
+  end
+
   include SharedRunnerTests
   
   def test_yielding_class_transform
@@ -12,12 +22,15 @@ class TestStreamingRunner < Kiba::Test
     destination_array = []
     
     job = Kiba.parse do
-      extend Kiba::DSLExtensions::Config
-
-      config :kiba, runner: Kiba::StreamingRunner
-
+      # provide a single row as the input
       source TestEnumerableSource, [input_row]
+
+      # explode tags in one row each
       transform TestYieldingTransform
+
+      # generate two rows out of each exploded tags row
+      transform TestDuplicateRowTranform
+
       destination TestArrayDestination, destination_array
     end
     
@@ -25,9 +38,33 @@ class TestStreamingRunner < Kiba::Test
   
     assert_equal [
       {item: 'one'},
+      {item: 'one'},
+
       {item: 'two'},
+      {item: 'two'},
+
+      {item: 'three'},
       {item: 'three'},
+
+      {item: 'classic-return-value'},
       {item: 'classic-return-value'}
     ], destination_array
   end
+  
+  def test_transform_yielding_from_close
+    destination_array = []
+    job = Kiba.parse do
+      transform CloseYieldingTransform, yield_on_close: [1, 2]
+      destination TestArrayDestination, destination_array
+    end
+    Kiba.run(job)
+    assert_equal [1, 2], destination_array
+  end
+
+  def test_transform_with_no_close_must_not_raise
+    job = Kiba.parse do
+      transform NonClosingTransform
+    end
+    Kiba.run(job)
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kiba
 version: !ruby/object:Gem::Version
-  version: 2.0.0.rc1
+  version: 3.6.0
 platform: ruby
 authors:
 - Thibaut Barrère
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-12-27 00:00:00.000000000 Z
+date: 2021-02-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -74,10 +74,13 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/FUNDING.yml"
 - ".gitignore"
 - ".travis.yml"
+- COMM-LICENSE.md
 - Changes.md
 - Gemfile
+- ISSUE_TEMPLATE.md
 - LICENSE
 - Pro-Changes.md
 - README.md
@@ -86,7 +89,6 @@ files:
 - bin/kiba
 - kiba.gemspec
 - lib/kiba.rb
-- lib/kiba/cli.rb
 - lib/kiba/context.rb
 - lib/kiba/control.rb
 - lib/kiba/dsl_extensions/config.rb
@@ -94,31 +96,36 @@ files:
 - lib/kiba/runner.rb
 - lib/kiba/streaming_runner.rb
 - lib/kiba/version.rb
-- test/common/runner.rb
-- test/fixtures/bogus.etl
-- test/fixtures/namespace_conflict.etl
-- test/fixtures/some_extension.rb
-- test/fixtures/valid.etl
 - test/helper.rb
+- test/shared_runner_tests.rb
 - test/support/shared_tests.rb
+- test/support/test_aggregate_transform.rb
 - test/support/test_array_destination.rb
+- test/support/test_close_yielding_transform.rb
 - test/support/test_csv_destination.rb
 - test/support/test_csv_source.rb
+- test/support/test_destination_returning_nil.rb
+- test/support/test_duplicate_row_transform.rb
 - test/support/test_enumerable_source.rb
+- test/support/test_keyword_arguments_component.rb
+- test/support/test_mixed_arguments_component.rb
+- test/support/test_non_closing_transform.rb
 - test/support/test_rename_field_transform.rb
 - test/support/test_source_that_reads_at_instantiation_time.rb
 - test/support/test_yielding_transform.rb
-- test/test_cli.rb
 - test/test_integration.rb
 - test/test_parser.rb
+- test/test_run.rb
 - test/test_runner.rb
 - test/test_streaming_runner.rb
 - test/tmp/.gitkeep
 homepage: http://thbar.github.io/kiba/
 licenses:
 - LGPL-3.0
-metadata: {}
-post_install_message: 
+metadata:
+  source_code_uri: https://github.com/thbar/kiba
+  documentation_uri: https://github.com/thbar/kiba/wiki
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -129,33 +136,35 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.14
-signing_key: 
+rubygems_version: 3.2.3
+signing_key:
 specification_version: 4
 summary: Lightweight ETL for Ruby
 test_files:
-- test/common/runner.rb
-- test/fixtures/bogus.etl
-- test/fixtures/namespace_conflict.etl
-- test/fixtures/some_extension.rb
-- test/fixtures/valid.etl
 - test/helper.rb
+- test/shared_runner_tests.rb
 - test/support/shared_tests.rb
+- test/support/test_aggregate_transform.rb
 - test/support/test_array_destination.rb
+- test/support/test_close_yielding_transform.rb
 - test/support/test_csv_destination.rb
 - test/support/test_csv_source.rb
+- test/support/test_destination_returning_nil.rb
+- test/support/test_duplicate_row_transform.rb
 - test/support/test_enumerable_source.rb
+- test/support/test_keyword_arguments_component.rb
+- test/support/test_mixed_arguments_component.rb
+- test/support/test_non_closing_transform.rb
 - test/support/test_rename_field_transform.rb
 - test/support/test_source_that_reads_at_instantiation_time.rb
 - test/support/test_yielding_transform.rb
-- test/test_cli.rb
 - test/test_integration.rb
 - test/test_parser.rb
+- test/test_run.rb
 - test/test_runner.rb
 - test/test_streaming_runner.rb
 - test/tmp/.gitkeep

data/lib/kiba/cli.rb

@@ -1,16 +0,0 @@
-require 'kiba'
-
-module Kiba
-  class Cli
-    def self.run(args)
-      unless args.size == 1
-        puts 'Syntax: kiba your-script.etl'
-        exit(-1)
-      end
-      filename = args[0]
-      script_content = IO.read(filename)
-      job_definition = Kiba.parse(script_content, filename)
-      Kiba.run(job_definition)
-    end
-  end
-end

data/test/fixtures/bogus.etl

@@ -1,2 +0,0 @@
-# this should fail because we have an unknown class
-source UnknownThing

data/test/fixtures/namespace_conflict.etl

@@ -1,9 +0,0 @@
-fail "Context should not be visible without Kiba namespace" if defined?(Context)
-fail "Control should not be visible without Kiba namespace" if defined?(Control)
-fail "Parser should not be visible without Kiba namespace" if defined?(Parser)
-fail "Config should not be visible without Kiba namespace" if defined?(DSLExtensions::Config)
-
-# verify Kiba config (namespaced under Kiba::DSLExtensions::Config)
-# isn't causing troubles to implementers using a top-level DSLExtensions module
-require_relative 'some_extension'
-extend DSLExtensions::SomeExtension