kiba 3.6.0 → 4.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f8569f0d68410f27d42f38dcb8182ef9e8f47a8c88f714dc64382943febad2c8
-  data.tar.gz: 4d7c6f8c5ef9256182cc51f85c614e383586dd600167e12190c110693d6963fb
+  metadata.gz: f6a466478d2837d085445593c36b2af804e53d29c8046fd4f71b6786f84fedc8
+  data.tar.gz: 99d67dd7c6ab706ba12d79031684bbd6b6f2099409beac0bf1cb9593e987a2e8
 SHA512:
-  metadata.gz: 9fb3c4fd7836e2863771a0086e98a7d8bf24f4df99310464dad6cbc84fef3963155cdac8af0d9a281b6470686334b163491ae9d15158fb19cd5269ef77102129
-  data.tar.gz: 25082194faf157dfa2398485e4cd993ba491871007a3a47c3b5f13db57e677f2f6b11621c2ec5e03ccdce5b44bb6a51e3bfee05968c8a72c99088dfbeb46cd7c
+  metadata.gz: 71765fe5a8dccd2b9db96b88919958c99b0010f117944bfffe8916ac81348c538306d58d0cdaa6e64e3605bd6714f2d7d223e32deed52446d94738d69895a2f9
+  data.tar.gz: ebb45df0fe69b0513efdfd07af38c63b15fa7f881068c11a53d5649d22d0f421aa1a17f2d48363188d41cfa30c1794d4c95ad752808ac2350f93d20523b1939e

data/.github/workflows/ci.yml

@@ -0,0 +1,41 @@
+name: CI build
+on: [push, pull_request]
+jobs:
+  build:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]
+        ruby:
+          - '2.5'
+          - '2.6'
+          - '2.7'
+          - '3.0'
+          - head
+          - jruby
+          - jruby-head
+          - truffleruby
+          - truffleruby-head
+    runs-on: ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@v2
+    - uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby }}
+        bundler-cache: true # runs 'bundle install' and caches installed gems automatically
+    - run: bundle exec rake
+    # NOTE: calling "bundle install" because the cache apparently did not update due to the gemspec injection
+    - run: bundle install && bundle exec standardrb
+      if: matrix.ruby == '3.0' # not using "head" because rubocop won't work there yet
+  # What's below helps having a single "status check" for mergeability, without
+  # having to add each new version to the list of expected status checks in GitHub.
+  # See https://github.community/t/status-check-for-a-matrix-jobs/127354/7 
+  global:
+    if: ${{ always() }}
+    runs-on: ubuntu-latest
+    name: build (matrix)
+    needs: build
+    steps:
+      - name: Check build matrix status
+        if: ${{ needs.build.result != 'success' }}
+        run: exit 1

data/Changes.md

@@ -1,5 +1,11 @@
-HEAD
-----
+4.0.0
+-----
+
+- Breaking: Ruby 2.4 (EOL since 2020-03-31) is not officially supported anymore.
+- CI changes: moving from Travis CI (EOL) to GitHub actions. The Windows CI has been removed for now (see [#97](https://github.com/thbar/kiba/pull/97))
+- Breaking: if your jobs use Kiba's "legacy runner" via `config :kiba, runner: Kiba::Runner`, be aware that this legacy runner has been removed in [#96](https://github.com/thbar/kiba/pull/96). The upgrade path is to remove this config line and let Kiba use the more modern `Kiba::StreamingRunner`, which is the default anyway since Kiba v3.0.0 (see [#83](https://github.com/thbar/kiba/pull/83) for context) and is normally fully backward-compatible.
+- Cleanup: in Kiba v3, the `kiba` shell command had been deprecated and replaced by a simple stub printing a warning to STDERR. It is now removed for good.
+- [StandardRB](https://github.com/testdouble/standard) has been added for formatting & linting the codebase.
 
 3.6.0
 -----

data/Gemfile

@@ -1,3 +1,3 @@
-source 'https://rubygems.org'
+source "https://rubygems.org"
 
 gemspec

data/README.md

@@ -1,7 +1,7 @@
 # Kiba ETL
 
 [![Gem Version](https://badge.fury.io/rb/kiba.svg)](http://badge.fury.io/rb/kiba)
-[![Build Status](https://travis-ci.org/thbar/kiba.svg?branch=master)](https://travis-ci.org/thbar/kiba) [![Build status](https://ci.appveyor.com/api/projects/status/v05jcyhpp1mueq9i?svg=true)](https://ci.appveyor.com/project/thbar/kiba) [![Code Climate](https://codeclimate.com/github/thbar/kiba/badges/gpa.svg)](https://codeclimate.com/github/thbar/kiba)
+[![Build Status](https://github.com/thbar/kiba/actions/workflows/ci.yml/badge.svg)](https://github.com/thbar/kiba/actions) [![Code Climate](https://codeclimate.com/github/thbar/kiba/badges/gpa.svg)](https://codeclimate.com/github/thbar/kiba)
 
 Writing reliable, concise, well-tested & maintainable data-processing code is tricky.
 
@@ -19,7 +19,7 @@ You can also check out the [author blog](https://thibautbarrere.com) and [StackO
 
 ## Supported Ruby versions
 
-Kiba currently supports Ruby 2.4+, JRuby 9.2+ and TruffleRuby. See [test matrix](https://travis-ci.org/thbar/kiba).
+Kiba currently supports Ruby 2.5+, JRuby 9.2+ and TruffleRuby. See [test matrix](https://github.com/thbar/kiba/actions).
 
 ## ETL consulting & commercial version
 

data/Rakefile

@@ -1,7 +1,7 @@
-require 'rake/testtask'
+require "rake/testtask"
 
 Rake::TestTask.new(:test) do |t|
-  t.pattern = 'test/test_*.rb'
+  t.pattern = "test/test_*.rb"
 end
 
 # A simple check to verify TruffleRuby installation trick is really in effect

data/kiba.gemspec

@@ -1,25 +1,24 @@
-# -*- encoding: utf-8 -*-
-require File.expand_path('../lib/kiba/version', __FILE__)
+require File.expand_path("../lib/kiba/version", __FILE__)
 
 Gem::Specification.new do |gem|
-  gem.authors       = ['Thibaut Barrère']
-  gem.email         = ['thibaut.barrere@gmail.com']
-  gem.description   = gem.summary = 'Lightweight ETL for Ruby'
-  gem.homepage      = 'http://thbar.github.io/kiba/'
-  gem.license       = 'LGPL-3.0'
-  gem.files         = `git ls-files | grep -Ev '^(examples)'`.split("\n")
-  gem.test_files    = `git ls-files -- test/*`.split("\n")
-  gem.name          = 'kiba'
-  gem.require_paths = ['lib']
-  gem.version       = Kiba::VERSION
-  gem.executables   = ['kiba']
-  gem.metadata      = {
-    'source_code_uri'   => 'https://github.com/thbar/kiba',
-    'documentation_uri' => 'https://github.com/thbar/kiba/wiki',
+  gem.authors = ["Thibaut Barrère"]
+  gem.email = ["thibaut.barrere@gmail.com"]
+  gem.description = gem.summary = "Lightweight ETL for Ruby"
+  gem.homepage = "https://www.kiba-etl.org"
+  gem.license = "LGPL-3.0"
+  gem.files = `git ls-files | grep -Ev '^(examples)'`.split("\n")
+  gem.test_files = `git ls-files -- test/*`.split("\n")
+  gem.name = "kiba"
+  gem.require_paths = ["lib"]
+  gem.version = Kiba::VERSION
+  gem.metadata = {
+    "source_code_uri" => "https://github.com/thbar/kiba",
+    "documentation_uri" => "https://github.com/thbar/kiba/wiki"
   }
 
-  gem.add_development_dependency 'rake'
-  gem.add_development_dependency 'minitest', '~> 5.9'
-  gem.add_development_dependency 'awesome_print'
-  gem.add_development_dependency 'minitest-focus'
+  gem.add_development_dependency "rake"
+  gem.add_development_dependency "minitest", "~> 5.9"
+  gem.add_development_dependency "awesome_print"
+  gem.add_development_dependency "minitest-focus"
+  gem.add_development_dependency "standard"
 end

data/lib/kiba.rb

@@ -1,24 +1,21 @@
-# encoding: utf-8
-require 'kiba/version'
+require "kiba/version"
 
-require 'kiba/control'
-require 'kiba/context'
-require 'kiba/parser'
-require 'kiba/runner'
-require 'kiba/streaming_runner'
-require 'kiba/dsl_extensions/config'
+require "kiba/control"
+require "kiba/context"
+require "kiba/parser"
+require "kiba/streaming_runner"
+require "kiba/dsl_extensions/config"
 
 Kiba.extend(Kiba::Parser)
 
 module Kiba
   def self.run(job = nil, &block)
-    unless (job.nil? ^ block.nil?)
+    unless job.nil? ^ block.nil?
       fail ArgumentError.new("Kiba.run takes either one argument (the job) or a block (defining the job)")
     end
 
     job ||= Kiba.parse { instance_exec(&block) }
 
-    # NOTE: use Hash#dig when Ruby 2.2 reaches EOL
     runner = job.config.fetch(:kiba, {}).fetch(:runner, Kiba::StreamingRunner)
     runner.run(job)
   end

data/lib/kiba/context.rb

@@ -5,23 +5,23 @@ module Kiba
     end
 
     def pre_process(&block)
-      @control.pre_processes << { block: block }
+      @control.pre_processes << {block: block}
     end
 
     def source(klass, *initialization_params)
-      @control.sources << { klass: klass, args: initialization_params }
+      @control.sources << {klass: klass, args: initialization_params}
     end
 
     def transform(klass = nil, *initialization_params, &block)
-      @control.transforms << { klass: klass, args: initialization_params, block: block }
+      @control.transforms << {klass: klass, args: initialization_params, block: block}
     end
 
     def destination(klass, *initialization_params)
-      @control.destinations << { klass: klass, args: initialization_params }
+      @control.destinations << {klass: klass, args: initialization_params}
     end
 
     def post_process(&block)
-      @control.post_processes << { block: block }
+      @control.post_processes << {block: block}
     end
 
     [:source, :transform, :destination].each do |m|

data/lib/kiba/control.rb

@@ -3,7 +3,7 @@ module Kiba
     def pre_processes
       @pre_processes ||= []
     end
-    
+
     def config
       @config ||= {}
     end

data/lib/kiba/dsl_extensions/config.rb

@@ -6,4 +6,4 @@ module Kiba
       end
     end
   end
-end
+end

data/lib/kiba/streaming_runner.rb

@@ -1,8 +1,37 @@
 module Kiba
   module StreamingRunner
-    include Runner
     extend self
-    
+
+    # allow to handle a block form just like a regular transform
+    class AliasingProc < Proc
+      alias_method :process, :call
+    end
+
+    def run(control)
+      run_pre_processes(control)
+      process_rows(
+        to_instances(control.sources),
+        to_instances(control.transforms, true),
+        destinations = to_instances(control.destinations)
+      )
+      close_destinations(destinations)
+      run_post_processes(control)
+    end
+
+    def run_pre_processes(control)
+      to_instances(control.pre_processes, true, false).each(&:call)
+    end
+
+    def run_post_processes(control)
+      to_instances(control.post_processes, true, false).each(&:call)
+    end
+
+    def close_destinations(destinations)
+      destinations
+        .find_all { |d| d.respond_to?(:close) }
+        .each(&:close)
+    end
+
     def transform_stream(stream, t)
       Enumerator.new do |y|
         stream.each do |input_row|
@@ -18,7 +47,7 @@ module Kiba
         end
       end
     end
-    
+
     def source_stream(sources)
       Enumerator.new do |y|
         sources.each do |source|
@@ -29,10 +58,33 @@ module Kiba
 
     def process_rows(sources, transforms, destinations)
       stream = source_stream(sources)
-      recurser = lambda { |s,t| transform_stream(s, t) }
+      recurser = lambda { |s, t| transform_stream(s, t) }
       transforms.inject(stream, &recurser).each do |r|
         destinations.each { |d| d.write(r) }
       end
     end
+
+    def to_instances(definitions, allow_block = false, allow_class = true)
+      definitions.map do |definition|
+        to_instance(
+          *definition.values_at(:klass, :args, :block),
+          allow_block, allow_class
+        )
+      end
+    end
+
+    def to_instance(klass, args, block, allow_block, allow_class)
+      if klass && block
+        fail "Class and block form cannot be used together at the moment"
+      elsif klass
+        fail "Class form is not allowed here" unless allow_class
+        klass.new(*args)
+      elsif block
+        fail "Block form is not allowed here" unless allow_block
+        AliasingProc.new(&block)
+      else
+        fail "Nil parameters not allowed here"
+      end
+    end
   end
-end
+end

data/lib/kiba/version.rb

@@ -1,3 +1,3 @@
 module Kiba
-  VERSION = '3.6.0'
+  VERSION = "4.0.0"
 end

data/test/helper.rb

@@ -1,9 +1,9 @@
-require 'minitest/autorun'
-require 'minitest/pride'
-require 'minitest/focus'
-require 'kiba'
+require "minitest/autorun"
+require "minitest/pride"
+require "minitest/focus"
+require "kiba"
 
-if ENV['CI'] == 'true'
+if ENV["CI"] == "true"
   puts "Running with MiniTest version #{MiniTest::VERSION}"
 end
 
@@ -15,10 +15,10 @@ class Kiba::Test < Minitest::Test
   end
 
   def fixture(file)
-    File.join(File.dirname(__FILE__), 'fixtures', file)
+    File.join(File.dirname(__FILE__), "fixtures", file)
   end
-  
-  unless self.method_defined?(:assert_mock)
+
+  unless method_defined?(:assert_mock)
     def assert_mock(mock)
       mock.verify
     end

data/test/shared_runner_tests.rb

@@ -1,15 +1,15 @@
-require 'minitest/mock'
-require_relative 'support/test_enumerable_source'
-require_relative 'support/test_destination_returning_nil'
+require "minitest/mock"
+require_relative "support/test_enumerable_source"
+require_relative "support/test_destination_returning_nil"
 
 module SharedRunnerTests
   def rows
     @rows ||= [
-      { identifier: 'first-row' },
-      { identifier: 'second-row' }
+      {identifier: "first-row"},
+      {identifier: "second-row"}
     ]
   end
-  
+
   def control
     @control ||= begin
       control = Kiba::Control.new
@@ -21,18 +21,18 @@ module SharedRunnerTests
       control
     end
   end
-  
+
   def test_block_transform_processing
     # is there a better way to assert a block was called in minitest?
-    control.transforms << { block: lambda { |r| @called = true; r } }
+    control.transforms << {block: lambda { |r| (@called = true) && (return r) }}
     kiba_run(control)
     assert_equal true, @called
   end
 
   def test_dismissed_row_not_passed_to_next_transform
     @called = nil
-    control.transforms << { block: lambda { |_| nil } }
-    control.transforms << { block: lambda { |_| @called = true; nil } }
+    control.transforms << {block: lambda { |_| }}
+    control.transforms << {block: lambda { |_| (@called = true) && (return nil) }}
     kiba_run(control)
     assert_nil @called
   end
@@ -40,23 +40,23 @@ module SharedRunnerTests
   def test_post_process_runs_once
     assert_equal 2, rows.size
     @called = 0
-    control.post_processes << { block: lambda { @called += 1 } }
+    control.post_processes << {block: lambda { @called += 1 }}
     kiba_run(control)
     assert_equal 1, @called
   end
 
   def test_post_process_not_called_after_row_failure
     @called = nil
-    control.transforms << { block: lambda { |_| fail 'FAIL' } }
-    control.post_processes << { block: lambda { @called = true } }
-    assert_raises(RuntimeError, 'FAIL') { kiba_run(control) }
+    control.transforms << {block: lambda { |_| fail "FAIL" }}
+    control.post_processes << {block: lambda { @called = true }}
+    assert_raises(RuntimeError, "FAIL") { kiba_run(control) }
     assert_nil @called
   end
-  
+
   def test_pre_process_runs_once
     assert_equal 2, rows.size
     @called = 0
-    control.pre_processes << { block: lambda { @called += 1 } }
+    control.pre_processes << {block: lambda { @called += 1 }}
     kiba_run(control)
     assert_equal 1, @called
   end
@@ -70,27 +70,27 @@ module SharedRunnerTests
     end
 
     control = Kiba::Control.new
-    control.pre_processes << { block: lambda { calls << :pre_processor_executed } }
-    control.sources << { klass: mock_source_class }
+    control.pre_processes << {block: lambda { calls << :pre_processor_executed }}
+    control.sources << {klass: mock_source_class}
     kiba_run(control)
 
     assert_equal [:pre_processor_executed, :source_instantiated], calls
     assert_mock mock_source_class
   end
-  
+
   def test_no_error_raised_if_destination_close_not_implemented
     # NOTE: this fake destination does not implement `close`
     destination_instance = MiniTest::Mock.new
 
     mock_destination_class = MiniTest::Mock.new
     mock_destination_class.expect(:new, destination_instance)
-    
+
     control = Kiba::Control.new
-    control.destinations << { klass: mock_destination_class }
+    control.destinations << {klass: mock_destination_class}
     kiba_run(control)
     assert_mock mock_destination_class
   end
-  
+
   def test_destination_close_called_if_defined
     destination_instance = MiniTest::Mock.new
     destination_instance.expect(:close, nil)
@@ -98,54 +98,53 @@ module SharedRunnerTests
     mock_destination_class.expect(:new, destination_instance)
 
     control = Kiba::Control.new
-    control.destinations << { klass: mock_destination_class }
+    control.destinations << {klass: mock_destination_class}
     kiba_run(control)
     assert_mock destination_instance
     assert_mock mock_destination_class
   end
-  
+
   def test_use_next_to_exit_early_from_block_transform
     assert_equal 2, rows.size
 
     # calling "return row" from a block is forbidden, but you can use "next" instead
     b = lambda do |row|
-      if row.fetch(:identifier) == 'first-row'
+      if row.fetch(:identifier) == "first-row"
         # demonstrate how to remove a row from the pipeline via next
         next
       else
         # demonstrate how you can reformat via next
         next({new_identifier: row.fetch(:identifier)})
       end
-      fail "This should not be called"
     end
-    control.transforms << { block: b }
+    control.transforms << {block: b}
 
     # keep track of the rows
     @remaining_rows = []
-    checker = lambda { |row| @remaining_rows << row; row }
-    control.transforms << { block: checker }
+    checker = lambda { |row| (@remaining_rows << row) && (return row) }
+    control.transforms << {block: checker}
 
     kiba_run(control)
-    
+
     # the first row should have been removed
     # and the second row should have been reformatted
-    assert_equal [{new_identifier: 'second-row'}], @remaining_rows
+    assert_equal [{new_identifier: "second-row"}], @remaining_rows
   end
-  
+
   def test_destination_returning_nil_does_not_remove_row_from_pipeline
     # safeguard to avoid modification on the support code
     assert_nil TestDestinationReturningNil.new.write("FOOBAR")
 
     destinations = []
     control = Kiba.parse do
-      source TestEnumerableSource, [{key: 'value'}]
+      source TestEnumerableSource, [{key: "value"}]
       2.times do
         destination TestDestinationReturningNil, on_init: lambda { |d| destinations << d }
       end
     end
     kiba_run(control)
     2.times do |i|
-      assert_equal [{key: 'value'}], destinations[i].instance_variable_get(:@written_rows)
+      assert_equal [{key: "value"}], destinations[i].instance_variable_get(:@written_rows)
     end
   end
 
@@ -153,11 +152,11 @@ module SharedRunnerTests
     control = Kiba.parse do
       transform
     end
-    assert_raises(RuntimeError, 'Nil parameters not allowed here') { kiba_run(control) }
+    assert_raises(RuntimeError, "Nil parameters not allowed here") { kiba_run(control) }
   end
 
   def test_ruby_3_source_kwargs
-    # NOTE: before Ruby 3 kwargs support, a Ruby warning would 
+    # NOTE: before Ruby 3 kwargs support, a Ruby warning would
     # be captured here with Ruby 2.7 & ensure we fail,
     # and an error would be raised with Ruby 2.8.0-dev
     # NOTE: only the first warning will be captured, though, but
@@ -167,7 +166,7 @@ module SharedRunnerTests
       Kiba.run(Kiba.parse do
         source TestKeywordArgumentsComponent,
           mandatory: "first",
-          on_init: -> (values) { storage = values }
+          on_init: ->(values) { storage = values }
       end)
     end
     assert_equal({
@@ -175,14 +174,14 @@ module SharedRunnerTests
       optional: nil
     }, storage)
   end
-  
+
   def test_ruby_3_transform_kwargs
     storage = nil
     assert_silent do
       Kiba.run(Kiba.parse do
         transform TestKeywordArgumentsComponent,
           mandatory: "first",
-          on_init: -> (values) { storage = values }
+          on_init: ->(values) { storage = values }
       end)
     end
     assert_equal({
@@ -190,14 +189,14 @@ module SharedRunnerTests
       optional: nil
     }, storage)
   end
-  
+
   def test_ruby_3_destination_kwargs
     storage = nil
     assert_silent do
       Kiba.run(Kiba.parse do
         destination TestKeywordArgumentsComponent,
           mandatory: "first",
-          on_init: -> (values) { storage = values }
+          on_init: ->(values) { storage = values }
       end)
     end
     assert_equal({
@@ -205,7 +204,7 @@ module SharedRunnerTests
       optional: nil
     }, storage)
   end
-      
+
   def test_positional_plus_keyword_arguments
     storage = nil
     assert_silent do
@@ -213,12 +212,12 @@ module SharedRunnerTests
         source TestMixedArgumentsComponent,
           "some positional argument",
           mandatory: "first",
-          on_init: -> (values) {
+          on_init: ->(values) {
             storage = values
           }
       end)
     end
-    
+
     assert_equal({
       some_value: "some positional argument",
       mandatory: "first",