kiba-common 0.7.0 → 1.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 73129b07c0c847b9a1ed5b2a88e0fe3b9449200589fea6e0b88696959c86448f
-  data.tar.gz: 1a73e5414c054fbb233c92c5459087153a6ad413bfa194c192d8ea54cd4ee3ef
+  metadata.gz: 7c572e8d0d2cea374d8ee86d54ada3d323dd1087d91c002ef30b565c8dd98199
+  data.tar.gz: 1153c67d2bff98455663f4b2859d8af8f723ddab135411e720bc4abdbca3da3b
 SHA512:
-  metadata.gz: f984de5843f926e3c153c381a1e05d171275fa185c3b0b2c7d855250b75c6ed85841ad78d87cb55b9cc4d6759e7d50c585765483ef27f0c3c64f80b8a739e957
-  data.tar.gz: 1e05cefc61c51f474f93b8f331e3d17a1de7c2ec3683212336796e3dcbc164bc9d5bc9049a8f255c25d237a8f8e38516d02db45f343a903d2bd76ad439515330
+  metadata.gz: 4cfccbc1d90c83bd202d5bbd996fe02a5599ca0b210921d1d3d7ff1d205ecb2484ebe1a686d3345773dc7298cd86616e8a89ceaa607312cfb495b4b5c41df010
+  data.tar.gz: 76fc871cc277824452e94db9545092a8333bf26404151c5a5d5367be6a0d7df0690c834b3a576eedcca79e6299a3947b393f5c502c6ad9041de9f88259ebc12f

data/.github/FUNDING.yml

@@ -0,0 +1 @@
+github: thbar

data/.github/workflows/ci.yml

@@ -0,0 +1,41 @@
+name: CI build
+on: [push, pull_request]
+jobs:
+  build:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]
+        ruby:
+          - '2.5'
+          - '2.6'
+          - '2.7'
+          - '3.0'
+          - head
+          - jruby
+          - jruby-head
+          - truffleruby
+          - truffleruby-head
+    runs-on: ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@v2
+    - uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby }}
+        bundler-cache: true # runs 'bundle install' and caches installed gems automatically
+    - run: bundle exec rake
+    # NOTE: calling "bundle install" because the cache apparently did not update due to the gemspec injection
+    - run: bundle install && bundle exec standardrb
+      if: matrix.ruby == '3.0' # not using "head" because rubocop won't work there yet
+  # What's below helps having a single "status check" for mergeability, without
+  # having to add each new version to the list of expected status checks in GitHub.
+  # See https://github.community/t/status-check-for-a-matrix-jobs/127354/7 
+  global:
+    if: ${{ always() }}
+    runs-on: ubuntu-latest
+    name: build (matrix)
+    needs: build
+    steps:
+      - name: Check build matrix status
+        if: ${{ needs.build.result != 'success' }}
+        run: exit 1

data/Changes.md

@@ -1,5 +1,55 @@
-HEAD
-----
+1.5.0
+-----
+
+- Support for Kiba 4
+- Breaking: Drop support for Ruby < 2.5
+- Breaking: Require Kiba 3+
+- [StandardRB](https://github.com/testdouble/standard) has been added for formatting & linting the codebase.
+
+
+1.1.0
+-----
+
+- Support for Ruby 2.7+ (affects CSV source and destination)
+- Breaking: `show_me!` is now compatible with both `awesome_print` and its modern replacement `amazing_print`. You will have to `require` the one you want to use in your code from now on.
+- Breaking: `SourceTransformAdapter` has been removed due to complexities with Ruby 2.7+ keyword arguments. The suggested replacement is to use `Enumerator` and `EnumerableExploder` like this:
+
+```ruby
+# before
+transform SourceTransformAdapter
+
+# after
+transform do |klass, args|
+  Enumerator.new do |y|
+    # NOTE: you may have to use double-splat (**) here instead
+    # if you provide keyword arguments, or other variants
+    klass.new(*args).each do |r|
+      y << r
+    end
+  end
+end
+
+transform Kiba::Common::Transforms::EnumerableExploder
+```
+
+1.0.0
+-----
+
+- Kiba ETL v3 compatibility
+- New: Kiba::Common::Destinations::Lambda lets you write block-form destinations (handy for one-off scripts).
+
+0.9.0
+-----
+
+- New: Kiba::Common::Sources::CSV provides a basic CSV source for simple needs.
+
+0.8.0
+-----
+
+- Bugfix: `show_me!` used with a block should not modify the processed row.
+
+0.7.0
+-----
 
 - New: Kiba::Common::Transforms::SourceTransformAdapter let you transform rows into parameters for source instantiation.
 

data/Gemfile

@@ -1,3 +1,3 @@
-source 'https://rubygems.org'
+source "https://rubygems.org"
 
 gemspec

data/README.md

@@ -1,6 +1,7 @@
 Kiba Common is a companion gem to [Kiba](https://github.com/thbar/kiba) and [Kiba Pro](https://github.com/thbar/kiba/blob/master/Pro-Changes.md) in which I'll share commonly used helpers.
 
-[![Build Status](https://travis-ci.org/thbar/kiba-common.svg?branch=master)](https://travis-ci.org/thbar/kiba-common) [![Gem Version](https://badge.fury.io/rb/kiba-common.svg)](https://badge.fury.io/rb/kiba-common)
+[![Gem Version](https://badge.fury.io/rb/kiba-common.svg)](https://badge.fury.io/rb/kiba-common)
+[![Build Status](https://github.com/thbar/kiba-common/actions/workflows/ci.yml/badge.svg)](https://github.com/thbar/kiba-common/actions) 
 
 ## Usage
 
@@ -14,7 +15,8 @@ Then see below for each module usage & require clause.
 
 ## Supported Ruby versions
 
-`kiba-common` currently supports Ruby 2.3+ and JRuby (with its default 1.9 syntax). See [test matrix](https://travis-ci.org/thbar/kiba-common).
+`kiba-common` currently supports Ruby 2.5+, JRuby 9.2+ and TruffleRuby. See [test matrix](https://github.com/thbar/kiba-common/actions).
+
 
 ## Available components
 
@@ -48,7 +50,7 @@ But since Kiba v2 introduction of `StreamingRunner`, it is possible for transfor
 
 Leveraging that possibility, you can use a `SourceTransformAdapter` to dynamically instantiate the source for each of your input rows.
 
-This allows to mix-and-match components in a much more versatile & powerful way.
+This allows you to mix-and-match components in a much more versatile & powerful way.
 
 Requirements: Kiba v2 with `StreamingRunner` enabled.
 
@@ -85,7 +87,7 @@ require 'kiba-common/transforms/enumerable_exploder'
 transform Kiba::Common::Transforms::EnumerableExploder
 ```
 
-For instance, this can help if you are reading XML/JSON documents from a source, and each input document contains multiple rows that you'd want to extract.
+This can help if you are reading XML/JSON documents from a source and each input document contains multiple rows that you want to extract.
 
 ```ruby
 source Kiba::Common::Sources::Enumerable, -> { Dir["input/*.xml"] }
@@ -104,7 +106,7 @@ Similarly, if you have a CSV document as your input:
 | ------------- | ------------- |
 | 00001  | John:Mary:Sally  |
 
-and you want to reformat it to get this instead:
+and you want to reformat it to the following:
 
 | po_number | buyer |
 |-------------|---------|
@@ -129,17 +131,81 @@ end
 transform Kiba::Common::Transforms::EnumerableExploder
 ```
 
+### Kiba::Common::Sources::CSV
+
+A CSV source for basic needs (in particular, it doesn't yield row metadata, which are useful in more advanced scenarios).
+
+Use the `csv_options` keyword to control the input format like when using [Ruby CSV class](http://ruby-doc.org/stdlib-2.4.0/libdoc/csv/rdoc/CSV.html#method-c-new).
+
+Usage:
+
+```ruby
+require 'kiba-common/sources/csv'
+
+# by defaults, csv_options are empty
+source Kiba::Common::Sources::CSV, filename: 'input.csv'
+
+# you can provide your own csv_options
+source Kiba::Common::Sources::CSV, filename: 'input.csv',
+  csv_options: { headers: true, header_converters: :symbol }
+```
+
+Note that the emitted rows are instances of [`CSV::Row`](http://ruby-doc.org/stdlib-2.5.3/libdoc/csv/rdoc/CSV/Row.html), part `Array` and part `Hash`, which retain order of fields and allow duplicates (unlike a regular `Hash`).
+
+If the rest of your pipeline expects `Hash` rows (like for instance `Kiba::Common::Destinations::CSV`), you'll want to transform the rows to `Hash` instances yourself using [`to_hash`](http://ruby-doc.org/stdlib-2.5.3/libdoc/csv/rdoc/CSV/Row.html#method-i-to_hash). This will "collapse the row into a simple Hash. Be warned that this discards field order and clobbers duplicate fields."
+
+```ruby
+transform { |r| r.to_hash }
+```
+
+#### Handling multiple input CSV files
+
+You can process multiple files by chaining the various components available in Kiba Common (see `test/test_integration#test_multiple_csv_inputs` for an actual demo):
+
+```ruby
+# create one row per input filename
+source Kiba::Common::Sources::Enumerable, -> { Dir[File.join(dir, '*.csv')] }
+
+# out of that row, create configuration for a CSV source
+transform do |r|
+  [
+    Kiba::Common::Sources::CSV,
+    filename: r,
+    csv_options: { headers: true, header_converters: :symbol }
+  ]
+end
+
+# instantiate & yield CSV rows for each configuration
+transform Kiba::Common::Transforms::SourceTransformAdapter
+```
+
+Alternatively, you can wrap this source in your own source like this:
+
+```ruby
+class MultipleCSVSource
+  def initialize(file_pattern:, csv_options:)
+
+  def each
+    Dir[file_pattern].each do |filename|
+      Kiba::Common::Sources::CSV.new(filename, csv_options).each do |row|
+        yield row
+      end
+    end
+  end
+end
+```
+
 ### Kiba::Common::Destinations::CSV
 
 A way to dump `Hash` rows as CSV.
 
 All rows are expected to have the exact same set of keys as the first row.
 
-The headers will be the first row keys, unless you pass an array of keys via `headers`.
+The headers will be the first row keys unless you pass an array of keys via `headers`.
 
-All keys are mandatory (although they can have a nil value).
+All keys are mandatory (although they can have a `nil` value).
 
-Use the `csv_options` keyword to control the output format like you would do when using [Ruby CSV class](http://ruby-doc.org/stdlib-2.4.0/libdoc/csv/rdoc/CSV.html#method-c-new).
+Use the `csv_options` keyword to control the output format like when using [Ruby CSV class](http://ruby-doc.org/stdlib-2.4.0/libdoc/csv/rdoc/CSV.html#method-c-new).
 
 Usage:
 
@@ -161,6 +227,30 @@ destination Kiba::Common::Destinations::CSV,
   headers: [:field, :other_field]
 ```
 
+### Kiba::Common::Destinations::Lambda
+
+At times, it can be convenient to use a block form for a destination (pretty much like Kiba's built-in "block transform"), especially for one-off scripts.
+
+The Lambda destination is there for that purpose.
+
+Example use:
+
+```ruby
+require 'kiba-common/destinations/lambda'
+
+destination Kiba::Common::Destinations::Lambda,
+  # called at destination instantiation time (once)
+  on_init: -> { ... },
+  # called for each row
+  on_write: -> (row) { ... },
+  # called after all the rows have been written
+  on_close: -> { ... }
+```
+
+Each "callback" (e.g. `on_init`) is optional.
+
+The callback code can refer to scope variables or instance variables you may have declared above.
+
 ### Kiba::Common::DSLExtensions::Logger
 
 A simple logging facility.
@@ -189,7 +279,7 @@ logger = Logger.new(xxx)
 
 ### Kiba::Common::DSLExtensions::ShowMe
 
-A way to color-dump rows on the screen, useful at development time while you are looking at the data (requires the `awesome_print` gem).
+A way to color-dump rows on the screen, useful during development when you are inspecting the data (requires either the `amazing_print` or the `awesome_print` gem).
 
 Usage:
 
@@ -212,6 +302,8 @@ show_me! { |r| r.except(:some_noisy_field) }
 
 ## Contributing & Legal
 
+See [LICENSE](LICENSE) for license.
+
 (agreement below borrowed from Sidekiq Legal)
 
 By submitting a Pull Request, you disavow any rights or claims to any changes submitted to the Kiba Common project and assign the copyright of those changes to LoGeek SARL.

data/Rakefile

@@ -1,7 +1,7 @@
-require 'rake/testtask'
+require "rake/testtask"
 
 Rake::TestTask.new(:test) do |t|
-  t.pattern = 'test/test_*.rb'
+  t.pattern = "test/test_*.rb"
 end
 
 task default: :test

data/kiba-common.gemspec

@@ -1,21 +1,25 @@
-# -*- encoding: utf-8 -*-
-require File.expand_path('../lib/kiba-common/version', __FILE__)
+require File.expand_path("../lib/kiba-common/version", __FILE__)
 
 Gem::Specification.new do |gem|
-  gem.authors       = ['Thibaut Barrère']
-  gem.email         = ['thibaut.barrere@gmail.com']
-  gem.description   = gem.summary = 'Commonly used helpers for Kiba ETL'
-  gem.homepage      = 'https://github.com/thbar/kiba-common'
-  gem.license       = 'LGPL-3.0'
-  gem.files         = `git ls-files | grep -Ev '^(examples)'`.split("\n")
-  gem.test_files    = `git ls-files -- test/*`.split("\n")
-  gem.name          = 'kiba-common'
-  gem.require_paths = ['lib']
-  gem.version       = Kiba::Common::VERSION
+  gem.authors = ["Thibaut Barrère"]
+  gem.email = ["thibaut.barrere@gmail.com"]
+  gem.description = gem.summary = "Commonly used helpers for Kiba ETL"
+  gem.homepage = "https://github.com/thbar/kiba-common"
+  gem.license = "LGPL-3.0"
+  gem.files = `git ls-files | grep -Ev '^(examples)'`.split("\n")
+  gem.test_files = `git ls-files -- test/*`.split("\n")
+  gem.name = "kiba-common"
+  gem.require_paths = ["lib"]
+  gem.version = Kiba::Common::VERSION
+  gem.metadata = {
+    "source_code_uri" => "https://github.com/thbar/kiba-common",
+    "documentation_uri" => "https://github.com/thbar/kiba-common/blob/master/README.md"
+  }
 
-  gem.add_dependency 'kiba', '>= 1.0.0', '< 3'
-  gem.add_development_dependency 'rake'
-  gem.add_development_dependency 'minitest'
-  gem.add_development_dependency 'awesome_print'
-  gem.add_development_dependency 'minitest-focus'
+  gem.add_dependency "kiba", ">= 3.0.0", "< 5"
+  gem.add_development_dependency "rake"
+  gem.add_development_dependency "minitest"
+  gem.add_development_dependency "amazing_print"
+  gem.add_development_dependency "minitest-focus"
+  gem.add_development_dependency "standard"
 end

data/lib/kiba-common/destinations/csv.rb

@@ -1,28 +1,31 @@
-require 'csv'
+require "csv"
 
 module Kiba
   module Common
     module Destinations
       class CSV
         attr_reader :filename, :csv_options, :csv, :headers
-        
+
         def initialize(filename:, csv_options: nil, headers: nil)
           @filename = filename
           @csv_options = csv_options || {}
           @headers = headers
         end
-        
+
         def write(row)
-          @csv ||= ::CSV.open(filename, 'wb', csv_options)
+          @csv ||= ::CSV.open(filename, "wb", **csv_options)
           @headers ||= row.keys
-          @headers_written ||= (csv << headers ; true)
+          @headers_written ||= begin
+            csv << headers
+            true
+          end
           csv << row.fetch_values(*@headers)
         end
-        
+
         def close
           csv&.close
         end
       end
     end
   end
-end
+end

data/lib/kiba-common/destinations/lambda.rb

@@ -0,0 +1,23 @@
+module Kiba
+  module Common
+    module Destinations
+      class Lambda
+        attr_reader :on_write, :on_close
+
+        def initialize(on_init: nil, on_write: nil, on_close: nil)
+          @on_write = on_write
+          @on_close = on_close
+          on_init&.call
+        end
+
+        def write(row)
+          on_write&.call(row)
+        end
+
+        def close
+          on_close&.call
+        end
+      end
+    end
+  end
+end

data/lib/kiba-common/dsl_extensions/logger.rb

@@ -1,4 +1,4 @@
-require 'logger'
+require "logger"
 
 module Kiba
   module Common
@@ -10,7 +10,7 @@ module Kiba
         end
 
         def logger
-          @logger ||= self.logger = ::Logger.new(STDOUT)
+          @logger ||= self.logger = ::Logger.new($stdout)
         end
       end
     end

data/lib/kiba-common/dsl_extensions/show_me.rb

@@ -1,17 +1,15 @@
-require 'awesome_print'
-
 module Kiba
   module Common
     module DSLExtensions
       # Color print your row.
       module ShowMe
         def show_me!(&pre_process)
-          transform do |row|
-            row = pre_process ? pre_process.call(row) : row
+          transform do |original_row|
+            row = pre_process ? pre_process.call(original_row) : original_row
             # NOTE: invoking Kernel.ap for testing since
             # ap itself is harder to mock (Kiba Context)
             Kernel.ap(row)
-            row
+            original_row
           end
         end
       end

data/lib/kiba-common/sources/csv.rb

@@ -0,0 +1,22 @@
+require "csv"
+
+module Kiba
+  module Common
+    module Sources
+      class CSV
+        attr_reader :filename, :csv_options
+
+        def initialize(filename:, csv_options: {})
+          @filename = filename
+          @csv_options = csv_options
+        end
+
+        def each
+          ::CSV.foreach(filename, **csv_options) do |row|
+            yield row
+          end
+        end
+      end
+    end
+  end
+end

data/lib/kiba-common/sources/enumerable.rb

@@ -7,7 +7,7 @@ module Kiba
         def initialize(enumerable)
           @enumerable = enumerable
         end
-        
+
         def unwrap_enumerable
           enumerable.respond_to?(:call) ? enumerable.call : enumerable
         end

data/lib/kiba-common/transforms/enumerable_exploder.rb

@@ -9,4 +9,4 @@ module Kiba
       end
     end
   end
-end
+end

data/lib/kiba-common/version.rb

@@ -1,5 +1,5 @@
 module Kiba
   module Common
-    VERSION = '0.7.0'
+    VERSION = "1.5.0"
   end
 end

data/test/helper.rb

@@ -1,6 +1,6 @@
-require 'minitest/autorun'
-require 'minitest/pride'
-require 'minitest/focus'
+require "minitest/autorun"
+require "minitest/pride"
+require "minitest/focus"
 
-require 'kiba-common/sources/enumerable'
-require_relative 'support/assert_called'
+require "kiba-common/sources/enumerable"
+require_relative "support/assert_called"

data/test/support/assert_called.rb

@@ -7,4 +7,4 @@ module AssertCalled
     end
     mock.verify
   end
-end
+end

data/test/support/test_array_destination.rb

@@ -2,8 +2,8 @@ class TestArrayDestination
   def initialize(array)
     @array = array
   end
-  
+
   def write(row)
     @array << row
   end
-end
+end

data/test/support/test_hash_configured_object.rb

@@ -0,0 +1,9 @@
+class TestHashConfiguredObject
+  def initialize(config)
+    @config = config
+  end
+
+  def each
+    yield(@config)
+  end
+end

data/test/support/test_keyword_proxy_source.rb

@@ -0,0 +1,13 @@
+class TestKeywordProxySource
+  def initialize(mandatory:, optional: nil)
+    @mandatory = mandatory
+    @optional = optional
+  end
+
+  def each
+    yield({
+      mandatory: @mandatory,
+      optional: @optional
+    })
+  end
+end

data/test/test_csv_destination.rb

@@ -1,10 +1,10 @@
-require_relative 'helper'
-require 'kiba'
-require 'kiba-common/destinations/csv'
+require_relative "helper"
+require "kiba"
+require "kiba-common/destinations/csv"
 
 class TestCSVDestination < Minitest::Test
-  TEST_FILENAME = 'output.csv'
-    
+  TEST_FILENAME = "output.csv"
+
   def teardown
     File.delete(TEST_FILENAME) if File.exist?(TEST_FILENAME)
   end
@@ -21,24 +21,24 @@ class TestCSVDestination < Minitest::Test
     end
 
     Kiba.run(job)
-    
+
     IO.read(TEST_FILENAME)
   end
-  
+
   def test_classic
     assert_equal <<~CSV, run_etl
       name,age
       world,999
     CSV
   end
-  
+
   def test_csv_options
-    assert_equal <<~CSV, run_etl(csv_options: {col_sep: ';'})
+    assert_equal <<~CSV, run_etl(csv_options: {col_sep: ";"})
       name;age
       world;999
     CSV
   end
-  
+
   def test_headers_provided
     assert_equal <<~CSV, run_etl(headers: [:age])
       age

data/test/test_csv_source.rb

@@ -0,0 +1,51 @@
+require_relative "helper"
+require "kiba"
+require "kiba-common/sources/csv"
+require_relative "support/test_array_destination"
+
+class TestCSVSource < MiniTest::Test
+  TEST_FILENAME = "input.csv"
+
+  def setup
+    CSV.open(TEST_FILENAME, "wb") do |csv|
+      csv << %w[first_name last_name]
+      csv << %w[John Barry]
+      csv << %w[Kate Bush]
+    end
+  end
+
+  def teardown
+    FileUtils.rm(TEST_FILENAME) if File.exist?(TEST_FILENAME)
+  end
+
+  def run_job(args)
+    rows = []
+    job = Kiba.parse do
+      source Kiba::Common::Sources::CSV, **args
+      destination TestArrayDestination, rows
+    end
+    Kiba.run(job)
+    rows
+  end
+
+  def test_with_csv_options
+    rows = run_job filename: TEST_FILENAME,
+                   csv_options: {headers: true, header_converters: :symbol}
+
+    assert_equal [CSV::Row], rows.map(&:class).uniq
+    assert_equal([
+      {first_name: "John", last_name: "Barry"},
+      {first_name: "Kate", last_name: "Bush"}
+    ], rows.map(&:to_h))
+  end
+
+  def test_without_csv_options
+    rows = run_job(filename: TEST_FILENAME)
+
+    assert_equal [
+      %w[first_name last_name],
+      %w[John Barry],
+      %w[Kate Bush]
+    ], rows
+  end
+end

data/test/test_enumerable_exploder_transform.rb

@@ -1,13 +1,13 @@
-require_relative 'helper'
-require 'kiba'
-require_relative 'support/test_array_destination'
-require 'kiba-common/transforms/enumerable_exploder'
-require 'kiba/dsl_extensions/config'
+require_relative "helper"
+require "kiba"
+require_relative "support/test_array_destination"
+require "kiba-common/transforms/enumerable_exploder"
+require "kiba/dsl_extensions/config"
 
 class TestEnumerableExploderTransform < Minitest::Test
   def test_exploder
     output = []
-    input = [[1,2],[3,4,5]]
+    input = [[1, 2], [3, 4, 5]]
 
     job = Kiba.parse do
       extend Kiba::DSLExtensions::Config
@@ -18,7 +18,7 @@ class TestEnumerableExploderTransform < Minitest::Test
       destination TestArrayDestination, output
     end
     Kiba.run(job)
-    
+
     assert_equal input.flatten, output
   end
-end
+end

data/test/test_enumerable_source.rb

@@ -1,6 +1,6 @@
-require_relative 'helper'
-require 'kiba'
-require_relative 'support/test_array_destination'
+require_relative "helper"
+require "kiba"
+require_relative "support/test_array_destination"
 
 class TestEnumerableSource < Minitest::Test
   def assert_enumerable_source(input:, expected_output:)
@@ -16,12 +16,12 @@ class TestEnumerableSource < Minitest::Test
     end
     Kiba.run(job)
   end
-    
+
   def test_source_as_enumerable
     assert_enumerable_source(input: (1..10), expected_output: (2..20).step(2).to_a)
   end
-  
+
   def test_source_as_callable
     assert_enumerable_source(input: -> { (1..10) }, expected_output: (2..20).step(2).to_a)
   end
-end
+end

data/test/test_integration.rb

@@ -0,0 +1,57 @@
+require_relative "helper"
+require "kiba"
+require_relative "support/test_array_destination"
+require "kiba-common/sources/csv"
+require "kiba-common/destinations/csv"
+
+# a testbed to verify & showcase how you can chain multiple components
+class TestIntegration < Minitest::Test
+  def write_csv(filename, data)
+    d = Kiba::Common::Destinations::CSV.new(filename: filename)
+    data.each { |r| d.write(r) }
+    d.close
+  end
+
+  def test_multiple_csv_inputs
+    Dir.mktmpdir do |dir|
+      write_csv File.join(dir, "001.csv"), [first_name: "John"]
+      write_csv File.join(dir, "002.csv"), [first_name: "Kate"]
+
+      rows = []
+      job = Kiba.parse do
+        # enable the new streaming-runner (for SourceTransformAdapter support)
+        extend Kiba::DSLExtensions::Config
+        config :kiba, runner: Kiba::StreamingRunner
+
+        # create one row per input file
+        source Kiba::Common::Sources::Enumerable, -> { Dir[File.join(dir, "*.csv")].sort }
+
+        # out of that row, create configuration for a CSV source
+        transform do |r|
+          [
+            Kiba::Common::Sources::CSV,
+            filename: r,
+            csv_options: {headers: true, header_converters: :symbol}
+          ]
+        end
+
+        transform do |klass, args|
+          Enumerator.new do |y|
+            klass.new(**args).each do |r|
+              y << r
+            end
+          end
+        end
+        transform Kiba::Common::Transforms::EnumerableExploder
+
+        destination TestArrayDestination, rows
+      end
+      Kiba.run(job)
+
+      assert_equal([
+        {first_name: "John"},
+        {first_name: "Kate"}
+      ], rows.map(&:to_h))
+    end
+  end
+end

data/test/test_lambda_destination.rb

@@ -0,0 +1,22 @@
+require_relative "helper"
+require "kiba"
+require "kiba-common/destinations/lambda"
+
+class TestLambdaDestination < MiniTest::Test
+  def test_lambda
+    accumulator = []
+    on_init_called = false
+    on_close_called = false
+    job = Kiba.parse do
+      source Kiba::Common::Sources::Enumerable, ["one", "two"]
+      destination Kiba::Common::Destinations::Lambda,
+        on_init: -> { on_init_called = true },
+        on_write: ->(r) { accumulator << r },
+        on_close: -> { on_close_called = true }
+    end
+    Kiba.run(job)
+    assert_equal ["one", "two"], accumulator
+    assert_equal true, on_init_called
+    assert_equal true, on_close_called
+  end
+end

data/test/test_logger.rb

@@ -1,6 +1,6 @@
-require_relative 'helper'
-require 'kiba'
-require 'kiba-common/dsl_extensions/logger'
+require_relative "helper"
+require "kiba"
+require "kiba-common/dsl_extensions/logger"
 
 class TestLogger < Minitest::Test
   def test_default_logger
@@ -20,11 +20,11 @@ class TestLogger < Minitest::Test
       logger = Logger.new(buffer)
 
       pre_process do
-        logger.info 'Logging from pre_process'
+        logger.info "Logging from pre_process"
       end
     end
     Kiba.run(job)
 
-    assert_includes buffer.string, 'Logging from pre_process'
+    assert_includes buffer.string, "Logging from pre_process"
   end
 end

data/test/test_show_me.rb

@@ -1,6 +1,7 @@
-require_relative 'helper'
-require 'kiba'
-require 'kiba-common/dsl_extensions/show_me'
+require_relative "helper"
+require "kiba"
+require "amazing_print"
+require "kiba-common/dsl_extensions/show_me"
 
 class TestShowMe < Minitest::Test
   include AssertCalled
@@ -8,24 +9,28 @@ class TestShowMe < Minitest::Test
   def test_show_me
     job = Kiba.parse do
       extend Kiba::Common::DSLExtensions::ShowMe
-      source Kiba::Common::Sources::Enumerable, ['row']
+      source Kiba::Common::Sources::Enumerable, ["row"]
       show_me!
     end
-    
-    assert_called(Kernel, :ap, ['row']) do
+
+    assert_called(Kernel, :ap, ["row"]) do
       Kiba.run(job)
     end
   end
-  
+
   def test_show_me_pre_process
+    output = []
     job = Kiba.parse do
       extend Kiba::Common::DSLExtensions::ShowMe
       source Kiba::Common::Sources::Enumerable, [{this: "OK", not_this: "KO"}]
       show_me! { |r| r.fetch(:this) }
+      destination TestArrayDestination, output
     end
 
-    assert_called(Kernel, :ap, ['OK']) do
+    assert_called(Kernel, :ap, ["OK"]) do
       Kiba.run(job)
     end
+
+    assert_equal [{this: "OK", not_this: "KO"}], output
   end
 end

data/test/test_source_transform_adapter.rb

@@ -1,6 +1,11 @@
-require_relative 'helper'
-require 'kiba-common/transforms/source_transform_adapter'
+require_relative "helper"
+require_relative "support/test_keyword_proxy_source"
+require_relative "support/test_hash_configured_object"
 
+# NOTE: the SourceTransformAdapter has been removed,
+# but I'm keeping these tests, patched to instead use
+# Enumerator, as a way to verify that the alternative
+# I provided in the change log still works.
 class TestSourceTransformAdapter < Minitest::Test
   include Kiba::Common::Sources
   include Kiba::DSLExtensions
@@ -12,13 +17,72 @@ class TestSourceTransformAdapter < Minitest::Test
       config :kiba, runner: Kiba::StreamingRunner
 
       source Enumerable, [
-        [ Enumerable, (1..10) ],
-        [ Enumerable, (11..20) ]
+        [Enumerable, (1..10)],
+        [Enumerable, (11..20)]
       ]
-      transform Kiba::Common::Transforms::SourceTransformAdapter
+
+      transform do |klass, args|
+        Enumerator.new do |y|
+          klass.new(args).each do |r|
+            y << r
+          end
+        end
+      end
+      transform Kiba::Common::Transforms::EnumerableExploder
+
       destination TestArrayDestination, rows
     end
     Kiba.run(job)
     assert_equal (1..20).to_a, rows
   end
-end
+
+  def test_instantiation_keyword_arguments
+    rows = []
+    job = Kiba.parse do
+      source Enumerable, [
+        # Test against a class that expects explicit keyword arguments
+        [TestKeywordProxySource, {mandatory: "some value"}]
+      ]
+
+      transform do |klass, args|
+        Enumerator.new do |y|
+          klass.new(**args).each do |r|
+            y << r
+          end
+        end
+      end
+      transform Kiba::Common::Transforms::EnumerableExploder
+
+      destination TestArrayDestination, rows
+    end
+    Kiba.run(job)
+    assert_equal([
+      {mandatory: "some value", optional: nil}
+    ], rows)
+  end
+
+  def test_hash_configured_object
+    rows = []
+    job = Kiba.parse do
+      source Enumerable, [
+        # Test against a class that takes a single Hash argument
+        [TestHashConfiguredObject, {mandatory: "some value"}]
+      ]
+
+      transform do |klass, args|
+        Enumerator.new do |y|
+          klass.new(**args).each do |r|
+            y << r
+          end
+        end
+      end
+      transform Kiba::Common::Transforms::EnumerableExploder
+
+      destination TestArrayDestination, rows
+    end
+    Kiba.run(job)
+    assert_equal([
+      {mandatory: "some value"}
+    ], rows)
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kiba-common
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 1.5.0
 platform: ruby
 authors:
 - Thibaut Barrère
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-02-06 00:00:00.000000000 Z
+date: 2021-03-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: kiba
@@ -16,20 +16,20 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: 3.0.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: '3'
+        version: '5'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: 3.0.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: '3'
+        version: '5'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -59,7 +59,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: awesome_print
+  name: amazing_print
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -86,6 +86,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: standard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Commonly used helpers for Kiba ETL
 email:
 - thibaut.barrere@gmail.com
@@ -93,8 +107,9 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/FUNDING.yml"
+- ".github/workflows/ci.yml"
 - ".gitignore"
-- ".travis.yml"
 - Changes.md
 - Gemfile
 - LICENSE
@@ -102,26 +117,34 @@ files:
 - Rakefile
 - kiba-common.gemspec
 - lib/kiba-common/destinations/csv.rb
+- lib/kiba-common/destinations/lambda.rb
 - lib/kiba-common/dsl_extensions/logger.rb
 - lib/kiba-common/dsl_extensions/show_me.rb
+- lib/kiba-common/sources/csv.rb
 - lib/kiba-common/sources/enumerable.rb
 - lib/kiba-common/transforms/enumerable_exploder.rb
-- lib/kiba-common/transforms/source_transform_adapter.rb
 - lib/kiba-common/version.rb
 - test/helper.rb
 - test/support/assert_called.rb
 - test/support/test_array_destination.rb
+- test/support/test_hash_configured_object.rb
+- test/support/test_keyword_proxy_source.rb
 - test/test_csv_destination.rb
+- test/test_csv_source.rb
 - test/test_enumerable_exploder_transform.rb
 - test/test_enumerable_source.rb
+- test/test_integration.rb
+- test/test_lambda_destination.rb
 - test/test_logger.rb
 - test/test_show_me.rb
 - test/test_source_transform_adapter.rb
 homepage: https://github.com/thbar/kiba-common
 licenses:
 - LGPL-3.0
-metadata: {}
-post_install_message: 
+metadata:
+  source_code_uri: https://github.com/thbar/kiba-common
+  documentation_uri: https://github.com/thbar/kiba-common/blob/master/README.md
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -136,18 +159,22 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.3
-signing_key: 
+rubygems_version: 3.2.3
+signing_key:
 specification_version: 4
 summary: Commonly used helpers for Kiba ETL
 test_files:
 - test/helper.rb
 - test/support/assert_called.rb
 - test/support/test_array_destination.rb
+- test/support/test_hash_configured_object.rb
+- test/support/test_keyword_proxy_source.rb
 - test/test_csv_destination.rb
+- test/test_csv_source.rb
 - test/test_enumerable_exploder_transform.rb
 - test/test_enumerable_source.rb
+- test/test_integration.rb
+- test/test_lambda_destination.rb
 - test/test_logger.rb
 - test/test_show_me.rb
 - test/test_source_transform_adapter.rb

data/.travis.yml

@@ -1,11 +0,0 @@
-language: ruby
-before_install:
-  # https://stackoverflow.com/a/47972768
-  - gem update --system
-  - gem update bundler
-rvm:
-  - ruby-head
-  - 2.5.0
-  - 2.4.3
-  - 2.3.6
-  - jruby-9.1.15.0

data/lib/kiba-common/transforms/source_transform_adapter.rb

@@ -1,14 +0,0 @@
-module Kiba
-  module Common
-    module Transforms
-      class SourceTransformAdapter
-        def process(args)
-          args.shift.new(*args).each do |row|
-            yield(row)
-          end
-          nil
-        end
-      end
-    end
-  end
-end