kiba-common 0.6.0 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1663d6106ac7cae429c6fba5a9bc3321222c957f44c21644eee77b73314f9e92
-  data.tar.gz: 92dfe90ee5c9f725ffad8635e4397a8da08bd86b879c1f56530afb5b0c206a39
+  metadata.gz: 73129b07c0c847b9a1ed5b2a88e0fe3b9449200589fea6e0b88696959c86448f
+  data.tar.gz: 1a73e5414c054fbb233c92c5459087153a6ad413bfa194c192d8ea54cd4ee3ef
 SHA512:
-  metadata.gz: 6f87775082169e7ce4f6bff67c4ba30c3504b5df81dc0c47a4f752ffdf96e6b752b7e9362e174f9b9ca4808ee0c2c042a0c93f3f2809ba734badcb212951e77e
-  data.tar.gz: 47263dfa9ed01ac9ad266624478f9767c504be7e47b7546e9725e0c6f0652e75ab76a67ad08f70dca618f18142e47156f876efca732bc770315e734ba3a911a7
+  metadata.gz: f984de5843f926e3c153c381a1e05d171275fa185c3b0b2c7d855250b75c6ed85841ad78d87cb55b9cc4d6759e7d50c585765483ef27f0c3c64f80b8a739e957
+  data.tar.gz: 1e05cefc61c51f474f93b8f331e3d17a1de7c2ec3683212336796e3dcbc164bc9d5bc9049a8f255c25d237a8f8e38516d02db45f343a903d2bd76ad439515330

data/Changes.md

@@ -1,6 +1,8 @@
 HEAD
 ----
 
+- New: Kiba::Common::Transforms::SourceTransformAdapter let you transform rows into parameters for source instantiation.
+
 0.6.0
 -----
 

data/README.md

@@ -38,6 +38,39 @@ You can pass a callable to make sure the evaluation will occur after your [pre-p
 source Kiba::Common::Sources::Enumerable, -> { Dir["input/*.json"] }
 ```
 
+### Kiba::Common::Transforms::SourceTransformAdapter
+
+Let's say you have a source (e.g. `CSVSource`), which you would like to instantiate for each input row (e.g. a list of filenames). 
+
+Normally you'd have to bake file iteration right inside your source. 
+
+But since Kiba v2 introduction of `StreamingRunner`, it is possible for transforms to yield an arbitrary (potentially infinite) amount of rows.
+
+Leveraging that possibility, you can use a `SourceTransformAdapter` to dynamically instantiate the source for each of your input rows.
+
+This allows to mix-and-match components in a much more versatile & powerful way.
+
+Requirements: Kiba v2 with `StreamingRunner` enabled.
+
+Usage:
+
+```ruby
+source Kiba::Common::Sources::Enumerable, -> { Dir["input/*.csv"] }
+
+transform do |r|
+  # build up the args that you would normally pass to your source, e.g.
+  # source MyCSV, filename: 'file.csv'
+  # but instead, using the input as a parameter
+  [MyCSVSource, filename: r]
+end
+
+# this will instantiate one source per input row, yielding rows
+# that your source would normally yield
+transform Kiba::Common::Transforms::SourceTransformAdapter
+```
+
+This can be used for a wide array of scenarios, including extracting data for N third-party accounts of a same system, with an array of API keys etc.
+
 ### Kiba::Common::Transforms::EnumerableExploder
 
 A transform calling `each` on input rows (assuming they are e.g. arrays of sub-rows) and yielding one output row per enumerated element.

data/kiba-common.gemspec

@@ -17,4 +17,5 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency 'rake'
   gem.add_development_dependency 'minitest'
   gem.add_development_dependency 'awesome_print'
+  gem.add_development_dependency 'minitest-focus'
 end

data/lib/kiba-common/transforms/source_transform_adapter.rb

@@ -0,0 +1,14 @@
+module Kiba
+  module Common
+    module Transforms
+      class SourceTransformAdapter
+        def process(args)
+          args.shift.new(*args).each do |row|
+            yield(row)
+          end
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/kiba-common/version.rb

@@ -1,5 +1,5 @@
 module Kiba
   module Common
-    VERSION = '0.6.0'
+    VERSION = '0.7.0'
   end
 end

data/test/helper.rb

@@ -1,5 +1,6 @@
 require 'minitest/autorun'
 require 'minitest/pride'
+require 'minitest/focus'
 
 require 'kiba-common/sources/enumerable'
 require_relative 'support/assert_called'

data/test/test_source_transform_adapter.rb

@@ -0,0 +1,24 @@
+require_relative 'helper'
+require 'kiba-common/transforms/source_transform_adapter'
+
+class TestSourceTransformAdapter < Minitest::Test
+  include Kiba::Common::Sources
+  include Kiba::DSLExtensions
+
+  def test_instantiation
+    rows = []
+    job = Kiba.parse do
+      extend Config
+      config :kiba, runner: Kiba::StreamingRunner
+
+      source Enumerable, [
+        [ Enumerable, (1..10) ],
+        [ Enumerable, (11..20) ]
+      ]
+      transform Kiba::Common::Transforms::SourceTransformAdapter
+      destination TestArrayDestination, rows
+    end
+    Kiba.run(job)
+    assert_equal (1..20).to_a, rows
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kiba-common
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.7.0
 platform: ruby
 authors:
 - Thibaut Barrère
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-01-26 00:00:00.000000000 Z
+date: 2018-02-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: kiba
@@ -72,6 +72,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest-focus
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Commonly used helpers for Kiba ETL
 email:
 - thibaut.barrere@gmail.com
@@ -92,6 +106,7 @@ files:
 - lib/kiba-common/dsl_extensions/show_me.rb
 - lib/kiba-common/sources/enumerable.rb
 - lib/kiba-common/transforms/enumerable_exploder.rb
+- lib/kiba-common/transforms/source_transform_adapter.rb
 - lib/kiba-common/version.rb
 - test/helper.rb
 - test/support/assert_called.rb
@@ -101,6 +116,7 @@ files:
 - test/test_enumerable_source.rb
 - test/test_logger.rb
 - test/test_show_me.rb
+- test/test_source_transform_adapter.rb
 homepage: https://github.com/thbar/kiba-common
 licenses:
 - LGPL-3.0
@@ -134,3 +150,4 @@ test_files:
 - test/test_enumerable_source.rb
 - test/test_logger.rb
 - test/test_show_me.rb
+- test/test_source_transform_adapter.rb