kiba-common 0.5.0 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 925c701dd086d91b0f9b9222aa575a634a10c6a6d56e62c3edbbfad3d58d81f2
-  data.tar.gz: ff158c83c13aa3378c9fc8b670aa9535867c68bad7c4f33b55c45e7cfd99ec57
+  metadata.gz: 1663d6106ac7cae429c6fba5a9bc3321222c957f44c21644eee77b73314f9e92
+  data.tar.gz: 92dfe90ee5c9f725ffad8635e4397a8da08bd86b879c1f56530afb5b0c206a39
 SHA512:
-  metadata.gz: b4836e640abeddf2a3cde529ef12ec52e405705b782c7ad462b7fd05ffa193a52338162e7c40425570a8bf332275141a3e2ed0132e1f558536996e60c2f3ebfc
-  data.tar.gz: 948bd196174bc02208438ebdb572bed401a01cfe461939728f3bb72c1770838dcd1d3ac48eeab36f04c25d0579ad445ef4d479f2103c6fc30257448ccb77f25b
+  metadata.gz: 6f87775082169e7ce4f6bff67c4ba30c3504b5df81dc0c47a4f752ffdf96e6b752b7e9362e174f9b9ca4808ee0c2c042a0c93f3f2809ba734badcb212951e77e
+  data.tar.gz: 47263dfa9ed01ac9ad266624478f9767c504be7e47b7546e9725e0c6f0652e75ab76a67ad08f70dca618f18142e47156f876efca732bc770315e734ba3a911a7

data/Changes.md

@@ -1,10 +1,15 @@
 HEAD
 ----
 
+0.6.0
+-----
+
+- New: Kiba::Common::Transforms::EnumerableExploder will explode each enumerable row (responding to `#each`) into N rows.
+
 0.5.0
 -----
 
-- New: Kiba::Common::Sources::Enumerable allow to use any Ruby instance responding to `#each` (or a `Proc` returning such an instance) as a source for rows.
+- New: Kiba::Common::Sources::Enumerable allows to use any Ruby instance responding to `#each` (or a `Proc` returning such an instance) as a source for rows.
 
 0.0.4
 -----

data/README.md

@@ -38,6 +38,64 @@ You can pass a callable to make sure the evaluation will occur after your [pre-p
 source Kiba::Common::Sources::Enumerable, -> { Dir["input/*.json"] }
 ```
 
+### Kiba::Common::Transforms::EnumerableExploder
+
+A transform calling `each` on input rows (assuming they are e.g. arrays of sub-rows) and yielding one output row per enumerated element.
+
+Requirements: [Kiba v2](https://github.com/thbar/kiba/releases/tag/v2.0.0) with `StreamingRunner` enabled.
+
+Usage:
+
+```ruby
+require 'kiba-common/transforms/enumerable_exploder'
+
+transform Kiba::Common::Transforms::EnumerableExploder
+```
+
+For instance, this can help if you are reading XML/JSON documents from a source, and each input document contains multiple rows that you'd want to extract.
+
+```ruby
+source Kiba::Common::Sources::Enumerable, -> { Dir["input/*.xml"] }
+
+transform { |r| IO.binread(r) }
+transform { |r| Nokogiri::XML(r) }
+# this will return an array of XML elements
+transform { |r| r.search('/orders') }
+# this will explode the array (one order per output row)
+transform Kiba::Common::Transforms::EnumerableExploder
+```
+
+Similarly, if you have a CSV document as your input:
+
+| po_number  | buyers |
+| ------------- | ------------- |
+| 00001  | John:Mary:Sally  |
+
+and you want to reformat it to get this instead:
+
+| po_number | buyer |
+|-------------|---------|
+| 00001 | John |
+| 00001 | Mary |
+| 00001 | Sally |
+
+then you can explode them again with:
+
+```ruby
+source MyCSVSource, filename: "input.csv"
+
+transform do |row|
+  row.fetch(:buyers).split(':').map do |buyer|
+    { 
+      po_number: row.fetch(:po_number),
+      buyer: buyer
+    }
+  end
+end
+
+transform Kiba::Common::Transforms::EnumerableExploder
+```
+
 ### Kiba::Common::Destinations::CSV
 
 A way to dump `Hash` rows as CSV.

data/lib/kiba-common/transforms/enumerable_exploder.rb

@@ -0,0 +1,12 @@
+module Kiba
+  module Common
+    module Transforms
+      class EnumerableExploder
+        def process(row)
+          row.each { |r| yield r }
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/kiba-common/version.rb

@@ -1,5 +1,5 @@
 module Kiba
   module Common
-    VERSION = '0.5.0'
+    VERSION = '0.6.0'
   end
 end

data/test/test_enumerable_exploder_transform.rb

@@ -0,0 +1,24 @@
+require_relative 'helper'
+require 'kiba'
+require_relative 'support/test_array_destination'
+require 'kiba-common/transforms/enumerable_exploder'
+require 'kiba/dsl_extensions/config'
+
+class TestEnumerableExploderTransform < Minitest::Test
+  def test_exploder
+    output = []
+    input = [[1,2],[3,4,5]]
+
+    job = Kiba.parse do
+      extend Kiba::DSLExtensions::Config
+      config :kiba, runner: Kiba::StreamingRunner
+
+      source Kiba::Common::Sources::Enumerable, input
+      transform Kiba::Common::Transforms::EnumerableExploder
+      destination TestArrayDestination, output
+    end
+    Kiba.run(job)
+    
+    assert_equal input.flatten, output
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kiba-common
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.6.0
 platform: ruby
 authors:
 - Thibaut Barrère
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-01-23 00:00:00.000000000 Z
+date: 2018-01-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: kiba
@@ -91,11 +91,13 @@ files:
 - lib/kiba-common/dsl_extensions/logger.rb
 - lib/kiba-common/dsl_extensions/show_me.rb
 - lib/kiba-common/sources/enumerable.rb
+- lib/kiba-common/transforms/enumerable_exploder.rb
 - lib/kiba-common/version.rb
 - test/helper.rb
 - test/support/assert_called.rb
 - test/support/test_array_destination.rb
 - test/test_csv_destination.rb
+- test/test_enumerable_exploder_transform.rb
 - test/test_enumerable_source.rb
 - test/test_logger.rb
 - test/test_show_me.rb
@@ -128,6 +130,7 @@ test_files:
 - test/support/assert_called.rb
 - test/support/test_array_destination.rb
 - test/test_csv_destination.rb
+- test/test_enumerable_exploder_transform.rb
 - test/test_enumerable_source.rb
 - test/test_logger.rb
 - test/test_show_me.rb