kiba-common 0.8.0 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 76aa97f6f777aa6e96f05a088346cdca52426732a0f033b8dcfd2856d78770b8
-  data.tar.gz: 820fddae1f7b853075e927a5fc878a0cbbf3cbda31ad61e2b06b7a446fe8531e
+  metadata.gz: 7823ab2b0363a8fd2a2dd69bd2c70fd75db837ab4a583464e46a330f3c489cbf
+  data.tar.gz: 2e61b3b7aaacfb930bd62f3a5714a86b511f35d4de8ccda71dbab3bc15b747eb
 SHA512:
-  metadata.gz: a5c036121f7955527bd983afcd37ca71547d99d5b6e48f3ef88351a082b9e7729ae118f8251896c6474410ad8a8674da96140f7ea47f04dabe024aca15e64eea
-  data.tar.gz: 0fcec0f0342151ce1a408292b80e41852ae0094aa20887ae23abdebdee73a6604cfa3086cc980bfc6d368667460765cb28a7d6e1b11464e147545ca6ee7e6764
+  metadata.gz: cdd431a4c090e7e59aea5dfc4ed3320f5643afeeaa729969a960bb8cb9ee35a1a0f77788ee9e1036f386f8cce064ae946330dc0592bce35a9388358bf6dcb39e
+  data.tar.gz: 93900ad025f97f0463f1acaf95bf003ac94cf0a0d49230c4b49c9d77efe23b0dd86e329f92593c970b777e19e98fe563f54028934495d533c3cef453fc33bc4d

data/Changes.md

@@ -1,6 +1,11 @@
 HEAD
 ----
 
+0.9.0
+-----
+
+- New: Kiba::Common::Sources::CSV provides a basic CSV source for simple needs.
+
 0.8.0
 -----
 

data/README.md

@@ -129,6 +129,62 @@ end
 transform Kiba::Common::Transforms::EnumerableExploder
 ```
 
+### Kiba::Common::Sources::CSV
+
+A CSV source for basic needs (in particular, it doesn't yield row metadata, which are useful in more advanced scenarios).
+
+Use the `csv_options` keyword to control the input format like when using [Ruby CSV class](http://ruby-doc.org/stdlib-2.4.0/libdoc/csv/rdoc/CSV.html#method-c-new).
+
+Usage:
+
+```ruby
+require 'kiba-common/sources/csv'
+
+# by defaults, csv_options are empty
+source Kiba::Common::Sources::CSV, filename: 'input.csv'
+
+# you can provide your own csv_options
+source Kiba::Common::Sources::CSV, filename: 'input.csv',
+  csv_options: { headers: true, header_converters: :symbol }
+```
+
+#### Handling multiple input CSV files
+
+You can process multiple files by chaining the various components available in Kiba Common (see `test/test_integration#test_multiple_csv_inputs` for an actual demo):
+
+```ruby
+# create one row per input filename
+source Kiba::Common::Sources::Enumerable, -> { Dir[File.join(dir, '*.csv')] }
+
+# out of that row, create configuration for a CSV source
+transform do |r|
+  [
+    Kiba::Common::Sources::CSV,
+    filename: r,
+    csv_options: { headers: true, header_converters: :symbol }
+  ]
+end
+
+# instantiate & yield CSV rows for each configuration
+transform Kiba::Common::Transforms::SourceTransformAdapter
+```
+
+Alternatively, you can wrap this source in your own source like this:
+
+```ruby
+class MultipleCSVSource
+  def initialize(file_pattern:, csv_options:)
+
+  def each
+    Dir[file_pattern].each do |filename|
+      Kiba::Common::Sources::CSV.new(filename, csv_options).each do |row|
+        yield row
+      end
+    end
+  end
+end
+```
+
 ### Kiba::Common::Destinations::CSV
 
 A way to dump `Hash` rows as CSV.

data/lib/kiba-common/sources/csv.rb

@@ -0,0 +1,22 @@
+require 'csv'
+
+module Kiba
+  module Common
+    module Sources
+      class CSV
+        attr_reader :filename, :csv_options
+
+        def initialize(filename:, csv_options: {})
+          @filename = filename
+          @csv_options = csv_options
+        end
+
+        def each
+          ::CSV.foreach(filename, csv_options) do |row|
+            yield row
+          end
+        end
+      end
+    end
+  end
+end

data/lib/kiba-common/version.rb

@@ -1,5 +1,5 @@
 module Kiba
   module Common
-    VERSION = '0.8.0'
+    VERSION = '0.9.0'
   end
 end

data/test/test_csv_source.rb

@@ -0,0 +1,51 @@
+require_relative 'helper'
+require 'kiba'
+require 'kiba-common/sources/csv'
+require_relative 'support/test_array_destination'
+
+class TestCSVSource < MiniTest::Test
+  TEST_FILENAME = 'input.csv'
+
+  def setup
+    CSV.open(TEST_FILENAME, 'wb') do |csv|
+      csv << %w(first_name last_name)
+      csv << %w(John Barry)
+      csv << %w(Kate Bush)
+    end
+  end
+
+  def teardown
+    FileUtils.rm(TEST_FILENAME) if File.exist?(TEST_FILENAME)
+  end
+
+  def run_job(*args)
+    rows = []
+    job = Kiba.parse do
+      source Kiba::Common::Sources::CSV, *args
+      destination TestArrayDestination, rows
+    end
+    Kiba.run(job)
+    rows
+  end
+
+  def test_with_csv_options
+    rows = run_job filename: TEST_FILENAME,
+      csv_options: { headers: true, header_converters: :symbol }
+
+    assert_equal [CSV::Row], rows.map(&:class).uniq
+    assert_equal([
+      {first_name: "John", last_name: "Barry" },
+      {first_name: "Kate", last_name: "Bush"}
+    ], rows.map(&:to_h))
+  end
+
+  def test_without_csv_options
+    rows = run_job(filename: TEST_FILENAME)
+
+    assert_equal [
+      %w(first_name last_name),
+      %w(John Barry),
+      %w(Kate Bush)
+    ], rows
+  end
+end

data/test/test_integration.rb

@@ -0,0 +1,52 @@
+require_relative 'helper'
+require 'kiba'
+require_relative 'support/test_array_destination'
+require 'kiba-common/transforms/source_transform_adapter'
+require 'kiba-common/sources/csv'
+require 'kiba-common/destinations/csv'
+
+# a testbed to verify & showcase how you can chain multiple components
+class TestIntegration < Minitest::Test
+  def write_csv(filename, data)
+    d = Kiba::Common::Destinations::CSV.new(filename: filename)
+    data.each { |r| d.write(r) }
+    d.close
+  end
+
+  def test_multiple_csv_inputs
+    Dir.mktmpdir do |dir|
+      write_csv File.join(dir, '001.csv'), [first_name: 'John']
+      write_csv File.join(dir, '002.csv'), [first_name: 'Kate']
+
+      rows = []
+      job = Kiba.parse do
+        # enable the new streaming-runner (for SourceTransformAdapter support)
+        extend Kiba::DSLExtensions::Config
+        config :kiba, runner: Kiba::StreamingRunner
+
+        # create one row per input file
+        source Kiba::Common::Sources::Enumerable, -> { Dir[File.join(dir, '*.csv')] }
+
+        # out of that row, create configuration for a CSV source
+        transform do |r|
+          [
+            Kiba::Common::Sources::CSV,
+            filename: r,
+            csv_options: { headers: true, header_converters: :symbol }
+          ]
+        end
+
+        # instantiate & yield CSV rows for each configuration
+        transform Kiba::Common::Transforms::SourceTransformAdapter
+
+        destination TestArrayDestination, rows
+      end
+      Kiba.run(job)
+
+      assert_equal([
+        { first_name: 'John' },
+        { first_name: 'Kate' }
+      ], rows.map(&:to_h))
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kiba-common
 version: !ruby/object:Gem::Version
-  version: 0.8.0
+  version: 0.9.0
 platform: ruby
 authors:
 - Thibaut Barrère
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-02-19 00:00:00.000000000 Z
+date: 2018-09-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: kiba
@@ -104,6 +104,7 @@ files:
 - lib/kiba-common/destinations/csv.rb
 - lib/kiba-common/dsl_extensions/logger.rb
 - lib/kiba-common/dsl_extensions/show_me.rb
+- lib/kiba-common/sources/csv.rb
 - lib/kiba-common/sources/enumerable.rb
 - lib/kiba-common/transforms/enumerable_exploder.rb
 - lib/kiba-common/transforms/source_transform_adapter.rb
@@ -112,8 +113,10 @@ files:
 - test/support/assert_called.rb
 - test/support/test_array_destination.rb
 - test/test_csv_destination.rb
+- test/test_csv_source.rb
 - test/test_enumerable_exploder_transform.rb
 - test/test_enumerable_source.rb
+- test/test_integration.rb
 - test/test_logger.rb
 - test/test_show_me.rb
 - test/test_source_transform_adapter.rb
@@ -137,7 +140,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.7.3
+rubygems_version: 2.7.6
 signing_key: 
 specification_version: 4
 summary: Commonly used helpers for Kiba ETL
@@ -146,8 +149,10 @@ test_files:
 - test/support/assert_called.rb
 - test/support/test_array_destination.rb
 - test/test_csv_destination.rb
+- test/test_csv_source.rb
 - test/test_enumerable_exploder_transform.rb
 - test/test_enumerable_source.rb
+- test/test_integration.rb
 - test/test_logger.rb
 - test/test_show_me.rb
 - test/test_source_transform_adapter.rb