kiba 2.5.0 → 3.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: aebe72316fa16e820c9c11b7a2a274b05b77abed7eda8799858fe00a4279e2db
-  data.tar.gz: 686b359e9b74684472d17491c98a32adddd30603186b4f6ec7b6a40263d47d19
+  metadata.gz: a5cf22f1013b65b59d9c91b9f629749b47b5474b3f500d5ca54118b6b6a46c4f
+  data.tar.gz: cae5e05fabc7c8d169d47d6aa6a7da8befd6711ad2d2471c2c0f5f1d3e604841
 SHA512:
-  metadata.gz: 41be9f07f5c7bef36d85c97bc3a9224c7f885de92f458407064d7fa153f8a0ed1443a626b82a4e97f1269154aac0aaca738b8ed63c596953cc1059ae0657f4d3
-  data.tar.gz: d03d4e66991f0b903ca6b86eda7ae33790218dc84d9de906337a0ddae709a7cbfa6adb9432bbb812396e6a122c942f2cecc8a95ee16d184714e6625cac411905
+  metadata.gz: 2ed99641ab799ee57f213525f6ca3ab4b9a9d5e0b7680115bef8e139d32dcfb350fb0117065b5be0ded6a886c21988413b2963e2cbea546a7a1d5691ed078f27
+  data.tar.gz: 24434d65dcf9bb81ce0f575860edbdf92598a1e990663347a599eb42155211c02b9c9eb0304c869cf85c8cd15df8c262afcf9cdf11bb7568323ea98b393b9518

data/.travis.yml

@@ -3,12 +3,10 @@ matrix:
   include:
     # see https://www.ruby-lang.org/en/downloads/branches/
     - rvm: ruby-head
+    - rvm: 2.7
     - rvm: 2.6
     - rvm: 2.5
     - rvm: 2.4
-    # NOTE: EOL since 2019-03-31
-    - rvm: 2.3
     # see https://www.jruby.org/download
-    - rvm: jruby-9.1
     - rvm: jruby-9.2
     - rvm: truffleruby

data/Changes.md

@@ -1,3 +1,11 @@
+3.0.0
+-----
+
+- Breaking: the `kiba` command line is deprecated to encourage using `Kiba.parse` API. See [#81](https://github.com/thbar/kiba/pull/81) and release notes for details & migration path.
+- Kiba now defaults to `StreamingRunner` (backward compatible & more powerful engine) [#83](https://github.com/thbar/kiba/pull/83).
+- Kiba now officially supports MRI Ruby 2.4+ (although 2.3 will still work for now), JRuby 9.2+ or TruffleRuby.
+- You may get warnings with Ruby 2.7 and errors with Ruby 2.8+. See [#85] for status on Ruby 3 keyword arguments support.
+
 2.5.0
 -----
 

data/Pro-Changes.md

@@ -3,8 +3,69 @@ Kiba Pro Changelog
 
 Kiba Pro is the commercial extension for Kiba. Documentation is available on the [Wiki](https://github.com/thbar/kiba/wiki).
 
-HEAD
--------
+1.5.0
+-----
+
+- Compatibility with Kiba v3
+- BREAKING CHANGE: deprecate non-live Sequel connection passing (https://github.com/thbar/kiba/issues/79). Do not use `database: "connection_string"`, instead pass your `Sequel` connection directly. This moves the connection management out of the destination, which is a better pattern & provides better (block-based) resources closing.
+- Official MySQL support:
+  - While the compatibility was already here, it is now tested for in our QA testing suite.
+  - MySQL 5.5-8.0 is supported & tested
+  - MariaDB should be supported (although not tested against in the QA testing suite)
+  - Amazon Aurora MySQL is also supposed to work (although not tested)
+  - `Kiba::Pro::Sources::SQL` supports for non-streaming + streaming use
+  - `Kiba::Pro::Destinations::SQLBulkInsert` supports:
+    - Bulk insert
+    - Bulk insert with ignore
+    - Bulk upsert (including with dynamically computed columns) via `ON DUPLICATE KEY UPDATE`
+  - Note that the `Kiba::Pro::Destinations::SQLUpsert` (row-by-row) is not MySQL compatible at the moment
+
+1.2.0
+-----
+
+- `SQL` source improvements:
+  - Deprecate use_cursor in favor of block query construct. The source could previously be configured with:
+
+    ```ruby
+    source Kiba::Pro::Sources::SQL,
+      query: "SELECT * FROM items",
+      use_cursor: true
+    ```
+
+    The `use_cursor` keyword is now deprecated. You can use the more powerful block query construct:
+
+    ```ruby
+    source Kiba::Pro::Sources::SQL,
+      query: -> (db) { db["SELECT * FROM items"].use_cursor },
+    ```
+
+  - Avoid bogus nested SQL calls when configuring the query via block/proc. A call with:
+  
+    ```ruby
+    source Kiba::Pro::Sources::SQL,
+      query: -> (db) { db["SELECT * FROM items"] },
+    ```
+    
+    would have previously generated a `SELECT * FROM (SELECT * FROM "items")`. This is now fixed.
+
+  - Add specs around streaming support (for both MySQL and Postgres).
+  
+    For Postgres, streaming was [recommended by the author of Sequel](https://groups.google.com/d/msg/sequel-talk/olznPcmEf8M/hd5Ris0pYNwJ) over `use_cursor: true` (but do compare on your actual cases!). To enable streaming for Postgres:
+    - Add `sequel_pg` to your `Gemfile`
+    - Enable the extension in your `db` instance & add `.stream` to your dataset e.g.:
+    
+    ```ruby
+    Sequel.connect(ENV.fetch('DATABASE_URL')) do |db|
+      db.extension(:pg_streaming)
+      Kiba.run(Kiba.parse do
+        source Kiba::Pro::Sources::SQL,
+          db: db,
+          query: -> (db) { db[:items].stream }
+        # SNIP
+      end)
+    ```
+    
+    For MySQL, just add `.stream` to your dataset like above (no extension required).
 
 1.1.0
 -----

data/bin/kiba

@@ -1,5 +1,15 @@
 #!/usr/bin/env ruby
 
-require_relative '../lib/kiba/cli'
+STDERR.puts <<DOC
 
-Kiba::Cli.run(ARGV)
+##########################################################################
+
+The 'kiba' CLI is deprecated and has been removed in Kiba ETL v3.
+
+See release notes / changelog for help.
+
+##########################################################################
+
+DOC
+
+exit(1)

data/lib/kiba.rb

@@ -13,7 +13,7 @@ Kiba.extend(Kiba::Parser)
 module Kiba
   def self.run(job)
     # NOTE: use Hash#dig when Ruby 2.2 reaches EOL
-    runner = job.config.fetch(:kiba, {}).fetch(:runner, Kiba::Runner)
+    runner = job.config.fetch(:kiba, {}).fetch(:runner, Kiba::StreamingRunner)
     runner.run(job)
   end
 end

data/lib/kiba/parser.rb

@@ -1,26 +1,10 @@
-# NOTE: using the "Kiba::Parser" declaration, as I discovered,
-# provides increased isolation to the declared ETL script, compared
-# to 2 nested modules.
-# Before that, a user creating entities named Control, Context
-# or DSLExtensions would see a conflict with Kiba own classes,
-# as by default instance_eval will resolve references by adding
-# the module containing the parser class (initially "Kiba").
-# Now, the classes appear to be further hidden from the user,
-# as Kiba::Parser is its own module.
-# This allows the user to create a Parser, Context, Control class
-# without it being interpreted as reopening Kiba::Parser, Kiba::Context,
-# etc.
-# See test in test_cli.rb (test_namespace_conflict)
-module Kiba::Parser
-  def parse(source_as_string = nil, source_file = nil, &source_as_block)
-    control = Kiba::Control.new
-    context = Kiba::Context.new(control)
-    if source_as_string
-      # this somewhat weird construct allows to remove a nil source_file
-      context.instance_eval(*[source_as_string, source_file].compact)
-    else
+module Kiba
+  module Parser
+    def parse(&source_as_block)
+      control = Kiba::Control.new
+      context = Kiba::Context.new(control)
       context.instance_eval(&source_as_block)
+      control
     end
-    control
   end
 end

data/lib/kiba/runner.rb

@@ -71,7 +71,7 @@ module Kiba
       elsif block
         fail 'Block form is not allowed here' unless allow_block
         AliasingProc.new(&block)
-      elsif
+      else
         fail 'Nil parameters not allowed here'
       end
     end

data/lib/kiba/version.rb

@@ -1,3 +1,3 @@
 module Kiba
-  VERSION = '2.5.0'
+  VERSION = '3.0.0'
 end

data/test/shared_runner_tests.rb

@@ -3,10 +3,6 @@ require_relative 'support/test_enumerable_source'
 require_relative 'support/test_destination_returning_nil'
 
 module SharedRunnerTests
-  def kiba_run(job)
-    Kiba.run(job)
-  end
-
   def rows
     @rows ||= [
       { identifier: 'first-row' },

data/test/test_parser.rb

@@ -68,39 +68,6 @@ class TestParser < Kiba::Test
     assert_instance_of Proc, control.pre_processes[0][:block]
   end
 
-  def test_source_as_string_parsing
-    control = Kiba.parse <<RUBY
-      source DummyClass, 'from', 'file'
-RUBY
-
-    assert_equal 1, control.sources.size
-    assert_equal DummyClass, control.sources[0][:klass]
-    assert_equal %w(from file), control.sources[0][:args]
-  end
-
-  def test_source_as_file_doing_require
-    IO.write 'test/tmp/etl-common.rb', <<RUBY
-      def common_source_declaration
-        source DummyClass, 'from', 'common'
-      end
-RUBY
-    IO.write 'test/tmp/etl-main.rb', <<RUBY
-      require './test/tmp/etl-common.rb'
-
-      source DummyClass, 'from', 'main'
-      common_source_declaration
-RUBY
-    control = Kiba.parse IO.read('test/tmp/etl-main.rb')
-
-    assert_equal 2, control.sources.size
-
-    assert_equal %w(from main), control.sources[0][:args]
-    assert_equal %w(from common), control.sources[1][:args]
-
-  ensure
-    remove_files('test/tmp/etl-common.rb', 'test/tmp/etl-main.rb')
-  end
-  
   def test_config
     control = Kiba.parse do
       extend Kiba::DSLExtensions::Config

data/test/test_run.rb

@@ -0,0 +1,12 @@
+require_relative 'helper'
+require 'minitest/mock'
+
+class TestRun < Kiba::Test
+  def test_ensure_kiba_defaults_to_streaming_runner
+    cb = -> (job) { "Streaming runner called" }
+    Kiba::StreamingRunner.stub(:run, cb) do
+      job = Kiba::Control.new
+      assert_equal "Streaming runner called", Kiba.run(job)
+    end
+  end
+end

data/test/test_runner.rb

@@ -2,5 +2,10 @@ require_relative 'helper'
 require_relative 'shared_runner_tests'
 
 class TestRunner < Kiba::Test
+  def kiba_run(job)
+    job.config[:kiba] = {runner: Kiba::Runner}
+    Kiba.run(job)
+  end
+
   include SharedRunnerTests
 end

data/test/test_streaming_runner.rb

@@ -8,6 +8,11 @@ require_relative 'support/test_non_closing_transform'
 require_relative 'shared_runner_tests'
 
 class TestStreamingRunner < Kiba::Test
+  def kiba_run(job)
+    job.config[:kiba] = {runner: Kiba::StreamingRunner}
+    Kiba.run(job)
+  end
+
   include SharedRunnerTests
   
   def test_yielding_class_transform
@@ -15,10 +20,6 @@ class TestStreamingRunner < Kiba::Test
     destination_array = []
     
     job = Kiba.parse do
-      extend Kiba::DSLExtensions::Config
-
-      config :kiba, runner: Kiba::StreamingRunner
-
       # provide a single row as the input
       source TestEnumerableSource, [input_row]
 
@@ -51,9 +52,6 @@ class TestStreamingRunner < Kiba::Test
   def test_transform_yielding_from_close
     destination_array = []
     job = Kiba.parse do
-      extend Kiba::DSLExtensions::Config
-      config :kiba, runner: Kiba::StreamingRunner
-
       transform CloseYieldingTransform, yield_on_close: [1, 2]
       destination TestArrayDestination, destination_array
     end
@@ -63,9 +61,6 @@ class TestStreamingRunner < Kiba::Test
 
   def test_transform_with_no_close_must_not_raise
     job = Kiba.parse do
-      extend Kiba::DSLExtensions::Config
-      config :kiba, runner: Kiba::StreamingRunner
-
       transform NonClosingTransform
     end
     Kiba.run(job)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kiba
 version: !ruby/object:Gem::Version
-  version: 2.5.0
+  version: 3.0.0
 platform: ruby
 authors:
 - Thibaut Barrère
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-05-29 00:00:00.000000000 Z
+date: 2020-02-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -88,7 +88,6 @@ files:
 - bin/kiba
 - kiba.gemspec
 - lib/kiba.rb
-- lib/kiba/cli.rb
 - lib/kiba/context.rb
 - lib/kiba/control.rb
 - lib/kiba/dsl_extensions/config.rb
@@ -96,10 +95,6 @@ files:
 - lib/kiba/runner.rb
 - lib/kiba/streaming_runner.rb
 - lib/kiba/version.rb
-- test/fixtures/bogus.etl
-- test/fixtures/namespace_conflict.etl
-- test/fixtures/some_extension.rb
-- test/fixtures/valid.etl
 - test/helper.rb
 - test/shared_runner_tests.rb
 - test/support/shared_tests.rb
@@ -115,9 +110,9 @@ files:
 - test/support/test_rename_field_transform.rb
 - test/support/test_source_that_reads_at_instantiation_time.rb
 - test/support/test_yielding_transform.rb
-- test/test_cli.rb
 - test/test_integration.rb
 - test/test_parser.rb
+- test/test_run.rb
 - test/test_runner.rb
 - test/test_streaming_runner.rb
 - test/tmp/.gitkeep
@@ -147,10 +142,6 @@ signing_key:
 specification_version: 4
 summary: Lightweight ETL for Ruby
 test_files:
-- test/fixtures/bogus.etl
-- test/fixtures/namespace_conflict.etl
-- test/fixtures/some_extension.rb
-- test/fixtures/valid.etl
 - test/helper.rb
 - test/shared_runner_tests.rb
 - test/support/shared_tests.rb
@@ -166,9 +157,9 @@ test_files:
 - test/support/test_rename_field_transform.rb
 - test/support/test_source_that_reads_at_instantiation_time.rb
 - test/support/test_yielding_transform.rb
-- test/test_cli.rb
 - test/test_integration.rb
 - test/test_parser.rb
+- test/test_run.rb
 - test/test_runner.rb
 - test/test_streaming_runner.rb
 - test/tmp/.gitkeep

data/lib/kiba/cli.rb

@@ -1,16 +0,0 @@
-require 'kiba'
-
-module Kiba
-  class Cli
-    def self.run(args)
-      unless args.size == 1
-        puts 'Syntax: kiba your-script.etl'
-        exit(-1)
-      end
-      filename = args[0]
-      script_content = IO.read(filename)
-      job_definition = Kiba.parse(script_content, filename)
-      Kiba.run(job_definition)
-    end
-  end
-end

data/test/fixtures/bogus.etl

@@ -1,2 +0,0 @@
-# this should fail because we have an unknown class
-source UnknownThing

data/test/fixtures/namespace_conflict.etl

@@ -1,9 +0,0 @@
-fail "Context should not be visible without Kiba namespace" if defined?(Context)
-fail "Control should not be visible without Kiba namespace" if defined?(Control)
-fail "Parser should not be visible without Kiba namespace" if defined?(Parser)
-fail "Config should not be visible without Kiba namespace" if defined?(DSLExtensions::Config)
-
-# verify Kiba config (namespaced under Kiba::DSLExtensions::Config)
-# isn't causing troubles to implementers using a top-level DSLExtensions module
-require_relative 'some_extension'
-extend DSLExtensions::SomeExtension

data/test/fixtures/some_extension.rb

@@ -1,4 +0,0 @@
-module DSLExtensions
-  module SomeExtension
-  end
-end

data/test/fixtures/valid.etl

@@ -1 +0,0 @@
-# this does nothing

data/test/test_cli.rb

@@ -1,21 +0,0 @@
-require_relative 'helper'
-require 'kiba/cli'
-
-class TestCli < Kiba::Test
-  def test_cli_launches
-    Kiba::Cli.run([fixture('valid.etl')])
-  end
-
-  def test_cli_reports_filename_and_lineno
-    exception = assert_raises(NameError) do
-      Kiba::Cli.run([fixture('bogus.etl')])
-    end
-
-    assert_match(/uninitialized constant(.*)UnknownThing/, exception.message)
-    assert_includes exception.backtrace.to_s, 'test/fixtures/bogus.etl:2:in'
-  end
-
-  def test_namespace_conflict
-    Kiba::Cli.run([fixture('namespace_conflict.etl')])
-  end
-end