kiba 2.0.0 → 4.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. checksums.yaml +5 -5
  2. data/.github/FUNDING.yml +1 -0
  3. data/.github/workflows/ci.yml +41 -0
  4. data/COMM-LICENSE.md +348 -0
  5. data/Changes.md +38 -2
  6. data/Gemfile +1 -1
  7. data/ISSUE_TEMPLATE.md +7 -0
  8. data/LICENSE +3 -1
  9. data/Pro-Changes.md +82 -5
  10. data/README.md +12 -65
  11. data/Rakefile +8 -3
  12. data/kiba.gemspec +20 -17
  13. data/lib/kiba.rb +14 -11
  14. data/lib/kiba/context.rb +9 -5
  15. data/lib/kiba/control.rb +1 -1
  16. data/lib/kiba/dsl_extensions/config.rb +1 -1
  17. data/lib/kiba/parser.rb +6 -22
  18. data/lib/kiba/streaming_runner.rb +62 -5
  19. data/lib/kiba/version.rb +1 -1
  20. data/test/helper.rb +15 -7
  21. data/test/shared_runner_tests.rb +227 -0
  22. data/test/support/shared_tests.rb +1 -1
  23. data/test/support/test_aggregate_transform.rb +19 -0
  24. data/test/support/test_array_destination.rb +2 -2
  25. data/test/support/test_close_yielding_transform.rb +11 -0
  26. data/test/support/test_csv_destination.rb +2 -2
  27. data/test/support/test_csv_source.rb +1 -1
  28. data/test/support/test_destination_returning_nil.rb +12 -0
  29. data/test/support/test_duplicate_row_transform.rb +1 -1
  30. data/test/support/test_keyword_arguments_component.rb +14 -0
  31. data/test/support/test_mixed_arguments_component.rb +14 -0
  32. data/test/support/test_non_closing_transform.rb +5 -0
  33. data/test/support/test_yielding_transform.rb +1 -1
  34. data/test/test_integration.rb +38 -33
  35. data/test/test_parser.rb +16 -50
  36. data/test/test_run.rb +37 -0
  37. data/test/test_streaming_runner.rb +44 -23
  38. metadata +45 -30
  39. data/.travis.yml +0 -15
  40. data/appveyor.yml +0 -26
  41. data/bin/kiba +0 -5
  42. data/lib/kiba/cli.rb +0 -16
  43. data/lib/kiba/runner.rb +0 -78
  44. data/test/common/runner.rb +0 -137
  45. data/test/fixtures/bogus.etl +0 -2
  46. data/test/fixtures/namespace_conflict.etl +0 -9
  47. data/test/fixtures/some_extension.rb +0 -4
  48. data/test/fixtures/valid.etl +0 -1
  49. data/test/test_cli.rb +0 -21
  50. data/test/test_runner.rb +0 -6
data/test/test_parser.rb CHANGED
@@ -1,6 +1,6 @@
1
- require_relative 'helper'
1
+ require_relative "helper"
2
2
 
3
- require_relative 'support/test_rename_field_transform'
3
+ require_relative "support/test_rename_field_transform"
4
4
 
5
5
  class DummyClass
6
6
  end
@@ -8,21 +8,20 @@ end
8
8
  class TestParser < Kiba::Test
9
9
  def test_source_definition
10
10
  control = Kiba.parse do
11
- source DummyClass, 'has', 'args'
11
+ source DummyClass, "has", "args"
12
12
  end
13
13
 
14
14
  assert_equal DummyClass, control.sources[0][:klass]
15
- assert_equal %w(has args), control.sources[0][:args]
15
+ assert_equal %w[has args], control.sources[0][:args]
16
16
  end
17
-
18
- # NOTE: useful for anything not using the CLI (e.g. sidekiq)
17
+
19
18
  def test_block_parsing_with_reference_to_outside_variable
20
19
  some_variable = Object.new
21
-
20
+
22
21
  control = Kiba.parse do
23
22
  source DummyClass, some_variable
24
23
  end
25
-
24
+
26
25
  assert_equal [some_variable], control.sources[0][:args]
27
26
  end
28
27
 
@@ -45,11 +44,11 @@ class TestParser < Kiba::Test
45
44
 
46
45
  def test_destination_definition
47
46
  control = Kiba.parse do
48
- destination DummyClass, 'has', 'args'
47
+ destination DummyClass, "has", "args"
49
48
  end
50
49
 
51
50
  assert_equal DummyClass, control.destinations[0][:klass]
52
- assert_equal %w(has args), control.destinations[0][:args]
51
+ assert_equal %w[has args], control.destinations[0][:args]
53
52
  end
54
53
 
55
54
  def test_block_post_process_definition
@@ -59,7 +58,7 @@ class TestParser < Kiba::Test
59
58
 
60
59
  assert_instance_of Proc, control.post_processes[0][:block]
61
60
  end
62
-
61
+
63
62
  def test_block_pre_process_definition
64
63
  control = Kiba.parse do
65
64
  pre_process {}
@@ -68,52 +67,19 @@ class TestParser < Kiba::Test
68
67
  assert_instance_of Proc, control.pre_processes[0][:block]
69
68
  end
70
69
 
71
- def test_source_as_string_parsing
72
- control = Kiba.parse <<RUBY
73
- source DummyClass, 'from', 'file'
74
- RUBY
75
-
76
- assert_equal 1, control.sources.size
77
- assert_equal DummyClass, control.sources[0][:klass]
78
- assert_equal %w(from file), control.sources[0][:args]
79
- end
80
-
81
- def test_source_as_file_doing_require
82
- IO.write 'test/tmp/etl-common.rb', <<RUBY
83
- def common_source_declaration
84
- source DummyClass, 'from', 'common'
85
- end
86
- RUBY
87
- IO.write 'test/tmp/etl-main.rb', <<RUBY
88
- require './test/tmp/etl-common.rb'
89
-
90
- source DummyClass, 'from', 'main'
91
- common_source_declaration
92
- RUBY
93
- control = Kiba.parse IO.read('test/tmp/etl-main.rb')
94
-
95
- assert_equal 2, control.sources.size
96
-
97
- assert_equal %w(from main), control.sources[0][:args]
98
- assert_equal %w(from common), control.sources[1][:args]
99
-
100
- ensure
101
- remove_files('test/tmp/etl-common.rb', 'test/tmp/etl-main.rb')
102
- end
103
-
104
70
  def test_config
105
71
  control = Kiba.parse do
106
72
  extend Kiba::DSLExtensions::Config
107
-
73
+
108
74
  config :context, key: "value", other_key: "other_value"
109
75
  end
110
-
111
- assert_equal({ context: {
76
+
77
+ assert_equal({context: {
112
78
  key: "value",
113
79
  other_key: "other_value"
114
80
  }}, control.config)
115
81
  end
116
-
82
+
117
83
  def test_config_override
118
84
  control = Kiba.parse do
119
85
  extend Kiba::DSLExtensions::Config
@@ -121,8 +87,8 @@ RUBY
121
87
  config :context, key: "value", other_key: "other_value"
122
88
  config :context, key: "new_value"
123
89
  end
124
-
125
- assert_equal({ context: {
90
+
91
+ assert_equal({context: {
126
92
  key: "new_value",
127
93
  other_key: "other_value"
128
94
  }}, control.config)
data/test/test_run.rb ADDED
@@ -0,0 +1,37 @@
1
+ require_relative "helper"
2
+ require "minitest/mock"
3
+ require_relative "support/test_enumerable_source"
4
+ require_relative "support/test_array_destination"
5
+
6
+ class TestRun < Kiba::Test
7
+ def test_ensure_kiba_defaults_to_streaming_runner
8
+ cb = ->(job) { "Streaming runner called" }
9
+ Kiba::StreamingRunner.stub(:run, cb) do
10
+ job = Kiba::Control.new
11
+ assert_equal "Streaming runner called", Kiba.run(job)
12
+ end
13
+ end
14
+
15
+ def test_run_allows_block_arg
16
+ rows = []
17
+ Kiba.run do
18
+ source TestEnumerableSource, (1..10)
19
+ destination TestArrayDestination, rows
20
+ end
21
+ assert_equal (1..10).to_a, rows
22
+ end
23
+
24
+ def test_forbids_no_arg
25
+ assert_raises ArgumentError do
26
+ Kiba.run
27
+ end
28
+ end
29
+
30
+ def test_forbids_multiple_args
31
+ assert_raises ArgumentError do
32
+ job = Kiba.parse {}
33
+ Kiba.run(job) do
34
+ end
35
+ end
36
+ end
37
+ end
@@ -1,23 +1,27 @@
1
- require_relative 'helper'
2
- require_relative 'support/test_enumerable_source'
3
- require_relative 'support/test_array_destination'
4
- require_relative 'support/test_yielding_transform'
5
- require_relative 'support/test_duplicate_row_transform'
6
-
7
- require_relative 'common/runner'
1
+ require_relative "helper"
2
+ require_relative "support/test_enumerable_source"
3
+ require_relative "support/test_array_destination"
4
+ require_relative "support/test_yielding_transform"
5
+ require_relative "support/test_duplicate_row_transform"
6
+ require_relative "support/test_close_yielding_transform"
7
+ require_relative "support/test_non_closing_transform"
8
+ require_relative "shared_runner_tests"
9
+ require_relative "support/test_keyword_arguments_component"
10
+ require_relative "support/test_mixed_arguments_component"
8
11
 
9
12
  class TestStreamingRunner < Kiba::Test
13
+ def kiba_run(job)
14
+ job.config[:kiba] = {runner: Kiba::StreamingRunner}
15
+ Kiba.run(job)
16
+ end
17
+
10
18
  include SharedRunnerTests
11
-
19
+
12
20
  def test_yielding_class_transform
13
21
  input_row = {tags: ["one", "two", "three"]}
14
22
  destination_array = []
15
-
16
- job = Kiba.parse do
17
- extend Kiba::DSLExtensions::Config
18
-
19
- config :kiba, runner: Kiba::StreamingRunner
20
23
 
24
+ job = Kiba.parse do
21
25
  # provide a single row as the input
22
26
  source TestEnumerableSource, [input_row]
23
27
 
@@ -29,21 +33,38 @@ class TestStreamingRunner < Kiba::Test
29
33
 
30
34
  destination TestArrayDestination, destination_array
31
35
  end
32
-
36
+
33
37
  kiba_run(job)
34
-
38
+
35
39
  assert_equal [
36
- {item: 'one'},
37
- {item: 'one'},
40
+ {item: "one"},
41
+ {item: "one"},
38
42
 
39
- {item: 'two'},
40
- {item: 'two'},
43
+ {item: "two"},
44
+ {item: "two"},
41
45
 
42
- {item: 'three'},
43
- {item: 'three'},
46
+ {item: "three"},
47
+ {item: "three"},
44
48
 
45
- {item: 'classic-return-value'},
46
- {item: 'classic-return-value'}
49
+ {item: "classic-return-value"},
50
+ {item: "classic-return-value"}
47
51
  ], destination_array
48
52
  end
53
+
54
+ def test_transform_yielding_from_close
55
+ destination_array = []
56
+ job = Kiba.parse do
57
+ transform CloseYieldingTransform, yield_on_close: [1, 2]
58
+ destination TestArrayDestination, destination_array
59
+ end
60
+ Kiba.run(job)
61
+ assert_equal [1, 2], destination_array
62
+ end
63
+
64
+ def test_transform_with_no_close_must_not_raise
65
+ job = Kiba.parse do
66
+ transform NonClosingTransform
67
+ end
68
+ Kiba.run(job)
69
+ end
49
70
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: kiba
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.0.0
4
+ version: 4.0.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Thibaut Barrère
8
- autorequire:
8
+ autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2018-01-05 00:00:00.000000000 Z
11
+ date: 2021-03-24 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: rake
@@ -66,60 +66,75 @@ dependencies:
66
66
  - - ">="
67
67
  - !ruby/object:Gem::Version
68
68
  version: '0'
69
+ - !ruby/object:Gem::Dependency
70
+ name: standard
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - ">="
74
+ - !ruby/object:Gem::Version
75
+ version: '0'
76
+ type: :development
77
+ prerelease: false
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - ">="
81
+ - !ruby/object:Gem::Version
82
+ version: '0'
69
83
  description: Lightweight ETL for Ruby
70
84
  email:
71
85
  - thibaut.barrere@gmail.com
72
- executables:
73
- - kiba
86
+ executables: []
74
87
  extensions: []
75
88
  extra_rdoc_files: []
76
89
  files:
90
+ - ".github/FUNDING.yml"
91
+ - ".github/workflows/ci.yml"
77
92
  - ".gitignore"
78
- - ".travis.yml"
93
+ - COMM-LICENSE.md
79
94
  - Changes.md
80
95
  - Gemfile
96
+ - ISSUE_TEMPLATE.md
81
97
  - LICENSE
82
98
  - Pro-Changes.md
83
99
  - README.md
84
100
  - Rakefile
85
- - appveyor.yml
86
- - bin/kiba
87
101
  - kiba.gemspec
88
102
  - lib/kiba.rb
89
- - lib/kiba/cli.rb
90
103
  - lib/kiba/context.rb
91
104
  - lib/kiba/control.rb
92
105
  - lib/kiba/dsl_extensions/config.rb
93
106
  - lib/kiba/parser.rb
94
- - lib/kiba/runner.rb
95
107
  - lib/kiba/streaming_runner.rb
96
108
  - lib/kiba/version.rb
97
- - test/common/runner.rb
98
- - test/fixtures/bogus.etl
99
- - test/fixtures/namespace_conflict.etl
100
- - test/fixtures/some_extension.rb
101
- - test/fixtures/valid.etl
102
109
  - test/helper.rb
110
+ - test/shared_runner_tests.rb
103
111
  - test/support/shared_tests.rb
112
+ - test/support/test_aggregate_transform.rb
104
113
  - test/support/test_array_destination.rb
114
+ - test/support/test_close_yielding_transform.rb
105
115
  - test/support/test_csv_destination.rb
106
116
  - test/support/test_csv_source.rb
117
+ - test/support/test_destination_returning_nil.rb
107
118
  - test/support/test_duplicate_row_transform.rb
108
119
  - test/support/test_enumerable_source.rb
120
+ - test/support/test_keyword_arguments_component.rb
121
+ - test/support/test_mixed_arguments_component.rb
122
+ - test/support/test_non_closing_transform.rb
109
123
  - test/support/test_rename_field_transform.rb
110
124
  - test/support/test_source_that_reads_at_instantiation_time.rb
111
125
  - test/support/test_yielding_transform.rb
112
- - test/test_cli.rb
113
126
  - test/test_integration.rb
114
127
  - test/test_parser.rb
115
- - test/test_runner.rb
128
+ - test/test_run.rb
116
129
  - test/test_streaming_runner.rb
117
130
  - test/tmp/.gitkeep
118
- homepage: http://thbar.github.io/kiba/
131
+ homepage: https://www.kiba-etl.org
119
132
  licenses:
120
133
  - LGPL-3.0
121
- metadata: {}
122
- post_install_message:
134
+ metadata:
135
+ source_code_uri: https://github.com/thbar/kiba
136
+ documentation_uri: https://github.com/thbar/kiba/wiki
137
+ post_install_message:
123
138
  rdoc_options: []
124
139
  require_paths:
125
140
  - lib
@@ -134,30 +149,30 @@ required_rubygems_version: !ruby/object:Gem::Requirement
134
149
  - !ruby/object:Gem::Version
135
150
  version: '0'
136
151
  requirements: []
137
- rubyforge_project:
138
- rubygems_version: 2.6.14
139
- signing_key:
152
+ rubygems_version: 3.2.3
153
+ signing_key:
140
154
  specification_version: 4
141
155
  summary: Lightweight ETL for Ruby
142
156
  test_files:
143
- - test/common/runner.rb
144
- - test/fixtures/bogus.etl
145
- - test/fixtures/namespace_conflict.etl
146
- - test/fixtures/some_extension.rb
147
- - test/fixtures/valid.etl
148
157
  - test/helper.rb
158
+ - test/shared_runner_tests.rb
149
159
  - test/support/shared_tests.rb
160
+ - test/support/test_aggregate_transform.rb
150
161
  - test/support/test_array_destination.rb
162
+ - test/support/test_close_yielding_transform.rb
151
163
  - test/support/test_csv_destination.rb
152
164
  - test/support/test_csv_source.rb
165
+ - test/support/test_destination_returning_nil.rb
153
166
  - test/support/test_duplicate_row_transform.rb
154
167
  - test/support/test_enumerable_source.rb
168
+ - test/support/test_keyword_arguments_component.rb
169
+ - test/support/test_mixed_arguments_component.rb
170
+ - test/support/test_non_closing_transform.rb
155
171
  - test/support/test_rename_field_transform.rb
156
172
  - test/support/test_source_that_reads_at_instantiation_time.rb
157
173
  - test/support/test_yielding_transform.rb
158
- - test/test_cli.rb
159
174
  - test/test_integration.rb
160
175
  - test/test_parser.rb
161
- - test/test_runner.rb
176
+ - test/test_run.rb
162
177
  - test/test_streaming_runner.rb
163
178
  - test/tmp/.gitkeep
data/.travis.yml DELETED
@@ -1,15 +0,0 @@
1
- language: ruby
2
- before_install:
3
- # https://stackoverflow.com/a/47972768
4
- - gem update --system
5
- - gem update bundler
6
- rvm:
7
- - ruby-head
8
- - 2.5.0
9
- - 2.4.3
10
- - 2.3
11
- - 2.2
12
- - 2.1
13
- - 2.0
14
- - jruby-1.7
15
- - jruby-9.1.15.0
data/appveyor.yml DELETED
@@ -1,26 +0,0 @@
1
- version: 1.0.{build}-{branch}
2
-
3
- cache:
4
- - vendor/bundle
5
-
6
- environment:
7
- matrix:
8
- - RUBY_VERSION: 24
9
- - RUBY_VERSION: 23
10
- - RUBY_VERSION: 22
11
- - RUBY_VERSION: 21
12
-
13
- install:
14
- - set PATH=C:\Ruby%RUBY_VERSION%\bin;%PATH%
15
- - bundle config --local path vendor/bundle
16
- - bundle install
17
-
18
- build: off
19
-
20
- before_test:
21
- - ruby -v
22
- - gem -v
23
- - bundle -v
24
-
25
- test_script:
26
- - bundle exec rake