kiba 2.0.0 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.github/FUNDING.yml +1 -0
- data/.github/workflows/ci.yml +41 -0
- data/COMM-LICENSE.md +348 -0
- data/Changes.md +38 -2
- data/Gemfile +1 -1
- data/ISSUE_TEMPLATE.md +7 -0
- data/LICENSE +3 -1
- data/Pro-Changes.md +82 -5
- data/README.md +12 -65
- data/Rakefile +8 -3
- data/kiba.gemspec +20 -17
- data/lib/kiba.rb +14 -11
- data/lib/kiba/context.rb +9 -5
- data/lib/kiba/control.rb +1 -1
- data/lib/kiba/dsl_extensions/config.rb +1 -1
- data/lib/kiba/parser.rb +6 -22
- data/lib/kiba/streaming_runner.rb +62 -5
- data/lib/kiba/version.rb +1 -1
- data/test/helper.rb +15 -7
- data/test/shared_runner_tests.rb +227 -0
- data/test/support/shared_tests.rb +1 -1
- data/test/support/test_aggregate_transform.rb +19 -0
- data/test/support/test_array_destination.rb +2 -2
- data/test/support/test_close_yielding_transform.rb +11 -0
- data/test/support/test_csv_destination.rb +2 -2
- data/test/support/test_csv_source.rb +1 -1
- data/test/support/test_destination_returning_nil.rb +12 -0
- data/test/support/test_duplicate_row_transform.rb +1 -1
- data/test/support/test_keyword_arguments_component.rb +14 -0
- data/test/support/test_mixed_arguments_component.rb +14 -0
- data/test/support/test_non_closing_transform.rb +5 -0
- data/test/support/test_yielding_transform.rb +1 -1
- data/test/test_integration.rb +38 -33
- data/test/test_parser.rb +16 -50
- data/test/test_run.rb +37 -0
- data/test/test_streaming_runner.rb +44 -23
- metadata +45 -30
- data/.travis.yml +0 -15
- data/appveyor.yml +0 -26
- data/bin/kiba +0 -5
- data/lib/kiba/cli.rb +0 -16
- data/lib/kiba/runner.rb +0 -78
- data/test/common/runner.rb +0 -137
- data/test/fixtures/bogus.etl +0 -2
- data/test/fixtures/namespace_conflict.etl +0 -9
- data/test/fixtures/some_extension.rb +0 -4
- data/test/fixtures/valid.etl +0 -1
- data/test/test_cli.rb +0 -21
- data/test/test_runner.rb +0 -6
data/test/test_parser.rb
CHANGED
@@ -1,6 +1,6 @@
|
|
1
|
-
require_relative
|
1
|
+
require_relative "helper"
|
2
2
|
|
3
|
-
require_relative
|
3
|
+
require_relative "support/test_rename_field_transform"
|
4
4
|
|
5
5
|
class DummyClass
|
6
6
|
end
|
@@ -8,21 +8,20 @@ end
|
|
8
8
|
class TestParser < Kiba::Test
|
9
9
|
def test_source_definition
|
10
10
|
control = Kiba.parse do
|
11
|
-
source DummyClass,
|
11
|
+
source DummyClass, "has", "args"
|
12
12
|
end
|
13
13
|
|
14
14
|
assert_equal DummyClass, control.sources[0][:klass]
|
15
|
-
assert_equal %w
|
15
|
+
assert_equal %w[has args], control.sources[0][:args]
|
16
16
|
end
|
17
|
-
|
18
|
-
# NOTE: useful for anything not using the CLI (e.g. sidekiq)
|
17
|
+
|
19
18
|
def test_block_parsing_with_reference_to_outside_variable
|
20
19
|
some_variable = Object.new
|
21
|
-
|
20
|
+
|
22
21
|
control = Kiba.parse do
|
23
22
|
source DummyClass, some_variable
|
24
23
|
end
|
25
|
-
|
24
|
+
|
26
25
|
assert_equal [some_variable], control.sources[0][:args]
|
27
26
|
end
|
28
27
|
|
@@ -45,11 +44,11 @@ class TestParser < Kiba::Test
|
|
45
44
|
|
46
45
|
def test_destination_definition
|
47
46
|
control = Kiba.parse do
|
48
|
-
destination DummyClass,
|
47
|
+
destination DummyClass, "has", "args"
|
49
48
|
end
|
50
49
|
|
51
50
|
assert_equal DummyClass, control.destinations[0][:klass]
|
52
|
-
assert_equal %w
|
51
|
+
assert_equal %w[has args], control.destinations[0][:args]
|
53
52
|
end
|
54
53
|
|
55
54
|
def test_block_post_process_definition
|
@@ -59,7 +58,7 @@ class TestParser < Kiba::Test
|
|
59
58
|
|
60
59
|
assert_instance_of Proc, control.post_processes[0][:block]
|
61
60
|
end
|
62
|
-
|
61
|
+
|
63
62
|
def test_block_pre_process_definition
|
64
63
|
control = Kiba.parse do
|
65
64
|
pre_process {}
|
@@ -68,52 +67,19 @@ class TestParser < Kiba::Test
|
|
68
67
|
assert_instance_of Proc, control.pre_processes[0][:block]
|
69
68
|
end
|
70
69
|
|
71
|
-
def test_source_as_string_parsing
|
72
|
-
control = Kiba.parse <<RUBY
|
73
|
-
source DummyClass, 'from', 'file'
|
74
|
-
RUBY
|
75
|
-
|
76
|
-
assert_equal 1, control.sources.size
|
77
|
-
assert_equal DummyClass, control.sources[0][:klass]
|
78
|
-
assert_equal %w(from file), control.sources[0][:args]
|
79
|
-
end
|
80
|
-
|
81
|
-
def test_source_as_file_doing_require
|
82
|
-
IO.write 'test/tmp/etl-common.rb', <<RUBY
|
83
|
-
def common_source_declaration
|
84
|
-
source DummyClass, 'from', 'common'
|
85
|
-
end
|
86
|
-
RUBY
|
87
|
-
IO.write 'test/tmp/etl-main.rb', <<RUBY
|
88
|
-
require './test/tmp/etl-common.rb'
|
89
|
-
|
90
|
-
source DummyClass, 'from', 'main'
|
91
|
-
common_source_declaration
|
92
|
-
RUBY
|
93
|
-
control = Kiba.parse IO.read('test/tmp/etl-main.rb')
|
94
|
-
|
95
|
-
assert_equal 2, control.sources.size
|
96
|
-
|
97
|
-
assert_equal %w(from main), control.sources[0][:args]
|
98
|
-
assert_equal %w(from common), control.sources[1][:args]
|
99
|
-
|
100
|
-
ensure
|
101
|
-
remove_files('test/tmp/etl-common.rb', 'test/tmp/etl-main.rb')
|
102
|
-
end
|
103
|
-
|
104
70
|
def test_config
|
105
71
|
control = Kiba.parse do
|
106
72
|
extend Kiba::DSLExtensions::Config
|
107
|
-
|
73
|
+
|
108
74
|
config :context, key: "value", other_key: "other_value"
|
109
75
|
end
|
110
|
-
|
111
|
-
assert_equal({
|
76
|
+
|
77
|
+
assert_equal({context: {
|
112
78
|
key: "value",
|
113
79
|
other_key: "other_value"
|
114
80
|
}}, control.config)
|
115
81
|
end
|
116
|
-
|
82
|
+
|
117
83
|
def test_config_override
|
118
84
|
control = Kiba.parse do
|
119
85
|
extend Kiba::DSLExtensions::Config
|
@@ -121,8 +87,8 @@ RUBY
|
|
121
87
|
config :context, key: "value", other_key: "other_value"
|
122
88
|
config :context, key: "new_value"
|
123
89
|
end
|
124
|
-
|
125
|
-
assert_equal({
|
90
|
+
|
91
|
+
assert_equal({context: {
|
126
92
|
key: "new_value",
|
127
93
|
other_key: "other_value"
|
128
94
|
}}, control.config)
|
data/test/test_run.rb
ADDED
@@ -0,0 +1,37 @@
|
|
1
|
+
require_relative "helper"
|
2
|
+
require "minitest/mock"
|
3
|
+
require_relative "support/test_enumerable_source"
|
4
|
+
require_relative "support/test_array_destination"
|
5
|
+
|
6
|
+
class TestRun < Kiba::Test
|
7
|
+
def test_ensure_kiba_defaults_to_streaming_runner
|
8
|
+
cb = ->(job) { "Streaming runner called" }
|
9
|
+
Kiba::StreamingRunner.stub(:run, cb) do
|
10
|
+
job = Kiba::Control.new
|
11
|
+
assert_equal "Streaming runner called", Kiba.run(job)
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
def test_run_allows_block_arg
|
16
|
+
rows = []
|
17
|
+
Kiba.run do
|
18
|
+
source TestEnumerableSource, (1..10)
|
19
|
+
destination TestArrayDestination, rows
|
20
|
+
end
|
21
|
+
assert_equal (1..10).to_a, rows
|
22
|
+
end
|
23
|
+
|
24
|
+
def test_forbids_no_arg
|
25
|
+
assert_raises ArgumentError do
|
26
|
+
Kiba.run
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
def test_forbids_multiple_args
|
31
|
+
assert_raises ArgumentError do
|
32
|
+
job = Kiba.parse {}
|
33
|
+
Kiba.run(job) do
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
@@ -1,23 +1,27 @@
|
|
1
|
-
require_relative
|
2
|
-
require_relative
|
3
|
-
require_relative
|
4
|
-
require_relative
|
5
|
-
require_relative
|
6
|
-
|
7
|
-
require_relative
|
1
|
+
require_relative "helper"
|
2
|
+
require_relative "support/test_enumerable_source"
|
3
|
+
require_relative "support/test_array_destination"
|
4
|
+
require_relative "support/test_yielding_transform"
|
5
|
+
require_relative "support/test_duplicate_row_transform"
|
6
|
+
require_relative "support/test_close_yielding_transform"
|
7
|
+
require_relative "support/test_non_closing_transform"
|
8
|
+
require_relative "shared_runner_tests"
|
9
|
+
require_relative "support/test_keyword_arguments_component"
|
10
|
+
require_relative "support/test_mixed_arguments_component"
|
8
11
|
|
9
12
|
class TestStreamingRunner < Kiba::Test
|
13
|
+
def kiba_run(job)
|
14
|
+
job.config[:kiba] = {runner: Kiba::StreamingRunner}
|
15
|
+
Kiba.run(job)
|
16
|
+
end
|
17
|
+
|
10
18
|
include SharedRunnerTests
|
11
|
-
|
19
|
+
|
12
20
|
def test_yielding_class_transform
|
13
21
|
input_row = {tags: ["one", "two", "three"]}
|
14
22
|
destination_array = []
|
15
|
-
|
16
|
-
job = Kiba.parse do
|
17
|
-
extend Kiba::DSLExtensions::Config
|
18
|
-
|
19
|
-
config :kiba, runner: Kiba::StreamingRunner
|
20
23
|
|
24
|
+
job = Kiba.parse do
|
21
25
|
# provide a single row as the input
|
22
26
|
source TestEnumerableSource, [input_row]
|
23
27
|
|
@@ -29,21 +33,38 @@ class TestStreamingRunner < Kiba::Test
|
|
29
33
|
|
30
34
|
destination TestArrayDestination, destination_array
|
31
35
|
end
|
32
|
-
|
36
|
+
|
33
37
|
kiba_run(job)
|
34
|
-
|
38
|
+
|
35
39
|
assert_equal [
|
36
|
-
{item:
|
37
|
-
{item:
|
40
|
+
{item: "one"},
|
41
|
+
{item: "one"},
|
38
42
|
|
39
|
-
{item:
|
40
|
-
{item:
|
43
|
+
{item: "two"},
|
44
|
+
{item: "two"},
|
41
45
|
|
42
|
-
{item:
|
43
|
-
{item:
|
46
|
+
{item: "three"},
|
47
|
+
{item: "three"},
|
44
48
|
|
45
|
-
{item:
|
46
|
-
{item:
|
49
|
+
{item: "classic-return-value"},
|
50
|
+
{item: "classic-return-value"}
|
47
51
|
], destination_array
|
48
52
|
end
|
53
|
+
|
54
|
+
def test_transform_yielding_from_close
|
55
|
+
destination_array = []
|
56
|
+
job = Kiba.parse do
|
57
|
+
transform CloseYieldingTransform, yield_on_close: [1, 2]
|
58
|
+
destination TestArrayDestination, destination_array
|
59
|
+
end
|
60
|
+
Kiba.run(job)
|
61
|
+
assert_equal [1, 2], destination_array
|
62
|
+
end
|
63
|
+
|
64
|
+
def test_transform_with_no_close_must_not_raise
|
65
|
+
job = Kiba.parse do
|
66
|
+
transform NonClosingTransform
|
67
|
+
end
|
68
|
+
Kiba.run(job)
|
69
|
+
end
|
49
70
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: kiba
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 4.0.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Thibaut Barrère
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2021-03-24 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rake
|
@@ -66,60 +66,75 @@ dependencies:
|
|
66
66
|
- - ">="
|
67
67
|
- !ruby/object:Gem::Version
|
68
68
|
version: '0'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: standard
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - ">="
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '0'
|
76
|
+
type: :development
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - ">="
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '0'
|
69
83
|
description: Lightweight ETL for Ruby
|
70
84
|
email:
|
71
85
|
- thibaut.barrere@gmail.com
|
72
|
-
executables:
|
73
|
-
- kiba
|
86
|
+
executables: []
|
74
87
|
extensions: []
|
75
88
|
extra_rdoc_files: []
|
76
89
|
files:
|
90
|
+
- ".github/FUNDING.yml"
|
91
|
+
- ".github/workflows/ci.yml"
|
77
92
|
- ".gitignore"
|
78
|
-
-
|
93
|
+
- COMM-LICENSE.md
|
79
94
|
- Changes.md
|
80
95
|
- Gemfile
|
96
|
+
- ISSUE_TEMPLATE.md
|
81
97
|
- LICENSE
|
82
98
|
- Pro-Changes.md
|
83
99
|
- README.md
|
84
100
|
- Rakefile
|
85
|
-
- appveyor.yml
|
86
|
-
- bin/kiba
|
87
101
|
- kiba.gemspec
|
88
102
|
- lib/kiba.rb
|
89
|
-
- lib/kiba/cli.rb
|
90
103
|
- lib/kiba/context.rb
|
91
104
|
- lib/kiba/control.rb
|
92
105
|
- lib/kiba/dsl_extensions/config.rb
|
93
106
|
- lib/kiba/parser.rb
|
94
|
-
- lib/kiba/runner.rb
|
95
107
|
- lib/kiba/streaming_runner.rb
|
96
108
|
- lib/kiba/version.rb
|
97
|
-
- test/common/runner.rb
|
98
|
-
- test/fixtures/bogus.etl
|
99
|
-
- test/fixtures/namespace_conflict.etl
|
100
|
-
- test/fixtures/some_extension.rb
|
101
|
-
- test/fixtures/valid.etl
|
102
109
|
- test/helper.rb
|
110
|
+
- test/shared_runner_tests.rb
|
103
111
|
- test/support/shared_tests.rb
|
112
|
+
- test/support/test_aggregate_transform.rb
|
104
113
|
- test/support/test_array_destination.rb
|
114
|
+
- test/support/test_close_yielding_transform.rb
|
105
115
|
- test/support/test_csv_destination.rb
|
106
116
|
- test/support/test_csv_source.rb
|
117
|
+
- test/support/test_destination_returning_nil.rb
|
107
118
|
- test/support/test_duplicate_row_transform.rb
|
108
119
|
- test/support/test_enumerable_source.rb
|
120
|
+
- test/support/test_keyword_arguments_component.rb
|
121
|
+
- test/support/test_mixed_arguments_component.rb
|
122
|
+
- test/support/test_non_closing_transform.rb
|
109
123
|
- test/support/test_rename_field_transform.rb
|
110
124
|
- test/support/test_source_that_reads_at_instantiation_time.rb
|
111
125
|
- test/support/test_yielding_transform.rb
|
112
|
-
- test/test_cli.rb
|
113
126
|
- test/test_integration.rb
|
114
127
|
- test/test_parser.rb
|
115
|
-
- test/
|
128
|
+
- test/test_run.rb
|
116
129
|
- test/test_streaming_runner.rb
|
117
130
|
- test/tmp/.gitkeep
|
118
|
-
homepage:
|
131
|
+
homepage: https://www.kiba-etl.org
|
119
132
|
licenses:
|
120
133
|
- LGPL-3.0
|
121
|
-
metadata:
|
122
|
-
|
134
|
+
metadata:
|
135
|
+
source_code_uri: https://github.com/thbar/kiba
|
136
|
+
documentation_uri: https://github.com/thbar/kiba/wiki
|
137
|
+
post_install_message:
|
123
138
|
rdoc_options: []
|
124
139
|
require_paths:
|
125
140
|
- lib
|
@@ -134,30 +149,30 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
134
149
|
- !ruby/object:Gem::Version
|
135
150
|
version: '0'
|
136
151
|
requirements: []
|
137
|
-
|
138
|
-
|
139
|
-
signing_key:
|
152
|
+
rubygems_version: 3.2.3
|
153
|
+
signing_key:
|
140
154
|
specification_version: 4
|
141
155
|
summary: Lightweight ETL for Ruby
|
142
156
|
test_files:
|
143
|
-
- test/common/runner.rb
|
144
|
-
- test/fixtures/bogus.etl
|
145
|
-
- test/fixtures/namespace_conflict.etl
|
146
|
-
- test/fixtures/some_extension.rb
|
147
|
-
- test/fixtures/valid.etl
|
148
157
|
- test/helper.rb
|
158
|
+
- test/shared_runner_tests.rb
|
149
159
|
- test/support/shared_tests.rb
|
160
|
+
- test/support/test_aggregate_transform.rb
|
150
161
|
- test/support/test_array_destination.rb
|
162
|
+
- test/support/test_close_yielding_transform.rb
|
151
163
|
- test/support/test_csv_destination.rb
|
152
164
|
- test/support/test_csv_source.rb
|
165
|
+
- test/support/test_destination_returning_nil.rb
|
153
166
|
- test/support/test_duplicate_row_transform.rb
|
154
167
|
- test/support/test_enumerable_source.rb
|
168
|
+
- test/support/test_keyword_arguments_component.rb
|
169
|
+
- test/support/test_mixed_arguments_component.rb
|
170
|
+
- test/support/test_non_closing_transform.rb
|
155
171
|
- test/support/test_rename_field_transform.rb
|
156
172
|
- test/support/test_source_that_reads_at_instantiation_time.rb
|
157
173
|
- test/support/test_yielding_transform.rb
|
158
|
-
- test/test_cli.rb
|
159
174
|
- test/test_integration.rb
|
160
175
|
- test/test_parser.rb
|
161
|
-
- test/
|
176
|
+
- test/test_run.rb
|
162
177
|
- test/test_streaming_runner.rb
|
163
178
|
- test/tmp/.gitkeep
|
data/.travis.yml
DELETED
data/appveyor.yml
DELETED
@@ -1,26 +0,0 @@
|
|
1
|
-
version: 1.0.{build}-{branch}
|
2
|
-
|
3
|
-
cache:
|
4
|
-
- vendor/bundle
|
5
|
-
|
6
|
-
environment:
|
7
|
-
matrix:
|
8
|
-
- RUBY_VERSION: 24
|
9
|
-
- RUBY_VERSION: 23
|
10
|
-
- RUBY_VERSION: 22
|
11
|
-
- RUBY_VERSION: 21
|
12
|
-
|
13
|
-
install:
|
14
|
-
- set PATH=C:\Ruby%RUBY_VERSION%\bin;%PATH%
|
15
|
-
- bundle config --local path vendor/bundle
|
16
|
-
- bundle install
|
17
|
-
|
18
|
-
build: off
|
19
|
-
|
20
|
-
before_test:
|
21
|
-
- ruby -v
|
22
|
-
- gem -v
|
23
|
-
- bundle -v
|
24
|
-
|
25
|
-
test_script:
|
26
|
-
- bundle exec rake
|