kiba 3.6.0 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/ci.yml +41 -0
- data/Changes.md +8 -2
- data/Gemfile +1 -1
- data/README.md +2 -2
- data/Rakefile +2 -2
- data/kiba.gemspec +19 -20
- data/lib/kiba.rb +7 -10
- data/lib/kiba/context.rb +5 -5
- data/lib/kiba/control.rb +1 -1
- data/lib/kiba/dsl_extensions/config.rb +1 -1
- data/lib/kiba/streaming_runner.rb +57 -5
- data/lib/kiba/version.rb +1 -1
- data/test/helper.rb +8 -8
- data/test/shared_runner_tests.rb +43 -44
- data/test/support/shared_tests.rb +1 -1
- data/test/support/test_aggregate_transform.rb +2 -2
- data/test/support/test_array_destination.rb +2 -2
- data/test/support/test_csv_destination.rb +2 -2
- data/test/support/test_csv_source.rb +1 -1
- data/test/support/test_destination_returning_nil.rb +3 -3
- data/test/support/test_duplicate_row_transform.rb +1 -1
- data/test/support/test_keyword_arguments_component.rb +1 -1
- data/test/support/test_mixed_arguments_component.rb +2 -2
- data/test/support/test_yielding_transform.rb +1 -1
- data/test/test_integration.rb +36 -31
- data/test/test_parser.rb +16 -17
- data/test/test_run.rb +6 -7
- data/test/test_streaming_runner.rb +23 -23
- metadata +19 -11
- data/.travis.yml +0 -13
- data/appveyor.yml +0 -29
- data/bin/kiba +0 -15
- data/lib/kiba/runner.rb +0 -74
- data/test/test_runner.rb +0 -11
data/.travis.yml
DELETED
|
@@ -1,13 +0,0 @@
|
|
|
1
|
-
language: ruby
|
|
2
|
-
matrix:
|
|
3
|
-
include:
|
|
4
|
-
# see https://www.ruby-lang.org/en/downloads/branches/
|
|
5
|
-
- rvm: ruby-head
|
|
6
|
-
- rvm: 3.0
|
|
7
|
-
- rvm: 2.7
|
|
8
|
-
- rvm: 2.6
|
|
9
|
-
- rvm: 2.5
|
|
10
|
-
- rvm: 2.4
|
|
11
|
-
# see https://www.jruby.org/download
|
|
12
|
-
- rvm: jruby-9.2
|
|
13
|
-
- rvm: truffleruby
|
data/appveyor.yml
DELETED
|
@@ -1,29 +0,0 @@
|
|
|
1
|
-
version: 1.0.{build}-{branch}
|
|
2
|
-
|
|
3
|
-
cache:
|
|
4
|
-
- vendor/bundle
|
|
5
|
-
|
|
6
|
-
environment:
|
|
7
|
-
matrix:
|
|
8
|
-
# TODO: add RUBY_VERSION=30 when available (https://www.appveyor.com/updates/)
|
|
9
|
-
- RUBY_VERSION: 27
|
|
10
|
-
- RUBY_VERSION: 26
|
|
11
|
-
- RUBY_VERSION: 25
|
|
12
|
-
- RUBY_VERSION: 24
|
|
13
|
-
# NOTE: jruby doesn't seem to be supported on default images
|
|
14
|
-
# see https://www.appveyor.com/docs/build-environment/#ruby
|
|
15
|
-
|
|
16
|
-
install:
|
|
17
|
-
- set PATH=C:\Ruby%RUBY_VERSION%\bin;%PATH%
|
|
18
|
-
- bundle config --local path vendor/bundle
|
|
19
|
-
- bundle install
|
|
20
|
-
|
|
21
|
-
build: off
|
|
22
|
-
|
|
23
|
-
before_test:
|
|
24
|
-
- ruby -v
|
|
25
|
-
- gem -v
|
|
26
|
-
- bundle -v
|
|
27
|
-
|
|
28
|
-
test_script:
|
|
29
|
-
- bundle exec rake
|
data/bin/kiba
DELETED
|
@@ -1,15 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env ruby
|
|
2
|
-
|
|
3
|
-
STDERR.puts <<DOC
|
|
4
|
-
|
|
5
|
-
##########################################################################
|
|
6
|
-
|
|
7
|
-
The 'kiba' CLI is deprecated and has been removed in Kiba ETL v3.
|
|
8
|
-
|
|
9
|
-
See release notes / changelog for help.
|
|
10
|
-
|
|
11
|
-
##########################################################################
|
|
12
|
-
|
|
13
|
-
DOC
|
|
14
|
-
|
|
15
|
-
exit(1)
|
data/lib/kiba/runner.rb
DELETED
|
@@ -1,74 +0,0 @@
|
|
|
1
|
-
module Kiba
|
|
2
|
-
module Runner
|
|
3
|
-
extend self
|
|
4
|
-
|
|
5
|
-
# allow to handle a block form just like a regular transform
|
|
6
|
-
class AliasingProc < Proc
|
|
7
|
-
alias_method :process, :call
|
|
8
|
-
end
|
|
9
|
-
|
|
10
|
-
def run(control)
|
|
11
|
-
run_pre_processes(control)
|
|
12
|
-
process_rows(
|
|
13
|
-
to_instances(control.sources),
|
|
14
|
-
to_instances(control.transforms, true),
|
|
15
|
-
destinations = to_instances(control.destinations)
|
|
16
|
-
)
|
|
17
|
-
close_destinations(destinations)
|
|
18
|
-
run_post_processes(control)
|
|
19
|
-
end
|
|
20
|
-
|
|
21
|
-
def run_pre_processes(control)
|
|
22
|
-
to_instances(control.pre_processes, true, false).each(&:call)
|
|
23
|
-
end
|
|
24
|
-
|
|
25
|
-
def run_post_processes(control)
|
|
26
|
-
to_instances(control.post_processes, true, false).each(&:call)
|
|
27
|
-
end
|
|
28
|
-
|
|
29
|
-
def close_destinations(destinations)
|
|
30
|
-
destinations
|
|
31
|
-
.find_all { |d| d.respond_to?(:close) }
|
|
32
|
-
.each(&:close)
|
|
33
|
-
end
|
|
34
|
-
|
|
35
|
-
def process_rows(sources, transforms, destinations)
|
|
36
|
-
sources.each do |source|
|
|
37
|
-
source.each do |row|
|
|
38
|
-
transforms.each do |transform|
|
|
39
|
-
row = transform.process(row)
|
|
40
|
-
break unless row
|
|
41
|
-
end
|
|
42
|
-
next unless row
|
|
43
|
-
destinations.each do |destination|
|
|
44
|
-
destination.write(row)
|
|
45
|
-
end
|
|
46
|
-
end
|
|
47
|
-
end
|
|
48
|
-
end
|
|
49
|
-
|
|
50
|
-
# not using keyword args because JRuby defaults to 1.9 syntax currently
|
|
51
|
-
def to_instances(definitions, allow_block = false, allow_class = true)
|
|
52
|
-
definitions.map do |definition|
|
|
53
|
-
to_instance(
|
|
54
|
-
*definition.values_at(:klass, :args, :block),
|
|
55
|
-
allow_block, allow_class
|
|
56
|
-
)
|
|
57
|
-
end
|
|
58
|
-
end
|
|
59
|
-
|
|
60
|
-
def to_instance(klass, args, block, allow_block, allow_class)
|
|
61
|
-
if klass && block
|
|
62
|
-
fail 'Class and block form cannot be used together at the moment'
|
|
63
|
-
elsif klass
|
|
64
|
-
fail 'Class form is not allowed here' unless allow_class
|
|
65
|
-
klass.new(*args)
|
|
66
|
-
elsif block
|
|
67
|
-
fail 'Block form is not allowed here' unless allow_block
|
|
68
|
-
AliasingProc.new(&block)
|
|
69
|
-
else
|
|
70
|
-
fail 'Nil parameters not allowed here'
|
|
71
|
-
end
|
|
72
|
-
end
|
|
73
|
-
end
|
|
74
|
-
end
|