kiba 3.6.0 → 4.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/ci.yml +41 -0
- data/Changes.md +8 -2
- data/Gemfile +1 -1
- data/README.md +2 -2
- data/Rakefile +2 -2
- data/kiba.gemspec +19 -20
- data/lib/kiba.rb +7 -10
- data/lib/kiba/context.rb +5 -5
- data/lib/kiba/control.rb +1 -1
- data/lib/kiba/dsl_extensions/config.rb +1 -1
- data/lib/kiba/streaming_runner.rb +57 -5
- data/lib/kiba/version.rb +1 -1
- data/test/helper.rb +8 -8
- data/test/shared_runner_tests.rb +43 -44
- data/test/support/shared_tests.rb +1 -1
- data/test/support/test_aggregate_transform.rb +2 -2
- data/test/support/test_array_destination.rb +2 -2
- data/test/support/test_csv_destination.rb +2 -2
- data/test/support/test_csv_source.rb +1 -1
- data/test/support/test_destination_returning_nil.rb +3 -3
- data/test/support/test_duplicate_row_transform.rb +1 -1
- data/test/support/test_keyword_arguments_component.rb +1 -1
- data/test/support/test_mixed_arguments_component.rb +2 -2
- data/test/support/test_yielding_transform.rb +1 -1
- data/test/test_integration.rb +36 -31
- data/test/test_parser.rb +16 -17
- data/test/test_run.rb +6 -7
- data/test/test_streaming_runner.rb +23 -23
- metadata +19 -11
- data/.travis.yml +0 -13
- data/appveyor.yml +0 -29
- data/bin/kiba +0 -15
- data/lib/kiba/runner.rb +0 -74
- data/test/test_runner.rb +0 -11
data/.travis.yml
DELETED
|
@@ -1,13 +0,0 @@
|
|
|
1
|
-
language: ruby
|
|
2
|
-
matrix:
|
|
3
|
-
include:
|
|
4
|
-
# see https://www.ruby-lang.org/en/downloads/branches/
|
|
5
|
-
- rvm: ruby-head
|
|
6
|
-
- rvm: 3.0
|
|
7
|
-
- rvm: 2.7
|
|
8
|
-
- rvm: 2.6
|
|
9
|
-
- rvm: 2.5
|
|
10
|
-
- rvm: 2.4
|
|
11
|
-
# see https://www.jruby.org/download
|
|
12
|
-
- rvm: jruby-9.2
|
|
13
|
-
- rvm: truffleruby
|
data/appveyor.yml
DELETED
|
@@ -1,29 +0,0 @@
|
|
|
1
|
-
version: 1.0.{build}-{branch}
|
|
2
|
-
|
|
3
|
-
cache:
|
|
4
|
-
- vendor/bundle
|
|
5
|
-
|
|
6
|
-
environment:
|
|
7
|
-
matrix:
|
|
8
|
-
# TODO: add RUBY_VERSION=30 when available (https://www.appveyor.com/updates/)
|
|
9
|
-
- RUBY_VERSION: 27
|
|
10
|
-
- RUBY_VERSION: 26
|
|
11
|
-
- RUBY_VERSION: 25
|
|
12
|
-
- RUBY_VERSION: 24
|
|
13
|
-
# NOTE: jruby doesn't seem to be supported on default images
|
|
14
|
-
# see https://www.appveyor.com/docs/build-environment/#ruby
|
|
15
|
-
|
|
16
|
-
install:
|
|
17
|
-
- set PATH=C:\Ruby%RUBY_VERSION%\bin;%PATH%
|
|
18
|
-
- bundle config --local path vendor/bundle
|
|
19
|
-
- bundle install
|
|
20
|
-
|
|
21
|
-
build: off
|
|
22
|
-
|
|
23
|
-
before_test:
|
|
24
|
-
- ruby -v
|
|
25
|
-
- gem -v
|
|
26
|
-
- bundle -v
|
|
27
|
-
|
|
28
|
-
test_script:
|
|
29
|
-
- bundle exec rake
|
data/bin/kiba
DELETED
|
@@ -1,15 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env ruby
|
|
2
|
-
|
|
3
|
-
STDERR.puts <<DOC
|
|
4
|
-
|
|
5
|
-
##########################################################################
|
|
6
|
-
|
|
7
|
-
The 'kiba' CLI is deprecated and has been removed in Kiba ETL v3.
|
|
8
|
-
|
|
9
|
-
See release notes / changelog for help.
|
|
10
|
-
|
|
11
|
-
##########################################################################
|
|
12
|
-
|
|
13
|
-
DOC
|
|
14
|
-
|
|
15
|
-
exit(1)
|
data/lib/kiba/runner.rb
DELETED
|
@@ -1,74 +0,0 @@
|
|
|
1
|
-
module Kiba
|
|
2
|
-
module Runner
|
|
3
|
-
extend self
|
|
4
|
-
|
|
5
|
-
# allow to handle a block form just like a regular transform
|
|
6
|
-
class AliasingProc < Proc
|
|
7
|
-
alias_method :process, :call
|
|
8
|
-
end
|
|
9
|
-
|
|
10
|
-
def run(control)
|
|
11
|
-
run_pre_processes(control)
|
|
12
|
-
process_rows(
|
|
13
|
-
to_instances(control.sources),
|
|
14
|
-
to_instances(control.transforms, true),
|
|
15
|
-
destinations = to_instances(control.destinations)
|
|
16
|
-
)
|
|
17
|
-
close_destinations(destinations)
|
|
18
|
-
run_post_processes(control)
|
|
19
|
-
end
|
|
20
|
-
|
|
21
|
-
def run_pre_processes(control)
|
|
22
|
-
to_instances(control.pre_processes, true, false).each(&:call)
|
|
23
|
-
end
|
|
24
|
-
|
|
25
|
-
def run_post_processes(control)
|
|
26
|
-
to_instances(control.post_processes, true, false).each(&:call)
|
|
27
|
-
end
|
|
28
|
-
|
|
29
|
-
def close_destinations(destinations)
|
|
30
|
-
destinations
|
|
31
|
-
.find_all { |d| d.respond_to?(:close) }
|
|
32
|
-
.each(&:close)
|
|
33
|
-
end
|
|
34
|
-
|
|
35
|
-
def process_rows(sources, transforms, destinations)
|
|
36
|
-
sources.each do |source|
|
|
37
|
-
source.each do |row|
|
|
38
|
-
transforms.each do |transform|
|
|
39
|
-
row = transform.process(row)
|
|
40
|
-
break unless row
|
|
41
|
-
end
|
|
42
|
-
next unless row
|
|
43
|
-
destinations.each do |destination|
|
|
44
|
-
destination.write(row)
|
|
45
|
-
end
|
|
46
|
-
end
|
|
47
|
-
end
|
|
48
|
-
end
|
|
49
|
-
|
|
50
|
-
# not using keyword args because JRuby defaults to 1.9 syntax currently
|
|
51
|
-
def to_instances(definitions, allow_block = false, allow_class = true)
|
|
52
|
-
definitions.map do |definition|
|
|
53
|
-
to_instance(
|
|
54
|
-
*definition.values_at(:klass, :args, :block),
|
|
55
|
-
allow_block, allow_class
|
|
56
|
-
)
|
|
57
|
-
end
|
|
58
|
-
end
|
|
59
|
-
|
|
60
|
-
def to_instance(klass, args, block, allow_block, allow_class)
|
|
61
|
-
if klass && block
|
|
62
|
-
fail 'Class and block form cannot be used together at the moment'
|
|
63
|
-
elsif klass
|
|
64
|
-
fail 'Class form is not allowed here' unless allow_class
|
|
65
|
-
klass.new(*args)
|
|
66
|
-
elsif block
|
|
67
|
-
fail 'Block form is not allowed here' unless allow_block
|
|
68
|
-
AliasingProc.new(&block)
|
|
69
|
-
else
|
|
70
|
-
fail 'Nil parameters not allowed here'
|
|
71
|
-
end
|
|
72
|
-
end
|
|
73
|
-
end
|
|
74
|
-
end
|