chronicle-etl 0.1.0 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.rubocop.yml +8 -0
- data/.ruby-version +1 -0
- data/.yardopts +1 -0
- data/CHANGELOG.md +23 -0
- data/Gemfile.lock +42 -10
- data/README.md +64 -11
- data/bin/console +16 -4
- data/chronicle-etl.gemspec +9 -7
- data/exe/chronicle-etl +2 -2
- data/lib/chronicle/etl.rb +5 -2
- data/lib/chronicle/etl/catalog.rb +62 -0
- data/lib/chronicle/etl/cli/connectors.rb +32 -0
- data/lib/chronicle/etl/cli/jobs.rb +111 -0
- data/lib/chronicle/etl/cli/main.rb +83 -0
- data/lib/chronicle/etl/cli/subcommand_base.rb +37 -0
- data/lib/chronicle/etl/config.rb +32 -0
- data/lib/chronicle/etl/extractors/{csv.rb → csv_extractor.rb} +3 -5
- data/lib/chronicle/etl/extractors/extractor.rb +23 -12
- data/lib/chronicle/etl/extractors/file_extractor.rb +52 -0
- data/lib/chronicle/etl/extractors/stdin_extractor.rb +11 -0
- data/lib/chronicle/etl/loaders/csv_loader.rb +29 -0
- data/lib/chronicle/etl/loaders/loader.rb +22 -16
- data/lib/chronicle/etl/loaders/stdout_loader.rb +9 -0
- data/lib/chronicle/etl/loaders/table_loader.rb +21 -0
- data/lib/chronicle/etl/runner.rb +33 -11
- data/lib/chronicle/etl/transformers/json_transformer.rb +11 -0
- data/lib/chronicle/etl/transformers/null_transformer.rb +10 -0
- data/lib/chronicle/etl/transformers/transformer.rb +27 -11
- data/lib/chronicle/etl/utils/progress_bar.rb +76 -0
- data/lib/chronicle/etl/version.rb +2 -2
- metadata +68 -30
- data/lib/chronicle/etl/cli.rb +0 -38
- data/lib/chronicle/etl/extractors/stdin.rb +0 -13
- data/lib/chronicle/etl/loaders/csv.rb +0 -31
- data/lib/chronicle/etl/loaders/stdout.rb +0 -13
- data/lib/chronicle/etl/loaders/table.rb +0 -22
- data/lib/chronicle/etl/transformers/json.rb +0 -13
- data/lib/chronicle/etl/transformers/null.rb +0 -11
- data/lib/chronicle/etl/utils/progress_bar_wrapper.rb +0 -43
@@ -1,22 +0,0 @@
|
|
1
|
-
require 'table_print'
|
2
|
-
|
3
|
-
module Chronicle
|
4
|
-
module Etl
|
5
|
-
module Loaders
|
6
|
-
class Table < Chronicle::Etl::Loaders::Loader
|
7
|
-
def initialize(options)
|
8
|
-
super(options)
|
9
|
-
@rows = []
|
10
|
-
end
|
11
|
-
|
12
|
-
def load(result)
|
13
|
-
@rows << result
|
14
|
-
end
|
15
|
-
|
16
|
-
def finish
|
17
|
-
tp @rows
|
18
|
-
end
|
19
|
-
end
|
20
|
-
end
|
21
|
-
end
|
22
|
-
end
|
@@ -1,43 +0,0 @@
|
|
1
|
-
require 'ruby-progressbar'
|
2
|
-
require 'colorize'
|
3
|
-
|
4
|
-
module Chronicle
|
5
|
-
module Etl
|
6
|
-
module Utils
|
7
|
-
class ProgressBarWrapper
|
8
|
-
def initialize(count)
|
9
|
-
return unless tty?
|
10
|
-
|
11
|
-
@pbar = ProgressBar.create(
|
12
|
-
format: '%b%i %c/%C (%P%%) %a %e Rate: %R',
|
13
|
-
remainder_mark: '░',
|
14
|
-
progress_mark: '▓'.colorize(:light_green),
|
15
|
-
starting_time: 0,
|
16
|
-
lenth: 200,
|
17
|
-
throttle_rate: 0.1,
|
18
|
-
total: count,
|
19
|
-
unknown_progress_animation_steps: ['▓░░░', '░▓░░', '░░▓░', '░░░▓']
|
20
|
-
)
|
21
|
-
end
|
22
|
-
|
23
|
-
def increment
|
24
|
-
@pbar&.increment
|
25
|
-
end
|
26
|
-
|
27
|
-
def log(message)
|
28
|
-
@pbar&.log message
|
29
|
-
end
|
30
|
-
|
31
|
-
def finish
|
32
|
-
@pbar&.finish
|
33
|
-
end
|
34
|
-
|
35
|
-
private
|
36
|
-
|
37
|
-
def tty?
|
38
|
-
$stdout.isatty
|
39
|
-
end
|
40
|
-
end
|
41
|
-
end
|
42
|
-
end
|
43
|
-
end
|