chronicle-etl 0.1.2 → 0.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +8 -0
  3. data/.ruby-version +1 -0
  4. data/.yardopts +1 -0
  5. data/CHANGELOG.md +11 -0
  6. data/Gemfile.lock +42 -10
  7. data/README.md +64 -11
  8. data/bin/console +16 -4
  9. data/chronicle-etl.gemspec +8 -6
  10. data/exe/chronicle-etl +2 -2
  11. data/lib/chronicle/etl.rb +6 -2
  12. data/lib/chronicle/etl/catalog.rb +102 -0
  13. data/lib/chronicle/etl/cli/connectors.rb +32 -0
  14. data/lib/chronicle/etl/cli/jobs.rb +110 -0
  15. data/lib/chronicle/etl/cli/main.rb +83 -0
  16. data/lib/chronicle/etl/cli/subcommand_base.rb +37 -0
  17. data/lib/chronicle/etl/config.rb +32 -0
  18. data/lib/chronicle/etl/exceptions.rb +17 -0
  19. data/lib/chronicle/etl/extractors/{csv.rb → csv_extractor.rb} +3 -3
  20. data/lib/chronicle/etl/extractors/extractor.rb +23 -12
  21. data/lib/chronicle/etl/extractors/file_extractor.rb +52 -0
  22. data/lib/chronicle/etl/extractors/stdin_extractor.rb +11 -0
  23. data/lib/chronicle/etl/loaders/csv_loader.rb +29 -0
  24. data/lib/chronicle/etl/loaders/loader.rb +23 -16
  25. data/lib/chronicle/etl/loaders/rest_loader.rb +30 -0
  26. data/lib/chronicle/etl/loaders/stdout_loader.rb +9 -0
  27. data/lib/chronicle/etl/loaders/table_loader.rb +21 -0
  28. data/lib/chronicle/etl/runner.rb +27 -38
  29. data/lib/chronicle/etl/transformers/json_transformer.rb +11 -0
  30. data/lib/chronicle/etl/transformers/null_transformer.rb +10 -0
  31. data/lib/chronicle/etl/transformers/transformer.rb +28 -11
  32. data/lib/chronicle/etl/utils/progress_bar.rb +76 -0
  33. data/lib/chronicle/etl/version.rb +2 -2
  34. metadata +68 -29
  35. data/lib/chronicle/etl/cli.rb +0 -38
  36. data/lib/chronicle/etl/extractors/stdin.rb +0 -13
  37. data/lib/chronicle/etl/loaders/csv.rb +0 -31
  38. data/lib/chronicle/etl/loaders/stdout.rb +0 -11
  39. data/lib/chronicle/etl/loaders/table.rb +0 -22
  40. data/lib/chronicle/etl/transformers/json.rb +0 -13
  41. data/lib/chronicle/etl/transformers/null.rb +0 -11
  42. data/lib/chronicle/etl/utils/progress_bar_wrapper.rb +0 -43
@@ -1,38 +0,0 @@
1
- require 'thor'
2
- require 'chronicle/etl'
3
-
4
- module Chronicle
5
- module Etl
6
- class CLI < Thor
7
- default_task :job
8
-
9
- desc 'job', 'Runs an ETL job'
10
- method_option :extractor, aliases: '-e', desc: 'Extractor class (available: stdin, csv, file)', default: 'stdin', banner: 'extractor-name'
11
- method_option :'extractor-opts', desc: 'Extractor options', type: :hash, default: {}
12
- method_option :transformer, aliases: '-t', desc: 'Transformer class (available: null)', default: 'null', banner: 'transformer-name'
13
- method_option :'transformer-opts', desc: 'Transformer options', type: :hash, default: {}
14
- method_option :loader, aliases: '-l', desc: 'Loader class (available: stdout, csv, table)', default: 'stdout', banner: 'loader-name'
15
- method_option :'loader-opts', desc: 'Loader options', type: :hash, default: {}
16
- method_option :job, aliases: '-j', desc: 'Job configuration file'
17
- def job
18
- runner_options = {
19
- extractor: {
20
- name: options[:extractor],
21
- options: options[:'extractor-opts']
22
- },
23
- transformer: {
24
- name: options[:transformer],
25
- options: options[:'transformer-opts']
26
- },
27
- loader: {
28
- name: options[:loader],
29
- options: options[:'loader-opts']
30
- }
31
- }
32
-
33
- runner = Runner.new(runner_options)
34
- runner.run!
35
- end
36
- end
37
- end
38
- end
@@ -1,13 +0,0 @@
1
- module Chronicle
2
- module Etl
3
- module Extractors
4
- class Stdin < Chronicle::Etl::Extractors::Extractor
5
- def extract
6
- $stdin.read.each_line do |line|
7
- yield line
8
- end
9
- end
10
- end
11
- end
12
- end
13
- end
@@ -1,31 +0,0 @@
1
- require 'csv'
2
-
3
- module Chronicle
4
- module Etl
5
- module Loaders
6
- class Csv < Chronicle::Etl::Loaders::Loader
7
- def initialize(options={})
8
- super(options)
9
- @rows = []
10
- end
11
-
12
- def load(result)
13
- if (result.values)
14
- @rows << result.values
15
- else
16
- @rows << result
17
- end
18
- end
19
-
20
- def finish
21
- z = $stdout
22
- CSV(z) do |csv|
23
- @rows.each do |row|
24
- csv << row
25
- end
26
- end
27
- end
28
- end
29
- end
30
- end
31
- end
@@ -1,11 +0,0 @@
1
- module Chronicle
2
- module Etl
3
- module Loaders
4
- class Stdout < Chronicle::Etl::Loaders::Loader
5
- def load(result)
6
- puts result.inspect
7
- end
8
- end
9
- end
10
- end
11
- end
@@ -1,22 +0,0 @@
1
- require 'table_print'
2
-
3
- module Chronicle
4
- module Etl
5
- module Loaders
6
- class Table < Chronicle::Etl::Loaders::Loader
7
- def initialize(options)
8
- super(options)
9
- @rows = []
10
- end
11
-
12
- def load(result)
13
- @rows << result
14
- end
15
-
16
- def finish
17
- tp @rows
18
- end
19
- end
20
- end
21
- end
22
- end
@@ -1,13 +0,0 @@
1
- require 'json'
2
-
3
- module Chronicle
4
- module Etl
5
- module Transformers
6
- class Json < Chronicle::Etl::Transformers::Transformer
7
- def transform data
8
- return JSON.parse(data)
9
- end
10
- end
11
- end
12
- end
13
- end
@@ -1,11 +0,0 @@
1
- module Chronicle
2
- module Etl
3
- module Transformers
4
- class Null < Chronicle::Etl::Transformers::Transformer
5
- def transform data
6
- return data
7
- end
8
- end
9
- end
10
- end
11
- end
@@ -1,43 +0,0 @@
1
- require 'ruby-progressbar'
2
- require 'colorize'
3
-
4
- module Chronicle
5
- module Etl
6
- module Utils
7
- class ProgressBarWrapper
8
- def initialize(count)
9
- return unless tty?
10
-
11
- @pbar = ProgressBar.create(
12
- format: '%b%i %c/%C (%P%%) %a %e Rate: %R',
13
- remainder_mark: '░',
14
- progress_mark: '▓'.colorize(:light_green),
15
- starting_time: 0,
16
- lenth: 200,
17
- throttle_rate: 0.1,
18
- total: count,
19
- unknown_progress_animation_steps: ['▓░░░', '░▓░░', '░░▓░', '░░░▓']
20
- )
21
- end
22
-
23
- def increment
24
- @pbar&.increment
25
- end
26
-
27
- def log(message)
28
- @pbar&.log message
29
- end
30
-
31
- def finish
32
- @pbar&.finish
33
- end
34
-
35
- private
36
-
37
- def tty?
38
- $stdout.isatty
39
- end
40
- end
41
- end
42
- end
43
- end