chronicle-etl 0.1.3 → 0.2.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.rubocop.yml +8 -0
- data/.yardopts +1 -0
- data/Gemfile.lock +15 -1
- data/README.md +62 -11
- data/chronicle-etl.gemspec +6 -1
- data/exe/chronicle-etl +2 -2
- data/lib/chronicle/etl.rb +9 -2
- data/lib/chronicle/etl/catalog.rb +68 -18
- data/lib/chronicle/etl/cli/connectors.rb +32 -0
- data/lib/chronicle/etl/cli/jobs.rb +116 -0
- data/lib/chronicle/etl/cli/main.rb +83 -0
- data/lib/chronicle/etl/cli/subcommand_base.rb +37 -0
- data/lib/chronicle/etl/config.rb +53 -0
- data/lib/chronicle/etl/exceptions.rb +17 -0
- data/lib/chronicle/etl/extractors/csv_extractor.rb +1 -1
- data/lib/chronicle/etl/extractors/extractor.rb +18 -5
- data/lib/chronicle/etl/extractors/file_extractor.rb +2 -2
- data/lib/chronicle/etl/extractors/stdin_extractor.rb +2 -2
- data/lib/chronicle/etl/job.rb +62 -0
- data/lib/chronicle/etl/job_definition.rb +51 -0
- data/lib/chronicle/etl/job_log.rb +79 -0
- data/lib/chronicle/etl/job_logger.rb +76 -0
- data/lib/chronicle/etl/loaders/csv_loader.rb +2 -2
- data/lib/chronicle/etl/loaders/loader.rb +13 -6
- data/lib/chronicle/etl/loaders/rest_loader.rb +30 -0
- data/lib/chronicle/etl/loaders/stdout_loader.rb +2 -2
- data/lib/chronicle/etl/loaders/table_loader.rb +6 -10
- data/lib/chronicle/etl/runner.rb +19 -51
- data/lib/chronicle/etl/transformers/json_transformer.rb +2 -2
- data/lib/chronicle/etl/transformers/null_transformer.rb +4 -4
- data/lib/chronicle/etl/transformers/transformer.rb +21 -4
- data/lib/chronicle/etl/utils/progress_bar.rb +1 -1
- data/lib/chronicle/etl/version.rb +2 -2
- metadata +85 -4
- data/CHANGELOG.md +0 -18
- data/lib/chronicle/etl/cli.rb +0 -48
data/CHANGELOG.md
DELETED
@@ -1,18 +0,0 @@
|
|
1
|
-
# Changelog
|
2
|
-
|
3
|
-
This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
4
|
-
|
5
|
-
## [0.1.3] - 2020-08-13
|
6
|
-
### Added
|
7
|
-
- Ability to list all available ETL classes
|
8
|
-
- Refactored E, T, L module and class structure
|
9
|
-
- Better progress bar
|
10
|
-
|
11
|
-
## [0.1.2] - 2020-08-02
|
12
|
-
### Added
|
13
|
-
- This changelog
|
14
|
-
- Ability to use extractors, transformers, and loaders from other gems
|
15
|
-
|
16
|
-
## [0.1.0] - 2020-08-01
|
17
|
-
### Added
|
18
|
-
- Basic job runner and ETL classes
|
data/lib/chronicle/etl/cli.rb
DELETED
@@ -1,48 +0,0 @@
|
|
1
|
-
require 'thor'
|
2
|
-
require 'chronicle/etl'
|
3
|
-
|
4
|
-
module Chronicle
|
5
|
-
module Etl
|
6
|
-
class CLI < Thor
|
7
|
-
default_task :job
|
8
|
-
|
9
|
-
desc 'job', 'Runs an ETL job'
|
10
|
-
method_option :extractor, aliases: '-e', desc: 'Extractor class (available: stdin, csv, file)', default: 'stdin', banner: 'extractor-name'
|
11
|
-
method_option :'extractor-opts', desc: 'Extractor options', type: :hash, default: {}
|
12
|
-
method_option :transformer, aliases: '-t', desc: 'Transformer class (available: null)', default: 'null', banner: 'transformer-name'
|
13
|
-
method_option :'transformer-opts', desc: 'Transformer options', type: :hash, default: {}
|
14
|
-
method_option :loader, aliases: '-l', desc: 'Loader class (available: stdout, csv, table)', default: 'stdout', banner: 'loader-name'
|
15
|
-
method_option :'loader-opts', desc: 'Loader options', type: :hash, default: {}
|
16
|
-
method_option :job, aliases: '-j', desc: 'Job configuration file'
|
17
|
-
|
18
|
-
def job
|
19
|
-
runner_options = {
|
20
|
-
extractor: {
|
21
|
-
name: options[:extractor],
|
22
|
-
options: options[:'extractor-opts']
|
23
|
-
},
|
24
|
-
transformer: {
|
25
|
-
name: options[:transformer],
|
26
|
-
options: options[:'transformer-opts']
|
27
|
-
},
|
28
|
-
loader: {
|
29
|
-
name: options[:loader],
|
30
|
-
options: options[:'loader-opts']
|
31
|
-
}
|
32
|
-
}
|
33
|
-
|
34
|
-
runner = Runner.new(runner_options)
|
35
|
-
runner.run!
|
36
|
-
end
|
37
|
-
|
38
|
-
# FIXME: namespace this differently
|
39
|
-
desc 'list', 'List all ETL classes'
|
40
|
-
def list
|
41
|
-
klasses = Chronicle::Etl::Catalog.available_classes
|
42
|
-
|
43
|
-
table = TTY::Table.new(['class_name', 'built_in?', 'provider', 'phase'], klasses.map(&:values))
|
44
|
-
puts table.render
|
45
|
-
end
|
46
|
-
end
|
47
|
-
end
|
48
|
-
end
|