chronicle-etl 0.1.3 → 0.2.3

Sign up to get free protection for your applications and to get access to all the features.
Files changed (37) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +8 -0
  3. data/.yardopts +1 -0
  4. data/Gemfile.lock +15 -1
  5. data/README.md +62 -11
  6. data/chronicle-etl.gemspec +6 -1
  7. data/exe/chronicle-etl +2 -2
  8. data/lib/chronicle/etl.rb +9 -2
  9. data/lib/chronicle/etl/catalog.rb +68 -18
  10. data/lib/chronicle/etl/cli/connectors.rb +32 -0
  11. data/lib/chronicle/etl/cli/jobs.rb +116 -0
  12. data/lib/chronicle/etl/cli/main.rb +83 -0
  13. data/lib/chronicle/etl/cli/subcommand_base.rb +37 -0
  14. data/lib/chronicle/etl/config.rb +53 -0
  15. data/lib/chronicle/etl/exceptions.rb +17 -0
  16. data/lib/chronicle/etl/extractors/csv_extractor.rb +1 -1
  17. data/lib/chronicle/etl/extractors/extractor.rb +18 -5
  18. data/lib/chronicle/etl/extractors/file_extractor.rb +2 -2
  19. data/lib/chronicle/etl/extractors/stdin_extractor.rb +2 -2
  20. data/lib/chronicle/etl/job.rb +62 -0
  21. data/lib/chronicle/etl/job_definition.rb +51 -0
  22. data/lib/chronicle/etl/job_log.rb +79 -0
  23. data/lib/chronicle/etl/job_logger.rb +76 -0
  24. data/lib/chronicle/etl/loaders/csv_loader.rb +2 -2
  25. data/lib/chronicle/etl/loaders/loader.rb +13 -6
  26. data/lib/chronicle/etl/loaders/rest_loader.rb +30 -0
  27. data/lib/chronicle/etl/loaders/stdout_loader.rb +2 -2
  28. data/lib/chronicle/etl/loaders/table_loader.rb +6 -10
  29. data/lib/chronicle/etl/runner.rb +19 -51
  30. data/lib/chronicle/etl/transformers/json_transformer.rb +2 -2
  31. data/lib/chronicle/etl/transformers/null_transformer.rb +4 -4
  32. data/lib/chronicle/etl/transformers/transformer.rb +21 -4
  33. data/lib/chronicle/etl/utils/progress_bar.rb +1 -1
  34. data/lib/chronicle/etl/version.rb +2 -2
  35. metadata +85 -4
  36. data/CHANGELOG.md +0 -18
  37. data/lib/chronicle/etl/cli.rb +0 -48
@@ -1,18 +0,0 @@
1
- # Changelog
2
-
3
- This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
4
-
5
- ## [0.1.3] - 2020-08-13
6
- ### Added
7
- - Ability to list all available ETL classes
8
- - Refactored E, T, L module and class structure
9
- - Better progress bar
10
-
11
- ## [0.1.2] - 2020-08-02
12
- ### Added
13
- - This changelog
14
- - Ability to use extractors, transformers, and loaders from other gems
15
-
16
- ## [0.1.0] - 2020-08-01
17
- ### Added
18
- - Basic job runner and ETL classes
@@ -1,48 +0,0 @@
1
- require 'thor'
2
- require 'chronicle/etl'
3
-
4
- module Chronicle
5
- module Etl
6
- class CLI < Thor
7
- default_task :job
8
-
9
- desc 'job', 'Runs an ETL job'
10
- method_option :extractor, aliases: '-e', desc: 'Extractor class (available: stdin, csv, file)', default: 'stdin', banner: 'extractor-name'
11
- method_option :'extractor-opts', desc: 'Extractor options', type: :hash, default: {}
12
- method_option :transformer, aliases: '-t', desc: 'Transformer class (available: null)', default: 'null', banner: 'transformer-name'
13
- method_option :'transformer-opts', desc: 'Transformer options', type: :hash, default: {}
14
- method_option :loader, aliases: '-l', desc: 'Loader class (available: stdout, csv, table)', default: 'stdout', banner: 'loader-name'
15
- method_option :'loader-opts', desc: 'Loader options', type: :hash, default: {}
16
- method_option :job, aliases: '-j', desc: 'Job configuration file'
17
-
18
- def job
19
- runner_options = {
20
- extractor: {
21
- name: options[:extractor],
22
- options: options[:'extractor-opts']
23
- },
24
- transformer: {
25
- name: options[:transformer],
26
- options: options[:'transformer-opts']
27
- },
28
- loader: {
29
- name: options[:loader],
30
- options: options[:'loader-opts']
31
- }
32
- }
33
-
34
- runner = Runner.new(runner_options)
35
- runner.run!
36
- end
37
-
38
- # FIXME: namespace this differently
39
- desc 'list', 'List all ETL classes'
40
- def list
41
- klasses = Chronicle::Etl::Catalog.available_classes
42
-
43
- table = TTY::Table.new(['class_name', 'built_in?', 'provider', 'phase'], klasses.map(&:values))
44
- puts table.render
45
- end
46
- end
47
- end
48
- end