chronicle-etl 0.1.3 → 0.2.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +8 -0
  3. data/.yardopts +1 -0
  4. data/Gemfile.lock +15 -1
  5. data/README.md +62 -11
  6. data/chronicle-etl.gemspec +6 -1
  7. data/exe/chronicle-etl +2 -2
  8. data/lib/chronicle/etl.rb +9 -2
  9. data/lib/chronicle/etl/catalog.rb +68 -18
  10. data/lib/chronicle/etl/cli/connectors.rb +32 -0
  11. data/lib/chronicle/etl/cli/jobs.rb +116 -0
  12. data/lib/chronicle/etl/cli/main.rb +83 -0
  13. data/lib/chronicle/etl/cli/subcommand_base.rb +37 -0
  14. data/lib/chronicle/etl/config.rb +53 -0
  15. data/lib/chronicle/etl/exceptions.rb +17 -0
  16. data/lib/chronicle/etl/extractors/csv_extractor.rb +1 -1
  17. data/lib/chronicle/etl/extractors/extractor.rb +18 -5
  18. data/lib/chronicle/etl/extractors/file_extractor.rb +2 -2
  19. data/lib/chronicle/etl/extractors/stdin_extractor.rb +2 -2
  20. data/lib/chronicle/etl/job.rb +62 -0
  21. data/lib/chronicle/etl/job_definition.rb +51 -0
  22. data/lib/chronicle/etl/job_log.rb +79 -0
  23. data/lib/chronicle/etl/job_logger.rb +76 -0
  24. data/lib/chronicle/etl/loaders/csv_loader.rb +2 -2
  25. data/lib/chronicle/etl/loaders/loader.rb +13 -6
  26. data/lib/chronicle/etl/loaders/rest_loader.rb +30 -0
  27. data/lib/chronicle/etl/loaders/stdout_loader.rb +2 -2
  28. data/lib/chronicle/etl/loaders/table_loader.rb +6 -10
  29. data/lib/chronicle/etl/runner.rb +19 -51
  30. data/lib/chronicle/etl/transformers/json_transformer.rb +2 -2
  31. data/lib/chronicle/etl/transformers/null_transformer.rb +4 -4
  32. data/lib/chronicle/etl/transformers/transformer.rb +21 -4
  33. data/lib/chronicle/etl/utils/progress_bar.rb +1 -1
  34. data/lib/chronicle/etl/version.rb +2 -2
  35. metadata +85 -4
  36. data/CHANGELOG.md +0 -18
  37. data/lib/chronicle/etl/cli.rb +0 -48
@@ -1,18 +0,0 @@
1
- # Changelog
2
-
3
- This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
4
-
5
- ## [0.1.3] - 2020-08-13
6
- ### Added
7
- - Ability to list all available ETL classes
8
- - Refactored E, T, L module and class structure
9
- - Better progress bar
10
-
11
- ## [0.1.2] - 2020-08-02
12
- ### Added
13
- - This changelog
14
- - Ability to use extractors, transformers, and loaders from other gems
15
-
16
- ## [0.1.0] - 2020-08-01
17
- ### Added
18
- - Basic job runner and ETL classes
@@ -1,48 +0,0 @@
1
- require 'thor'
2
- require 'chronicle/etl'
3
-
4
- module Chronicle
5
- module Etl
6
- class CLI < Thor
7
- default_task :job
8
-
9
- desc 'job', 'Runs an ETL job'
10
- method_option :extractor, aliases: '-e', desc: 'Extractor class (available: stdin, csv, file)', default: 'stdin', banner: 'extractor-name'
11
- method_option :'extractor-opts', desc: 'Extractor options', type: :hash, default: {}
12
- method_option :transformer, aliases: '-t', desc: 'Transformer class (available: null)', default: 'null', banner: 'transformer-name'
13
- method_option :'transformer-opts', desc: 'Transformer options', type: :hash, default: {}
14
- method_option :loader, aliases: '-l', desc: 'Loader class (available: stdout, csv, table)', default: 'stdout', banner: 'loader-name'
15
- method_option :'loader-opts', desc: 'Loader options', type: :hash, default: {}
16
- method_option :job, aliases: '-j', desc: 'Job configuration file'
17
-
18
- def job
19
- runner_options = {
20
- extractor: {
21
- name: options[:extractor],
22
- options: options[:'extractor-opts']
23
- },
24
- transformer: {
25
- name: options[:transformer],
26
- options: options[:'transformer-opts']
27
- },
28
- loader: {
29
- name: options[:loader],
30
- options: options[:'loader-opts']
31
- }
32
- }
33
-
34
- runner = Runner.new(runner_options)
35
- runner.run!
36
- end
37
-
38
- # FIXME: namespace this differently
39
- desc 'list', 'List all ETL classes'
40
- def list
41
- klasses = Chronicle::Etl::Catalog.available_classes
42
-
43
- table = TTY::Table.new(['class_name', 'built_in?', 'provider', 'phase'], klasses.map(&:values))
44
- puts table.render
45
- end
46
- end
47
- end
48
- end