chronicle-etl 0.1.4 → 0.2.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.rubocop.yml +8 -0
- data/.yardopts +1 -0
- data/Gemfile.lock +15 -1
- data/README.md +31 -13
- data/chronicle-etl.gemspec +6 -1
- data/exe/chronicle-etl +2 -2
- data/lib/chronicle/etl.rb +15 -2
- data/lib/chronicle/etl/catalog.rb +67 -17
- data/lib/chronicle/etl/cli/connectors.rb +32 -0
- data/lib/chronicle/etl/cli/jobs.rb +116 -0
- data/lib/chronicle/etl/cli/main.rb +83 -0
- data/lib/chronicle/etl/cli/subcommand_base.rb +37 -0
- data/lib/chronicle/etl/config.rb +53 -0
- data/lib/chronicle/etl/exceptions.rb +19 -0
- data/lib/chronicle/etl/extractors/csv_extractor.rb +2 -3
- data/lib/chronicle/etl/extractors/extractor.rb +21 -5
- data/lib/chronicle/etl/extractors/file_extractor.rb +2 -2
- data/lib/chronicle/etl/extractors/stdin_extractor.rb +2 -2
- data/lib/chronicle/etl/job.rb +71 -0
- data/lib/chronicle/etl/job_definition.rb +51 -0
- data/lib/chronicle/etl/job_log.rb +85 -0
- data/lib/chronicle/etl/job_logger.rb +78 -0
- data/lib/chronicle/etl/loaders/csv_loader.rb +4 -8
- data/lib/chronicle/etl/loaders/loader.rb +11 -2
- data/lib/chronicle/etl/loaders/rest_loader.rb +33 -0
- data/lib/chronicle/etl/loaders/stdout_loader.rb +5 -5
- data/lib/chronicle/etl/loaders/table_loader.rb +7 -6
- data/lib/chronicle/etl/models/activity.rb +15 -0
- data/lib/chronicle/etl/models/base.rb +103 -0
- data/lib/chronicle/etl/models/entity.rb +15 -0
- data/lib/chronicle/etl/models/generic.rb +23 -0
- data/lib/chronicle/etl/runner.rb +24 -46
- data/lib/chronicle/etl/transformers/null_transformer.rb +5 -6
- data/lib/chronicle/etl/transformers/transformer.rb +23 -7
- data/lib/chronicle/etl/utils/hash_utilities.rb +19 -0
- data/lib/chronicle/etl/utils/jsonapi.rb +28 -0
- data/lib/chronicle/etl/utils/progress_bar.rb +2 -2
- data/lib/chronicle/etl/version.rb +2 -2
- metadata +91 -5
- data/CHANGELOG.md +0 -23
- data/lib/chronicle/etl/cli.rb +0 -56
- data/lib/chronicle/etl/transformers/json_transformer.rb +0 -11
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7a02a2377d0e8d4135f3b931bc73641eac28058d736d9c1dba0a97107c1d4c0e
|
4
|
+
data.tar.gz: 810d5bff80e852fa08ef9824ed6b313aa309bb69e84228bc1fbb7595069e043b
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 0d5fbea3c63349bb3f566e6137755f6cc8a4060d0e401abf5a0e7d8b44a4c4278089c10ffb8bb9cf2d783a238449140e5e54d90f3ad158aa362c6335eedca5aa
|
7
|
+
data.tar.gz: bf6fa83b1d5e55760e62d3cc090bf09bb69a7c761ae4a9358fb4d82192c7efc7500b6db361f39adac3581982862654aa4603a78dfbb3aed53b51d01137ffd736
|
data/.rubocop.yml
ADDED
data/.yardopts
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
--markup=markdown
|
data/Gemfile.lock
CHANGED
@@ -1,8 +1,10 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
chronicle-etl (0.
|
4
|
+
chronicle-etl (0.2.4)
|
5
5
|
colorize (~> 0.8.1)
|
6
|
+
deep_merge (~> 1.2)
|
7
|
+
sequel (~> 5.35)
|
6
8
|
thor (~> 0.20)
|
7
9
|
tty-progressbar (~> 0.17)
|
8
10
|
tty-table (~> 0.11)
|
@@ -13,6 +15,7 @@ GEM
|
|
13
15
|
byebug (11.1.3)
|
14
16
|
coderay (1.1.3)
|
15
17
|
colorize (0.8.1)
|
18
|
+
deep_merge (1.2.1)
|
16
19
|
diff-lcs (1.4.4)
|
17
20
|
equatable (0.6.1)
|
18
21
|
method_source (1.0.0)
|
@@ -27,6 +30,8 @@ GEM
|
|
27
30
|
byebug (~> 11.0)
|
28
31
|
pry (~> 0.13.0)
|
29
32
|
rake (13.0.1)
|
33
|
+
redcarpet (3.5.0)
|
34
|
+
refinements (7.7.0)
|
30
35
|
rspec (3.9.0)
|
31
36
|
rspec-core (~> 3.9.0)
|
32
37
|
rspec-expectations (~> 3.9.0)
|
@@ -40,6 +45,11 @@ GEM
|
|
40
45
|
diff-lcs (>= 1.2.0, < 2.0)
|
41
46
|
rspec-support (~> 3.9.0)
|
42
47
|
rspec-support (3.9.3)
|
48
|
+
runcom (6.2.0)
|
49
|
+
refinements (~> 7.4)
|
50
|
+
xdg (~> 4.0)
|
51
|
+
sequel (5.36.0)
|
52
|
+
sqlite3 (1.4.2)
|
43
53
|
strings (0.1.8)
|
44
54
|
strings-ansi (~> 0.1)
|
45
55
|
unicode-display_width (~> 1.5)
|
@@ -62,6 +72,7 @@ GEM
|
|
62
72
|
tty-screen (~> 0.7)
|
63
73
|
unicode-display_width (1.7.0)
|
64
74
|
unicode_utils (1.4.0)
|
75
|
+
xdg (4.2.0)
|
65
76
|
|
66
77
|
PLATFORMS
|
67
78
|
ruby
|
@@ -71,7 +82,10 @@ DEPENDENCIES
|
|
71
82
|
chronicle-etl!
|
72
83
|
pry-byebug (~> 3.9)
|
73
84
|
rake (~> 13.0)
|
85
|
+
redcarpet (~> 3.5)
|
74
86
|
rspec (~> 3.9)
|
87
|
+
runcom (~> 6.2)
|
88
|
+
sqlite3 (~> 1.4)
|
75
89
|
|
76
90
|
BUNDLED WITH
|
77
91
|
2.1.4
|
data/README.md
CHANGED
@@ -1,10 +1,10 @@
|
|
1
|
-
# Chronicle::
|
1
|
+
# Chronicle::ETL
|
2
2
|
|
3
3
|
[![Gem Version](https://badge.fury.io/rb/chronicle-etl.svg)](https://badge.fury.io/rb/chronicle-etl)
|
4
4
|
|
5
|
-
Chronicle ETL is a utility
|
5
|
+
Chronicle ETL is a utility that helps you archive and processes personal data. You can *extract* it from a variety of sources, *transform* it, and *load* it to an external API, file, or stdout.
|
6
6
|
|
7
|
-
This
|
7
|
+
This tool is an adaptation of Andrew Louis's experimental [Memex project](https://hyfen.net/memex) and the dozens of existing importers are being migrated to Chronicle.
|
8
8
|
|
9
9
|
## Installation
|
10
10
|
|
@@ -16,12 +16,24 @@ $ gem install chronicle-etl
|
|
16
16
|
|
17
17
|
After installing the gem, `chronicle-etl` is available to run in your shell.
|
18
18
|
|
19
|
+
```bash
|
20
|
+
# read test.csv and display it as a table
|
21
|
+
$ chronicle-etl jobs:run --extractor csv --extractor-opts filename:test.csv --loader table
|
22
|
+
|
23
|
+
# Display help for the jobs:run command
|
24
|
+
$ chronicle-etl jobs help run
|
19
25
|
```
|
20
|
-
|
21
|
-
|
26
|
+
|
27
|
+
## Connectors
|
28
|
+
|
29
|
+
Connectors are available to read, process, and load data from different formats or external services.
|
30
|
+
|
31
|
+
```bash
|
32
|
+
# List all available connectors
|
33
|
+
$ chronicle-etl connectors:list
|
22
34
|
```
|
23
35
|
|
24
|
-
|
36
|
+
Built in connectors:
|
25
37
|
|
26
38
|
### Extractors
|
27
39
|
- `stdin` - (default) Load records from line-separated stdin
|
@@ -40,7 +52,7 @@ cat test.csv | chronicle-etl --extractor csv --loader table
|
|
40
52
|
|
41
53
|
In addition to the built-in importers, importers for third-party platforms are available. They are packaged as individual Ruby gems.
|
42
54
|
|
43
|
-
- [email](https://github.com/chronicle-app/chronicle-email). Extractors for `mbox` files. Transformers for chronicle schema
|
55
|
+
- [email](https://github.com/chronicle-app/chronicle-email). Extractors for `mbox` and other email files. Transformers for chronicle schema
|
44
56
|
- [bash](https://github.com/chronicle-app/chronicle-bash). Extract bash history from `~/.bash_history`. Transform it for chronicle schema
|
45
57
|
|
46
58
|
To install any of these, run `gem install chronicle-PROVIDER`.
|
@@ -54,17 +66,23 @@ I'll be open-sourcing more importers. Please [contact me](mailto:andrew@hyfen.ne
|
|
54
66
|
```
|
55
67
|
$ chronicle-etl help
|
56
68
|
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
69
|
+
ALL COMMANDS
|
70
|
+
help # This help menu
|
71
|
+
connectors help [COMMAND] # Describe subcommands or one specific subcommand
|
72
|
+
connectors:install NAME # Installs connector NAME
|
73
|
+
connectors:list # Lists available connectors
|
74
|
+
jobs help [COMMAND] # Describe subcommands or one specific subcommand
|
75
|
+
jobs:create # Create a job
|
76
|
+
jobs:list # List all available jobs
|
77
|
+
jobs:run # Start a job
|
78
|
+
jobs:show # Show a job
|
61
79
|
```
|
62
80
|
|
63
81
|
### Job options
|
64
82
|
|
65
83
|
```
|
66
84
|
Usage:
|
67
|
-
chronicle-etl
|
85
|
+
chronicle-etl jobs:run
|
68
86
|
|
69
87
|
Options:
|
70
88
|
-e, [--extractor=extractor-name] # Extractor class (available: stdin, csv, file)
|
@@ -97,4 +115,4 @@ The gem is available as open source under the terms of the [MIT License](https:/
|
|
97
115
|
|
98
116
|
## Code of Conduct
|
99
117
|
|
100
|
-
Everyone interacting in the Chronicle::
|
118
|
+
Everyone interacting in the Chronicle::ETL project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/chronicle-app/chronicle-etl/blob/master/CODE_OF_CONDUCT.md).
|
data/chronicle-etl.gemspec
CHANGED
@@ -5,7 +5,7 @@ require "chronicle/etl/version"
|
|
5
5
|
|
6
6
|
Gem::Specification.new do |spec|
|
7
7
|
spec.name = "chronicle-etl"
|
8
|
-
spec.version = Chronicle::
|
8
|
+
spec.version = Chronicle::ETL::VERSION
|
9
9
|
spec.authors = ["Andrew Louis"]
|
10
10
|
spec.email = ["andrew@hyfen.net"]
|
11
11
|
|
@@ -40,9 +40,14 @@ Gem::Specification.new do |spec|
|
|
40
40
|
spec.add_dependency "colorize", "~> 0.8.1"
|
41
41
|
spec.add_dependency "tty-table", "~> 0.11"
|
42
42
|
spec.add_dependency "tty-progressbar", "~> 0.17"
|
43
|
+
spec.add_dependency 'sequel', '~> 5.35'
|
44
|
+
spec.add_dependency 'deep_merge', '~> 1.2'
|
43
45
|
|
44
46
|
spec.add_development_dependency "bundler", "~> 2.1"
|
45
47
|
spec.add_development_dependency "rake", "~> 13.0"
|
46
48
|
spec.add_development_dependency "rspec", "~> 3.9"
|
47
49
|
spec.add_development_dependency "pry-byebug", "~> 3.9"
|
50
|
+
spec.add_development_dependency 'runcom', '~> 6.2'
|
51
|
+
spec.add_development_dependency 'redcarpet', '~> 3.5'
|
52
|
+
spec.add_development_dependency 'sqlite3', '~> 1.4'
|
48
53
|
end
|
data/exe/chronicle-etl
CHANGED
data/lib/chronicle/etl.rb
CHANGED
@@ -1,6 +1,19 @@
|
|
1
1
|
require_relative 'etl/catalog'
|
2
|
+
require_relative 'etl/config'
|
3
|
+
require_relative 'etl/exceptions'
|
2
4
|
require_relative 'etl/extractors/extractor'
|
3
|
-
require_relative 'etl/
|
5
|
+
require_relative 'etl/job_definition'
|
6
|
+
require_relative 'etl/job_log'
|
7
|
+
require_relative 'etl/job_logger'
|
8
|
+
require_relative 'etl/job'
|
4
9
|
require_relative 'etl/loaders/loader'
|
5
|
-
require_relative 'etl/
|
10
|
+
require_relative 'etl/models/activity'
|
11
|
+
require_relative 'etl/models/base'
|
12
|
+
require_relative 'etl/models/entity'
|
13
|
+
require_relative 'etl/models/generic'
|
6
14
|
require_relative 'etl/runner'
|
15
|
+
require_relative 'etl/transformers/transformer'
|
16
|
+
require_relative 'etl/utils/hash_utilities'
|
17
|
+
require_relative 'etl/utils/jsonapi'
|
18
|
+
require_relative 'etl/utils/progress_bar'
|
19
|
+
require_relative 'etl/version'
|
@@ -1,30 +1,37 @@
|
|
1
1
|
module Chronicle
|
2
|
-
module
|
2
|
+
module ETL
|
3
3
|
# Utility methods to catalogue which Extractor, Transformer, and
|
4
|
-
# Loader classes are available to chronicle-etl
|
4
|
+
# Loader connector classes are available to chronicle-etl
|
5
5
|
module Catalog
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
]
|
6
|
+
PHASES = [:extractor, :transformer, :loader]
|
7
|
+
PLUGINS = ['email', 'bash']
|
8
|
+
BUILTIN = {
|
9
|
+
extractor: ['stdin', 'json', 'csv', 'file'],
|
10
|
+
transformer: ['null'],
|
11
|
+
loader: ['stdout', 'csv', 'table', 'rest']
|
12
|
+
}.freeze
|
12
13
|
|
14
|
+
# Return which ETL connectors are available, both built in and externally-defined
|
15
|
+
def self.available_classes
|
13
16
|
# TODO: have a registry of plugins
|
14
|
-
plugins = ['email', 'bash']
|
15
17
|
|
16
18
|
# Attempt to load each chronicle plugin that we might know about so
|
17
19
|
# that we can later search for subclasses to build our list of
|
18
20
|
# available classes
|
19
|
-
|
21
|
+
PLUGINS.each do |plugin|
|
20
22
|
require "chronicle/#{plugin}"
|
21
23
|
rescue LoadError
|
22
|
-
# this will happen if the gem isn't available globally
|
24
|
+
# this will happen if the gem isn't available globally
|
23
25
|
end
|
24
26
|
|
27
|
+
parent_klasses = [
|
28
|
+
::Chronicle::ETL::Extractor,
|
29
|
+
::Chronicle::ETL::Transformer,
|
30
|
+
::Chronicle::ETL::Loader
|
31
|
+
]
|
25
32
|
klasses = []
|
26
|
-
parent_klasses.
|
27
|
-
klasses += ObjectSpace.each_object(Class).select { |klass| klass < parent }
|
33
|
+
parent_klasses.map do |parent|
|
34
|
+
klasses += ::ObjectSpace.each_object(::Class).select { |klass| klass < parent }
|
28
35
|
end
|
29
36
|
|
30
37
|
klasses.map do |klass|
|
@@ -37,21 +44,64 @@ module Chronicle
|
|
37
44
|
end
|
38
45
|
end
|
39
46
|
|
47
|
+
# Take a phase (e, t, or l) and an identifier and return the right class
|
48
|
+
def self.phase_and_identifier_to_klass(phase, identifier)
|
49
|
+
Chronicle::ETL::Catalog.identifier_to_klass(phase: phase, identifier: identifier)
|
50
|
+
end
|
51
|
+
|
52
|
+
# For a given connector identifier, return the class (either builtin, or from a
|
53
|
+
# external chronicle gem)
|
54
|
+
def self.identifier_to_klass(identifier:, phase:)
|
55
|
+
if BUILTIN[phase].include? identifier
|
56
|
+
load_builtin_klass(name: identifier, phase: phase)
|
57
|
+
else
|
58
|
+
provider, name = identifier.split(':')
|
59
|
+
name ||= ''
|
60
|
+
load_provider_klass(provider: provider, name: name, phase: phase)
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
# Returns whether a class is an Extractor, Transformer, or Loader
|
40
65
|
def phase
|
41
66
|
ancestors = self.ancestors
|
42
|
-
return :extractor if ancestors.include? Chronicle::
|
43
|
-
return :transformer if ancestors.include? Chronicle::
|
44
|
-
return :loader if ancestors.include? Chronicle::
|
67
|
+
return :extractor if ancestors.include? Chronicle::ETL::Extractor
|
68
|
+
return :transformer if ancestors.include? Chronicle::ETL::Transformer
|
69
|
+
return :loader if ancestors.include? Chronicle::ETL::Loader
|
45
70
|
end
|
46
71
|
|
72
|
+
# Returns which third-party provider this connector is associated wtih
|
47
73
|
def provider
|
48
74
|
# TODO: needs better convention for a gem reporting its provider name
|
49
75
|
provider = to_s.split('::')[1].downcase
|
50
76
|
provider == 'etl' ? 'chronicle' : provider
|
51
77
|
end
|
52
78
|
|
79
|
+
# Returns whether this connector is a built-in one
|
53
80
|
def built_in?
|
54
|
-
to_s.include? 'Chronicle::
|
81
|
+
to_s.include? 'Chronicle::ETL'
|
82
|
+
end
|
83
|
+
|
84
|
+
private
|
85
|
+
|
86
|
+
def self.load_builtin_klass(name:, phase:)
|
87
|
+
klass_str = "Chronicle::ETL::#{name.capitalize}#{phase.capitalize}"
|
88
|
+
begin
|
89
|
+
Object.const_get(klass_str)
|
90
|
+
rescue NameError => e
|
91
|
+
raise ConnectorNotAvailableError.new("Connector not found", name: name)
|
92
|
+
end
|
93
|
+
end
|
94
|
+
|
95
|
+
def self.load_provider_klass(name: '', phase:, provider:)
|
96
|
+
begin
|
97
|
+
require "chronicle/#{provider}"
|
98
|
+
klass_str = "Chronicle::#{provider.capitalize}::#{name.capitalize}#{phase.capitalize}"
|
99
|
+
Object.const_get(klass_str)
|
100
|
+
rescue LoadError => e
|
101
|
+
raise ProviderNotAvailableError.new("Provider '#{provider.capitalize}' could not be loaded", provider: provider)
|
102
|
+
rescue NameError => e
|
103
|
+
raise ProviderConnectorNotAvailableError.new("Connector '#{name}' in '#{provider}' could not be found", provider: provider, name: name)
|
104
|
+
end
|
55
105
|
end
|
56
106
|
end
|
57
107
|
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
module Chronicle
|
2
|
+
module ETL
|
3
|
+
module CLI
|
4
|
+
# CLI commands for working with ETL connectors
|
5
|
+
class Connectors < SubcommandBase
|
6
|
+
default_task 'list'
|
7
|
+
namespace :connectors
|
8
|
+
|
9
|
+
desc "install NAME", "Installs connector NAME"
|
10
|
+
def install
|
11
|
+
puts "Installing"
|
12
|
+
end
|
13
|
+
|
14
|
+
desc "list", "Lists available connectors"
|
15
|
+
# Display all available connectors that chronicle-etl has access to
|
16
|
+
def list
|
17
|
+
klasses = Chronicle::ETL::Catalog.available_classes
|
18
|
+
klasses = klasses.sort_by do |a|
|
19
|
+
[a[:built_in].to_s, a[:provider], a[:phase]]
|
20
|
+
end
|
21
|
+
|
22
|
+
headers = klasses.first.keys.map do |key|
|
23
|
+
key.to_s.upcase.bold
|
24
|
+
end
|
25
|
+
|
26
|
+
table = TTY::Table.new(headers, klasses.map(&:values))
|
27
|
+
puts table.render(indent: 0, padding: [0, 2])
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
@@ -0,0 +1,116 @@
|
|
1
|
+
require 'pp'
|
2
|
+
module Chronicle
|
3
|
+
module ETL
|
4
|
+
module CLI
|
5
|
+
# CLI commands for working with ETL jobs
|
6
|
+
class Jobs < SubcommandBase
|
7
|
+
default_task "start"
|
8
|
+
namespace :jobs
|
9
|
+
|
10
|
+
class_option :extractor, aliases: '-e', desc: 'Extractor class (available: stdin, csv, file)', default: 'stdin', banner: 'extractor-name'
|
11
|
+
class_option :'extractor-opts', desc: 'Extractor options', type: :hash, default: {}
|
12
|
+
class_option :transformer, aliases: '-t', desc: 'Transformer class (available: null)', default: 'null', banner: 'transformer-name'
|
13
|
+
class_option :'transformer-opts', desc: 'Transformer options', type: :hash, default: {}
|
14
|
+
class_option :loader, aliases: '-l', desc: 'Loader class (available: stdout, csv, table)', default: 'stdout', banner: 'loader-name'
|
15
|
+
class_option :'loader-opts', desc: 'Loader options', type: :hash, default: {}
|
16
|
+
class_option :name, aliases: '-j', desc: 'Job configuration name'
|
17
|
+
|
18
|
+
map run: :start # Thor doesn't like `run` as a command name
|
19
|
+
desc "run", "Start a job"
|
20
|
+
long_desc <<-LONG_DESC
|
21
|
+
This will run an ETL job. Each job needs three parts:
|
22
|
+
|
23
|
+
1. #{'Extractor'.underline}: pulls data from an external source. By default, this is stdout. Other common options including pulling data from an API or reading JSON from a file.
|
24
|
+
|
25
|
+
2. #{'Transformer'.underline}: transforms data into a new format. If none is specified, we use the `null` transformer which does nothing to the data.
|
26
|
+
|
27
|
+
3. #{'Loader'.underline}: takes that transformed data and loads it externally. This can be an API, flat files, (or by default), stdout.
|
28
|
+
|
29
|
+
If you do not want to use the command line flags, you can also configure a job with a .yml config file. You can either specify the path to this file or use the filename and place the file in ~/.config/chronicle/etl/jobs/NAME.yml and call it with `--job NAME`
|
30
|
+
LONG_DESC
|
31
|
+
# Run an ETL job
|
32
|
+
def start
|
33
|
+
job_definition = build_job_definition(options)
|
34
|
+
job = Chronicle::ETL::Job.new(job_definition)
|
35
|
+
runner = Chronicle::ETL::Runner.new(job)
|
36
|
+
runner.run!
|
37
|
+
rescue Chronicle::ETL::ProviderNotAvailableError => e
|
38
|
+
warn(e.message.red)
|
39
|
+
warn(" Perhaps you haven't installed it yet: `$ gem install chronicle-#{e.provider}`")
|
40
|
+
exit(false)
|
41
|
+
rescue Chronicle::ETL::ConnectorNotAvailableError => e
|
42
|
+
warn(e.message.red)
|
43
|
+
exit(false)
|
44
|
+
end
|
45
|
+
|
46
|
+
desc "create", "Create a job"
|
47
|
+
# Create an ETL job
|
48
|
+
def create
|
49
|
+
job_definition = build_job_definition(options)
|
50
|
+
path = File.join('chronicle', 'etl', 'jobs', options[:name])
|
51
|
+
Chronicle::ETL::Config.write(path, job_definition)
|
52
|
+
end
|
53
|
+
|
54
|
+
desc "show", "Show details about a job"
|
55
|
+
# Show an ETL job
|
56
|
+
def show
|
57
|
+
job_config = build_job_definition(options)
|
58
|
+
pp job_config
|
59
|
+
end
|
60
|
+
|
61
|
+
desc "list", "List all available jobs"
|
62
|
+
# List available ETL jobs
|
63
|
+
def list
|
64
|
+
jobs = Chronicle::ETL::Config.available_jobs
|
65
|
+
|
66
|
+
job_details = jobs.map do |job|
|
67
|
+
r = Chronicle::ETL::Config.load("chronicle/etl/jobs/#{job}.yml")
|
68
|
+
|
69
|
+
extractor = r[:extractor][:name] if r[:extractor]
|
70
|
+
transformer = r[:transformer][:name] if r[:transformer]
|
71
|
+
loader = r[:loader][:name] if r[:loader]
|
72
|
+
|
73
|
+
[job, extractor, transformer, loader]
|
74
|
+
end
|
75
|
+
|
76
|
+
headers = ['name', 'extractor', 'transformer', 'loader'].map{|h| h.upcase.bold }
|
77
|
+
|
78
|
+
table = TTY::Table.new(headers, job_details)
|
79
|
+
puts table.render(indent: 0, padding: [0, 2])
|
80
|
+
end
|
81
|
+
|
82
|
+
private
|
83
|
+
|
84
|
+
# Create job definition by reading config file and then overwriting with flag options
|
85
|
+
def build_job_definition(options)
|
86
|
+
definition = Chronicle::ETL::JobDefinition.new
|
87
|
+
definition.add_config(process_flag_options(options))
|
88
|
+
definition.add_config(load_job_config(options[:name]))
|
89
|
+
definition
|
90
|
+
end
|
91
|
+
|
92
|
+
def load_job_config name
|
93
|
+
Chronicle::ETL::Config.load_job_from_config(name)
|
94
|
+
end
|
95
|
+
|
96
|
+
# Takes flag options and turns them into a runner config
|
97
|
+
def process_flag_options options
|
98
|
+
{
|
99
|
+
extractor: {
|
100
|
+
name: options[:extractor],
|
101
|
+
options: options[:'extractor-opts']
|
102
|
+
},
|
103
|
+
transformer: {
|
104
|
+
name: options[:transformer],
|
105
|
+
options: options[:'transformer-opts']
|
106
|
+
},
|
107
|
+
loader: {
|
108
|
+
name: options[:loader],
|
109
|
+
options: options[:'loader-opts']
|
110
|
+
}
|
111
|
+
}
|
112
|
+
end
|
113
|
+
end
|
114
|
+
end
|
115
|
+
end
|
116
|
+
end
|