chronicle-etl 0.1.2 → 0.1.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.ruby-version +1 -0
- data/CHANGELOG.md +6 -0
- data/Gemfile.lock +34 -10
- data/README.md +2 -0
- data/bin/console +16 -4
- data/chronicle-etl.gemspec +5 -5
- data/lib/chronicle/etl.rb +2 -1
- data/lib/chronicle/etl/catalog.rb +58 -0
- data/lib/chronicle/etl/cli.rb +10 -0
- data/lib/chronicle/etl/extractors/{csv.rb → csv_extractor.rb} +3 -3
- data/lib/chronicle/etl/extractors/extractor.rb +16 -11
- data/lib/chronicle/etl/extractors/file_extractor.rb +52 -0
- data/lib/chronicle/etl/extractors/stdin_extractor.rb +11 -0
- data/lib/chronicle/etl/loaders/csv_loader.rb +29 -0
- data/lib/chronicle/etl/loaders/loader.rb +15 -15
- data/lib/chronicle/etl/loaders/stdout_loader.rb +9 -0
- data/lib/chronicle/etl/loaders/table_loader.rb +25 -0
- data/lib/chronicle/etl/runner.rb +15 -8
- data/lib/chronicle/etl/transformers/json_transformer.rb +11 -0
- data/lib/chronicle/etl/transformers/null_transformer.rb +10 -0
- data/lib/chronicle/etl/transformers/transformer.rb +10 -10
- data/lib/chronicle/etl/utils/progress_bar.rb +76 -0
- data/lib/chronicle/etl/version.rb +1 -1
- metadata +31 -28
- data/lib/chronicle/etl/extractors/stdin.rb +0 -13
- data/lib/chronicle/etl/loaders/csv.rb +0 -31
- data/lib/chronicle/etl/loaders/stdout.rb +0 -11
- data/lib/chronicle/etl/loaders/table.rb +0 -22
- data/lib/chronicle/etl/transformers/json.rb +0 -13
- data/lib/chronicle/etl/transformers/null.rb +0 -11
- data/lib/chronicle/etl/utils/progress_bar_wrapper.rb +0 -43
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: bf96bda84a74a4b43166060ba7fc5ec9a0aa3f129a7717c4d47d2672b9d0b654
|
4
|
+
data.tar.gz: eedf76e4684970f1c5e45fc68a86086e4e81be3080489c86410c8eb255f9eb68
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ad274453c44b94be793025bdd96767f9db2d84f1ae769d41bab8af4280dc95379dd4b710024c1519403b4cab13a55fb6f1657b7ba1f77ef4e4701159a80c0460
|
7
|
+
data.tar.gz: 8e3c68d4f29e2e35b6bda9dce7e6d7f79e4a6d15fec93154a66cf18cc13045fe45b2abe7b37e370ab413cd2ff561f4dcaad6c18bb09de4f60da8527509b2e94a
|
data/.ruby-version
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
2.7.1
|
data/CHANGELOG.md
CHANGED
@@ -2,6 +2,12 @@
|
|
2
2
|
|
3
3
|
This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
4
4
|
|
5
|
+
## [0.1.3] - 2020-08-13
|
6
|
+
### Added
|
7
|
+
- Ability to list all available ETL classes
|
8
|
+
- Refactored E, T, L module and class structure
|
9
|
+
- Better progress bar
|
10
|
+
|
5
11
|
## [0.1.2] - 2020-08-02
|
6
12
|
### Added
|
7
13
|
- This changelog
|
data/Gemfile.lock
CHANGED
@@ -1,11 +1,11 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
chronicle-etl (0.1.
|
4
|
+
chronicle-etl (0.1.3)
|
5
5
|
colorize (~> 0.8.1)
|
6
|
-
ruby-progressbar (~> 1.10)
|
7
|
-
table_print
|
8
6
|
thor (~> 0.20)
|
7
|
+
tty-progressbar (~> 0.17)
|
8
|
+
tty-table (~> 0.11)
|
9
9
|
|
10
10
|
GEM
|
11
11
|
remote: https://rubygems.org/
|
@@ -14,14 +14,19 @@ GEM
|
|
14
14
|
coderay (1.1.3)
|
15
15
|
colorize (0.8.1)
|
16
16
|
diff-lcs (1.4.4)
|
17
|
+
equatable (0.6.1)
|
17
18
|
method_source (1.0.0)
|
19
|
+
necromancer (0.6.0)
|
20
|
+
pastel (0.7.4)
|
21
|
+
equatable (~> 0.6)
|
22
|
+
tty-color (~> 0.5)
|
18
23
|
pry (0.13.1)
|
19
24
|
coderay (~> 1.1)
|
20
25
|
method_source (~> 1.0)
|
21
26
|
pry-byebug (3.9.0)
|
22
27
|
byebug (~> 11.0)
|
23
28
|
pry (~> 0.13.0)
|
24
|
-
rake (
|
29
|
+
rake (13.0.1)
|
25
30
|
rspec (3.9.0)
|
26
31
|
rspec-core (~> 3.9.0)
|
27
32
|
rspec-expectations (~> 3.9.0)
|
@@ -35,19 +40,38 @@ GEM
|
|
35
40
|
diff-lcs (>= 1.2.0, < 2.0)
|
36
41
|
rspec-support (~> 3.9.0)
|
37
42
|
rspec-support (3.9.3)
|
38
|
-
|
39
|
-
|
43
|
+
strings (0.1.8)
|
44
|
+
strings-ansi (~> 0.1)
|
45
|
+
unicode-display_width (~> 1.5)
|
46
|
+
unicode_utils (~> 1.4)
|
47
|
+
strings-ansi (0.1.0)
|
40
48
|
thor (0.20.3)
|
49
|
+
tty-color (0.5.2)
|
50
|
+
tty-cursor (0.7.1)
|
51
|
+
tty-progressbar (0.17.0)
|
52
|
+
strings-ansi (~> 0.1.0)
|
53
|
+
tty-cursor (~> 0.7)
|
54
|
+
tty-screen (~> 0.7)
|
55
|
+
unicode-display_width (~> 1.6)
|
56
|
+
tty-screen (0.8.1)
|
57
|
+
tty-table (0.11.0)
|
58
|
+
equatable (~> 0.6)
|
59
|
+
necromancer (~> 0.5)
|
60
|
+
pastel (~> 0.7.2)
|
61
|
+
strings (~> 0.1.5)
|
62
|
+
tty-screen (~> 0.7)
|
63
|
+
unicode-display_width (1.7.0)
|
64
|
+
unicode_utils (1.4.0)
|
41
65
|
|
42
66
|
PLATFORMS
|
43
67
|
ruby
|
44
68
|
|
45
69
|
DEPENDENCIES
|
46
|
-
bundler (~> 1
|
70
|
+
bundler (~> 2.1)
|
47
71
|
chronicle-etl!
|
48
72
|
pry-byebug (~> 3.9)
|
49
|
-
rake (~>
|
50
|
-
rspec (~> 3.
|
73
|
+
rake (~> 13.0)
|
74
|
+
rspec (~> 3.9)
|
51
75
|
|
52
76
|
BUNDLED WITH
|
53
|
-
1.
|
77
|
+
2.1.4
|
data/README.md
CHANGED
@@ -1,5 +1,7 @@
|
|
1
1
|
# Chronicle::Etl
|
2
2
|
|
3
|
+
[![Gem Version](https://badge.fury.io/rb/chronicle-etl.svg)](https://badge.fury.io/rb/chronicle-etl)
|
4
|
+
|
3
5
|
Chronicle ETL is a utility tool for manipulating personal data. You can extract it from a variety of source, transform it, and load it to different APIs or file formats.
|
4
6
|
|
5
7
|
## Installation
|
data/bin/console
CHANGED
@@ -7,8 +7,20 @@ require "chronicle/etl"
|
|
7
7
|
# with your gem easier. You can also use a different console, if you like.
|
8
8
|
|
9
9
|
# (If you use this, don't forget to add pry to your Gemfile!)
|
10
|
-
|
11
|
-
|
10
|
+
require "pry"
|
11
|
+
Pry.start
|
12
|
+
|
13
|
+
def reload!(print = true)
|
14
|
+
puts 'Reloading ...' if print
|
15
|
+
# Main project directory.
|
16
|
+
root_dir = File.expand_path('..', __dir__)
|
17
|
+
# Directories within the project that should be reloaded.
|
18
|
+
reload_dirs = %w{lib}
|
19
|
+
# Loop through and reload every file in all relevant project directories.
|
20
|
+
reload_dirs.each do |dir|
|
21
|
+
Dir.glob("#{root_dir}/#{dir}/**/*.rb").each { |f| load(f) }
|
22
|
+
end
|
23
|
+
# Return true when complete.
|
24
|
+
true
|
25
|
+
end
|
12
26
|
|
13
|
-
require "irb"
|
14
|
-
IRB.start(__FILE__)
|
data/chronicle-etl.gemspec
CHANGED
@@ -38,11 +38,11 @@ Gem::Specification.new do |spec|
|
|
38
38
|
|
39
39
|
spec.add_dependency "thor", "~> 0.20"
|
40
40
|
spec.add_dependency "colorize", "~> 0.8.1"
|
41
|
-
spec.add_dependency "
|
42
|
-
spec.add_dependency "
|
41
|
+
spec.add_dependency "tty-table", "~> 0.11"
|
42
|
+
spec.add_dependency "tty-progressbar", "~> 0.17"
|
43
43
|
|
44
|
-
spec.add_development_dependency "bundler", "~> 1
|
45
|
-
spec.add_development_dependency "rake", "~>
|
46
|
-
spec.add_development_dependency "rspec", "~> 3.
|
44
|
+
spec.add_development_dependency "bundler", "~> 2.1"
|
45
|
+
spec.add_development_dependency "rake", "~> 13.0"
|
46
|
+
spec.add_development_dependency "rspec", "~> 3.9"
|
47
47
|
spec.add_development_dependency "pry-byebug", "~> 3.9"
|
48
48
|
end
|
data/lib/chronicle/etl.rb
CHANGED
@@ -1,5 +1,6 @@
|
|
1
|
+
require_relative 'etl/catalog'
|
1
2
|
require_relative 'etl/extractors/extractor'
|
2
3
|
require_relative 'etl/transformers/transformer'
|
3
4
|
require_relative 'etl/loaders/loader'
|
4
|
-
require_relative 'etl/utils/
|
5
|
+
require_relative 'etl/utils/progress_bar'
|
5
6
|
require_relative 'etl/runner'
|
@@ -0,0 +1,58 @@
|
|
1
|
+
module Chronicle
|
2
|
+
module Etl
|
3
|
+
# Utility methods to catalogue which Extractor, Transformer, and
|
4
|
+
# Loader classes are available to chronicle-etl
|
5
|
+
module Catalog
|
6
|
+
def self.available_classes
|
7
|
+
parent_klasses = [
|
8
|
+
Chronicle::Etl::Extractor,
|
9
|
+
Chronicle::Etl::Transformer,
|
10
|
+
Chronicle::Etl::Loader
|
11
|
+
]
|
12
|
+
|
13
|
+
# TODO: have a registry of plugins
|
14
|
+
plugins = ['email', 'bash']
|
15
|
+
|
16
|
+
# Attempt to load each chronicle plugin that we might know about so
|
17
|
+
# that we can later search for subclasses to build our list of
|
18
|
+
# available classes
|
19
|
+
plugins.each do |plugin|
|
20
|
+
require "chronicle/#{plugin}"
|
21
|
+
rescue LoadError
|
22
|
+
# this will happen if the gem isn't available globally
|
23
|
+
end
|
24
|
+
|
25
|
+
klasses = []
|
26
|
+
parent_klasses.each do |parent|
|
27
|
+
klasses += ObjectSpace.each_object(Class).select { |klass| klass < parent }
|
28
|
+
end
|
29
|
+
|
30
|
+
klasses.map do |klass|
|
31
|
+
{
|
32
|
+
name: klass.name,
|
33
|
+
built_in: klass.built_in?,
|
34
|
+
provider: klass.provider,
|
35
|
+
phase: klass.phase
|
36
|
+
}
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
def phase
|
41
|
+
ancestors = self.ancestors
|
42
|
+
return :extractor if ancestors.include? Chronicle::Etl::Extractor
|
43
|
+
return :transformer if ancestors.include? Chronicle::Etl::Transformer
|
44
|
+
return :loader if ancestors.include? Chronicle::Etl::Loader
|
45
|
+
end
|
46
|
+
|
47
|
+
def provider
|
48
|
+
# TODO: needs better convention for a gem reporting its provider name
|
49
|
+
provider = to_s.split('::')[1].downcase
|
50
|
+
return provider unless provider == 'etl'
|
51
|
+
end
|
52
|
+
|
53
|
+
def built_in?
|
54
|
+
to_s.include? 'Chronicle::Etl'
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
data/lib/chronicle/etl/cli.rb
CHANGED
@@ -14,6 +14,7 @@ module Chronicle
|
|
14
14
|
method_option :loader, aliases: '-l', desc: 'Loader class (available: stdout, csv, table)', default: 'stdout', banner: 'loader-name'
|
15
15
|
method_option :'loader-opts', desc: 'Loader options', type: :hash, default: {}
|
16
16
|
method_option :job, aliases: '-j', desc: 'Job configuration file'
|
17
|
+
|
17
18
|
def job
|
18
19
|
runner_options = {
|
19
20
|
extractor: {
|
@@ -33,6 +34,15 @@ module Chronicle
|
|
33
34
|
runner = Runner.new(runner_options)
|
34
35
|
runner.run!
|
35
36
|
end
|
37
|
+
|
38
|
+
# FIXME: namespace this differently
|
39
|
+
desc 'list', 'List all ETL classes'
|
40
|
+
def list
|
41
|
+
klasses = Chronicle::Etl::Catalog.available_classes
|
42
|
+
|
43
|
+
table = TTY::Table.new(['class_name', 'built_in?', 'provider', 'phase'], klasses.map(&:values))
|
44
|
+
puts table.render
|
45
|
+
end
|
36
46
|
end
|
37
47
|
end
|
38
48
|
end
|
@@ -1,5 +1,5 @@
|
|
1
1
|
require 'csv'
|
2
|
-
class Chronicle::Etl::
|
2
|
+
class Chronicle::Etl::CsvExtractor < Chronicle::Etl::Extractor
|
3
3
|
DEFAULT_OPTIONS = {
|
4
4
|
headers: true,
|
5
5
|
filename: $stdin
|
@@ -18,7 +18,7 @@ class Chronicle::Etl::Extractors::Csv < Chronicle::Etl::Extractors::Extractor
|
|
18
18
|
end
|
19
19
|
|
20
20
|
def results_count
|
21
|
-
CSV.read(@options[:filename],
|
21
|
+
CSV.read(@options[:filename], headers: @options[:headers]).count if read_from_file?
|
22
22
|
end
|
23
23
|
|
24
24
|
private
|
@@ -33,7 +33,7 @@ class Chronicle::Etl::Extractors::Csv < Chronicle::Etl::Extractors::Extractor
|
|
33
33
|
}
|
34
34
|
|
35
35
|
stream = read_from_file? ? File.open(@options[:filename]) : @options[:filename]
|
36
|
-
CSV.new(stream, csv_options)
|
36
|
+
CSV.new(stream, **csv_options)
|
37
37
|
end
|
38
38
|
|
39
39
|
def read_from_file?
|
@@ -1,20 +1,25 @@
|
|
1
|
+
require 'chronicle/etl'
|
2
|
+
|
1
3
|
module Chronicle
|
2
4
|
module Etl
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
end
|
5
|
+
class Extractor
|
6
|
+
extend Chronicle::Etl::Catalog
|
7
|
+
|
8
|
+
ETL_PHASE = :extractor
|
8
9
|
|
9
|
-
|
10
|
-
|
11
|
-
|
10
|
+
def initialize(options = {})
|
11
|
+
@options = options.transform_keys!(&:to_sym)
|
12
|
+
end
|
12
13
|
|
13
|
-
|
14
|
+
def extract
|
15
|
+
raise NotImplementedError
|
14
16
|
end
|
17
|
+
|
18
|
+
def results_count; end
|
15
19
|
end
|
16
20
|
end
|
17
21
|
end
|
18
22
|
|
19
|
-
require_relative '
|
20
|
-
require_relative '
|
23
|
+
require_relative 'csv_extractor'
|
24
|
+
require_relative 'file_extractor'
|
25
|
+
require_relative 'stdin_extractor'
|
@@ -0,0 +1,52 @@
|
|
1
|
+
require 'pathname'
|
2
|
+
|
3
|
+
module Chronicle
|
4
|
+
module Etl
|
5
|
+
class FileExtractor < Chronicle::Etl::Extractor
|
6
|
+
def extract
|
7
|
+
if file?
|
8
|
+
extract_file do |data, metadata|
|
9
|
+
yield(data, metadata)
|
10
|
+
end
|
11
|
+
elsif directory?
|
12
|
+
extract_from_directory do |data, metadata|
|
13
|
+
yield(data, metadata)
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
def results_count
|
19
|
+
if file?
|
20
|
+
return 1
|
21
|
+
else
|
22
|
+
search_pattern = File.join(@options[:filename], '**/*.eml')
|
23
|
+
Dir.glob(search_pattern).count
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
private
|
28
|
+
|
29
|
+
def extract_from_directory
|
30
|
+
search_pattern = File.join(@options[:filename], '**/*.eml')
|
31
|
+
filenames = Dir.glob(search_pattern)
|
32
|
+
filenames.each do |filename|
|
33
|
+
file = File.open(filename)
|
34
|
+
yield(file.read, {filename: file})
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
def extract_file
|
39
|
+
file = File.open(@options[:filename])
|
40
|
+
yield(file.read, {filename: @options[:filename]})
|
41
|
+
end
|
42
|
+
|
43
|
+
def directory?
|
44
|
+
Pathname.new(@options[:filename]).directory?
|
45
|
+
end
|
46
|
+
|
47
|
+
def file?
|
48
|
+
Pathname.new(@options[:filename]).file?
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
require 'csv'
|
2
|
+
|
3
|
+
module Chronicle
|
4
|
+
module Etl
|
5
|
+
class CsvLoader < Chronicle::Etl::Loader
|
6
|
+
def initialize(options={})
|
7
|
+
super(options)
|
8
|
+
@rows = []
|
9
|
+
end
|
10
|
+
|
11
|
+
def load(result)
|
12
|
+
if (result.is_a? Hash)
|
13
|
+
@rows << result.values
|
14
|
+
else
|
15
|
+
@rows << result
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
def finish
|
20
|
+
z = $stdout
|
21
|
+
CSV(z) do |csv|
|
22
|
+
@rows.each do |row|
|
23
|
+
csv << row
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
@@ -1,25 +1,25 @@
|
|
1
1
|
module Chronicle
|
2
2
|
module Etl
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
def start; end
|
3
|
+
class Loader
|
4
|
+
extend Chronicle::Etl::Catalog
|
5
|
+
|
6
|
+
def initialize(options = {})
|
7
|
+
@options = options
|
8
|
+
end
|
10
9
|
|
11
|
-
|
10
|
+
def start; end
|
12
11
|
|
13
|
-
|
14
|
-
raise NotImplementedError
|
15
|
-
end
|
12
|
+
def first_load result; end
|
16
13
|
|
17
|
-
|
14
|
+
def load
|
15
|
+
raise NotImplementedError
|
18
16
|
end
|
17
|
+
|
18
|
+
def finish; end
|
19
19
|
end
|
20
20
|
end
|
21
21
|
end
|
22
22
|
|
23
|
-
require_relative '
|
24
|
-
require_relative '
|
25
|
-
require_relative '
|
23
|
+
require_relative 'csv_loader'
|
24
|
+
require_relative 'stdout_loader'
|
25
|
+
require_relative 'table_loader'
|
@@ -0,0 +1,25 @@
|
|
1
|
+
require 'tty/table'
|
2
|
+
|
3
|
+
module Chronicle
|
4
|
+
module Etl
|
5
|
+
class TableLoader < Chronicle::Etl::Loader
|
6
|
+
def initialize(options)
|
7
|
+
super(options)
|
8
|
+
end
|
9
|
+
|
10
|
+
# defer creating table until we get first result and can determine headers
|
11
|
+
def first_load(result)
|
12
|
+
headers = result.keys
|
13
|
+
@table = TTY::Table.new(header: headers)
|
14
|
+
end
|
15
|
+
|
16
|
+
def load(result)
|
17
|
+
@table << result
|
18
|
+
end
|
19
|
+
|
20
|
+
def finish
|
21
|
+
puts @table.render(:ascii)
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
data/lib/chronicle/etl/runner.rb
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
class Chronicle::Etl::Runner
|
2
2
|
BUILTIN = {
|
3
|
-
extractor: ['stdin', 'json', 'csv'],
|
3
|
+
extractor: ['stdin', 'json', 'csv', 'file'],
|
4
4
|
transformer: ['null'],
|
5
5
|
loader: ['stdout', 'csv', 'table']
|
6
6
|
}.freeze
|
@@ -12,16 +12,23 @@ class Chronicle::Etl::Runner
|
|
12
12
|
end
|
13
13
|
|
14
14
|
def run!
|
15
|
-
progress_bar = Chronicle::Etl::Utils::
|
16
|
-
|
15
|
+
progress_bar = Chronicle::Etl::Utils::ProgressBar.new(title: "Running job", total: @extractor.results_count)
|
16
|
+
count = 0
|
17
17
|
|
18
|
-
@
|
19
|
-
@loader.first_load(result) if i == 0
|
18
|
+
@loader.start
|
20
19
|
|
21
|
-
|
20
|
+
@extractor.extract do |data, metadata|
|
21
|
+
transformed_data = @transformer.transform(data)
|
22
|
+
|
23
|
+
@loader.first_load(transformed_data) if count == 0
|
22
24
|
@loader.load(transformed_data)
|
23
25
|
|
24
26
|
progress_bar.increment
|
27
|
+
count += 1
|
28
|
+
# rescue StandardError => e
|
29
|
+
# require 'pry'
|
30
|
+
# binding.pry
|
31
|
+
# progress_bar.log "Error processing; #{e.inspect}"
|
25
32
|
end
|
26
33
|
|
27
34
|
progress_bar.finish
|
@@ -38,7 +45,7 @@ class Chronicle::Etl::Runner
|
|
38
45
|
|
39
46
|
def load_etl_class(phase, name)
|
40
47
|
if BUILTIN[phase].include? name
|
41
|
-
klass_name = "Chronicle::Etl::#{
|
48
|
+
klass_name = "Chronicle::Etl::#{name.capitalize}#{phase.to_s.capitalize}"
|
42
49
|
else
|
43
50
|
# TODO: come up with syntax for specifying a particular extractor in a provider library
|
44
51
|
# provider, extractor = name.split(":")
|
@@ -50,7 +57,7 @@ class Chronicle::Etl::Runner
|
|
50
57
|
warn(" Perhaps you haven't installed it yet: `$ gem install chronicle-#{provider}`")
|
51
58
|
exit(false)
|
52
59
|
end
|
53
|
-
klass_name = "Chronicle::#{name.capitalize}::
|
60
|
+
klass_name = "Chronicle::#{name.capitalize}::ChronicleTransformer"
|
54
61
|
end
|
55
62
|
Object.const_get(klass_name)
|
56
63
|
end
|
@@ -1,18 +1,18 @@
|
|
1
1
|
module Chronicle
|
2
2
|
module Etl
|
3
|
-
|
4
|
-
|
5
|
-
def initialize(options = {})
|
6
|
-
@options = options
|
7
|
-
end
|
3
|
+
class Transformer
|
4
|
+
extend Chronicle::Etl::Catalog
|
8
5
|
|
9
|
-
|
10
|
-
|
11
|
-
|
6
|
+
def initialize(options = {})
|
7
|
+
@options = options
|
8
|
+
end
|
9
|
+
|
10
|
+
def transform data
|
11
|
+
raise NotImplementedError
|
12
12
|
end
|
13
13
|
end
|
14
14
|
end
|
15
15
|
end
|
16
16
|
|
17
|
-
require_relative '
|
18
|
-
require_relative '
|
17
|
+
require_relative 'json_transformer'
|
18
|
+
require_relative 'null_transformer'
|
@@ -0,0 +1,76 @@
|
|
1
|
+
require 'tty/progressbar'
|
2
|
+
require 'colorize'
|
3
|
+
|
4
|
+
module Chronicle
|
5
|
+
module Etl
|
6
|
+
module Utils
|
7
|
+
|
8
|
+
class ProgressBar
|
9
|
+
FORMAT_WITH_TOTAL = [
|
10
|
+
':bar ',
|
11
|
+
':percent'.light_white,
|
12
|
+
' | '.light_black,
|
13
|
+
':current'.light_white,
|
14
|
+
'/'.light_black,
|
15
|
+
':total'.light_white,
|
16
|
+
' ('.light_black,
|
17
|
+
'ELAPSED:'.light_black,
|
18
|
+
':elapsed'.light_white,
|
19
|
+
' | ETA:'.light_black,
|
20
|
+
':eta'.light_white,
|
21
|
+
' | RATE: '.light_black,
|
22
|
+
':mean_rate'.light_white,
|
23
|
+
'/s) '.light_black
|
24
|
+
].join.freeze
|
25
|
+
|
26
|
+
FORMAT_WITHOUT_TOTAL = [
|
27
|
+
':current'.light_white,
|
28
|
+
'/'.light_black,
|
29
|
+
'???'.light_white,
|
30
|
+
' ('.light_black,
|
31
|
+
'ELAPSED:'.light_black,
|
32
|
+
':elapsed'.light_white,
|
33
|
+
' | ETA:'.light_black,
|
34
|
+
'??:??'.light_white,
|
35
|
+
' | RATE: '.light_black,
|
36
|
+
':mean_rate'.light_white,
|
37
|
+
'/s) '.light_black
|
38
|
+
].join.freeze
|
39
|
+
|
40
|
+
def initialize(title: 'Loading', total:)
|
41
|
+
opts = {
|
42
|
+
clear: true,
|
43
|
+
complete: '▓'.light_blue,
|
44
|
+
incomplete: '░'.blue,
|
45
|
+
frequency: 10
|
46
|
+
}
|
47
|
+
|
48
|
+
if total
|
49
|
+
opts[:total] = total
|
50
|
+
format_str = "#{title} #{FORMAT_WITH_TOTAL}"
|
51
|
+
@pbar = TTY::ProgressBar.new(FORMAT_WITH_TOTAL, opts)
|
52
|
+
else
|
53
|
+
format_str = "#{title} #{FORMAT_WITHOUT_TOTAL}"
|
54
|
+
opts[:no_width] = true
|
55
|
+
end
|
56
|
+
|
57
|
+
@pbar = TTY::ProgressBar.new(format_str, opts)
|
58
|
+
|
59
|
+
@pbar.resize
|
60
|
+
end
|
61
|
+
|
62
|
+
def increment
|
63
|
+
@pbar.advance(1)
|
64
|
+
end
|
65
|
+
|
66
|
+
def log(message)
|
67
|
+
@pbar.log message.inspect
|
68
|
+
end
|
69
|
+
|
70
|
+
def finish
|
71
|
+
@pbar.finish
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|
76
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: chronicle-etl
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Andrew Louis
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-08-
|
11
|
+
date: 2020-08-13 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: thor
|
@@ -39,75 +39,75 @@ dependencies:
|
|
39
39
|
- !ruby/object:Gem::Version
|
40
40
|
version: 0.8.1
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
|
-
name:
|
42
|
+
name: tty-table
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
|
-
- - "
|
45
|
+
- - "~>"
|
46
46
|
- !ruby/object:Gem::Version
|
47
|
-
version: '0'
|
47
|
+
version: '0.11'
|
48
48
|
type: :runtime
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
|
-
- - "
|
52
|
+
- - "~>"
|
53
53
|
- !ruby/object:Gem::Version
|
54
|
-
version: '0'
|
54
|
+
version: '0.11'
|
55
55
|
- !ruby/object:Gem::Dependency
|
56
|
-
name:
|
56
|
+
name: tty-progressbar
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
58
58
|
requirements:
|
59
59
|
- - "~>"
|
60
60
|
- !ruby/object:Gem::Version
|
61
|
-
version: '
|
61
|
+
version: '0.17'
|
62
62
|
type: :runtime
|
63
63
|
prerelease: false
|
64
64
|
version_requirements: !ruby/object:Gem::Requirement
|
65
65
|
requirements:
|
66
66
|
- - "~>"
|
67
67
|
- !ruby/object:Gem::Version
|
68
|
-
version: '
|
68
|
+
version: '0.17'
|
69
69
|
- !ruby/object:Gem::Dependency
|
70
70
|
name: bundler
|
71
71
|
requirement: !ruby/object:Gem::Requirement
|
72
72
|
requirements:
|
73
73
|
- - "~>"
|
74
74
|
- !ruby/object:Gem::Version
|
75
|
-
version: '1
|
75
|
+
version: '2.1'
|
76
76
|
type: :development
|
77
77
|
prerelease: false
|
78
78
|
version_requirements: !ruby/object:Gem::Requirement
|
79
79
|
requirements:
|
80
80
|
- - "~>"
|
81
81
|
- !ruby/object:Gem::Version
|
82
|
-
version: '1
|
82
|
+
version: '2.1'
|
83
83
|
- !ruby/object:Gem::Dependency
|
84
84
|
name: rake
|
85
85
|
requirement: !ruby/object:Gem::Requirement
|
86
86
|
requirements:
|
87
87
|
- - "~>"
|
88
88
|
- !ruby/object:Gem::Version
|
89
|
-
version: '
|
89
|
+
version: '13.0'
|
90
90
|
type: :development
|
91
91
|
prerelease: false
|
92
92
|
version_requirements: !ruby/object:Gem::Requirement
|
93
93
|
requirements:
|
94
94
|
- - "~>"
|
95
95
|
- !ruby/object:Gem::Version
|
96
|
-
version: '
|
96
|
+
version: '13.0'
|
97
97
|
- !ruby/object:Gem::Dependency
|
98
98
|
name: rspec
|
99
99
|
requirement: !ruby/object:Gem::Requirement
|
100
100
|
requirements:
|
101
101
|
- - "~>"
|
102
102
|
- !ruby/object:Gem::Version
|
103
|
-
version: '3.
|
103
|
+
version: '3.9'
|
104
104
|
type: :development
|
105
105
|
prerelease: false
|
106
106
|
version_requirements: !ruby/object:Gem::Requirement
|
107
107
|
requirements:
|
108
108
|
- - "~>"
|
109
109
|
- !ruby/object:Gem::Version
|
110
|
-
version: '3.
|
110
|
+
version: '3.9'
|
111
111
|
- !ruby/object:Gem::Dependency
|
112
112
|
name: pry-byebug
|
113
113
|
requirement: !ruby/object:Gem::Requirement
|
@@ -133,6 +133,7 @@ extra_rdoc_files: []
|
|
133
133
|
files:
|
134
134
|
- ".gitignore"
|
135
135
|
- ".rspec"
|
136
|
+
- ".ruby-version"
|
136
137
|
- ".travis.yml"
|
137
138
|
- CHANGELOG.md
|
138
139
|
- CODE_OF_CONDUCT.md
|
@@ -146,19 +147,21 @@ files:
|
|
146
147
|
- chronicle-etl.gemspec
|
147
148
|
- exe/chronicle-etl
|
148
149
|
- lib/chronicle/etl.rb
|
150
|
+
- lib/chronicle/etl/catalog.rb
|
149
151
|
- lib/chronicle/etl/cli.rb
|
150
|
-
- lib/chronicle/etl/extractors/
|
152
|
+
- lib/chronicle/etl/extractors/csv_extractor.rb
|
151
153
|
- lib/chronicle/etl/extractors/extractor.rb
|
152
|
-
- lib/chronicle/etl/extractors/
|
153
|
-
- lib/chronicle/etl/
|
154
|
+
- lib/chronicle/etl/extractors/file_extractor.rb
|
155
|
+
- lib/chronicle/etl/extractors/stdin_extractor.rb
|
156
|
+
- lib/chronicle/etl/loaders/csv_loader.rb
|
154
157
|
- lib/chronicle/etl/loaders/loader.rb
|
155
|
-
- lib/chronicle/etl/loaders/
|
156
|
-
- lib/chronicle/etl/loaders/
|
158
|
+
- lib/chronicle/etl/loaders/stdout_loader.rb
|
159
|
+
- lib/chronicle/etl/loaders/table_loader.rb
|
157
160
|
- lib/chronicle/etl/runner.rb
|
158
|
-
- lib/chronicle/etl/transformers/
|
159
|
-
- lib/chronicle/etl/transformers/
|
161
|
+
- lib/chronicle/etl/transformers/json_transformer.rb
|
162
|
+
- lib/chronicle/etl/transformers/null_transformer.rb
|
160
163
|
- lib/chronicle/etl/transformers/transformer.rb
|
161
|
-
- lib/chronicle/etl/utils/
|
164
|
+
- lib/chronicle/etl/utils/progress_bar.rb
|
162
165
|
- lib/chronicle/etl/version.rb
|
163
166
|
homepage: https://github.com/chronicle-app
|
164
167
|
licenses:
|
@@ -167,7 +170,7 @@ metadata:
|
|
167
170
|
homepage_uri: https://github.com/chronicle-app
|
168
171
|
source_code_uri: https://github.com/chronicle-app/chronicle-etl
|
169
172
|
changelog_uri: https://github.com/chronicle-app/chronicle-etl/blob/master/CHANGELOG.md
|
170
|
-
post_install_message:
|
173
|
+
post_install_message:
|
171
174
|
rdoc_options: []
|
172
175
|
require_paths:
|
173
176
|
- lib
|
@@ -182,8 +185,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
182
185
|
- !ruby/object:Gem::Version
|
183
186
|
version: '0'
|
184
187
|
requirements: []
|
185
|
-
rubygems_version: 3.
|
186
|
-
signing_key:
|
188
|
+
rubygems_version: 3.1.2
|
189
|
+
signing_key:
|
187
190
|
specification_version: 4
|
188
191
|
summary: ETL tool for personal data
|
189
192
|
test_files: []
|
@@ -1,31 +0,0 @@
|
|
1
|
-
require 'csv'
|
2
|
-
|
3
|
-
module Chronicle
|
4
|
-
module Etl
|
5
|
-
module Loaders
|
6
|
-
class Csv < Chronicle::Etl::Loaders::Loader
|
7
|
-
def initialize(options={})
|
8
|
-
super(options)
|
9
|
-
@rows = []
|
10
|
-
end
|
11
|
-
|
12
|
-
def load(result)
|
13
|
-
if (result.values)
|
14
|
-
@rows << result.values
|
15
|
-
else
|
16
|
-
@rows << result
|
17
|
-
end
|
18
|
-
end
|
19
|
-
|
20
|
-
def finish
|
21
|
-
z = $stdout
|
22
|
-
CSV(z) do |csv|
|
23
|
-
@rows.each do |row|
|
24
|
-
csv << row
|
25
|
-
end
|
26
|
-
end
|
27
|
-
end
|
28
|
-
end
|
29
|
-
end
|
30
|
-
end
|
31
|
-
end
|
@@ -1,22 +0,0 @@
|
|
1
|
-
require 'table_print'
|
2
|
-
|
3
|
-
module Chronicle
|
4
|
-
module Etl
|
5
|
-
module Loaders
|
6
|
-
class Table < Chronicle::Etl::Loaders::Loader
|
7
|
-
def initialize(options)
|
8
|
-
super(options)
|
9
|
-
@rows = []
|
10
|
-
end
|
11
|
-
|
12
|
-
def load(result)
|
13
|
-
@rows << result
|
14
|
-
end
|
15
|
-
|
16
|
-
def finish
|
17
|
-
tp @rows
|
18
|
-
end
|
19
|
-
end
|
20
|
-
end
|
21
|
-
end
|
22
|
-
end
|
@@ -1,43 +0,0 @@
|
|
1
|
-
require 'ruby-progressbar'
|
2
|
-
require 'colorize'
|
3
|
-
|
4
|
-
module Chronicle
|
5
|
-
module Etl
|
6
|
-
module Utils
|
7
|
-
class ProgressBarWrapper
|
8
|
-
def initialize(count)
|
9
|
-
return unless tty?
|
10
|
-
|
11
|
-
@pbar = ProgressBar.create(
|
12
|
-
format: '%b%i %c/%C (%P%%) %a %e Rate: %R',
|
13
|
-
remainder_mark: '░',
|
14
|
-
progress_mark: '▓'.colorize(:light_green),
|
15
|
-
starting_time: 0,
|
16
|
-
lenth: 200,
|
17
|
-
throttle_rate: 0.1,
|
18
|
-
total: count,
|
19
|
-
unknown_progress_animation_steps: ['▓░░░', '░▓░░', '░░▓░', '░░░▓']
|
20
|
-
)
|
21
|
-
end
|
22
|
-
|
23
|
-
def increment
|
24
|
-
@pbar&.increment
|
25
|
-
end
|
26
|
-
|
27
|
-
def log(message)
|
28
|
-
@pbar&.log message
|
29
|
-
end
|
30
|
-
|
31
|
-
def finish
|
32
|
-
@pbar&.finish
|
33
|
-
end
|
34
|
-
|
35
|
-
private
|
36
|
-
|
37
|
-
def tty?
|
38
|
-
$stdout.isatty
|
39
|
-
end
|
40
|
-
end
|
41
|
-
end
|
42
|
-
end
|
43
|
-
end
|