jetel 0.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +37 -0
- data/.rspec +3 -0
- data/.rubocop.yml +35 -0
- data/Gemfile +4 -0
- data/Gemfile.lock +112 -0
- data/LICENSE +22 -0
- data/LICENSE.txt +22 -0
- data/README.md +53 -0
- data/Rakefile +1 -0
- data/bin/jetel +5 -0
- data/data/.gitkeepme +1 -0
- data/jetel.gemspec +39 -0
- data/lib/jetel.rb +3 -0
- data/lib/jetel/cli/app.rb +35 -0
- data/lib/jetel/cli/cli.rb +7 -0
- data/lib/jetel/cli/cmd/config_cmd.rb +19 -0
- data/lib/jetel/cli/cmd/modules_cmd.rb +80 -0
- data/lib/jetel/cli/cmd/version_cmd.rb +16 -0
- data/lib/jetel/cli/shared.rb +12 -0
- data/lib/jetel/config/config.rb +7 -0
- data/lib/jetel/data/.gitignore +10 -0
- data/lib/jetel/downloader/backends/aria.rb +13 -0
- data/lib/jetel/downloader/backends/base_downloader.rb +11 -0
- data/lib/jetel/downloader/downloader.rb +28 -0
- data/lib/jetel/etl/etl.rb +4 -0
- data/lib/jetel/extensions/extensions.rb +12 -0
- data/lib/jetel/extensions/kernel.rb +9 -0
- data/lib/jetel/jetel/jetel.rb +4 -0
- data/lib/jetel/lib.rb +6 -0
- data/lib/jetel/modules/ip/ip.rb +53 -0
- data/lib/jetel/modules/module.rb +19 -0
- data/lib/jetel/modules/modules.rb +42 -0
- data/lib/jetel/version.rb +5 -0
- data/test/spec_helper.rb +40 -0
- metadata +291 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: f23144e9d1ac4c40055409e8723d264c1d3f7462
|
4
|
+
data.tar.gz: a3cea834902181127a30a2ea7c488689f89378ba
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 6dd4f76bce5461558dd7aea2d237391d4a18c5c347ffb87d56e53050f5bcc2bbf1d72c2adf8395d5dbaadca86ad953b2c6e63047d42db894f22ff3a7dba8c355
|
7
|
+
data.tar.gz: 56ed1fda59fa4e809ec6036c77f9fd421f139e0ef6b49856f2588584ee22650e118ba0aa050f252e9cfaf1106b649fb7968ed24cf1c8b1c77699b17f6733a7cc
|
data/.gitignore
ADDED
@@ -0,0 +1,37 @@
|
|
1
|
+
.idea
|
2
|
+
|
3
|
+
*.gem
|
4
|
+
*.rbc
|
5
|
+
/.config
|
6
|
+
/coverage/
|
7
|
+
/InstalledFiles
|
8
|
+
/pkg/
|
9
|
+
/spec/reports/
|
10
|
+
/test/tmp/
|
11
|
+
/test/version_tmp/
|
12
|
+
/tmp/
|
13
|
+
|
14
|
+
## Specific to RubyMotion:
|
15
|
+
.dat*
|
16
|
+
.repl_history
|
17
|
+
build/
|
18
|
+
|
19
|
+
## Documentation cache and generated files:
|
20
|
+
/.yardoc/
|
21
|
+
/_yardoc/
|
22
|
+
/doc/
|
23
|
+
/rdoc/
|
24
|
+
|
25
|
+
## Environment normalisation:
|
26
|
+
/.bundle/
|
27
|
+
/vendor/bundle
|
28
|
+
/lib/bundler/man/
|
29
|
+
|
30
|
+
# for a library or gem, you might want to ignore these files since the code is
|
31
|
+
# intended to run in multiple environments; otherwise, check them in:
|
32
|
+
# Gemfile.lock
|
33
|
+
# .ruby-version
|
34
|
+
# .ruby-gemset
|
35
|
+
|
36
|
+
# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
|
37
|
+
.rvmrc
|
data/.rspec
ADDED
data/.rubocop.yml
ADDED
@@ -0,0 +1,35 @@
|
|
1
|
+
AbcSize:
|
2
|
+
Enabled: false
|
3
|
+
|
4
|
+
ClassLength:
|
5
|
+
Enabled: false
|
6
|
+
|
7
|
+
ModuleLength:
|
8
|
+
Enabled: false
|
9
|
+
|
10
|
+
CyclomaticComplexity:
|
11
|
+
Enabled: false
|
12
|
+
|
13
|
+
Documentation:
|
14
|
+
Enabled: false
|
15
|
+
|
16
|
+
EachWithObject:
|
17
|
+
Enabled: false
|
18
|
+
|
19
|
+
Eval:
|
20
|
+
Enabled: false
|
21
|
+
|
22
|
+
FormatString:
|
23
|
+
Enabled: false
|
24
|
+
|
25
|
+
HashSyntax:
|
26
|
+
Enabled: false
|
27
|
+
|
28
|
+
LineLength:
|
29
|
+
Max: 323
|
30
|
+
|
31
|
+
MethodLength:
|
32
|
+
Enabled: false
|
33
|
+
|
34
|
+
PerceivedComplexity:
|
35
|
+
Enabled: false
|
data/Gemfile
ADDED
data/Gemfile.lock
ADDED
@@ -0,0 +1,112 @@
|
|
1
|
+
PATH
|
2
|
+
remote: .
|
3
|
+
specs:
|
4
|
+
jetel (0.0.3)
|
5
|
+
activesupport
|
6
|
+
gli
|
7
|
+
json_pure
|
8
|
+
multi_json
|
9
|
+
nokogiri
|
10
|
+
pg
|
11
|
+
pmap
|
12
|
+
terminal-table
|
13
|
+
zip
|
14
|
+
|
15
|
+
GEM
|
16
|
+
remote: https://rubygems.org/
|
17
|
+
specs:
|
18
|
+
activesupport (4.2.4)
|
19
|
+
i18n (~> 0.7)
|
20
|
+
json (~> 1.7, >= 1.7.7)
|
21
|
+
minitest (~> 5.1)
|
22
|
+
thread_safe (~> 0.3, >= 0.3.4)
|
23
|
+
tzinfo (~> 1.1)
|
24
|
+
ast (2.1.0)
|
25
|
+
astrolabe (1.3.1)
|
26
|
+
parser (~> 2.2)
|
27
|
+
coveralls (0.8.3)
|
28
|
+
json (~> 1.8)
|
29
|
+
rest-client (>= 1.6.8, < 2)
|
30
|
+
simplecov (~> 0.10.0)
|
31
|
+
term-ansicolor (~> 1.3)
|
32
|
+
thor (~> 0.19.1)
|
33
|
+
diff-lcs (1.2.5)
|
34
|
+
docile (1.1.5)
|
35
|
+
domain_name (0.5.25)
|
36
|
+
unf (>= 0.0.5, < 1.0.0)
|
37
|
+
gli (2.13.2)
|
38
|
+
http-cookie (1.0.2)
|
39
|
+
domain_name (~> 0.5)
|
40
|
+
i18n (0.7.0)
|
41
|
+
json (1.8.3)
|
42
|
+
json_pure (1.8.3)
|
43
|
+
mime-types (2.6.2)
|
44
|
+
mini_portile (0.6.2)
|
45
|
+
minitest (5.8.2)
|
46
|
+
multi_json (1.11.2)
|
47
|
+
netrc (0.11.0)
|
48
|
+
nokogiri (1.6.6.2)
|
49
|
+
mini_portile (~> 0.6.0)
|
50
|
+
parser (2.2.3.0)
|
51
|
+
ast (>= 1.1, < 3.0)
|
52
|
+
pg (0.18.3)
|
53
|
+
pmap (1.0.2)
|
54
|
+
powerpack (0.1.1)
|
55
|
+
rainbow (2.0.0)
|
56
|
+
rake (10.4.2)
|
57
|
+
rest-client (1.8.0)
|
58
|
+
http-cookie (>= 1.0.2, < 2.0)
|
59
|
+
mime-types (>= 1.16, < 3.0)
|
60
|
+
netrc (~> 0.7)
|
61
|
+
rspec (3.3.0)
|
62
|
+
rspec-core (~> 3.3.0)
|
63
|
+
rspec-expectations (~> 3.3.0)
|
64
|
+
rspec-mocks (~> 3.3.0)
|
65
|
+
rspec-core (3.3.2)
|
66
|
+
rspec-support (~> 3.3.0)
|
67
|
+
rspec-expectations (3.3.1)
|
68
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
69
|
+
rspec-support (~> 3.3.0)
|
70
|
+
rspec-mocks (3.3.2)
|
71
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
72
|
+
rspec-support (~> 3.3.0)
|
73
|
+
rspec-support (3.3.0)
|
74
|
+
rubocop (0.34.2)
|
75
|
+
astrolabe (~> 1.3)
|
76
|
+
parser (>= 2.2.2.5, < 3.0)
|
77
|
+
powerpack (~> 0.1)
|
78
|
+
rainbow (>= 1.99.1, < 3.0)
|
79
|
+
ruby-progressbar (~> 1.4)
|
80
|
+
ruby-progressbar (1.7.5)
|
81
|
+
simplecov (0.10.0)
|
82
|
+
docile (~> 1.1.0)
|
83
|
+
json (~> 1.8)
|
84
|
+
simplecov-html (~> 0.10.0)
|
85
|
+
simplecov-html (0.10.0)
|
86
|
+
term-ansicolor (1.3.2)
|
87
|
+
tins (~> 1.0)
|
88
|
+
terminal-table (1.5.2)
|
89
|
+
thor (0.19.1)
|
90
|
+
thread_safe (0.3.5)
|
91
|
+
tins (1.6.0)
|
92
|
+
tzinfo (1.2.2)
|
93
|
+
thread_safe (~> 0.1)
|
94
|
+
unf (0.1.4)
|
95
|
+
unf_ext
|
96
|
+
unf_ext (0.0.7.1)
|
97
|
+
zip (2.0.2)
|
98
|
+
|
99
|
+
PLATFORMS
|
100
|
+
ruby
|
101
|
+
|
102
|
+
DEPENDENCIES
|
103
|
+
bundler (~> 1.5)
|
104
|
+
coveralls
|
105
|
+
jetel!
|
106
|
+
rake
|
107
|
+
rspec
|
108
|
+
rubocop
|
109
|
+
simplecov
|
110
|
+
|
111
|
+
BUNDLED WITH
|
112
|
+
1.10.6
|
data/LICENSE
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
The MIT License (MIT)
|
2
|
+
|
3
|
+
Copyright (c) 2015 Tomas Korcak
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
7
|
+
in the Software without restriction, including without limitation the rights
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
10
|
+
furnished to do so, subject to the following conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
13
|
+
copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
21
|
+
SOFTWARE.
|
22
|
+
|
data/LICENSE.txt
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2015 Tomas Korcak
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,53 @@
|
|
1
|
+
# jetel
|
2
|
+
|
3
|
+
## CLI - Command Line Interface
|
4
|
+
|
5
|
+
Run `jetel`
|
6
|
+
|
7
|
+
```
|
8
|
+
tomaskorcak@kx-mac:~$ jetel
|
9
|
+
NAME
|
10
|
+
jetel - Jetel CLI 0.0.1
|
11
|
+
|
12
|
+
SYNOPSIS
|
13
|
+
jetel [global options] command [command options] [arguments...]
|
14
|
+
|
15
|
+
GLOBAL OPTIONS
|
16
|
+
--help - Show this message
|
17
|
+
|
18
|
+
COMMANDS
|
19
|
+
help - Shows a list of commands or help for one command
|
20
|
+
ip, Ip - Module ip
|
21
|
+
modules - Print modules info
|
22
|
+
version - Print version info
|
23
|
+
```
|
24
|
+
|
25
|
+
## Structure
|
26
|
+
|
27
|
+
```
|
28
|
+
.
|
29
|
+
├── bin
|
30
|
+
├── lib
|
31
|
+
│ └── jetel
|
32
|
+
│ ├── cli
|
33
|
+
│ │ └── cmd
|
34
|
+
│ ├── downloader
|
35
|
+
│ ├── etl
|
36
|
+
│ ├── extensions
|
37
|
+
│ └── modules
|
38
|
+
│ └── ip
|
39
|
+
├── pkg
|
40
|
+
└── test
|
41
|
+
```
|
42
|
+
|
43
|
+
## Executables
|
44
|
+
|
45
|
+
### Rake
|
46
|
+
|
47
|
+
```
|
48
|
+
tomaskorcak@kx-mac:~/dev/jetel$ bundle exec rake -T
|
49
|
+
rake build # Build jetel-0.0.1.gem into the pkg directory
|
50
|
+
rake install # Build and install jetel-0.0.1.gem into system gems
|
51
|
+
rake install:local # Build and install jetel-0.0.1.gem into system gems without network access
|
52
|
+
rake release # Create tag v0.0.1 and build and push jetel-0.0.1.gem to Rubygems
|
53
|
+
```
|
data/Rakefile
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
require 'bundler/gem_tasks'
|
data/bin/jetel
ADDED
data/data/.gitkeepme
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
# KEEP ME, PLEASE !!!
|
data/jetel.gemspec
ADDED
@@ -0,0 +1,39 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
|
3
|
+
lib = File.expand_path('../lib', __FILE__)
|
4
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
5
|
+
require 'jetel/version'
|
6
|
+
|
7
|
+
Gem::Specification.new do |spec|
|
8
|
+
spec.name = 'jetel'
|
9
|
+
spec.version = Jetel::VERSION
|
10
|
+
spec.authors = ['Tomas Korcak']
|
11
|
+
spec.email = ['korczis@gmail.com']
|
12
|
+
spec.summary = 'Jetel'
|
13
|
+
spec.description = 'Jetel - Custom made ETL for specific needs'
|
14
|
+
spec.homepage = 'https://github.com/korczis/jetel'
|
15
|
+
spec.license = 'MIT'
|
16
|
+
|
17
|
+
spec.files = `git ls-files`.split("\n")
|
18
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
19
|
+
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
20
|
+
spec.require_paths = ['lib']
|
21
|
+
|
22
|
+
spec.add_dependency 'activesupport'
|
23
|
+
spec.add_dependency 'gli'
|
24
|
+
spec.add_dependency 'json_pure'
|
25
|
+
spec.add_dependency 'multi_json'
|
26
|
+
spec.add_dependency 'nokogiri'
|
27
|
+
spec.add_dependency 'pg'
|
28
|
+
spec.add_dependency 'pmap'
|
29
|
+
spec.add_dependency 'terminal-table'
|
30
|
+
spec.add_dependency 'zip'
|
31
|
+
|
32
|
+
spec.add_development_dependency 'bundler', '~> 1.5'
|
33
|
+
spec.add_development_dependency 'rake'
|
34
|
+
|
35
|
+
spec.add_development_dependency 'coveralls'
|
36
|
+
spec.add_development_dependency 'rspec'
|
37
|
+
spec.add_development_dependency 'rubocop'
|
38
|
+
spec.add_development_dependency 'simplecov'
|
39
|
+
end
|
data/lib/jetel.rb
ADDED
@@ -0,0 +1,35 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
3
|
+
require 'gli'
|
4
|
+
require 'pathname'
|
5
|
+
require 'pp'
|
6
|
+
|
7
|
+
require_relative 'shared'
|
8
|
+
require_relative '../version'
|
9
|
+
|
10
|
+
def launch(argv = ARGV)
|
11
|
+
run(argv)
|
12
|
+
end
|
13
|
+
|
14
|
+
include GLI::App
|
15
|
+
|
16
|
+
program_desc "Jetel CLI #{Jetel::VERSION}"
|
17
|
+
|
18
|
+
module Esri
|
19
|
+
# ESRI CLI
|
20
|
+
module Cli
|
21
|
+
# CLI Application
|
22
|
+
class App
|
23
|
+
cmds = File.absolute_path(File.join(File.dirname(__FILE__), 'cmd'))
|
24
|
+
Dir.glob(cmds + '/*.rb').each do |file|
|
25
|
+
require file
|
26
|
+
end
|
27
|
+
|
28
|
+
def main(argv = ARGV)
|
29
|
+
launch(argv)
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
launch
|
@@ -0,0 +1,19 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
3
|
+
require 'gli'
|
4
|
+
require 'pp'
|
5
|
+
|
6
|
+
include GLI::App
|
7
|
+
|
8
|
+
require_relative '../../version'
|
9
|
+
|
10
|
+
require_relative '../shared'
|
11
|
+
|
12
|
+
require_relative '../../config/config'
|
13
|
+
|
14
|
+
desc 'Show config'
|
15
|
+
command :config do |c|
|
16
|
+
c.action do |_global_options, _options, _args|
|
17
|
+
pp Jetel::Config
|
18
|
+
end
|
19
|
+
end
|
@@ -0,0 +1,80 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
3
|
+
require 'gli'
|
4
|
+
require 'terminal-table'
|
5
|
+
|
6
|
+
include GLI::App
|
7
|
+
|
8
|
+
require_relative '../../version'
|
9
|
+
|
10
|
+
require_relative '../shared'
|
11
|
+
|
12
|
+
require_relative '../../modules/modules'
|
13
|
+
|
14
|
+
MODULES = Jetel::Modules.modules
|
15
|
+
|
16
|
+
MODULES_ACTIONS = {
|
17
|
+
download: nil,
|
18
|
+
extract: nil,
|
19
|
+
transform: nil,
|
20
|
+
load: nil
|
21
|
+
}
|
22
|
+
|
23
|
+
# Gets module name
|
24
|
+
#
|
25
|
+
# @param m [Hash] Module info
|
26
|
+
# @return [String] Module name
|
27
|
+
def get_module_name(m)
|
28
|
+
m[:name]
|
29
|
+
end
|
30
|
+
|
31
|
+
# Register module (CLI) action
|
32
|
+
#
|
33
|
+
# @param c [] Parent command
|
34
|
+
# @param m [Module] Module instance
|
35
|
+
# @param action_command [String] Nested command name
|
36
|
+
# @param action_desc [String] Nested command action description
|
37
|
+
# @return [Object] Return value
|
38
|
+
def register_module_action(c, _m, action_command, action_desc, &block)
|
39
|
+
c.desc(action_desc)
|
40
|
+
c.command(action_command) do |cmd|
|
41
|
+
cmd.action(&block)
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
def register_module(m)
|
46
|
+
module_name = get_module_name(m)
|
47
|
+
|
48
|
+
desc "Module #{module_name}"
|
49
|
+
command(m[:name], m[:class_name]) do |c|
|
50
|
+
module_instance = m[:klass].new
|
51
|
+
module_name = m[:name]
|
52
|
+
|
53
|
+
MODULES_ACTIONS.each do |k, v|
|
54
|
+
next unless module_instance.respond_to?(k)
|
55
|
+
|
56
|
+
action_name = k
|
57
|
+
action_description = v || "#{action_name} #{module_name}"
|
58
|
+
register_module_action(c, m, action_name, action_description) do
|
59
|
+
module_instance.send(k)
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
desc 'Print modules info'
|
66
|
+
command :modules do |c|
|
67
|
+
c.action do |_global_options, _options, _args|
|
68
|
+
rows = MODULES.map do |m|
|
69
|
+
[m[:name], m[:klass]]
|
70
|
+
end
|
71
|
+
|
72
|
+
table = Terminal::Table.new :headings => %w(Name Class), :rows => rows
|
73
|
+
puts table
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
# Register all modules
|
78
|
+
MODULES.each do |m|
|
79
|
+
register_module(m)
|
80
|
+
end
|
@@ -0,0 +1,16 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
3
|
+
require 'gli'
|
4
|
+
|
5
|
+
include GLI::App
|
6
|
+
|
7
|
+
require_relative '../../version'
|
8
|
+
|
9
|
+
require_relative '../shared'
|
10
|
+
|
11
|
+
desc 'Print version info'
|
12
|
+
command :version do |c|
|
13
|
+
c.action do |_global_options, _options, _args|
|
14
|
+
puts Jetel::VERSION
|
15
|
+
end
|
16
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
3
|
+
require_relative 'backends/aria'
|
4
|
+
|
5
|
+
module Jetel
|
6
|
+
class Downloader
|
7
|
+
attr_reader :downloader
|
8
|
+
|
9
|
+
def initialize
|
10
|
+
@downloader = Downloaders::Aria.new
|
11
|
+
end
|
12
|
+
|
13
|
+
def download(file)
|
14
|
+
file.is_a?(Array) ? download_files(file) : download_file(file)
|
15
|
+
end
|
16
|
+
|
17
|
+
def download_files(files)
|
18
|
+
files.map do |file|
|
19
|
+
download_file(file)
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
def download_file(file)
|
24
|
+
puts "Downloading file #{file}"
|
25
|
+
downloader.download(file)
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,12 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
#
|
3
|
+
# Copyright (c) 2010-2015 GoodData Corporation. All rights reserved.
|
4
|
+
# This source code is licensed under the BSD-style license found in the
|
5
|
+
# LICENSE file in the root directory of this source tree.
|
6
|
+
|
7
|
+
require 'pathname'
|
8
|
+
|
9
|
+
base = Pathname(__FILE__).dirname.expand_path
|
10
|
+
Dir.glob(base + '*.rb').each do |file|
|
11
|
+
require_relative file
|
12
|
+
end
|
data/lib/jetel/lib.rb
ADDED
@@ -0,0 +1,53 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
3
|
+
require 'pmap'
|
4
|
+
|
5
|
+
require_relative '../../modules/module'
|
6
|
+
|
7
|
+
module Jetel
|
8
|
+
module Modules
|
9
|
+
class Ip < Module
|
10
|
+
SOURCES = [
|
11
|
+
{
|
12
|
+
name: 'afrinic',
|
13
|
+
url: 'ftp://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-latest'
|
14
|
+
},
|
15
|
+
{
|
16
|
+
name: 'apnic',
|
17
|
+
url: 'ftp://ftp.apnic.net/pub/stats/apnic/delegated-apnic-latest'
|
18
|
+
},
|
19
|
+
{
|
20
|
+
name: 'arin',
|
21
|
+
url: 'ftp://ftp.arin.net/pub/stats/arin/delegated-arin-latest'
|
22
|
+
},
|
23
|
+
{
|
24
|
+
name: 'lacnic',
|
25
|
+
url: 'ftp://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-latest'
|
26
|
+
},
|
27
|
+
{
|
28
|
+
name: 'ripencc',
|
29
|
+
url: 'ftp://ftp.ripe.net/ripe/stats/delegated-ripencc-latest'
|
30
|
+
},
|
31
|
+
{
|
32
|
+
name: 'iana',
|
33
|
+
url: 'ftp://ftp.apnic.net/pub/stats/iana/delegated-iana-latest'
|
34
|
+
}
|
35
|
+
]
|
36
|
+
|
37
|
+
def download
|
38
|
+
SOURCES.pmap do |source|
|
39
|
+
downloader.download(source[:url])
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
def extract
|
44
|
+
end
|
45
|
+
|
46
|
+
def transform
|
47
|
+
end
|
48
|
+
|
49
|
+
def load
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
3
|
+
require_relative '../downloader/downloader'
|
4
|
+
|
5
|
+
module Jetel
|
6
|
+
module Modules
|
7
|
+
class Module
|
8
|
+
attr_reader :downloader
|
9
|
+
|
10
|
+
def initialize
|
11
|
+
@downloader = Downloader.new
|
12
|
+
end
|
13
|
+
|
14
|
+
def download(*args)
|
15
|
+
downloader.download(*args)
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
require 'active_support/inflector'
|
2
|
+
require_relative '../extensions/extensions'
|
3
|
+
|
4
|
+
module Jetel
|
5
|
+
module Modules
|
6
|
+
class << self
|
7
|
+
def modules(modules_dir = File.join(File.dirname(__FILE__)), auto_require = true, require_only = false)
|
8
|
+
dir = File.absolute_path(modules_dir)
|
9
|
+
res = Dir.entries(dir).map do |entry|
|
10
|
+
dir_path = File.join(dir, entry)
|
11
|
+
|
12
|
+
next unless File.directory?(dir_path)
|
13
|
+
next if entry == '.' || entry == '..'
|
14
|
+
|
15
|
+
full_path = File.join(dir_path, "#{entry}.rb")
|
16
|
+
next unless File.exist?(full_path)
|
17
|
+
|
18
|
+
# Require file if auto_require true
|
19
|
+
require(full_path) if auto_require
|
20
|
+
|
21
|
+
# Go to next file if require_only mode
|
22
|
+
next if require_only
|
23
|
+
|
24
|
+
class_name = entry.camelize
|
25
|
+
qualified_class_name = "Jetel::Modules::#{class_name}"
|
26
|
+
|
27
|
+
# Return value from map iteration
|
28
|
+
{
|
29
|
+
name: entry,
|
30
|
+
path: full_path,
|
31
|
+
class_name: class_name,
|
32
|
+
qualified_class_name: qualified_class_name,
|
33
|
+
klass: auto_require ? Kernel.qualified_const_get(qualified_class_name) : nil
|
34
|
+
}
|
35
|
+
end
|
36
|
+
|
37
|
+
# Remove nil values and return
|
38
|
+
res.compact
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
data/test/spec_helper.rb
ADDED
@@ -0,0 +1,40 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
|
3
|
+
require 'simplecov'
|
4
|
+
require 'rspec'
|
5
|
+
require 'pathname'
|
6
|
+
|
7
|
+
# Automagically include all helpers/*_helper.rb
|
8
|
+
|
9
|
+
base = Pathname(__FILE__).dirname.expand_path
|
10
|
+
Dir.glob(base + 'helpers/*_helper.rb').each do |file|
|
11
|
+
require file
|
12
|
+
end
|
13
|
+
|
14
|
+
RSpec.configure do |config|
|
15
|
+
config.filter_run_excluding :broken => true
|
16
|
+
|
17
|
+
config.before(:all) do
|
18
|
+
# TODO: Fully setup global environment
|
19
|
+
end
|
20
|
+
|
21
|
+
config.after(:all) do
|
22
|
+
# TODO: Fully setup global environment
|
23
|
+
end
|
24
|
+
|
25
|
+
config.before(:suite) do
|
26
|
+
# TODO: Setup test project
|
27
|
+
end
|
28
|
+
|
29
|
+
config.after(:suite) do
|
30
|
+
# TODO: Delete test project
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
SimpleCov.formatter = SimpleCov::Formatter::MultiFormatter[
|
35
|
+
SimpleCov::Formatter::HTMLFormatter
|
36
|
+
]
|
37
|
+
|
38
|
+
SimpleCov.start do
|
39
|
+
add_filter 'test/'
|
40
|
+
end
|
metadata
ADDED
@@ -0,0 +1,291 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: jetel
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.3
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Tomas Korcak
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2015-11-03 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: activesupport
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ">="
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ">="
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: gli
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: json_pure
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">="
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: multi_json
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ">="
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
type: :runtime
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - ">="
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '0'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: nokogiri
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - ">="
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '0'
|
76
|
+
type: :runtime
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - ">="
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '0'
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: pg
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - ">="
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '0'
|
90
|
+
type: :runtime
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - ">="
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '0'
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: pmap
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - ">="
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '0'
|
104
|
+
type: :runtime
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - ">="
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: '0'
|
111
|
+
- !ruby/object:Gem::Dependency
|
112
|
+
name: terminal-table
|
113
|
+
requirement: !ruby/object:Gem::Requirement
|
114
|
+
requirements:
|
115
|
+
- - ">="
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: '0'
|
118
|
+
type: :runtime
|
119
|
+
prerelease: false
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
121
|
+
requirements:
|
122
|
+
- - ">="
|
123
|
+
- !ruby/object:Gem::Version
|
124
|
+
version: '0'
|
125
|
+
- !ruby/object:Gem::Dependency
|
126
|
+
name: zip
|
127
|
+
requirement: !ruby/object:Gem::Requirement
|
128
|
+
requirements:
|
129
|
+
- - ">="
|
130
|
+
- !ruby/object:Gem::Version
|
131
|
+
version: '0'
|
132
|
+
type: :runtime
|
133
|
+
prerelease: false
|
134
|
+
version_requirements: !ruby/object:Gem::Requirement
|
135
|
+
requirements:
|
136
|
+
- - ">="
|
137
|
+
- !ruby/object:Gem::Version
|
138
|
+
version: '0'
|
139
|
+
- !ruby/object:Gem::Dependency
|
140
|
+
name: bundler
|
141
|
+
requirement: !ruby/object:Gem::Requirement
|
142
|
+
requirements:
|
143
|
+
- - "~>"
|
144
|
+
- !ruby/object:Gem::Version
|
145
|
+
version: '1.5'
|
146
|
+
type: :development
|
147
|
+
prerelease: false
|
148
|
+
version_requirements: !ruby/object:Gem::Requirement
|
149
|
+
requirements:
|
150
|
+
- - "~>"
|
151
|
+
- !ruby/object:Gem::Version
|
152
|
+
version: '1.5'
|
153
|
+
- !ruby/object:Gem::Dependency
|
154
|
+
name: rake
|
155
|
+
requirement: !ruby/object:Gem::Requirement
|
156
|
+
requirements:
|
157
|
+
- - ">="
|
158
|
+
- !ruby/object:Gem::Version
|
159
|
+
version: '0'
|
160
|
+
type: :development
|
161
|
+
prerelease: false
|
162
|
+
version_requirements: !ruby/object:Gem::Requirement
|
163
|
+
requirements:
|
164
|
+
- - ">="
|
165
|
+
- !ruby/object:Gem::Version
|
166
|
+
version: '0'
|
167
|
+
- !ruby/object:Gem::Dependency
|
168
|
+
name: coveralls
|
169
|
+
requirement: !ruby/object:Gem::Requirement
|
170
|
+
requirements:
|
171
|
+
- - ">="
|
172
|
+
- !ruby/object:Gem::Version
|
173
|
+
version: '0'
|
174
|
+
type: :development
|
175
|
+
prerelease: false
|
176
|
+
version_requirements: !ruby/object:Gem::Requirement
|
177
|
+
requirements:
|
178
|
+
- - ">="
|
179
|
+
- !ruby/object:Gem::Version
|
180
|
+
version: '0'
|
181
|
+
- !ruby/object:Gem::Dependency
|
182
|
+
name: rspec
|
183
|
+
requirement: !ruby/object:Gem::Requirement
|
184
|
+
requirements:
|
185
|
+
- - ">="
|
186
|
+
- !ruby/object:Gem::Version
|
187
|
+
version: '0'
|
188
|
+
type: :development
|
189
|
+
prerelease: false
|
190
|
+
version_requirements: !ruby/object:Gem::Requirement
|
191
|
+
requirements:
|
192
|
+
- - ">="
|
193
|
+
- !ruby/object:Gem::Version
|
194
|
+
version: '0'
|
195
|
+
- !ruby/object:Gem::Dependency
|
196
|
+
name: rubocop
|
197
|
+
requirement: !ruby/object:Gem::Requirement
|
198
|
+
requirements:
|
199
|
+
- - ">="
|
200
|
+
- !ruby/object:Gem::Version
|
201
|
+
version: '0'
|
202
|
+
type: :development
|
203
|
+
prerelease: false
|
204
|
+
version_requirements: !ruby/object:Gem::Requirement
|
205
|
+
requirements:
|
206
|
+
- - ">="
|
207
|
+
- !ruby/object:Gem::Version
|
208
|
+
version: '0'
|
209
|
+
- !ruby/object:Gem::Dependency
|
210
|
+
name: simplecov
|
211
|
+
requirement: !ruby/object:Gem::Requirement
|
212
|
+
requirements:
|
213
|
+
- - ">="
|
214
|
+
- !ruby/object:Gem::Version
|
215
|
+
version: '0'
|
216
|
+
type: :development
|
217
|
+
prerelease: false
|
218
|
+
version_requirements: !ruby/object:Gem::Requirement
|
219
|
+
requirements:
|
220
|
+
- - ">="
|
221
|
+
- !ruby/object:Gem::Version
|
222
|
+
version: '0'
|
223
|
+
description: Jetel - Custom made ETL for specific needs
|
224
|
+
email:
|
225
|
+
- korczis@gmail.com
|
226
|
+
executables:
|
227
|
+
- jetel
|
228
|
+
extensions: []
|
229
|
+
extra_rdoc_files: []
|
230
|
+
files:
|
231
|
+
- ".gitignore"
|
232
|
+
- ".idea/vcs.xml"
|
233
|
+
- ".rspec"
|
234
|
+
- ".rubocop.yml"
|
235
|
+
- Gemfile
|
236
|
+
- Gemfile.lock
|
237
|
+
- LICENSE
|
238
|
+
- LICENSE.txt
|
239
|
+
- README.md
|
240
|
+
- Rakefile
|
241
|
+
- bin/jetel
|
242
|
+
- data/.gitkeepme
|
243
|
+
- jetel.gemspec
|
244
|
+
- lib/jetel.rb
|
245
|
+
- lib/jetel/cli/app.rb
|
246
|
+
- lib/jetel/cli/cli.rb
|
247
|
+
- lib/jetel/cli/cmd/config_cmd.rb
|
248
|
+
- lib/jetel/cli/cmd/modules_cmd.rb
|
249
|
+
- lib/jetel/cli/cmd/version_cmd.rb
|
250
|
+
- lib/jetel/cli/shared.rb
|
251
|
+
- lib/jetel/config/config.rb
|
252
|
+
- lib/jetel/data/.gitignore
|
253
|
+
- lib/jetel/downloader/backends/aria.rb
|
254
|
+
- lib/jetel/downloader/backends/base_downloader.rb
|
255
|
+
- lib/jetel/downloader/downloader.rb
|
256
|
+
- lib/jetel/etl/etl.rb
|
257
|
+
- lib/jetel/extensions/extensions.rb
|
258
|
+
- lib/jetel/extensions/kernel.rb
|
259
|
+
- lib/jetel/jetel/jetel.rb
|
260
|
+
- lib/jetel/lib.rb
|
261
|
+
- lib/jetel/modules/ip/ip.rb
|
262
|
+
- lib/jetel/modules/module.rb
|
263
|
+
- lib/jetel/modules/modules.rb
|
264
|
+
- lib/jetel/version.rb
|
265
|
+
- test/spec_helper.rb
|
266
|
+
homepage: https://github.com/korczis/jetel
|
267
|
+
licenses:
|
268
|
+
- MIT
|
269
|
+
metadata: {}
|
270
|
+
post_install_message:
|
271
|
+
rdoc_options: []
|
272
|
+
require_paths:
|
273
|
+
- lib
|
274
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
275
|
+
requirements:
|
276
|
+
- - ">="
|
277
|
+
- !ruby/object:Gem::Version
|
278
|
+
version: '0'
|
279
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
280
|
+
requirements:
|
281
|
+
- - ">="
|
282
|
+
- !ruby/object:Gem::Version
|
283
|
+
version: '0'
|
284
|
+
requirements: []
|
285
|
+
rubyforge_project:
|
286
|
+
rubygems_version: 2.4.6
|
287
|
+
signing_key:
|
288
|
+
specification_version: 4
|
289
|
+
summary: Jetel
|
290
|
+
test_files:
|
291
|
+
- test/spec_helper.rb
|