jetel 0.0.5 → 0.0.6
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile.lock +4 -1
- data/README.md +16 -7
- data/docks/postgresql/pull.sh +3 -0
- data/docks/postgresql/run.sh +3 -0
- data/jetel.gemspec +2 -0
- data/lib/jetel/cli/app.rb +1 -3
- data/lib/jetel/cli/cli.rb +0 -2
- data/lib/jetel/cli/cmd/config_cmd.rb +0 -3
- data/lib/jetel/cli/cmd/modules_cmd.rb +2 -5
- data/lib/jetel/cli/cmd/version_cmd.rb +0 -4
- data/lib/jetel/cli/shared.rb +38 -8
- data/lib/jetel/downloader/backends/aria.rb +1 -1
- data/lib/jetel/downloader/backends/base_downloader.rb +2 -1
- data/lib/jetel/helpers/general_helper.rb +18 -2
- data/lib/jetel/loaders/loader.rb +18 -0
- data/lib/jetel/loaders/loaders.rb +42 -0
- data/lib/jetel/loaders/pg/pg.rb +36 -0
- data/lib/jetel/modules/ip/ip.rb +56 -6
- data/lib/jetel/modules/iso3166/iso3166.rb +82 -0
- data/lib/jetel/modules/module.rb +63 -2
- data/lib/jetel/modules/nga/nga.rb +93 -6
- data/lib/jetel/modules/sfpd/sfpd.rb +5 -7
- data/lib/jetel/version.rb +1 -1
- metadata +35 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 51f8e1d58f1480832113887664c59bc1fee6f142
|
4
|
+
data.tar.gz: c144f84c9664e281299bdc9016e81411a5f6c7b3
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 8fde78020fdcadbf6b6eb6fe5a172607c3177f3a2099ee0de095748d7896d1811c0cf8c27ab9cef16fa2c67a244367c689aafa5b8cf681629949b320777b7f5e
|
7
|
+
data.tar.gz: fecde5a4930c26d987a4ff14b93755eb6401304f52944b3cf2ffb2e81fe82bdaf5d867c38f1ebd741def9c9b9fa7305bcd93541c80ac7ffe64ba541d58da698e
|
data/Gemfile.lock
CHANGED
@@ -1,14 +1,16 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
jetel (0.0.
|
4
|
+
jetel (0.0.6)
|
5
5
|
activesupport
|
6
6
|
gli
|
7
|
+
i18n
|
7
8
|
json_pure
|
8
9
|
multi_json
|
9
10
|
nokogiri
|
10
11
|
pg
|
11
12
|
pmap
|
13
|
+
rubyzip
|
12
14
|
terminal-table
|
13
15
|
zip
|
14
16
|
|
@@ -78,6 +80,7 @@ GEM
|
|
78
80
|
rainbow (>= 1.99.1, < 3.0)
|
79
81
|
ruby-progressbar (~> 1.4)
|
80
82
|
ruby-progressbar (1.7.5)
|
83
|
+
rubyzip (1.1.7)
|
81
84
|
simplecov (0.10.0)
|
82
85
|
docile (~> 1.1.0)
|
83
86
|
json (~> 1.8)
|
data/README.md
CHANGED
@@ -5,9 +5,9 @@
|
|
5
5
|
Run `jetel`
|
6
6
|
|
7
7
|
```
|
8
|
-
|
8
|
+
$ jetel
|
9
9
|
NAME
|
10
|
-
jetel - Jetel CLI 0.0.
|
10
|
+
jetel - Jetel CLI 0.0.5
|
11
11
|
|
12
12
|
SYNOPSIS
|
13
13
|
jetel [global options] command [command options] [arguments...]
|
@@ -16,10 +16,13 @@ GLOBAL OPTIONS
|
|
16
16
|
--help - Show this message
|
17
17
|
|
18
18
|
COMMANDS
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
19
|
+
config - Show config
|
20
|
+
help - Shows a list of commands or help for one command
|
21
|
+
ip, Ip - Module ip
|
22
|
+
modules - Print modules info
|
23
|
+
nga, Nga - Module nga
|
24
|
+
sfpd, Sfpd - Module sfpd
|
25
|
+
version - Print version info
|
23
26
|
```
|
24
27
|
|
25
28
|
## Structure
|
@@ -31,11 +34,17 @@ COMMANDS
|
|
31
34
|
│ └── jetel
|
32
35
|
│ ├── cli
|
33
36
|
│ │ └── cmd
|
37
|
+
│ ├── config
|
34
38
|
│ ├── downloader
|
39
|
+
│ │ └── backends
|
35
40
|
│ ├── etl
|
36
41
|
│ ├── extensions
|
42
|
+
│ ├── helpers
|
43
|
+
│ ├── jetel
|
37
44
|
│ └── modules
|
38
|
-
│
|
45
|
+
│ ├── ip
|
46
|
+
│ ├── nga
|
47
|
+
│ └── sfpd
|
39
48
|
├── pkg
|
40
49
|
└── test
|
41
50
|
```
|
data/jetel.gemspec
CHANGED
@@ -21,11 +21,13 @@ Gem::Specification.new do |spec|
|
|
21
21
|
|
22
22
|
spec.add_dependency 'activesupport'
|
23
23
|
spec.add_dependency 'gli'
|
24
|
+
spec.add_dependency 'i18n'
|
24
25
|
spec.add_dependency 'json_pure'
|
25
26
|
spec.add_dependency 'multi_json'
|
26
27
|
spec.add_dependency 'nokogiri'
|
27
28
|
spec.add_dependency 'pg'
|
28
29
|
spec.add_dependency 'pmap'
|
30
|
+
spec.add_dependency 'rubyzip'
|
29
31
|
spec.add_dependency 'terminal-table'
|
30
32
|
spec.add_dependency 'zip'
|
31
33
|
|
data/lib/jetel/cli/app.rb
CHANGED
data/lib/jetel/cli/cli.rb
CHANGED
@@ -1,10 +1,7 @@
|
|
1
1
|
# encoding: utf-8
|
2
2
|
|
3
|
-
require 'gli'
|
4
3
|
require 'terminal-table'
|
5
4
|
|
6
|
-
include GLI::App
|
7
|
-
|
8
5
|
require_relative '../../version'
|
9
6
|
|
10
7
|
require_relative '../shared'
|
@@ -55,8 +52,8 @@ def register_module(m)
|
|
55
52
|
|
56
53
|
action_name = k
|
57
54
|
action_description = v || "#{action_name} #{module_name}"
|
58
|
-
register_module_action(c, m, action_name, action_description) do
|
59
|
-
module_instance.send(k)
|
55
|
+
register_module_action(c, m, action_name, action_description) do |global_options, options, args|
|
56
|
+
module_instance.send(k, global_options, options, args)
|
60
57
|
end
|
61
58
|
end
|
62
59
|
end
|
data/lib/jetel/cli/shared.rb
CHANGED
@@ -1,12 +1,42 @@
|
|
1
1
|
# encoding: UTF-8
|
2
2
|
|
3
3
|
require 'gli'
|
4
|
+
require 'pathname'
|
4
5
|
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
6
|
+
require_relative '../version'
|
7
|
+
|
8
|
+
require_relative '../downloader/backends/base_downloader'
|
9
|
+
|
10
|
+
include GLI::App
|
11
|
+
|
12
|
+
# Program description
|
13
|
+
program_desc 'Simple custom made tool for data download and basic ETL'
|
14
|
+
|
15
|
+
# Version info
|
16
|
+
version Jetel::VERSION
|
17
|
+
|
18
|
+
# Download directory
|
19
|
+
desc 'Download directory'
|
20
|
+
default_value 'data' # File.absolute_path(File.join(File.dirname(__FILE__), '..', '..', '..', 'data'))
|
21
|
+
arg_name 'download-dir'
|
22
|
+
flag [:d, :download_dir]
|
23
|
+
|
24
|
+
# Download timeout
|
25
|
+
desc 'Download timeout'
|
26
|
+
default_value Jetel::Downloaders::BaseDownloader::OPTS_DOWNLOAD[:timeout]
|
27
|
+
arg_name 'download-timeout'
|
28
|
+
flag [:t, :timeout]
|
29
|
+
|
30
|
+
# Data loader
|
31
|
+
desc 'Data Loader'
|
32
|
+
default_value 'pg://jetel:jetel@localhost:5432/jetel'
|
33
|
+
|
34
|
+
arg_name 'data-loader'
|
35
|
+
flag [:l, :data_loader]
|
36
|
+
|
37
|
+
# On Error handler
|
38
|
+
on_error do |exception|
|
39
|
+
pp exception.backtrace
|
40
|
+
pp exception
|
41
|
+
true
|
42
|
+
end
|
@@ -2,11 +2,27 @@
|
|
2
2
|
|
3
3
|
require_relative '../config/config'
|
4
4
|
|
5
|
+
require_relative '../loaders/loaders'
|
6
|
+
|
7
|
+
require 'i18n'
|
8
|
+
|
5
9
|
module Jetel
|
6
10
|
module Helper
|
7
11
|
class << self
|
8
|
-
def target_dir(modul, source)
|
9
|
-
|
12
|
+
def target_dir(modul, dir, source)
|
13
|
+
klass = I18n.transliterate(modul.class.name.split('::').last).gsub(/[^0-9a-z_\-]/i, '_')
|
14
|
+
source_name = I18n.transliterate(source[:name]).gsub(/[^0-9a-z_\-]/i, '_')
|
15
|
+
File.join(dir || Config[:DATA_DIRECTORY], klass, source_name)
|
16
|
+
end
|
17
|
+
|
18
|
+
def get_loader(uri)
|
19
|
+
loaders = Loaders.loaders
|
20
|
+
loader_schema = uri.split(':/').first.downcase
|
21
|
+
res = loaders.find do |loader|
|
22
|
+
loader[:class_name].downcase === loader_schema
|
23
|
+
end
|
24
|
+
|
25
|
+
res[:klass].new(uri)
|
10
26
|
end
|
11
27
|
end
|
12
28
|
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
require 'active_support/inflector'
|
2
|
+
require_relative '../extensions/extensions'
|
3
|
+
|
4
|
+
module Jetel
|
5
|
+
module Loaders
|
6
|
+
class << self
|
7
|
+
def loaders(loaders_dir = File.join(File.dirname(__FILE__)), auto_require = true, require_only = false)
|
8
|
+
dir = File.absolute_path(loaders_dir)
|
9
|
+
res = Dir.entries(dir).map do |entry|
|
10
|
+
dir_path = File.join(dir, entry)
|
11
|
+
|
12
|
+
next unless File.directory?(dir_path)
|
13
|
+
next if entry == '.' || entry == '..'
|
14
|
+
|
15
|
+
full_path = File.join(dir_path, "#{entry}.rb")
|
16
|
+
next unless File.exist?(full_path)
|
17
|
+
|
18
|
+
# Require file if auto_require true
|
19
|
+
require(full_path) if auto_require
|
20
|
+
|
21
|
+
# Go to next file if require_only mode
|
22
|
+
next if require_only
|
23
|
+
|
24
|
+
class_name = entry.camelize
|
25
|
+
qualified_class_name = "Jetel::Loaders::#{class_name}"
|
26
|
+
|
27
|
+
# Return value from map iteration
|
28
|
+
{
|
29
|
+
name: entry,
|
30
|
+
path: full_path,
|
31
|
+
class_name: class_name,
|
32
|
+
qualified_class_name: qualified_class_name,
|
33
|
+
klass: auto_require ? Kernel.qualified_const_get(qualified_class_name) : nil
|
34
|
+
}
|
35
|
+
end
|
36
|
+
|
37
|
+
# Remove nil values and return
|
38
|
+
res.compact
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
require_relative '../loader'
|
2
|
+
|
3
|
+
require 'pg'
|
4
|
+
|
5
|
+
module Jetel
|
6
|
+
module Loaders
|
7
|
+
class Pg < Loader
|
8
|
+
def initialize(uri)
|
9
|
+
super
|
10
|
+
|
11
|
+
tmp = uri.split('://')
|
12
|
+
tmp = tmp[1].split('@')
|
13
|
+
|
14
|
+
parts = tmp[0].split(':')
|
15
|
+
user = parts[0]
|
16
|
+
password = parts[1]
|
17
|
+
|
18
|
+
parts = tmp[1].split('/')
|
19
|
+
host, port = parts[0].split(':')
|
20
|
+
dbname = parts[1]
|
21
|
+
|
22
|
+
opts = {
|
23
|
+
:host => host,
|
24
|
+
:port => port.to_i,
|
25
|
+
# :options => '',
|
26
|
+
# :tty => '',
|
27
|
+
:dbname => dbname,
|
28
|
+
:user => user,
|
29
|
+
:password => password
|
30
|
+
}
|
31
|
+
|
32
|
+
@client = PG.connect(opts)
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
data/lib/jetel/modules/ip/ip.rb
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
# encoding: utf-8
|
2
2
|
|
3
3
|
require 'pmap'
|
4
|
+
require 'csv'
|
4
5
|
|
5
6
|
require_relative '../../config/config'
|
6
7
|
require_relative '../../helpers/helpers'
|
@@ -36,20 +37,69 @@ module Jetel
|
|
36
37
|
}
|
37
38
|
]
|
38
39
|
|
39
|
-
def download
|
40
|
+
def download(global_options, options, args)
|
40
41
|
SOURCES.pmap do |source|
|
41
|
-
|
42
|
-
downloader.download(source[:url], {:dir => target_dir})
|
42
|
+
download_source(source, global_options.merge(options))
|
43
43
|
end
|
44
44
|
end
|
45
45
|
|
46
|
-
def extract
|
46
|
+
def extract(global_options, options, args)
|
47
|
+
SOURCES.pmap do |source|
|
48
|
+
downloaded_file = downloaded_file(source, global_options.merge(options))
|
49
|
+
dest_dir = extract_dir(source, global_options.merge(options))
|
50
|
+
|
51
|
+
puts "Extracting #{downloaded_file}"
|
52
|
+
|
53
|
+
FileUtils.mkdir_p(dest_dir)
|
54
|
+
FileUtils.cp(downloaded_file, dest_dir)
|
55
|
+
end
|
47
56
|
end
|
48
57
|
|
49
|
-
def transform
|
58
|
+
def transform(global_options, options, args)
|
59
|
+
SOURCES.pmap do |source|
|
60
|
+
opts = global_options.merge(options)
|
61
|
+
|
62
|
+
extracted_file = extracted_file(source, opts)
|
63
|
+
transformed_file = transformed_file(source, opts)
|
64
|
+
|
65
|
+
FileUtils.mkdir_p(transform_dir(source, opts))
|
66
|
+
|
67
|
+
csv_opts = {
|
68
|
+
:col_sep => '|'
|
69
|
+
}
|
70
|
+
|
71
|
+
puts "Transforming #{extracted_file}"
|
72
|
+
CSV.open(extracted_file, 'r', csv_opts) do |csv|
|
73
|
+
headers = %w(
|
74
|
+
registry
|
75
|
+
cc
|
76
|
+
type
|
77
|
+
start
|
78
|
+
value
|
79
|
+
date
|
80
|
+
status
|
81
|
+
)
|
82
|
+
CSV.open(transformed_file, 'w', :write_headers => true, :headers => headers) do |csv_out|
|
83
|
+
csv.each do |row|
|
84
|
+
next if row.length < 7
|
85
|
+
next if row[0] != source[:name]
|
86
|
+
csv_out << row
|
87
|
+
end
|
88
|
+
end
|
89
|
+
end
|
90
|
+
end
|
50
91
|
end
|
51
92
|
|
52
|
-
def load
|
93
|
+
def load(global_options, options, args)
|
94
|
+
SOURCES.map do |source|
|
95
|
+
opts = global_options.merge(options)
|
96
|
+
|
97
|
+
transformed_file = transformed_file(source, opts)
|
98
|
+
|
99
|
+
loader = Helper.get_loader(opts['data_loader'])
|
100
|
+
|
101
|
+
loader.load(self, source, transformed_file, opts)
|
102
|
+
end
|
53
103
|
end
|
54
104
|
end
|
55
105
|
end
|
@@ -0,0 +1,82 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
3
|
+
require 'pmap'
|
4
|
+
require 'zip'
|
5
|
+
require 'csv'
|
6
|
+
|
7
|
+
require_relative '../../config/config'
|
8
|
+
require_relative '../../modules/module'
|
9
|
+
|
10
|
+
module Jetel
|
11
|
+
module Modules
|
12
|
+
class Iso3166 < Module
|
13
|
+
SOURCES = [
|
14
|
+
{
|
15
|
+
name: 'iso366',
|
16
|
+
filename_extracted: 'IP2LOCATION-ISO3166-2.CSV',
|
17
|
+
filename_transformed: 'IP2LOCATION-ISO3166-2.CSV',
|
18
|
+
url: 'http://www.ip2location.com/downloads/Hi3sL9bnXfe/IP2LOCATION-ISO3166-2.ZIP'
|
19
|
+
}
|
20
|
+
]
|
21
|
+
|
22
|
+
def download(global_options, options, args)
|
23
|
+
SOURCES.pmap do |source|
|
24
|
+
download_source(source, global_options.merge(options))
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
def extract(global_options, options, args)
|
29
|
+
SOURCES.pmap do |source|
|
30
|
+
downloaded_file = downloaded_file(source, global_options.merge(options))
|
31
|
+
dest_dir = extract_dir(source, global_options.merge(options))
|
32
|
+
|
33
|
+
FileUtils.mkdir_p(dest_dir)
|
34
|
+
|
35
|
+
Zip::ZipFile.open(downloaded_file) do |zip_file|
|
36
|
+
# Handle entries one by one
|
37
|
+
zip_file.each do |entry|
|
38
|
+
# Extract to file/directory/symlink
|
39
|
+
puts "Extracting #{entry.name}"
|
40
|
+
dest_file = File.join(dest_dir, entry.name.split('/').last)
|
41
|
+
FileUtils.rm_rf(dest_file)
|
42
|
+
entry.extract(dest_file)
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
def transform(global_options, options, args)
|
49
|
+
SOURCES.pmap do |source|
|
50
|
+
opts = global_options.merge(options)
|
51
|
+
|
52
|
+
extracted_file = extracted_file(source, opts)
|
53
|
+
transformed_file = transformed_file(source, opts)
|
54
|
+
|
55
|
+
FileUtils.mkdir_p(transform_dir(source, opts))
|
56
|
+
|
57
|
+
csv_opts = {
|
58
|
+
:headers => true
|
59
|
+
}
|
60
|
+
|
61
|
+
puts "Transforming #{extracted_file}"
|
62
|
+
CSV.open(extracted_file, 'r', csv_opts) do |csv|
|
63
|
+
headers = %w(
|
64
|
+
country_code
|
65
|
+
subdivision_name
|
66
|
+
code
|
67
|
+
)
|
68
|
+
CSV.open(transformed_file, 'w', :write_headers => true, :headers => headers) do |csv_out|
|
69
|
+
csv.each do |row|
|
70
|
+
next if row.length < 3
|
71
|
+
csv_out << row
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
def load(global_options, options, args)
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
data/lib/jetel/modules/module.rb
CHANGED
@@ -1,18 +1,79 @@
|
|
1
1
|
# encoding: utf-8
|
2
2
|
|
3
3
|
require_relative '../downloader/downloader'
|
4
|
+
require_relative '../helpers/helpers'
|
4
5
|
|
5
6
|
module Jetel
|
6
7
|
module Modules
|
7
8
|
class Module
|
8
9
|
attr_reader :downloader
|
9
10
|
|
11
|
+
class << self
|
12
|
+
def target_dir(modul, source, dir, *path)
|
13
|
+
klass = I18n.transliterate(modul.class.name.split('::').last).gsub(/[^0-9a-z_\-]/i, '_')
|
14
|
+
source_name = I18n.transliterate(source[:name]).gsub(/[^0-9a-z_\-]/i, '_')
|
15
|
+
File.join(dir.kind_of?(String) ? dir : dir['download_dir'] || Config[:DATA_DIRECTORY], klass, source_name, path)
|
16
|
+
end
|
17
|
+
|
18
|
+
def download_dir(modul, source, opts)
|
19
|
+
Module.target_dir(modul, source, opts, 'downloaded')
|
20
|
+
end
|
21
|
+
|
22
|
+
def extract_dir(modul, source, opts)
|
23
|
+
Module.target_dir(modul, source, opts, 'extracted')
|
24
|
+
end
|
25
|
+
|
26
|
+
def transform_dir(modul, source, opts)
|
27
|
+
Module.target_dir(modul, source, opts, 'transformed')
|
28
|
+
end
|
29
|
+
|
30
|
+
def downloaded_file(modul, source, opts)
|
31
|
+
File.join(download_dir(modul, source, opts), source[:filename_downloaded] || source[:url].split('/').last)
|
32
|
+
end
|
33
|
+
|
34
|
+
def extracted_file(modul, source, opts)
|
35
|
+
File.join(extract_dir(modul, source, opts), source[:filename_extracted] || source[:url].split('/').last)
|
36
|
+
end
|
37
|
+
|
38
|
+
def transformed_file(modul, source, opts)
|
39
|
+
File.join(transform_dir(modul, source, opts), source[:filename_transformed] || source[:url].split('/').last)
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
10
43
|
def initialize
|
11
44
|
@downloader = Downloader.new
|
12
45
|
end
|
13
46
|
|
14
|
-
def
|
15
|
-
downloader.download(
|
47
|
+
def download_source(source, opts)
|
48
|
+
downloader.download(source[:url], {:dir => download_dir(source, opts)})
|
49
|
+
end
|
50
|
+
|
51
|
+
def target_dir(source, opts, *path)
|
52
|
+
Module.target_dir(self, source, opts['download_dir'], *path)
|
53
|
+
end
|
54
|
+
|
55
|
+
def download_dir(source, opts)
|
56
|
+
Module.download_dir(self, source, opts)
|
57
|
+
end
|
58
|
+
|
59
|
+
def extract_dir(source, opts)
|
60
|
+
Module.extract_dir(self, source, opts)
|
61
|
+
end
|
62
|
+
|
63
|
+
def transform_dir(source, opts)
|
64
|
+
Module.transform_dir(self, source, opts)
|
65
|
+
end
|
66
|
+
|
67
|
+
def downloaded_file(source, opts)
|
68
|
+
Module.downloaded_file(self, source, opts)
|
69
|
+
end
|
70
|
+
|
71
|
+
def extracted_file(source, opts)
|
72
|
+
Module.extracted_file(self, source, opts)
|
73
|
+
end
|
74
|
+
|
75
|
+
def transformed_file(source, opts)
|
76
|
+
Module.transformed_file(self, source, opts)
|
16
77
|
end
|
17
78
|
end
|
18
79
|
end
|
@@ -1038,20 +1038,107 @@ module Jetel
|
|
1038
1038
|
}
|
1039
1039
|
]
|
1040
1040
|
|
1041
|
-
def download
|
1041
|
+
def download(global_options, options, args)
|
1042
1042
|
SOURCES.pmap do |source|
|
1043
|
-
|
1044
|
-
downloader.download(source[:url], {:dir => target_dir})
|
1043
|
+
download_source(source, global_options.merge(options))
|
1045
1044
|
end
|
1046
1045
|
end
|
1047
1046
|
|
1048
|
-
def extract
|
1047
|
+
def extract(global_options, options, args)
|
1048
|
+
SOURCES.pmap do |source|
|
1049
|
+
downloaded_file = downloaded_file(source, global_options.merge(options))
|
1050
|
+
dest_dir = extract_dir(source, global_options.merge(options))
|
1051
|
+
|
1052
|
+
FileUtils.mkdir_p(dest_dir)
|
1053
|
+
|
1054
|
+
Zip::ZipFile.open(downloaded_file) do |zip_file|
|
1055
|
+
# Handle entries one by one
|
1056
|
+
zip_file.each do |entry|
|
1057
|
+
# Extract to file/directory/symlink
|
1058
|
+
puts "Extracting #{entry.name}"
|
1059
|
+
dest_file = File.join(dest_dir, entry.name.split('/').last)
|
1060
|
+
FileUtils.rm_rf(dest_file)
|
1061
|
+
entry.extract(dest_file)
|
1062
|
+
end
|
1063
|
+
end
|
1064
|
+
end
|
1065
|
+
end
|
1066
|
+
|
1067
|
+
def transform(global_options, options, args)
|
1068
|
+
SOURCES.pmap do |source|
|
1069
|
+
opts = global_options.merge(options)
|
1070
|
+
|
1071
|
+
extracted_file = extracted_file(source, opts)
|
1072
|
+
transformed_file = transformed_file(source, opts)
|
1073
|
+
|
1074
|
+
FileUtils.mkdir_p(transform_dir(source, opts))
|
1075
|
+
|
1076
|
+
csv_opts = {
|
1077
|
+
:headers => true,
|
1078
|
+
:col_sep => "\t",
|
1079
|
+
:quote_char => "\x00"
|
1080
|
+
}
|
1081
|
+
|
1082
|
+
puts "Transforming #{extracted_file}"
|
1083
|
+
CSV.open(extracted_file, 'r', csv_opts) do |csv|
|
1084
|
+
headers = %w(
|
1085
|
+
RC
|
1086
|
+
UFI
|
1087
|
+
UNI
|
1088
|
+
LAT
|
1089
|
+
LONG
|
1090
|
+
DMS_LAT
|
1091
|
+
DMS_LONG
|
1092
|
+
MGRS
|
1093
|
+
JOG
|
1094
|
+
FC
|
1095
|
+
DSG
|
1096
|
+
PC
|
1097
|
+
CC1
|
1098
|
+
ADM1
|
1099
|
+
POP
|
1100
|
+
ELEV
|
1101
|
+
CC2
|
1102
|
+
NT
|
1103
|
+
LC
|
1104
|
+
SHORT_FORM
|
1105
|
+
GENERIC
|
1106
|
+
SORT_NAME_RO
|
1107
|
+
FULL_NAME_RO
|
1108
|
+
FULL_NAME_ND_RO
|
1109
|
+
SORT_NAME_RG
|
1110
|
+
FULL_NAME_RG
|
1111
|
+
FULL_NAME_ND_RG
|
1112
|
+
NOTE
|
1113
|
+
MODIFY_DATE
|
1114
|
+
DISPLAY
|
1115
|
+
NAME_RANK
|
1116
|
+
NAME_LINK
|
1117
|
+
TRANSL_CD
|
1118
|
+
NM_MODIFY_DATE
|
1119
|
+
F_EFCTV_DT
|
1120
|
+
F_TERM_DT
|
1121
|
+
)
|
1122
|
+
CSV.open(transformed_file, 'w', :write_headers => true, :headers => headers, :quote_char => '"') do |csv_out|
|
1123
|
+
csv.each do |row|
|
1124
|
+
csv_out << row
|
1125
|
+
end
|
1126
|
+
end
|
1127
|
+
end
|
1128
|
+
end
|
1129
|
+
end
|
1130
|
+
|
1131
|
+
def load(global_options, options, args)
|
1049
1132
|
end
|
1050
1133
|
|
1051
|
-
def
|
1134
|
+
def extracted_file(source, opts)
|
1135
|
+
res = super(source, opts)
|
1136
|
+
res.gsub('.zip', '.txt')
|
1052
1137
|
end
|
1053
1138
|
|
1054
|
-
def
|
1139
|
+
def transformed_file(source, opts)
|
1140
|
+
res = super(source, opts)
|
1141
|
+
res.gsub('.zip', '.txt')
|
1055
1142
|
end
|
1056
1143
|
end
|
1057
1144
|
end
|
@@ -3,7 +3,6 @@
|
|
3
3
|
require 'pmap'
|
4
4
|
|
5
5
|
require_relative '../../config/config'
|
6
|
-
require_relative '../../helpers/helpers'
|
7
6
|
require_relative '../../modules/module'
|
8
7
|
|
9
8
|
module Jetel
|
@@ -16,20 +15,19 @@ module Jetel
|
|
16
15
|
}
|
17
16
|
]
|
18
17
|
|
19
|
-
def download
|
18
|
+
def download(global_options, options, args)
|
20
19
|
SOURCES.pmap do |source|
|
21
|
-
|
22
|
-
downloader.download(source[:url], {:dir => target_dir})
|
20
|
+
download_source(source, global_options.merge(options))
|
23
21
|
end
|
24
22
|
end
|
25
23
|
|
26
|
-
def extract
|
24
|
+
def extract(global_options, options, args)
|
27
25
|
end
|
28
26
|
|
29
|
-
def transform
|
27
|
+
def transform(global_options, options, args)
|
30
28
|
end
|
31
29
|
|
32
|
-
def load
|
30
|
+
def load(global_options, options, args)
|
33
31
|
end
|
34
32
|
end
|
35
33
|
end
|
data/lib/jetel/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: jetel
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Tomas Korcak
|
@@ -38,6 +38,20 @@ dependencies:
|
|
38
38
|
- - ">="
|
39
39
|
- !ruby/object:Gem::Version
|
40
40
|
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: i18n
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">="
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
41
55
|
- !ruby/object:Gem::Dependency
|
42
56
|
name: json_pure
|
43
57
|
requirement: !ruby/object:Gem::Requirement
|
@@ -108,6 +122,20 @@ dependencies:
|
|
108
122
|
- - ">="
|
109
123
|
- !ruby/object:Gem::Version
|
110
124
|
version: '0'
|
125
|
+
- !ruby/object:Gem::Dependency
|
126
|
+
name: rubyzip
|
127
|
+
requirement: !ruby/object:Gem::Requirement
|
128
|
+
requirements:
|
129
|
+
- - ">="
|
130
|
+
- !ruby/object:Gem::Version
|
131
|
+
version: '0'
|
132
|
+
type: :runtime
|
133
|
+
prerelease: false
|
134
|
+
version_requirements: !ruby/object:Gem::Requirement
|
135
|
+
requirements:
|
136
|
+
- - ">="
|
137
|
+
- !ruby/object:Gem::Version
|
138
|
+
version: '0'
|
111
139
|
- !ruby/object:Gem::Dependency
|
112
140
|
name: terminal-table
|
113
141
|
requirement: !ruby/object:Gem::Requirement
|
@@ -240,6 +268,8 @@ files:
|
|
240
268
|
- Rakefile
|
241
269
|
- bin/jetel
|
242
270
|
- data/.gitkeepme
|
271
|
+
- docks/postgresql/pull.sh
|
272
|
+
- docks/postgresql/run.sh
|
243
273
|
- jetel.gemspec
|
244
274
|
- lib/jetel.rb
|
245
275
|
- lib/jetel/cli/app.rb
|
@@ -259,7 +289,11 @@ files:
|
|
259
289
|
- lib/jetel/helpers/helpers.rb
|
260
290
|
- lib/jetel/jetel/jetel.rb
|
261
291
|
- lib/jetel/lib.rb
|
292
|
+
- lib/jetel/loaders/loader.rb
|
293
|
+
- lib/jetel/loaders/loaders.rb
|
294
|
+
- lib/jetel/loaders/pg/pg.rb
|
262
295
|
- lib/jetel/modules/ip/ip.rb
|
296
|
+
- lib/jetel/modules/iso3166/iso3166.rb
|
263
297
|
- lib/jetel/modules/module.rb
|
264
298
|
- lib/jetel/modules/modules.rb
|
265
299
|
- lib/jetel/modules/nga/nga.rb
|