jetel 0.0.5 → 0.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile.lock +4 -1
- data/README.md +16 -7
- data/docks/postgresql/pull.sh +3 -0
- data/docks/postgresql/run.sh +3 -0
- data/jetel.gemspec +2 -0
- data/lib/jetel/cli/app.rb +1 -3
- data/lib/jetel/cli/cli.rb +0 -2
- data/lib/jetel/cli/cmd/config_cmd.rb +0 -3
- data/lib/jetel/cli/cmd/modules_cmd.rb +2 -5
- data/lib/jetel/cli/cmd/version_cmd.rb +0 -4
- data/lib/jetel/cli/shared.rb +38 -8
- data/lib/jetel/downloader/backends/aria.rb +1 -1
- data/lib/jetel/downloader/backends/base_downloader.rb +2 -1
- data/lib/jetel/helpers/general_helper.rb +18 -2
- data/lib/jetel/loaders/loader.rb +18 -0
- data/lib/jetel/loaders/loaders.rb +42 -0
- data/lib/jetel/loaders/pg/pg.rb +36 -0
- data/lib/jetel/modules/ip/ip.rb +56 -6
- data/lib/jetel/modules/iso3166/iso3166.rb +82 -0
- data/lib/jetel/modules/module.rb +63 -2
- data/lib/jetel/modules/nga/nga.rb +93 -6
- data/lib/jetel/modules/sfpd/sfpd.rb +5 -7
- data/lib/jetel/version.rb +1 -1
- metadata +35 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 51f8e1d58f1480832113887664c59bc1fee6f142
|
4
|
+
data.tar.gz: c144f84c9664e281299bdc9016e81411a5f6c7b3
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 8fde78020fdcadbf6b6eb6fe5a172607c3177f3a2099ee0de095748d7896d1811c0cf8c27ab9cef16fa2c67a244367c689aafa5b8cf681629949b320777b7f5e
|
7
|
+
data.tar.gz: fecde5a4930c26d987a4ff14b93755eb6401304f52944b3cf2ffb2e81fe82bdaf5d867c38f1ebd741def9c9b9fa7305bcd93541c80ac7ffe64ba541d58da698e
|
data/Gemfile.lock
CHANGED
@@ -1,14 +1,16 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
jetel (0.0.
|
4
|
+
jetel (0.0.6)
|
5
5
|
activesupport
|
6
6
|
gli
|
7
|
+
i18n
|
7
8
|
json_pure
|
8
9
|
multi_json
|
9
10
|
nokogiri
|
10
11
|
pg
|
11
12
|
pmap
|
13
|
+
rubyzip
|
12
14
|
terminal-table
|
13
15
|
zip
|
14
16
|
|
@@ -78,6 +80,7 @@ GEM
|
|
78
80
|
rainbow (>= 1.99.1, < 3.0)
|
79
81
|
ruby-progressbar (~> 1.4)
|
80
82
|
ruby-progressbar (1.7.5)
|
83
|
+
rubyzip (1.1.7)
|
81
84
|
simplecov (0.10.0)
|
82
85
|
docile (~> 1.1.0)
|
83
86
|
json (~> 1.8)
|
data/README.md
CHANGED
@@ -5,9 +5,9 @@
|
|
5
5
|
Run `jetel`
|
6
6
|
|
7
7
|
```
|
8
|
-
|
8
|
+
$ jetel
|
9
9
|
NAME
|
10
|
-
jetel - Jetel CLI 0.0.
|
10
|
+
jetel - Jetel CLI 0.0.5
|
11
11
|
|
12
12
|
SYNOPSIS
|
13
13
|
jetel [global options] command [command options] [arguments...]
|
@@ -16,10 +16,13 @@ GLOBAL OPTIONS
|
|
16
16
|
--help - Show this message
|
17
17
|
|
18
18
|
COMMANDS
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
19
|
+
config - Show config
|
20
|
+
help - Shows a list of commands or help for one command
|
21
|
+
ip, Ip - Module ip
|
22
|
+
modules - Print modules info
|
23
|
+
nga, Nga - Module nga
|
24
|
+
sfpd, Sfpd - Module sfpd
|
25
|
+
version - Print version info
|
23
26
|
```
|
24
27
|
|
25
28
|
## Structure
|
@@ -31,11 +34,17 @@ COMMANDS
|
|
31
34
|
│ └── jetel
|
32
35
|
│ ├── cli
|
33
36
|
│ │ └── cmd
|
37
|
+
│ ├── config
|
34
38
|
│ ├── downloader
|
39
|
+
│ │ └── backends
|
35
40
|
│ ├── etl
|
36
41
|
│ ├── extensions
|
42
|
+
│ ├── helpers
|
43
|
+
│ ├── jetel
|
37
44
|
│ └── modules
|
38
|
-
│
|
45
|
+
│ ├── ip
|
46
|
+
│ ├── nga
|
47
|
+
│ └── sfpd
|
39
48
|
├── pkg
|
40
49
|
└── test
|
41
50
|
```
|
data/jetel.gemspec
CHANGED
@@ -21,11 +21,13 @@ Gem::Specification.new do |spec|
|
|
21
21
|
|
22
22
|
spec.add_dependency 'activesupport'
|
23
23
|
spec.add_dependency 'gli'
|
24
|
+
spec.add_dependency 'i18n'
|
24
25
|
spec.add_dependency 'json_pure'
|
25
26
|
spec.add_dependency 'multi_json'
|
26
27
|
spec.add_dependency 'nokogiri'
|
27
28
|
spec.add_dependency 'pg'
|
28
29
|
spec.add_dependency 'pmap'
|
30
|
+
spec.add_dependency 'rubyzip'
|
29
31
|
spec.add_dependency 'terminal-table'
|
30
32
|
spec.add_dependency 'zip'
|
31
33
|
|
data/lib/jetel/cli/app.rb
CHANGED
data/lib/jetel/cli/cli.rb
CHANGED
@@ -1,10 +1,7 @@
|
|
1
1
|
# encoding: utf-8
|
2
2
|
|
3
|
-
require 'gli'
|
4
3
|
require 'terminal-table'
|
5
4
|
|
6
|
-
include GLI::App
|
7
|
-
|
8
5
|
require_relative '../../version'
|
9
6
|
|
10
7
|
require_relative '../shared'
|
@@ -55,8 +52,8 @@ def register_module(m)
|
|
55
52
|
|
56
53
|
action_name = k
|
57
54
|
action_description = v || "#{action_name} #{module_name}"
|
58
|
-
register_module_action(c, m, action_name, action_description) do
|
59
|
-
module_instance.send(k)
|
55
|
+
register_module_action(c, m, action_name, action_description) do |global_options, options, args|
|
56
|
+
module_instance.send(k, global_options, options, args)
|
60
57
|
end
|
61
58
|
end
|
62
59
|
end
|
data/lib/jetel/cli/shared.rb
CHANGED
@@ -1,12 +1,42 @@
|
|
1
1
|
# encoding: UTF-8
|
2
2
|
|
3
3
|
require 'gli'
|
4
|
+
require 'pathname'
|
4
5
|
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
6
|
+
require_relative '../version'
|
7
|
+
|
8
|
+
require_relative '../downloader/backends/base_downloader'
|
9
|
+
|
10
|
+
include GLI::App
|
11
|
+
|
12
|
+
# Program description
|
13
|
+
program_desc 'Simple custom made tool for data download and basic ETL'
|
14
|
+
|
15
|
+
# Version info
|
16
|
+
version Jetel::VERSION
|
17
|
+
|
18
|
+
# Download directory
|
19
|
+
desc 'Download directory'
|
20
|
+
default_value 'data' # File.absolute_path(File.join(File.dirname(__FILE__), '..', '..', '..', 'data'))
|
21
|
+
arg_name 'download-dir'
|
22
|
+
flag [:d, :download_dir]
|
23
|
+
|
24
|
+
# Download timeout
|
25
|
+
desc 'Download timeout'
|
26
|
+
default_value Jetel::Downloaders::BaseDownloader::OPTS_DOWNLOAD[:timeout]
|
27
|
+
arg_name 'download-timeout'
|
28
|
+
flag [:t, :timeout]
|
29
|
+
|
30
|
+
# Data loader
|
31
|
+
desc 'Data Loader'
|
32
|
+
default_value 'pg://jetel:jetel@localhost:5432/jetel'
|
33
|
+
|
34
|
+
arg_name 'data-loader'
|
35
|
+
flag [:l, :data_loader]
|
36
|
+
|
37
|
+
# On Error handler
|
38
|
+
on_error do |exception|
|
39
|
+
pp exception.backtrace
|
40
|
+
pp exception
|
41
|
+
true
|
42
|
+
end
|
@@ -2,11 +2,27 @@
|
|
2
2
|
|
3
3
|
require_relative '../config/config'
|
4
4
|
|
5
|
+
require_relative '../loaders/loaders'
|
6
|
+
|
7
|
+
require 'i18n'
|
8
|
+
|
5
9
|
module Jetel
|
6
10
|
module Helper
|
7
11
|
class << self
|
8
|
-
def target_dir(modul, source)
|
9
|
-
|
12
|
+
def target_dir(modul, dir, source)
|
13
|
+
klass = I18n.transliterate(modul.class.name.split('::').last).gsub(/[^0-9a-z_\-]/i, '_')
|
14
|
+
source_name = I18n.transliterate(source[:name]).gsub(/[^0-9a-z_\-]/i, '_')
|
15
|
+
File.join(dir || Config[:DATA_DIRECTORY], klass, source_name)
|
16
|
+
end
|
17
|
+
|
18
|
+
def get_loader(uri)
|
19
|
+
loaders = Loaders.loaders
|
20
|
+
loader_schema = uri.split(':/').first.downcase
|
21
|
+
res = loaders.find do |loader|
|
22
|
+
loader[:class_name].downcase === loader_schema
|
23
|
+
end
|
24
|
+
|
25
|
+
res[:klass].new(uri)
|
10
26
|
end
|
11
27
|
end
|
12
28
|
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
require 'active_support/inflector'
|
2
|
+
require_relative '../extensions/extensions'
|
3
|
+
|
4
|
+
module Jetel
|
5
|
+
module Loaders
|
6
|
+
class << self
|
7
|
+
def loaders(loaders_dir = File.join(File.dirname(__FILE__)), auto_require = true, require_only = false)
|
8
|
+
dir = File.absolute_path(loaders_dir)
|
9
|
+
res = Dir.entries(dir).map do |entry|
|
10
|
+
dir_path = File.join(dir, entry)
|
11
|
+
|
12
|
+
next unless File.directory?(dir_path)
|
13
|
+
next if entry == '.' || entry == '..'
|
14
|
+
|
15
|
+
full_path = File.join(dir_path, "#{entry}.rb")
|
16
|
+
next unless File.exist?(full_path)
|
17
|
+
|
18
|
+
# Require file if auto_require true
|
19
|
+
require(full_path) if auto_require
|
20
|
+
|
21
|
+
# Go to next file if require_only mode
|
22
|
+
next if require_only
|
23
|
+
|
24
|
+
class_name = entry.camelize
|
25
|
+
qualified_class_name = "Jetel::Loaders::#{class_name}"
|
26
|
+
|
27
|
+
# Return value from map iteration
|
28
|
+
{
|
29
|
+
name: entry,
|
30
|
+
path: full_path,
|
31
|
+
class_name: class_name,
|
32
|
+
qualified_class_name: qualified_class_name,
|
33
|
+
klass: auto_require ? Kernel.qualified_const_get(qualified_class_name) : nil
|
34
|
+
}
|
35
|
+
end
|
36
|
+
|
37
|
+
# Remove nil values and return
|
38
|
+
res.compact
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
require_relative '../loader'
|
2
|
+
|
3
|
+
require 'pg'
|
4
|
+
|
5
|
+
module Jetel
|
6
|
+
module Loaders
|
7
|
+
class Pg < Loader
|
8
|
+
def initialize(uri)
|
9
|
+
super
|
10
|
+
|
11
|
+
tmp = uri.split('://')
|
12
|
+
tmp = tmp[1].split('@')
|
13
|
+
|
14
|
+
parts = tmp[0].split(':')
|
15
|
+
user = parts[0]
|
16
|
+
password = parts[1]
|
17
|
+
|
18
|
+
parts = tmp[1].split('/')
|
19
|
+
host, port = parts[0].split(':')
|
20
|
+
dbname = parts[1]
|
21
|
+
|
22
|
+
opts = {
|
23
|
+
:host => host,
|
24
|
+
:port => port.to_i,
|
25
|
+
# :options => '',
|
26
|
+
# :tty => '',
|
27
|
+
:dbname => dbname,
|
28
|
+
:user => user,
|
29
|
+
:password => password
|
30
|
+
}
|
31
|
+
|
32
|
+
@client = PG.connect(opts)
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
data/lib/jetel/modules/ip/ip.rb
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
# encoding: utf-8
|
2
2
|
|
3
3
|
require 'pmap'
|
4
|
+
require 'csv'
|
4
5
|
|
5
6
|
require_relative '../../config/config'
|
6
7
|
require_relative '../../helpers/helpers'
|
@@ -36,20 +37,69 @@ module Jetel
|
|
36
37
|
}
|
37
38
|
]
|
38
39
|
|
39
|
-
def download
|
40
|
+
def download(global_options, options, args)
|
40
41
|
SOURCES.pmap do |source|
|
41
|
-
|
42
|
-
downloader.download(source[:url], {:dir => target_dir})
|
42
|
+
download_source(source, global_options.merge(options))
|
43
43
|
end
|
44
44
|
end
|
45
45
|
|
46
|
-
def extract
|
46
|
+
def extract(global_options, options, args)
|
47
|
+
SOURCES.pmap do |source|
|
48
|
+
downloaded_file = downloaded_file(source, global_options.merge(options))
|
49
|
+
dest_dir = extract_dir(source, global_options.merge(options))
|
50
|
+
|
51
|
+
puts "Extracting #{downloaded_file}"
|
52
|
+
|
53
|
+
FileUtils.mkdir_p(dest_dir)
|
54
|
+
FileUtils.cp(downloaded_file, dest_dir)
|
55
|
+
end
|
47
56
|
end
|
48
57
|
|
49
|
-
def transform
|
58
|
+
def transform(global_options, options, args)
|
59
|
+
SOURCES.pmap do |source|
|
60
|
+
opts = global_options.merge(options)
|
61
|
+
|
62
|
+
extracted_file = extracted_file(source, opts)
|
63
|
+
transformed_file = transformed_file(source, opts)
|
64
|
+
|
65
|
+
FileUtils.mkdir_p(transform_dir(source, opts))
|
66
|
+
|
67
|
+
csv_opts = {
|
68
|
+
:col_sep => '|'
|
69
|
+
}
|
70
|
+
|
71
|
+
puts "Transforming #{extracted_file}"
|
72
|
+
CSV.open(extracted_file, 'r', csv_opts) do |csv|
|
73
|
+
headers = %w(
|
74
|
+
registry
|
75
|
+
cc
|
76
|
+
type
|
77
|
+
start
|
78
|
+
value
|
79
|
+
date
|
80
|
+
status
|
81
|
+
)
|
82
|
+
CSV.open(transformed_file, 'w', :write_headers => true, :headers => headers) do |csv_out|
|
83
|
+
csv.each do |row|
|
84
|
+
next if row.length < 7
|
85
|
+
next if row[0] != source[:name]
|
86
|
+
csv_out << row
|
87
|
+
end
|
88
|
+
end
|
89
|
+
end
|
90
|
+
end
|
50
91
|
end
|
51
92
|
|
52
|
-
def load
|
93
|
+
def load(global_options, options, args)
|
94
|
+
SOURCES.map do |source|
|
95
|
+
opts = global_options.merge(options)
|
96
|
+
|
97
|
+
transformed_file = transformed_file(source, opts)
|
98
|
+
|
99
|
+
loader = Helper.get_loader(opts['data_loader'])
|
100
|
+
|
101
|
+
loader.load(self, source, transformed_file, opts)
|
102
|
+
end
|
53
103
|
end
|
54
104
|
end
|
55
105
|
end
|
@@ -0,0 +1,82 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
3
|
+
require 'pmap'
|
4
|
+
require 'zip'
|
5
|
+
require 'csv'
|
6
|
+
|
7
|
+
require_relative '../../config/config'
|
8
|
+
require_relative '../../modules/module'
|
9
|
+
|
10
|
+
module Jetel
|
11
|
+
module Modules
|
12
|
+
class Iso3166 < Module
|
13
|
+
SOURCES = [
|
14
|
+
{
|
15
|
+
name: 'iso366',
|
16
|
+
filename_extracted: 'IP2LOCATION-ISO3166-2.CSV',
|
17
|
+
filename_transformed: 'IP2LOCATION-ISO3166-2.CSV',
|
18
|
+
url: 'http://www.ip2location.com/downloads/Hi3sL9bnXfe/IP2LOCATION-ISO3166-2.ZIP'
|
19
|
+
}
|
20
|
+
]
|
21
|
+
|
22
|
+
def download(global_options, options, args)
|
23
|
+
SOURCES.pmap do |source|
|
24
|
+
download_source(source, global_options.merge(options))
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
def extract(global_options, options, args)
|
29
|
+
SOURCES.pmap do |source|
|
30
|
+
downloaded_file = downloaded_file(source, global_options.merge(options))
|
31
|
+
dest_dir = extract_dir(source, global_options.merge(options))
|
32
|
+
|
33
|
+
FileUtils.mkdir_p(dest_dir)
|
34
|
+
|
35
|
+
Zip::ZipFile.open(downloaded_file) do |zip_file|
|
36
|
+
# Handle entries one by one
|
37
|
+
zip_file.each do |entry|
|
38
|
+
# Extract to file/directory/symlink
|
39
|
+
puts "Extracting #{entry.name}"
|
40
|
+
dest_file = File.join(dest_dir, entry.name.split('/').last)
|
41
|
+
FileUtils.rm_rf(dest_file)
|
42
|
+
entry.extract(dest_file)
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
def transform(global_options, options, args)
|
49
|
+
SOURCES.pmap do |source|
|
50
|
+
opts = global_options.merge(options)
|
51
|
+
|
52
|
+
extracted_file = extracted_file(source, opts)
|
53
|
+
transformed_file = transformed_file(source, opts)
|
54
|
+
|
55
|
+
FileUtils.mkdir_p(transform_dir(source, opts))
|
56
|
+
|
57
|
+
csv_opts = {
|
58
|
+
:headers => true
|
59
|
+
}
|
60
|
+
|
61
|
+
puts "Transforming #{extracted_file}"
|
62
|
+
CSV.open(extracted_file, 'r', csv_opts) do |csv|
|
63
|
+
headers = %w(
|
64
|
+
country_code
|
65
|
+
subdivision_name
|
66
|
+
code
|
67
|
+
)
|
68
|
+
CSV.open(transformed_file, 'w', :write_headers => true, :headers => headers) do |csv_out|
|
69
|
+
csv.each do |row|
|
70
|
+
next if row.length < 3
|
71
|
+
csv_out << row
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
def load(global_options, options, args)
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
data/lib/jetel/modules/module.rb
CHANGED
@@ -1,18 +1,79 @@
|
|
1
1
|
# encoding: utf-8
|
2
2
|
|
3
3
|
require_relative '../downloader/downloader'
|
4
|
+
require_relative '../helpers/helpers'
|
4
5
|
|
5
6
|
module Jetel
|
6
7
|
module Modules
|
7
8
|
class Module
|
8
9
|
attr_reader :downloader
|
9
10
|
|
11
|
+
class << self
|
12
|
+
def target_dir(modul, source, dir, *path)
|
13
|
+
klass = I18n.transliterate(modul.class.name.split('::').last).gsub(/[^0-9a-z_\-]/i, '_')
|
14
|
+
source_name = I18n.transliterate(source[:name]).gsub(/[^0-9a-z_\-]/i, '_')
|
15
|
+
File.join(dir.kind_of?(String) ? dir : dir['download_dir'] || Config[:DATA_DIRECTORY], klass, source_name, path)
|
16
|
+
end
|
17
|
+
|
18
|
+
def download_dir(modul, source, opts)
|
19
|
+
Module.target_dir(modul, source, opts, 'downloaded')
|
20
|
+
end
|
21
|
+
|
22
|
+
def extract_dir(modul, source, opts)
|
23
|
+
Module.target_dir(modul, source, opts, 'extracted')
|
24
|
+
end
|
25
|
+
|
26
|
+
def transform_dir(modul, source, opts)
|
27
|
+
Module.target_dir(modul, source, opts, 'transformed')
|
28
|
+
end
|
29
|
+
|
30
|
+
def downloaded_file(modul, source, opts)
|
31
|
+
File.join(download_dir(modul, source, opts), source[:filename_downloaded] || source[:url].split('/').last)
|
32
|
+
end
|
33
|
+
|
34
|
+
def extracted_file(modul, source, opts)
|
35
|
+
File.join(extract_dir(modul, source, opts), source[:filename_extracted] || source[:url].split('/').last)
|
36
|
+
end
|
37
|
+
|
38
|
+
def transformed_file(modul, source, opts)
|
39
|
+
File.join(transform_dir(modul, source, opts), source[:filename_transformed] || source[:url].split('/').last)
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
10
43
|
def initialize
|
11
44
|
@downloader = Downloader.new
|
12
45
|
end
|
13
46
|
|
14
|
-
def
|
15
|
-
downloader.download(
|
47
|
+
def download_source(source, opts)
|
48
|
+
downloader.download(source[:url], {:dir => download_dir(source, opts)})
|
49
|
+
end
|
50
|
+
|
51
|
+
def target_dir(source, opts, *path)
|
52
|
+
Module.target_dir(self, source, opts['download_dir'], *path)
|
53
|
+
end
|
54
|
+
|
55
|
+
def download_dir(source, opts)
|
56
|
+
Module.download_dir(self, source, opts)
|
57
|
+
end
|
58
|
+
|
59
|
+
def extract_dir(source, opts)
|
60
|
+
Module.extract_dir(self, source, opts)
|
61
|
+
end
|
62
|
+
|
63
|
+
def transform_dir(source, opts)
|
64
|
+
Module.transform_dir(self, source, opts)
|
65
|
+
end
|
66
|
+
|
67
|
+
def downloaded_file(source, opts)
|
68
|
+
Module.downloaded_file(self, source, opts)
|
69
|
+
end
|
70
|
+
|
71
|
+
def extracted_file(source, opts)
|
72
|
+
Module.extracted_file(self, source, opts)
|
73
|
+
end
|
74
|
+
|
75
|
+
def transformed_file(source, opts)
|
76
|
+
Module.transformed_file(self, source, opts)
|
16
77
|
end
|
17
78
|
end
|
18
79
|
end
|
@@ -1038,20 +1038,107 @@ module Jetel
|
|
1038
1038
|
}
|
1039
1039
|
]
|
1040
1040
|
|
1041
|
-
def download
|
1041
|
+
def download(global_options, options, args)
|
1042
1042
|
SOURCES.pmap do |source|
|
1043
|
-
|
1044
|
-
downloader.download(source[:url], {:dir => target_dir})
|
1043
|
+
download_source(source, global_options.merge(options))
|
1045
1044
|
end
|
1046
1045
|
end
|
1047
1046
|
|
1048
|
-
def extract
|
1047
|
+
def extract(global_options, options, args)
|
1048
|
+
SOURCES.pmap do |source|
|
1049
|
+
downloaded_file = downloaded_file(source, global_options.merge(options))
|
1050
|
+
dest_dir = extract_dir(source, global_options.merge(options))
|
1051
|
+
|
1052
|
+
FileUtils.mkdir_p(dest_dir)
|
1053
|
+
|
1054
|
+
Zip::ZipFile.open(downloaded_file) do |zip_file|
|
1055
|
+
# Handle entries one by one
|
1056
|
+
zip_file.each do |entry|
|
1057
|
+
# Extract to file/directory/symlink
|
1058
|
+
puts "Extracting #{entry.name}"
|
1059
|
+
dest_file = File.join(dest_dir, entry.name.split('/').last)
|
1060
|
+
FileUtils.rm_rf(dest_file)
|
1061
|
+
entry.extract(dest_file)
|
1062
|
+
end
|
1063
|
+
end
|
1064
|
+
end
|
1065
|
+
end
|
1066
|
+
|
1067
|
+
def transform(global_options, options, args)
|
1068
|
+
SOURCES.pmap do |source|
|
1069
|
+
opts = global_options.merge(options)
|
1070
|
+
|
1071
|
+
extracted_file = extracted_file(source, opts)
|
1072
|
+
transformed_file = transformed_file(source, opts)
|
1073
|
+
|
1074
|
+
FileUtils.mkdir_p(transform_dir(source, opts))
|
1075
|
+
|
1076
|
+
csv_opts = {
|
1077
|
+
:headers => true,
|
1078
|
+
:col_sep => "\t",
|
1079
|
+
:quote_char => "\x00"
|
1080
|
+
}
|
1081
|
+
|
1082
|
+
puts "Transforming #{extracted_file}"
|
1083
|
+
CSV.open(extracted_file, 'r', csv_opts) do |csv|
|
1084
|
+
headers = %w(
|
1085
|
+
RC
|
1086
|
+
UFI
|
1087
|
+
UNI
|
1088
|
+
LAT
|
1089
|
+
LONG
|
1090
|
+
DMS_LAT
|
1091
|
+
DMS_LONG
|
1092
|
+
MGRS
|
1093
|
+
JOG
|
1094
|
+
FC
|
1095
|
+
DSG
|
1096
|
+
PC
|
1097
|
+
CC1
|
1098
|
+
ADM1
|
1099
|
+
POP
|
1100
|
+
ELEV
|
1101
|
+
CC2
|
1102
|
+
NT
|
1103
|
+
LC
|
1104
|
+
SHORT_FORM
|
1105
|
+
GENERIC
|
1106
|
+
SORT_NAME_RO
|
1107
|
+
FULL_NAME_RO
|
1108
|
+
FULL_NAME_ND_RO
|
1109
|
+
SORT_NAME_RG
|
1110
|
+
FULL_NAME_RG
|
1111
|
+
FULL_NAME_ND_RG
|
1112
|
+
NOTE
|
1113
|
+
MODIFY_DATE
|
1114
|
+
DISPLAY
|
1115
|
+
NAME_RANK
|
1116
|
+
NAME_LINK
|
1117
|
+
TRANSL_CD
|
1118
|
+
NM_MODIFY_DATE
|
1119
|
+
F_EFCTV_DT
|
1120
|
+
F_TERM_DT
|
1121
|
+
)
|
1122
|
+
CSV.open(transformed_file, 'w', :write_headers => true, :headers => headers, :quote_char => '"') do |csv_out|
|
1123
|
+
csv.each do |row|
|
1124
|
+
csv_out << row
|
1125
|
+
end
|
1126
|
+
end
|
1127
|
+
end
|
1128
|
+
end
|
1129
|
+
end
|
1130
|
+
|
1131
|
+
def load(global_options, options, args)
|
1049
1132
|
end
|
1050
1133
|
|
1051
|
-
def
|
1134
|
+
def extracted_file(source, opts)
|
1135
|
+
res = super(source, opts)
|
1136
|
+
res.gsub('.zip', '.txt')
|
1052
1137
|
end
|
1053
1138
|
|
1054
|
-
def
|
1139
|
+
def transformed_file(source, opts)
|
1140
|
+
res = super(source, opts)
|
1141
|
+
res.gsub('.zip', '.txt')
|
1055
1142
|
end
|
1056
1143
|
end
|
1057
1144
|
end
|
@@ -3,7 +3,6 @@
|
|
3
3
|
require 'pmap'
|
4
4
|
|
5
5
|
require_relative '../../config/config'
|
6
|
-
require_relative '../../helpers/helpers'
|
7
6
|
require_relative '../../modules/module'
|
8
7
|
|
9
8
|
module Jetel
|
@@ -16,20 +15,19 @@ module Jetel
|
|
16
15
|
}
|
17
16
|
]
|
18
17
|
|
19
|
-
def download
|
18
|
+
def download(global_options, options, args)
|
20
19
|
SOURCES.pmap do |source|
|
21
|
-
|
22
|
-
downloader.download(source[:url], {:dir => target_dir})
|
20
|
+
download_source(source, global_options.merge(options))
|
23
21
|
end
|
24
22
|
end
|
25
23
|
|
26
|
-
def extract
|
24
|
+
def extract(global_options, options, args)
|
27
25
|
end
|
28
26
|
|
29
|
-
def transform
|
27
|
+
def transform(global_options, options, args)
|
30
28
|
end
|
31
29
|
|
32
|
-
def load
|
30
|
+
def load(global_options, options, args)
|
33
31
|
end
|
34
32
|
end
|
35
33
|
end
|
data/lib/jetel/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: jetel
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Tomas Korcak
|
@@ -38,6 +38,20 @@ dependencies:
|
|
38
38
|
- - ">="
|
39
39
|
- !ruby/object:Gem::Version
|
40
40
|
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: i18n
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">="
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
41
55
|
- !ruby/object:Gem::Dependency
|
42
56
|
name: json_pure
|
43
57
|
requirement: !ruby/object:Gem::Requirement
|
@@ -108,6 +122,20 @@ dependencies:
|
|
108
122
|
- - ">="
|
109
123
|
- !ruby/object:Gem::Version
|
110
124
|
version: '0'
|
125
|
+
- !ruby/object:Gem::Dependency
|
126
|
+
name: rubyzip
|
127
|
+
requirement: !ruby/object:Gem::Requirement
|
128
|
+
requirements:
|
129
|
+
- - ">="
|
130
|
+
- !ruby/object:Gem::Version
|
131
|
+
version: '0'
|
132
|
+
type: :runtime
|
133
|
+
prerelease: false
|
134
|
+
version_requirements: !ruby/object:Gem::Requirement
|
135
|
+
requirements:
|
136
|
+
- - ">="
|
137
|
+
- !ruby/object:Gem::Version
|
138
|
+
version: '0'
|
111
139
|
- !ruby/object:Gem::Dependency
|
112
140
|
name: terminal-table
|
113
141
|
requirement: !ruby/object:Gem::Requirement
|
@@ -240,6 +268,8 @@ files:
|
|
240
268
|
- Rakefile
|
241
269
|
- bin/jetel
|
242
270
|
- data/.gitkeepme
|
271
|
+
- docks/postgresql/pull.sh
|
272
|
+
- docks/postgresql/run.sh
|
243
273
|
- jetel.gemspec
|
244
274
|
- lib/jetel.rb
|
245
275
|
- lib/jetel/cli/app.rb
|
@@ -259,7 +289,11 @@ files:
|
|
259
289
|
- lib/jetel/helpers/helpers.rb
|
260
290
|
- lib/jetel/jetel/jetel.rb
|
261
291
|
- lib/jetel/lib.rb
|
292
|
+
- lib/jetel/loaders/loader.rb
|
293
|
+
- lib/jetel/loaders/loaders.rb
|
294
|
+
- lib/jetel/loaders/pg/pg.rb
|
262
295
|
- lib/jetel/modules/ip/ip.rb
|
296
|
+
- lib/jetel/modules/iso3166/iso3166.rb
|
263
297
|
- lib/jetel/modules/module.rb
|
264
298
|
- lib/jetel/modules/modules.rb
|
265
299
|
- lib/jetel/modules/nga/nga.rb
|