jetel 0.0.5 → 0.0.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 02a00403f513d81a9ca96ed06023ba0afb572b1a
4
- data.tar.gz: cb3bb40fbe7523a6c528b672f250fff747564f58
3
+ metadata.gz: 51f8e1d58f1480832113887664c59bc1fee6f142
4
+ data.tar.gz: c144f84c9664e281299bdc9016e81411a5f6c7b3
5
5
  SHA512:
6
- metadata.gz: 9721b74d52fc670071dc3e1b43e85b59354b38e295c8f66a094ac426e060284a6713787b249a92f470283477cf7d730e7c40961bce760eef4c895d44c87625d1
7
- data.tar.gz: 87ebe4dc08f902010e221a3c0588347a36083fa98dec5ca9c32ce9587d35a6e3fb3d1aafaee2a1e483762769cdd322111f3cad43c1445dec8c2eb8db47c20646
6
+ metadata.gz: 8fde78020fdcadbf6b6eb6fe5a172607c3177f3a2099ee0de095748d7896d1811c0cf8c27ab9cef16fa2c67a244367c689aafa5b8cf681629949b320777b7f5e
7
+ data.tar.gz: fecde5a4930c26d987a4ff14b93755eb6401304f52944b3cf2ffb2e81fe82bdaf5d867c38f1ebd741def9c9b9fa7305bcd93541c80ac7ffe64ba541d58da698e
data/Gemfile.lock CHANGED
@@ -1,14 +1,16 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- jetel (0.0.5)
4
+ jetel (0.0.6)
5
5
  activesupport
6
6
  gli
7
+ i18n
7
8
  json_pure
8
9
  multi_json
9
10
  nokogiri
10
11
  pg
11
12
  pmap
13
+ rubyzip
12
14
  terminal-table
13
15
  zip
14
16
 
@@ -78,6 +80,7 @@ GEM
78
80
  rainbow (>= 1.99.1, < 3.0)
79
81
  ruby-progressbar (~> 1.4)
80
82
  ruby-progressbar (1.7.5)
83
+ rubyzip (1.1.7)
81
84
  simplecov (0.10.0)
82
85
  docile (~> 1.1.0)
83
86
  json (~> 1.8)
data/README.md CHANGED
@@ -5,9 +5,9 @@
5
5
  Run `jetel`
6
6
 
7
7
  ```
8
- tomaskorcak@kx-mac:~$ jetel
8
+ $ jetel
9
9
  NAME
10
- jetel - Jetel CLI 0.0.1
10
+ jetel - Jetel CLI 0.0.5
11
11
 
12
12
  SYNOPSIS
13
13
  jetel [global options] command [command options] [arguments...]
@@ -16,10 +16,13 @@ GLOBAL OPTIONS
16
16
  --help - Show this message
17
17
 
18
18
  COMMANDS
19
- help - Shows a list of commands or help for one command
20
- ip, Ip - Module ip
21
- modules - Print modules info
22
- version - Print version info
19
+ config - Show config
20
+ help - Shows a list of commands or help for one command
21
+ ip, Ip - Module ip
22
+ modules - Print modules info
23
+ nga, Nga - Module nga
24
+ sfpd, Sfpd - Module sfpd
25
+ version - Print version info
23
26
  ```
24
27
 
25
28
  ## Structure
@@ -31,11 +34,17 @@ COMMANDS
31
34
  │   └── jetel
32
35
  │   ├── cli
33
36
  │   │   └── cmd
37
+ │   ├── config
34
38
  │   ├── downloader
39
+ │   │   └── backends
35
40
  │   ├── etl
36
41
  │   ├── extensions
42
+ │   ├── helpers
43
+ │   ├── jetel
37
44
  │   └── modules
38
- │   └── ip
45
+ │   ├── ip
46
+ │   ├── nga
47
+ │   └── sfpd
39
48
  ├── pkg
40
49
  └── test
41
50
  ```
@@ -0,0 +1,3 @@
1
+ #! /usr/bin/env bash
2
+
3
+ docker pull postgres
@@ -0,0 +1,3 @@
1
+ #! /usr/bin/env bash
2
+
3
+ docker run --name jetel-postgres -e POSTGRES_USER=jetel -e POSTGRES_PASSWORD=jetel -p 5432:5432 -d postgres
data/jetel.gemspec CHANGED
@@ -21,11 +21,13 @@ Gem::Specification.new do |spec|
21
21
 
22
22
  spec.add_dependency 'activesupport'
23
23
  spec.add_dependency 'gli'
24
+ spec.add_dependency 'i18n'
24
25
  spec.add_dependency 'json_pure'
25
26
  spec.add_dependency 'multi_json'
26
27
  spec.add_dependency 'nokogiri'
27
28
  spec.add_dependency 'pg'
28
29
  spec.add_dependency 'pmap'
30
+ spec.add_dependency 'rubyzip'
29
31
  spec.add_dependency 'terminal-table'
30
32
  spec.add_dependency 'zip'
31
33
 
data/lib/jetel/cli/app.rb CHANGED
@@ -13,9 +13,7 @@ end
13
13
 
14
14
  include GLI::App
15
15
 
16
- program_desc "Jetel CLI #{Jetel::VERSION}"
17
-
18
- module Esri
16
+ module Jetel
19
17
  # ESRI CLI
20
18
  module Cli
21
19
  # CLI Application
data/lib/jetel/cli/cli.rb CHANGED
@@ -2,6 +2,4 @@
2
2
 
3
3
  require 'pathname'
4
4
 
5
- require_relative 'shared'
6
-
7
5
  require_relative 'app'
@@ -1,10 +1,7 @@
1
1
  # encoding: utf-8
2
2
 
3
- require 'gli'
4
3
  require 'pp'
5
4
 
6
- include GLI::App
7
-
8
5
  require_relative '../../version'
9
6
 
10
7
  require_relative '../shared'
@@ -1,10 +1,7 @@
1
1
  # encoding: utf-8
2
2
 
3
- require 'gli'
4
3
  require 'terminal-table'
5
4
 
6
- include GLI::App
7
-
8
5
  require_relative '../../version'
9
6
 
10
7
  require_relative '../shared'
@@ -55,8 +52,8 @@ def register_module(m)
55
52
 
56
53
  action_name = k
57
54
  action_description = v || "#{action_name} #{module_name}"
58
- register_module_action(c, m, action_name, action_description) do
59
- module_instance.send(k)
55
+ register_module_action(c, m, action_name, action_description) do |global_options, options, args|
56
+ module_instance.send(k, global_options, options, args)
60
57
  end
61
58
  end
62
59
  end
@@ -1,9 +1,5 @@
1
1
  # encoding: utf-8
2
2
 
3
- require 'gli'
4
-
5
- include GLI::App
6
-
7
3
  require_relative '../../version'
8
4
 
9
5
  require_relative '../shared'
@@ -1,12 +1,42 @@
1
1
  # encoding: UTF-8
2
2
 
3
3
  require 'gli'
4
+ require 'pathname'
4
5
 
5
- module Esri
6
- module Cli
7
- # Shared CLI Stuff
8
- module Shared
9
- extend GLI::App
10
- end
11
- end
12
- end
6
+ require_relative '../version'
7
+
8
+ require_relative '../downloader/backends/base_downloader'
9
+
10
+ include GLI::App
11
+
12
+ # Program description
13
+ program_desc 'Simple custom made tool for data download and basic ETL'
14
+
15
+ # Version info
16
+ version Jetel::VERSION
17
+
18
+ # Download directory
19
+ desc 'Download directory'
20
+ default_value 'data' # File.absolute_path(File.join(File.dirname(__FILE__), '..', '..', '..', 'data'))
21
+ arg_name 'download-dir'
22
+ flag [:d, :download_dir]
23
+
24
+ # Download timeout
25
+ desc 'Download timeout'
26
+ default_value Jetel::Downloaders::BaseDownloader::OPTS_DOWNLOAD[:timeout]
27
+ arg_name 'download-timeout'
28
+ flag [:t, :timeout]
29
+
30
+ # Data loader
31
+ desc 'Data Loader'
32
+ default_value 'pg://jetel:jetel@localhost:5432/jetel'
33
+
34
+ arg_name 'data-loader'
35
+ flag [:l, :data_loader]
36
+
37
+ # On Error handler
38
+ on_error do |exception|
39
+ pp exception.backtrace
40
+ pp exception
41
+ true
42
+ end
@@ -14,7 +14,7 @@ module Jetel
14
14
 
15
15
  opts = BaseDownloader::OPTS_DOWNLOAD.merge(opts)
16
16
 
17
- cmd = "aria2c -j 4 -t 600 -d \"#{opts[:dir]}\" #{url}"
17
+ cmd = "aria2c -j 4 -t #{opts[:timeout]} -d \"#{opts[:dir]}\" #{url}"
18
18
  puts(cmd)
19
19
 
20
20
  PTY.spawn(cmd) do |stdout, stdin, pid|
@@ -9,7 +9,8 @@ module Jetel
9
9
  DATA_DIRECTORY = Config[:DATA_DIRECTORY]
10
10
 
11
11
  OPTS_DOWNLOAD = {
12
- :dir => DATA_DIRECTORY
12
+ :dir => DATA_DIRECTORY,
13
+ :timeout => 600
13
14
  }
14
15
 
15
16
  def download(url, opts = OPTS_DOWNLOAD)
@@ -2,11 +2,27 @@
2
2
 
3
3
  require_relative '../config/config'
4
4
 
5
+ require_relative '../loaders/loaders'
6
+
7
+ require 'i18n'
8
+
5
9
  module Jetel
6
10
  module Helper
7
11
  class << self
8
- def target_dir(modul, source)
9
- "#{File.join(Config[:DATA_DIRECTORY], modul.class.name.split('::').last, source[:name])}"
12
+ def target_dir(modul, dir, source)
13
+ klass = I18n.transliterate(modul.class.name.split('::').last).gsub(/[^0-9a-z_\-]/i, '_')
14
+ source_name = I18n.transliterate(source[:name]).gsub(/[^0-9a-z_\-]/i, '_')
15
+ File.join(dir || Config[:DATA_DIRECTORY], klass, source_name)
16
+ end
17
+
18
+ def get_loader(uri)
19
+ loaders = Loaders.loaders
20
+ loader_schema = uri.split(':/').first.downcase
21
+ res = loaders.find do |loader|
22
+ loader[:class_name].downcase === loader_schema
23
+ end
24
+
25
+ res[:klass].new(uri)
10
26
  end
11
27
  end
12
28
  end
@@ -0,0 +1,18 @@
1
+ module Jetel
2
+ module Loaders
3
+ class Loader
4
+ attr_reader :uri
5
+ def initialize(uri)
6
+ @uri = uri
7
+ end
8
+
9
+ def load(modul, source, file, opts)
10
+ {
11
+ :file => file,
12
+ :module => modul,
13
+ :source => source
14
+ }
15
+ end
16
+ end
17
+ end
18
+ end
@@ -0,0 +1,42 @@
1
+ require 'active_support/inflector'
2
+ require_relative '../extensions/extensions'
3
+
4
+ module Jetel
5
+ module Loaders
6
+ class << self
7
+ def loaders(loaders_dir = File.join(File.dirname(__FILE__)), auto_require = true, require_only = false)
8
+ dir = File.absolute_path(loaders_dir)
9
+ res = Dir.entries(dir).map do |entry|
10
+ dir_path = File.join(dir, entry)
11
+
12
+ next unless File.directory?(dir_path)
13
+ next if entry == '.' || entry == '..'
14
+
15
+ full_path = File.join(dir_path, "#{entry}.rb")
16
+ next unless File.exist?(full_path)
17
+
18
+ # Require file if auto_require true
19
+ require(full_path) if auto_require
20
+
21
+ # Go to next file if require_only mode
22
+ next if require_only
23
+
24
+ class_name = entry.camelize
25
+ qualified_class_name = "Jetel::Loaders::#{class_name}"
26
+
27
+ # Return value from map iteration
28
+ {
29
+ name: entry,
30
+ path: full_path,
31
+ class_name: class_name,
32
+ qualified_class_name: qualified_class_name,
33
+ klass: auto_require ? Kernel.qualified_const_get(qualified_class_name) : nil
34
+ }
35
+ end
36
+
37
+ # Remove nil values and return
38
+ res.compact
39
+ end
40
+ end
41
+ end
42
+ end
@@ -0,0 +1,36 @@
1
+ require_relative '../loader'
2
+
3
+ require 'pg'
4
+
5
+ module Jetel
6
+ module Loaders
7
+ class Pg < Loader
8
+ def initialize(uri)
9
+ super
10
+
11
+ tmp = uri.split('://')
12
+ tmp = tmp[1].split('@')
13
+
14
+ parts = tmp[0].split(':')
15
+ user = parts[0]
16
+ password = parts[1]
17
+
18
+ parts = tmp[1].split('/')
19
+ host, port = parts[0].split(':')
20
+ dbname = parts[1]
21
+
22
+ opts = {
23
+ :host => host,
24
+ :port => port.to_i,
25
+ # :options => '',
26
+ # :tty => '',
27
+ :dbname => dbname,
28
+ :user => user,
29
+ :password => password
30
+ }
31
+
32
+ @client = PG.connect(opts)
33
+ end
34
+ end
35
+ end
36
+ end
@@ -1,6 +1,7 @@
1
1
  # encoding: utf-8
2
2
 
3
3
  require 'pmap'
4
+ require 'csv'
4
5
 
5
6
  require_relative '../../config/config'
6
7
  require_relative '../../helpers/helpers'
@@ -36,20 +37,69 @@ module Jetel
36
37
  }
37
38
  ]
38
39
 
39
- def download
40
+ def download(global_options, options, args)
40
41
  SOURCES.pmap do |source|
41
- target_dir = Helper.target_dir(self, source)
42
- downloader.download(source[:url], {:dir => target_dir})
42
+ download_source(source, global_options.merge(options))
43
43
  end
44
44
  end
45
45
 
46
- def extract
46
+ def extract(global_options, options, args)
47
+ SOURCES.pmap do |source|
48
+ downloaded_file = downloaded_file(source, global_options.merge(options))
49
+ dest_dir = extract_dir(source, global_options.merge(options))
50
+
51
+ puts "Extracting #{downloaded_file}"
52
+
53
+ FileUtils.mkdir_p(dest_dir)
54
+ FileUtils.cp(downloaded_file, dest_dir)
55
+ end
47
56
  end
48
57
 
49
- def transform
58
+ def transform(global_options, options, args)
59
+ SOURCES.pmap do |source|
60
+ opts = global_options.merge(options)
61
+
62
+ extracted_file = extracted_file(source, opts)
63
+ transformed_file = transformed_file(source, opts)
64
+
65
+ FileUtils.mkdir_p(transform_dir(source, opts))
66
+
67
+ csv_opts = {
68
+ :col_sep => '|'
69
+ }
70
+
71
+ puts "Transforming #{extracted_file}"
72
+ CSV.open(extracted_file, 'r', csv_opts) do |csv|
73
+ headers = %w(
74
+ registry
75
+ cc
76
+ type
77
+ start
78
+ value
79
+ date
80
+ status
81
+ )
82
+ CSV.open(transformed_file, 'w', :write_headers => true, :headers => headers) do |csv_out|
83
+ csv.each do |row|
84
+ next if row.length < 7
85
+ next if row[0] != source[:name]
86
+ csv_out << row
87
+ end
88
+ end
89
+ end
90
+ end
50
91
  end
51
92
 
52
- def load
93
+ def load(global_options, options, args)
94
+ SOURCES.map do |source|
95
+ opts = global_options.merge(options)
96
+
97
+ transformed_file = transformed_file(source, opts)
98
+
99
+ loader = Helper.get_loader(opts['data_loader'])
100
+
101
+ loader.load(self, source, transformed_file, opts)
102
+ end
53
103
  end
54
104
  end
55
105
  end
@@ -0,0 +1,82 @@
1
+ # encoding: utf-8
2
+
3
+ require 'pmap'
4
+ require 'zip'
5
+ require 'csv'
6
+
7
+ require_relative '../../config/config'
8
+ require_relative '../../modules/module'
9
+
10
+ module Jetel
11
+ module Modules
12
+ class Iso3166 < Module
13
+ SOURCES = [
14
+ {
15
+ name: 'iso366',
16
+ filename_extracted: 'IP2LOCATION-ISO3166-2.CSV',
17
+ filename_transformed: 'IP2LOCATION-ISO3166-2.CSV',
18
+ url: 'http://www.ip2location.com/downloads/Hi3sL9bnXfe/IP2LOCATION-ISO3166-2.ZIP'
19
+ }
20
+ ]
21
+
22
+ def download(global_options, options, args)
23
+ SOURCES.pmap do |source|
24
+ download_source(source, global_options.merge(options))
25
+ end
26
+ end
27
+
28
+ def extract(global_options, options, args)
29
+ SOURCES.pmap do |source|
30
+ downloaded_file = downloaded_file(source, global_options.merge(options))
31
+ dest_dir = extract_dir(source, global_options.merge(options))
32
+
33
+ FileUtils.mkdir_p(dest_dir)
34
+
35
+ Zip::ZipFile.open(downloaded_file) do |zip_file|
36
+ # Handle entries one by one
37
+ zip_file.each do |entry|
38
+ # Extract to file/directory/symlink
39
+ puts "Extracting #{entry.name}"
40
+ dest_file = File.join(dest_dir, entry.name.split('/').last)
41
+ FileUtils.rm_rf(dest_file)
42
+ entry.extract(dest_file)
43
+ end
44
+ end
45
+ end
46
+ end
47
+
48
+ def transform(global_options, options, args)
49
+ SOURCES.pmap do |source|
50
+ opts = global_options.merge(options)
51
+
52
+ extracted_file = extracted_file(source, opts)
53
+ transformed_file = transformed_file(source, opts)
54
+
55
+ FileUtils.mkdir_p(transform_dir(source, opts))
56
+
57
+ csv_opts = {
58
+ :headers => true
59
+ }
60
+
61
+ puts "Transforming #{extracted_file}"
62
+ CSV.open(extracted_file, 'r', csv_opts) do |csv|
63
+ headers = %w(
64
+ country_code
65
+ subdivision_name
66
+ code
67
+ )
68
+ CSV.open(transformed_file, 'w', :write_headers => true, :headers => headers) do |csv_out|
69
+ csv.each do |row|
70
+ next if row.length < 3
71
+ csv_out << row
72
+ end
73
+ end
74
+ end
75
+ end
76
+ end
77
+
78
+ def load(global_options, options, args)
79
+ end
80
+ end
81
+ end
82
+ end
@@ -1,18 +1,79 @@
1
1
  # encoding: utf-8
2
2
 
3
3
  require_relative '../downloader/downloader'
4
+ require_relative '../helpers/helpers'
4
5
 
5
6
  module Jetel
6
7
  module Modules
7
8
  class Module
8
9
  attr_reader :downloader
9
10
 
11
+ class << self
12
+ def target_dir(modul, source, dir, *path)
13
+ klass = I18n.transliterate(modul.class.name.split('::').last).gsub(/[^0-9a-z_\-]/i, '_')
14
+ source_name = I18n.transliterate(source[:name]).gsub(/[^0-9a-z_\-]/i, '_')
15
+ File.join(dir.kind_of?(String) ? dir : dir['download_dir'] || Config[:DATA_DIRECTORY], klass, source_name, path)
16
+ end
17
+
18
+ def download_dir(modul, source, opts)
19
+ Module.target_dir(modul, source, opts, 'downloaded')
20
+ end
21
+
22
+ def extract_dir(modul, source, opts)
23
+ Module.target_dir(modul, source, opts, 'extracted')
24
+ end
25
+
26
+ def transform_dir(modul, source, opts)
27
+ Module.target_dir(modul, source, opts, 'transformed')
28
+ end
29
+
30
+ def downloaded_file(modul, source, opts)
31
+ File.join(download_dir(modul, source, opts), source[:filename_downloaded] || source[:url].split('/').last)
32
+ end
33
+
34
+ def extracted_file(modul, source, opts)
35
+ File.join(extract_dir(modul, source, opts), source[:filename_extracted] || source[:url].split('/').last)
36
+ end
37
+
38
+ def transformed_file(modul, source, opts)
39
+ File.join(transform_dir(modul, source, opts), source[:filename_transformed] || source[:url].split('/').last)
40
+ end
41
+ end
42
+
10
43
  def initialize
11
44
  @downloader = Downloader.new
12
45
  end
13
46
 
14
- def download(*args)
15
- downloader.download(*args)
47
+ def download_source(source, opts)
48
+ downloader.download(source[:url], {:dir => download_dir(source, opts)})
49
+ end
50
+
51
+ def target_dir(source, opts, *path)
52
+ Module.target_dir(self, source, opts['download_dir'], *path)
53
+ end
54
+
55
+ def download_dir(source, opts)
56
+ Module.download_dir(self, source, opts)
57
+ end
58
+
59
+ def extract_dir(source, opts)
60
+ Module.extract_dir(self, source, opts)
61
+ end
62
+
63
+ def transform_dir(source, opts)
64
+ Module.transform_dir(self, source, opts)
65
+ end
66
+
67
+ def downloaded_file(source, opts)
68
+ Module.downloaded_file(self, source, opts)
69
+ end
70
+
71
+ def extracted_file(source, opts)
72
+ Module.extracted_file(self, source, opts)
73
+ end
74
+
75
+ def transformed_file(source, opts)
76
+ Module.transformed_file(self, source, opts)
16
77
  end
17
78
  end
18
79
  end
@@ -1038,20 +1038,107 @@ module Jetel
1038
1038
  }
1039
1039
  ]
1040
1040
 
1041
- def download
1041
+ def download(global_options, options, args)
1042
1042
  SOURCES.pmap do |source|
1043
- target_dir = Helper.target_dir(self, source)
1044
- downloader.download(source[:url], {:dir => target_dir})
1043
+ download_source(source, global_options.merge(options))
1045
1044
  end
1046
1045
  end
1047
1046
 
1048
- def extract
1047
+ def extract(global_options, options, args)
1048
+ SOURCES.pmap do |source|
1049
+ downloaded_file = downloaded_file(source, global_options.merge(options))
1050
+ dest_dir = extract_dir(source, global_options.merge(options))
1051
+
1052
+ FileUtils.mkdir_p(dest_dir)
1053
+
1054
+ Zip::ZipFile.open(downloaded_file) do |zip_file|
1055
+ # Handle entries one by one
1056
+ zip_file.each do |entry|
1057
+ # Extract to file/directory/symlink
1058
+ puts "Extracting #{entry.name}"
1059
+ dest_file = File.join(dest_dir, entry.name.split('/').last)
1060
+ FileUtils.rm_rf(dest_file)
1061
+ entry.extract(dest_file)
1062
+ end
1063
+ end
1064
+ end
1065
+ end
1066
+
1067
+ def transform(global_options, options, args)
1068
+ SOURCES.pmap do |source|
1069
+ opts = global_options.merge(options)
1070
+
1071
+ extracted_file = extracted_file(source, opts)
1072
+ transformed_file = transformed_file(source, opts)
1073
+
1074
+ FileUtils.mkdir_p(transform_dir(source, opts))
1075
+
1076
+ csv_opts = {
1077
+ :headers => true,
1078
+ :col_sep => "\t",
1079
+ :quote_char => "\x00"
1080
+ }
1081
+
1082
+ puts "Transforming #{extracted_file}"
1083
+ CSV.open(extracted_file, 'r', csv_opts) do |csv|
1084
+ headers = %w(
1085
+ RC
1086
+ UFI
1087
+ UNI
1088
+ LAT
1089
+ LONG
1090
+ DMS_LAT
1091
+ DMS_LONG
1092
+ MGRS
1093
+ JOG
1094
+ FC
1095
+ DSG
1096
+ PC
1097
+ CC1
1098
+ ADM1
1099
+ POP
1100
+ ELEV
1101
+ CC2
1102
+ NT
1103
+ LC
1104
+ SHORT_FORM
1105
+ GENERIC
1106
+ SORT_NAME_RO
1107
+ FULL_NAME_RO
1108
+ FULL_NAME_ND_RO
1109
+ SORT_NAME_RG
1110
+ FULL_NAME_RG
1111
+ FULL_NAME_ND_RG
1112
+ NOTE
1113
+ MODIFY_DATE
1114
+ DISPLAY
1115
+ NAME_RANK
1116
+ NAME_LINK
1117
+ TRANSL_CD
1118
+ NM_MODIFY_DATE
1119
+ F_EFCTV_DT
1120
+ F_TERM_DT
1121
+ )
1122
+ CSV.open(transformed_file, 'w', :write_headers => true, :headers => headers, :quote_char => '"') do |csv_out|
1123
+ csv.each do |row|
1124
+ csv_out << row
1125
+ end
1126
+ end
1127
+ end
1128
+ end
1129
+ end
1130
+
1131
+ def load(global_options, options, args)
1049
1132
  end
1050
1133
 
1051
- def transform
1134
+ def extracted_file(source, opts)
1135
+ res = super(source, opts)
1136
+ res.gsub('.zip', '.txt')
1052
1137
  end
1053
1138
 
1054
- def load
1139
+ def transformed_file(source, opts)
1140
+ res = super(source, opts)
1141
+ res.gsub('.zip', '.txt')
1055
1142
  end
1056
1143
  end
1057
1144
  end
@@ -3,7 +3,6 @@
3
3
  require 'pmap'
4
4
 
5
5
  require_relative '../../config/config'
6
- require_relative '../../helpers/helpers'
7
6
  require_relative '../../modules/module'
8
7
 
9
8
  module Jetel
@@ -16,20 +15,19 @@ module Jetel
16
15
  }
17
16
  ]
18
17
 
19
- def download
18
+ def download(global_options, options, args)
20
19
  SOURCES.pmap do |source|
21
- target_dir = Helper.target_dir(self, source)
22
- downloader.download(source[:url], {:dir => target_dir})
20
+ download_source(source, global_options.merge(options))
23
21
  end
24
22
  end
25
23
 
26
- def extract
24
+ def extract(global_options, options, args)
27
25
  end
28
26
 
29
- def transform
27
+ def transform(global_options, options, args)
30
28
  end
31
29
 
32
- def load
30
+ def load(global_options, options, args)
33
31
  end
34
32
  end
35
33
  end
data/lib/jetel/version.rb CHANGED
@@ -1,5 +1,5 @@
1
1
  # encoding: utf-8
2
2
 
3
3
  module Jetel
4
- VERSION = '0.0.5'
4
+ VERSION = '0.0.6'
5
5
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: jetel
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.5
4
+ version: 0.0.6
5
5
  platform: ruby
6
6
  authors:
7
7
  - Tomas Korcak
@@ -38,6 +38,20 @@ dependencies:
38
38
  - - ">="
39
39
  - !ruby/object:Gem::Version
40
40
  version: '0'
41
+ - !ruby/object:Gem::Dependency
42
+ name: i18n
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - ">="
46
+ - !ruby/object:Gem::Version
47
+ version: '0'
48
+ type: :runtime
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - ">="
53
+ - !ruby/object:Gem::Version
54
+ version: '0'
41
55
  - !ruby/object:Gem::Dependency
42
56
  name: json_pure
43
57
  requirement: !ruby/object:Gem::Requirement
@@ -108,6 +122,20 @@ dependencies:
108
122
  - - ">="
109
123
  - !ruby/object:Gem::Version
110
124
  version: '0'
125
+ - !ruby/object:Gem::Dependency
126
+ name: rubyzip
127
+ requirement: !ruby/object:Gem::Requirement
128
+ requirements:
129
+ - - ">="
130
+ - !ruby/object:Gem::Version
131
+ version: '0'
132
+ type: :runtime
133
+ prerelease: false
134
+ version_requirements: !ruby/object:Gem::Requirement
135
+ requirements:
136
+ - - ">="
137
+ - !ruby/object:Gem::Version
138
+ version: '0'
111
139
  - !ruby/object:Gem::Dependency
112
140
  name: terminal-table
113
141
  requirement: !ruby/object:Gem::Requirement
@@ -240,6 +268,8 @@ files:
240
268
  - Rakefile
241
269
  - bin/jetel
242
270
  - data/.gitkeepme
271
+ - docks/postgresql/pull.sh
272
+ - docks/postgresql/run.sh
243
273
  - jetel.gemspec
244
274
  - lib/jetel.rb
245
275
  - lib/jetel/cli/app.rb
@@ -259,7 +289,11 @@ files:
259
289
  - lib/jetel/helpers/helpers.rb
260
290
  - lib/jetel/jetel/jetel.rb
261
291
  - lib/jetel/lib.rb
292
+ - lib/jetel/loaders/loader.rb
293
+ - lib/jetel/loaders/loaders.rb
294
+ - lib/jetel/loaders/pg/pg.rb
262
295
  - lib/jetel/modules/ip/ip.rb
296
+ - lib/jetel/modules/iso3166/iso3166.rb
263
297
  - lib/jetel/modules/module.rb
264
298
  - lib/jetel/modules/modules.rb
265
299
  - lib/jetel/modules/nga/nga.rb