jetel 0.0.5 → 0.0.6

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 02a00403f513d81a9ca96ed06023ba0afb572b1a
4
- data.tar.gz: cb3bb40fbe7523a6c528b672f250fff747564f58
3
+ metadata.gz: 51f8e1d58f1480832113887664c59bc1fee6f142
4
+ data.tar.gz: c144f84c9664e281299bdc9016e81411a5f6c7b3
5
5
  SHA512:
6
- metadata.gz: 9721b74d52fc670071dc3e1b43e85b59354b38e295c8f66a094ac426e060284a6713787b249a92f470283477cf7d730e7c40961bce760eef4c895d44c87625d1
7
- data.tar.gz: 87ebe4dc08f902010e221a3c0588347a36083fa98dec5ca9c32ce9587d35a6e3fb3d1aafaee2a1e483762769cdd322111f3cad43c1445dec8c2eb8db47c20646
6
+ metadata.gz: 8fde78020fdcadbf6b6eb6fe5a172607c3177f3a2099ee0de095748d7896d1811c0cf8c27ab9cef16fa2c67a244367c689aafa5b8cf681629949b320777b7f5e
7
+ data.tar.gz: fecde5a4930c26d987a4ff14b93755eb6401304f52944b3cf2ffb2e81fe82bdaf5d867c38f1ebd741def9c9b9fa7305bcd93541c80ac7ffe64ba541d58da698e
data/Gemfile.lock CHANGED
@@ -1,14 +1,16 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- jetel (0.0.5)
4
+ jetel (0.0.6)
5
5
  activesupport
6
6
  gli
7
+ i18n
7
8
  json_pure
8
9
  multi_json
9
10
  nokogiri
10
11
  pg
11
12
  pmap
13
+ rubyzip
12
14
  terminal-table
13
15
  zip
14
16
 
@@ -78,6 +80,7 @@ GEM
78
80
  rainbow (>= 1.99.1, < 3.0)
79
81
  ruby-progressbar (~> 1.4)
80
82
  ruby-progressbar (1.7.5)
83
+ rubyzip (1.1.7)
81
84
  simplecov (0.10.0)
82
85
  docile (~> 1.1.0)
83
86
  json (~> 1.8)
data/README.md CHANGED
@@ -5,9 +5,9 @@
5
5
  Run `jetel`
6
6
 
7
7
  ```
8
- tomaskorcak@kx-mac:~$ jetel
8
+ $ jetel
9
9
  NAME
10
- jetel - Jetel CLI 0.0.1
10
+ jetel - Jetel CLI 0.0.5
11
11
 
12
12
  SYNOPSIS
13
13
  jetel [global options] command [command options] [arguments...]
@@ -16,10 +16,13 @@ GLOBAL OPTIONS
16
16
  --help - Show this message
17
17
 
18
18
  COMMANDS
19
- help - Shows a list of commands or help for one command
20
- ip, Ip - Module ip
21
- modules - Print modules info
22
- version - Print version info
19
+ config - Show config
20
+ help - Shows a list of commands or help for one command
21
+ ip, Ip - Module ip
22
+ modules - Print modules info
23
+ nga, Nga - Module nga
24
+ sfpd, Sfpd - Module sfpd
25
+ version - Print version info
23
26
  ```
24
27
 
25
28
  ## Structure
@@ -31,11 +34,17 @@ COMMANDS
31
34
  │   └── jetel
32
35
  │   ├── cli
33
36
  │   │   └── cmd
37
+ │   ├── config
34
38
  │   ├── downloader
39
+ │   │   └── backends
35
40
  │   ├── etl
36
41
  │   ├── extensions
42
+ │   ├── helpers
43
+ │   ├── jetel
37
44
  │   └── modules
38
- │   └── ip
45
+ │   ├── ip
46
+ │   ├── nga
47
+ │   └── sfpd
39
48
  ├── pkg
40
49
  └── test
41
50
  ```
@@ -0,0 +1,3 @@
1
+ #! /usr/bin/env bash
2
+
3
+ docker pull postgres
@@ -0,0 +1,3 @@
1
+ #! /usr/bin/env bash
2
+
3
+ docker run --name jetel-postgres -e POSTGRES_USER=jetel -e POSTGRES_PASSWORD=jetel -p 5432:5432 -d postgres
data/jetel.gemspec CHANGED
@@ -21,11 +21,13 @@ Gem::Specification.new do |spec|
21
21
 
22
22
  spec.add_dependency 'activesupport'
23
23
  spec.add_dependency 'gli'
24
+ spec.add_dependency 'i18n'
24
25
  spec.add_dependency 'json_pure'
25
26
  spec.add_dependency 'multi_json'
26
27
  spec.add_dependency 'nokogiri'
27
28
  spec.add_dependency 'pg'
28
29
  spec.add_dependency 'pmap'
30
+ spec.add_dependency 'rubyzip'
29
31
  spec.add_dependency 'terminal-table'
30
32
  spec.add_dependency 'zip'
31
33
 
data/lib/jetel/cli/app.rb CHANGED
@@ -13,9 +13,7 @@ end
13
13
 
14
14
  include GLI::App
15
15
 
16
- program_desc "Jetel CLI #{Jetel::VERSION}"
17
-
18
- module Esri
16
+ module Jetel
19
17
  # ESRI CLI
20
18
  module Cli
21
19
  # CLI Application
data/lib/jetel/cli/cli.rb CHANGED
@@ -2,6 +2,4 @@
2
2
 
3
3
  require 'pathname'
4
4
 
5
- require_relative 'shared'
6
-
7
5
  require_relative 'app'
@@ -1,10 +1,7 @@
1
1
  # encoding: utf-8
2
2
 
3
- require 'gli'
4
3
  require 'pp'
5
4
 
6
- include GLI::App
7
-
8
5
  require_relative '../../version'
9
6
 
10
7
  require_relative '../shared'
@@ -1,10 +1,7 @@
1
1
  # encoding: utf-8
2
2
 
3
- require 'gli'
4
3
  require 'terminal-table'
5
4
 
6
- include GLI::App
7
-
8
5
  require_relative '../../version'
9
6
 
10
7
  require_relative '../shared'
@@ -55,8 +52,8 @@ def register_module(m)
55
52
 
56
53
  action_name = k
57
54
  action_description = v || "#{action_name} #{module_name}"
58
- register_module_action(c, m, action_name, action_description) do
59
- module_instance.send(k)
55
+ register_module_action(c, m, action_name, action_description) do |global_options, options, args|
56
+ module_instance.send(k, global_options, options, args)
60
57
  end
61
58
  end
62
59
  end
@@ -1,9 +1,5 @@
1
1
  # encoding: utf-8
2
2
 
3
- require 'gli'
4
-
5
- include GLI::App
6
-
7
3
  require_relative '../../version'
8
4
 
9
5
  require_relative '../shared'
@@ -1,12 +1,42 @@
1
1
  # encoding: UTF-8
2
2
 
3
3
  require 'gli'
4
+ require 'pathname'
4
5
 
5
- module Esri
6
- module Cli
7
- # Shared CLI Stuff
8
- module Shared
9
- extend GLI::App
10
- end
11
- end
12
- end
6
+ require_relative '../version'
7
+
8
+ require_relative '../downloader/backends/base_downloader'
9
+
10
+ include GLI::App
11
+
12
+ # Program description
13
+ program_desc 'Simple custom made tool for data download and basic ETL'
14
+
15
+ # Version info
16
+ version Jetel::VERSION
17
+
18
+ # Download directory
19
+ desc 'Download directory'
20
+ default_value 'data' # File.absolute_path(File.join(File.dirname(__FILE__), '..', '..', '..', 'data'))
21
+ arg_name 'download-dir'
22
+ flag [:d, :download_dir]
23
+
24
+ # Download timeout
25
+ desc 'Download timeout'
26
+ default_value Jetel::Downloaders::BaseDownloader::OPTS_DOWNLOAD[:timeout]
27
+ arg_name 'download-timeout'
28
+ flag [:t, :timeout]
29
+
30
+ # Data loader
31
+ desc 'Data Loader'
32
+ default_value 'pg://jetel:jetel@localhost:5432/jetel'
33
+
34
+ arg_name 'data-loader'
35
+ flag [:l, :data_loader]
36
+
37
+ # On Error handler
38
+ on_error do |exception|
39
+ pp exception.backtrace
40
+ pp exception
41
+ true
42
+ end
@@ -14,7 +14,7 @@ module Jetel
14
14
 
15
15
  opts = BaseDownloader::OPTS_DOWNLOAD.merge(opts)
16
16
 
17
- cmd = "aria2c -j 4 -t 600 -d \"#{opts[:dir]}\" #{url}"
17
+ cmd = "aria2c -j 4 -t #{opts[:timeout]} -d \"#{opts[:dir]}\" #{url}"
18
18
  puts(cmd)
19
19
 
20
20
  PTY.spawn(cmd) do |stdout, stdin, pid|
@@ -9,7 +9,8 @@ module Jetel
9
9
  DATA_DIRECTORY = Config[:DATA_DIRECTORY]
10
10
 
11
11
  OPTS_DOWNLOAD = {
12
- :dir => DATA_DIRECTORY
12
+ :dir => DATA_DIRECTORY,
13
+ :timeout => 600
13
14
  }
14
15
 
15
16
  def download(url, opts = OPTS_DOWNLOAD)
@@ -2,11 +2,27 @@
2
2
 
3
3
  require_relative '../config/config'
4
4
 
5
+ require_relative '../loaders/loaders'
6
+
7
+ require 'i18n'
8
+
5
9
  module Jetel
6
10
  module Helper
7
11
  class << self
8
- def target_dir(modul, source)
9
- "#{File.join(Config[:DATA_DIRECTORY], modul.class.name.split('::').last, source[:name])}"
12
+ def target_dir(modul, dir, source)
13
+ klass = I18n.transliterate(modul.class.name.split('::').last).gsub(/[^0-9a-z_\-]/i, '_')
14
+ source_name = I18n.transliterate(source[:name]).gsub(/[^0-9a-z_\-]/i, '_')
15
+ File.join(dir || Config[:DATA_DIRECTORY], klass, source_name)
16
+ end
17
+
18
+ def get_loader(uri)
19
+ loaders = Loaders.loaders
20
+ loader_schema = uri.split(':/').first.downcase
21
+ res = loaders.find do |loader|
22
+ loader[:class_name].downcase === loader_schema
23
+ end
24
+
25
+ res[:klass].new(uri)
10
26
  end
11
27
  end
12
28
  end
@@ -0,0 +1,18 @@
1
+ module Jetel
2
+ module Loaders
3
+ class Loader
4
+ attr_reader :uri
5
+ def initialize(uri)
6
+ @uri = uri
7
+ end
8
+
9
+ def load(modul, source, file, opts)
10
+ {
11
+ :file => file,
12
+ :module => modul,
13
+ :source => source
14
+ }
15
+ end
16
+ end
17
+ end
18
+ end
@@ -0,0 +1,42 @@
1
+ require 'active_support/inflector'
2
+ require_relative '../extensions/extensions'
3
+
4
+ module Jetel
5
+ module Loaders
6
+ class << self
7
+ def loaders(loaders_dir = File.join(File.dirname(__FILE__)), auto_require = true, require_only = false)
8
+ dir = File.absolute_path(loaders_dir)
9
+ res = Dir.entries(dir).map do |entry|
10
+ dir_path = File.join(dir, entry)
11
+
12
+ next unless File.directory?(dir_path)
13
+ next if entry == '.' || entry == '..'
14
+
15
+ full_path = File.join(dir_path, "#{entry}.rb")
16
+ next unless File.exist?(full_path)
17
+
18
+ # Require file if auto_require true
19
+ require(full_path) if auto_require
20
+
21
+ # Go to next file if require_only mode
22
+ next if require_only
23
+
24
+ class_name = entry.camelize
25
+ qualified_class_name = "Jetel::Loaders::#{class_name}"
26
+
27
+ # Return value from map iteration
28
+ {
29
+ name: entry,
30
+ path: full_path,
31
+ class_name: class_name,
32
+ qualified_class_name: qualified_class_name,
33
+ klass: auto_require ? Kernel.qualified_const_get(qualified_class_name) : nil
34
+ }
35
+ end
36
+
37
+ # Remove nil values and return
38
+ res.compact
39
+ end
40
+ end
41
+ end
42
+ end
@@ -0,0 +1,36 @@
1
+ require_relative '../loader'
2
+
3
+ require 'pg'
4
+
5
+ module Jetel
6
+ module Loaders
7
+ class Pg < Loader
8
+ def initialize(uri)
9
+ super
10
+
11
+ tmp = uri.split('://')
12
+ tmp = tmp[1].split('@')
13
+
14
+ parts = tmp[0].split(':')
15
+ user = parts[0]
16
+ password = parts[1]
17
+
18
+ parts = tmp[1].split('/')
19
+ host, port = parts[0].split(':')
20
+ dbname = parts[1]
21
+
22
+ opts = {
23
+ :host => host,
24
+ :port => port.to_i,
25
+ # :options => '',
26
+ # :tty => '',
27
+ :dbname => dbname,
28
+ :user => user,
29
+ :password => password
30
+ }
31
+
32
+ @client = PG.connect(opts)
33
+ end
34
+ end
35
+ end
36
+ end
@@ -1,6 +1,7 @@
1
1
  # encoding: utf-8
2
2
 
3
3
  require 'pmap'
4
+ require 'csv'
4
5
 
5
6
  require_relative '../../config/config'
6
7
  require_relative '../../helpers/helpers'
@@ -36,20 +37,69 @@ module Jetel
36
37
  }
37
38
  ]
38
39
 
39
- def download
40
+ def download(global_options, options, args)
40
41
  SOURCES.pmap do |source|
41
- target_dir = Helper.target_dir(self, source)
42
- downloader.download(source[:url], {:dir => target_dir})
42
+ download_source(source, global_options.merge(options))
43
43
  end
44
44
  end
45
45
 
46
- def extract
46
+ def extract(global_options, options, args)
47
+ SOURCES.pmap do |source|
48
+ downloaded_file = downloaded_file(source, global_options.merge(options))
49
+ dest_dir = extract_dir(source, global_options.merge(options))
50
+
51
+ puts "Extracting #{downloaded_file}"
52
+
53
+ FileUtils.mkdir_p(dest_dir)
54
+ FileUtils.cp(downloaded_file, dest_dir)
55
+ end
47
56
  end
48
57
 
49
- def transform
58
+ def transform(global_options, options, args)
59
+ SOURCES.pmap do |source|
60
+ opts = global_options.merge(options)
61
+
62
+ extracted_file = extracted_file(source, opts)
63
+ transformed_file = transformed_file(source, opts)
64
+
65
+ FileUtils.mkdir_p(transform_dir(source, opts))
66
+
67
+ csv_opts = {
68
+ :col_sep => '|'
69
+ }
70
+
71
+ puts "Transforming #{extracted_file}"
72
+ CSV.open(extracted_file, 'r', csv_opts) do |csv|
73
+ headers = %w(
74
+ registry
75
+ cc
76
+ type
77
+ start
78
+ value
79
+ date
80
+ status
81
+ )
82
+ CSV.open(transformed_file, 'w', :write_headers => true, :headers => headers) do |csv_out|
83
+ csv.each do |row|
84
+ next if row.length < 7
85
+ next if row[0] != source[:name]
86
+ csv_out << row
87
+ end
88
+ end
89
+ end
90
+ end
50
91
  end
51
92
 
52
- def load
93
+ def load(global_options, options, args)
94
+ SOURCES.map do |source|
95
+ opts = global_options.merge(options)
96
+
97
+ transformed_file = transformed_file(source, opts)
98
+
99
+ loader = Helper.get_loader(opts['data_loader'])
100
+
101
+ loader.load(self, source, transformed_file, opts)
102
+ end
53
103
  end
54
104
  end
55
105
  end
@@ -0,0 +1,82 @@
1
+ # encoding: utf-8
2
+
3
+ require 'pmap'
4
+ require 'zip'
5
+ require 'csv'
6
+
7
+ require_relative '../../config/config'
8
+ require_relative '../../modules/module'
9
+
10
+ module Jetel
11
+ module Modules
12
+ class Iso3166 < Module
13
+ SOURCES = [
14
+ {
15
+ name: 'iso366',
16
+ filename_extracted: 'IP2LOCATION-ISO3166-2.CSV',
17
+ filename_transformed: 'IP2LOCATION-ISO3166-2.CSV',
18
+ url: 'http://www.ip2location.com/downloads/Hi3sL9bnXfe/IP2LOCATION-ISO3166-2.ZIP'
19
+ }
20
+ ]
21
+
22
+ def download(global_options, options, args)
23
+ SOURCES.pmap do |source|
24
+ download_source(source, global_options.merge(options))
25
+ end
26
+ end
27
+
28
+ def extract(global_options, options, args)
29
+ SOURCES.pmap do |source|
30
+ downloaded_file = downloaded_file(source, global_options.merge(options))
31
+ dest_dir = extract_dir(source, global_options.merge(options))
32
+
33
+ FileUtils.mkdir_p(dest_dir)
34
+
35
+ Zip::ZipFile.open(downloaded_file) do |zip_file|
36
+ # Handle entries one by one
37
+ zip_file.each do |entry|
38
+ # Extract to file/directory/symlink
39
+ puts "Extracting #{entry.name}"
40
+ dest_file = File.join(dest_dir, entry.name.split('/').last)
41
+ FileUtils.rm_rf(dest_file)
42
+ entry.extract(dest_file)
43
+ end
44
+ end
45
+ end
46
+ end
47
+
48
+ def transform(global_options, options, args)
49
+ SOURCES.pmap do |source|
50
+ opts = global_options.merge(options)
51
+
52
+ extracted_file = extracted_file(source, opts)
53
+ transformed_file = transformed_file(source, opts)
54
+
55
+ FileUtils.mkdir_p(transform_dir(source, opts))
56
+
57
+ csv_opts = {
58
+ :headers => true
59
+ }
60
+
61
+ puts "Transforming #{extracted_file}"
62
+ CSV.open(extracted_file, 'r', csv_opts) do |csv|
63
+ headers = %w(
64
+ country_code
65
+ subdivision_name
66
+ code
67
+ )
68
+ CSV.open(transformed_file, 'w', :write_headers => true, :headers => headers) do |csv_out|
69
+ csv.each do |row|
70
+ next if row.length < 3
71
+ csv_out << row
72
+ end
73
+ end
74
+ end
75
+ end
76
+ end
77
+
78
+ def load(global_options, options, args)
79
+ end
80
+ end
81
+ end
82
+ end
@@ -1,18 +1,79 @@
1
1
  # encoding: utf-8
2
2
 
3
3
  require_relative '../downloader/downloader'
4
+ require_relative '../helpers/helpers'
4
5
 
5
6
  module Jetel
6
7
  module Modules
7
8
  class Module
8
9
  attr_reader :downloader
9
10
 
11
+ class << self
12
+ def target_dir(modul, source, dir, *path)
13
+ klass = I18n.transliterate(modul.class.name.split('::').last).gsub(/[^0-9a-z_\-]/i, '_')
14
+ source_name = I18n.transliterate(source[:name]).gsub(/[^0-9a-z_\-]/i, '_')
15
+ File.join(dir.kind_of?(String) ? dir : dir['download_dir'] || Config[:DATA_DIRECTORY], klass, source_name, path)
16
+ end
17
+
18
+ def download_dir(modul, source, opts)
19
+ Module.target_dir(modul, source, opts, 'downloaded')
20
+ end
21
+
22
+ def extract_dir(modul, source, opts)
23
+ Module.target_dir(modul, source, opts, 'extracted')
24
+ end
25
+
26
+ def transform_dir(modul, source, opts)
27
+ Module.target_dir(modul, source, opts, 'transformed')
28
+ end
29
+
30
+ def downloaded_file(modul, source, opts)
31
+ File.join(download_dir(modul, source, opts), source[:filename_downloaded] || source[:url].split('/').last)
32
+ end
33
+
34
+ def extracted_file(modul, source, opts)
35
+ File.join(extract_dir(modul, source, opts), source[:filename_extracted] || source[:url].split('/').last)
36
+ end
37
+
38
+ def transformed_file(modul, source, opts)
39
+ File.join(transform_dir(modul, source, opts), source[:filename_transformed] || source[:url].split('/').last)
40
+ end
41
+ end
42
+
10
43
  def initialize
11
44
  @downloader = Downloader.new
12
45
  end
13
46
 
14
- def download(*args)
15
- downloader.download(*args)
47
+ def download_source(source, opts)
48
+ downloader.download(source[:url], {:dir => download_dir(source, opts)})
49
+ end
50
+
51
+ def target_dir(source, opts, *path)
52
+ Module.target_dir(self, source, opts['download_dir'], *path)
53
+ end
54
+
55
+ def download_dir(source, opts)
56
+ Module.download_dir(self, source, opts)
57
+ end
58
+
59
+ def extract_dir(source, opts)
60
+ Module.extract_dir(self, source, opts)
61
+ end
62
+
63
+ def transform_dir(source, opts)
64
+ Module.transform_dir(self, source, opts)
65
+ end
66
+
67
+ def downloaded_file(source, opts)
68
+ Module.downloaded_file(self, source, opts)
69
+ end
70
+
71
+ def extracted_file(source, opts)
72
+ Module.extracted_file(self, source, opts)
73
+ end
74
+
75
+ def transformed_file(source, opts)
76
+ Module.transformed_file(self, source, opts)
16
77
  end
17
78
  end
18
79
  end
@@ -1038,20 +1038,107 @@ module Jetel
1038
1038
  }
1039
1039
  ]
1040
1040
 
1041
- def download
1041
+ def download(global_options, options, args)
1042
1042
  SOURCES.pmap do |source|
1043
- target_dir = Helper.target_dir(self, source)
1044
- downloader.download(source[:url], {:dir => target_dir})
1043
+ download_source(source, global_options.merge(options))
1045
1044
  end
1046
1045
  end
1047
1046
 
1048
- def extract
1047
+ def extract(global_options, options, args)
1048
+ SOURCES.pmap do |source|
1049
+ downloaded_file = downloaded_file(source, global_options.merge(options))
1050
+ dest_dir = extract_dir(source, global_options.merge(options))
1051
+
1052
+ FileUtils.mkdir_p(dest_dir)
1053
+
1054
+ Zip::ZipFile.open(downloaded_file) do |zip_file|
1055
+ # Handle entries one by one
1056
+ zip_file.each do |entry|
1057
+ # Extract to file/directory/symlink
1058
+ puts "Extracting #{entry.name}"
1059
+ dest_file = File.join(dest_dir, entry.name.split('/').last)
1060
+ FileUtils.rm_rf(dest_file)
1061
+ entry.extract(dest_file)
1062
+ end
1063
+ end
1064
+ end
1065
+ end
1066
+
1067
+ def transform(global_options, options, args)
1068
+ SOURCES.pmap do |source|
1069
+ opts = global_options.merge(options)
1070
+
1071
+ extracted_file = extracted_file(source, opts)
1072
+ transformed_file = transformed_file(source, opts)
1073
+
1074
+ FileUtils.mkdir_p(transform_dir(source, opts))
1075
+
1076
+ csv_opts = {
1077
+ :headers => true,
1078
+ :col_sep => "\t",
1079
+ :quote_char => "\x00"
1080
+ }
1081
+
1082
+ puts "Transforming #{extracted_file}"
1083
+ CSV.open(extracted_file, 'r', csv_opts) do |csv|
1084
+ headers = %w(
1085
+ RC
1086
+ UFI
1087
+ UNI
1088
+ LAT
1089
+ LONG
1090
+ DMS_LAT
1091
+ DMS_LONG
1092
+ MGRS
1093
+ JOG
1094
+ FC
1095
+ DSG
1096
+ PC
1097
+ CC1
1098
+ ADM1
1099
+ POP
1100
+ ELEV
1101
+ CC2
1102
+ NT
1103
+ LC
1104
+ SHORT_FORM
1105
+ GENERIC
1106
+ SORT_NAME_RO
1107
+ FULL_NAME_RO
1108
+ FULL_NAME_ND_RO
1109
+ SORT_NAME_RG
1110
+ FULL_NAME_RG
1111
+ FULL_NAME_ND_RG
1112
+ NOTE
1113
+ MODIFY_DATE
1114
+ DISPLAY
1115
+ NAME_RANK
1116
+ NAME_LINK
1117
+ TRANSL_CD
1118
+ NM_MODIFY_DATE
1119
+ F_EFCTV_DT
1120
+ F_TERM_DT
1121
+ )
1122
+ CSV.open(transformed_file, 'w', :write_headers => true, :headers => headers, :quote_char => '"') do |csv_out|
1123
+ csv.each do |row|
1124
+ csv_out << row
1125
+ end
1126
+ end
1127
+ end
1128
+ end
1129
+ end
1130
+
1131
+ def load(global_options, options, args)
1049
1132
  end
1050
1133
 
1051
- def transform
1134
+ def extracted_file(source, opts)
1135
+ res = super(source, opts)
1136
+ res.gsub('.zip', '.txt')
1052
1137
  end
1053
1138
 
1054
- def load
1139
+ def transformed_file(source, opts)
1140
+ res = super(source, opts)
1141
+ res.gsub('.zip', '.txt')
1055
1142
  end
1056
1143
  end
1057
1144
  end
@@ -3,7 +3,6 @@
3
3
  require 'pmap'
4
4
 
5
5
  require_relative '../../config/config'
6
- require_relative '../../helpers/helpers'
7
6
  require_relative '../../modules/module'
8
7
 
9
8
  module Jetel
@@ -16,20 +15,19 @@ module Jetel
16
15
  }
17
16
  ]
18
17
 
19
- def download
18
+ def download(global_options, options, args)
20
19
  SOURCES.pmap do |source|
21
- target_dir = Helper.target_dir(self, source)
22
- downloader.download(source[:url], {:dir => target_dir})
20
+ download_source(source, global_options.merge(options))
23
21
  end
24
22
  end
25
23
 
26
- def extract
24
+ def extract(global_options, options, args)
27
25
  end
28
26
 
29
- def transform
27
+ def transform(global_options, options, args)
30
28
  end
31
29
 
32
- def load
30
+ def load(global_options, options, args)
33
31
  end
34
32
  end
35
33
  end
data/lib/jetel/version.rb CHANGED
@@ -1,5 +1,5 @@
1
1
  # encoding: utf-8
2
2
 
3
3
  module Jetel
4
- VERSION = '0.0.5'
4
+ VERSION = '0.0.6'
5
5
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: jetel
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.5
4
+ version: 0.0.6
5
5
  platform: ruby
6
6
  authors:
7
7
  - Tomas Korcak
@@ -38,6 +38,20 @@ dependencies:
38
38
  - - ">="
39
39
  - !ruby/object:Gem::Version
40
40
  version: '0'
41
+ - !ruby/object:Gem::Dependency
42
+ name: i18n
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - ">="
46
+ - !ruby/object:Gem::Version
47
+ version: '0'
48
+ type: :runtime
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - ">="
53
+ - !ruby/object:Gem::Version
54
+ version: '0'
41
55
  - !ruby/object:Gem::Dependency
42
56
  name: json_pure
43
57
  requirement: !ruby/object:Gem::Requirement
@@ -108,6 +122,20 @@ dependencies:
108
122
  - - ">="
109
123
  - !ruby/object:Gem::Version
110
124
  version: '0'
125
+ - !ruby/object:Gem::Dependency
126
+ name: rubyzip
127
+ requirement: !ruby/object:Gem::Requirement
128
+ requirements:
129
+ - - ">="
130
+ - !ruby/object:Gem::Version
131
+ version: '0'
132
+ type: :runtime
133
+ prerelease: false
134
+ version_requirements: !ruby/object:Gem::Requirement
135
+ requirements:
136
+ - - ">="
137
+ - !ruby/object:Gem::Version
138
+ version: '0'
111
139
  - !ruby/object:Gem::Dependency
112
140
  name: terminal-table
113
141
  requirement: !ruby/object:Gem::Requirement
@@ -240,6 +268,8 @@ files:
240
268
  - Rakefile
241
269
  - bin/jetel
242
270
  - data/.gitkeepme
271
+ - docks/postgresql/pull.sh
272
+ - docks/postgresql/run.sh
243
273
  - jetel.gemspec
244
274
  - lib/jetel.rb
245
275
  - lib/jetel/cli/app.rb
@@ -259,7 +289,11 @@ files:
259
289
  - lib/jetel/helpers/helpers.rb
260
290
  - lib/jetel/jetel/jetel.rb
261
291
  - lib/jetel/lib.rb
292
+ - lib/jetel/loaders/loader.rb
293
+ - lib/jetel/loaders/loaders.rb
294
+ - lib/jetel/loaders/pg/pg.rb
262
295
  - lib/jetel/modules/ip/ip.rb
296
+ - lib/jetel/modules/iso3166/iso3166.rb
263
297
  - lib/jetel/modules/module.rb
264
298
  - lib/jetel/modules/modules.rb
265
299
  - lib/jetel/modules/nga/nga.rb