jetel 0.0.14 → 0.0.15

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ea0e9c2a01bc364e5e46006540f86d4addda47be
-  data.tar.gz: 71217244d4494fcaa12f480a0ea6180f35e5c418
+  metadata.gz: 08a134a7446127a80ff17bc70f7ee1312db161e0
+  data.tar.gz: 6c6b65b87afd99a4f05672d421eb33619b100420
 SHA512:
-  metadata.gz: b07351ad0b02a17d45973de7491f55d68a6a8c47d288c034fec083fb2896cf5084ae380fd86cb478bcf41633c5ee12440e9db0a52a1f0ccde0b8f73f716c7ad4
-  data.tar.gz: 907f2c48bfa520617933722899bd95c2428861cc891f482613ec12ff58fbf6f2cf467a0a7869cd6b19f70ab7621716ac1528669ef7d36ddcec7f8edbe0b62b68
+  metadata.gz: fa616721b387cbf35f0f586e9bac07dd2119e6f1133a7ee0a9574ab82efa2f285e3e16dc12dccda643e77ada5e7fa59b38c2f215d3737908200bc4ef19db1183
+  data.tar.gz: 4009e9ca9867ce8d2eb9ca2d6322cd5254b73334e729d283a4d1d81231bc34c982114a7b2926ba66e005578e1d3a014786bc23b02b76c64d0327e030290f4c73

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    jetel (0.0.14)
+    jetel (0.0.15)
       activesupport
       aws-sdk (~> 2)
       couchbase (~> 1.3, >= 1.3.14)
@@ -84,7 +84,7 @@ GEM
     multi_json (1.11.2)
     multipart-post (2.0.0)
     netrc (0.7.9)
-    nokogiri (1.6.6.2)
+    nokogiri (1.6.6.3)
       mini_portile (~> 0.6.0)
     parser (2.2.3.0)
       ast (>= 1.1, < 3.0)

data/README.md

@@ -102,6 +102,7 @@ $ jetel downloaders
 | aria | Jetel::Downloaders::Aria |
 | curl | Jetel::Downloaders::Curl |
 | ruby | Jetel::Downloaders::Ruby |
+| wget | Jetel::Downloaders::Wget |
 +------+--------------------------+
 ```
 
@@ -200,14 +201,16 @@ COPY "geolite"
 ```
 .
 ├── bin
-── lib
+├── lib
 │   └── jetel
 │       ├── cli
 │       │   └── cmd
 │       ├── config
 │       ├── downloaders
 │       │   ├── aria
-│       │   └── ruby
+│       │   ├── curl
+│       │   ├── ruby
+│       │   └── wget
 │       ├── extensions
 │       ├── helpers
 │       ├── loaders

data/lib/jetel/modules/alexa/alexa.rb

@@ -1,6 +1,7 @@
 # encoding: utf-8
 
 require 'pmap'
+require 'pty'
 
 require_relative '../../config/config'
 require_relative '../../modules/module'

data/lib/jetel/modules/gadm/gadm.rb

@@ -0,0 +1,72 @@
+# encoding: utf-8
+
+require 'pmap'
+
+require_relative '../../config/config'
+require_relative '../../modules/module'
+
+require 'nokogiri'
+require 'open-uri'
+
+module Jetel
+  module Modules
+    class Gadm < Module
+      class << self
+        def sources
+          page = Nokogiri::HTML(open('http://www.gadm.org/country'))
+
+          options = page.css('select[name="cnt"] > option')
+
+          res = options.map do |option|
+            full_name = option['value']
+            name = "#{full_name.split('_').first}"
+            filename = "#{name}_adm_shp.zip"
+            {
+              name: name,
+              url: "http://biogeo.ucdavis.edu/data/gadm2.8/shp/#{filename}",
+              filename_downloaded: filename
+            }
+          end
+
+          res
+        end
+      end
+
+      def download(global_options, options, args)
+        self.class.sources.pmap do |source|
+          download_source(source, global_options.merge(options))
+        end
+      end
+
+      def extract(global_options, options, args)
+        self.class.sources.pmap do |source|
+          unzip(source, global_options.merge(options))
+        end
+      end
+
+      def transform(global_options, options, args)
+        self.class.sources.pmap(8) do |source|
+          extracted_file = extracted_file(source, global_options.merge(options))
+          transformed_file = transformed_file(source, global_options.merge(options))
+          dest_dir = transform_dir(source, global_options.merge(options))
+          FileUtils.mkdir_p(dest_dir)
+
+          extracted_dir = extract_dir(source, global_options.merge(options))
+          Dir.glob("#{extracted_dir}/*.shp") do |shapefile|
+            puts "Transforming #{shapefile}"
+
+            # "topojson data/Gadm/AFG/extracted/AFG_adm0.shp -o data/Gadm/AFG/transformed/AFG_adm0.topo.json"
+            cmd = "topojson #{shapefile} -o #{shapefile.gsub(extracted_dir, dest_dir).gsub('.shp', '.topo.json')}"
+            puts cmd
+            PTY.spawn(cmd) do |stdout, stdin, pid|
+              begin
+                # Do stuff with the output here. Just printing to show it works
+                stdout.each { |line| print line }
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/jetel/modules/iso3166/iso3166.rb

@@ -1,5 +1,7 @@
 # encoding: utf-8
 
+require 'nokogiri'
+require 'open-uri'
 require 'pmap'
 require 'zip'
 require 'csv'
@@ -14,10 +16,8 @@ module Jetel
         def sources
           [
             {
-              name: 'iso366',
-              filename_extracted: 'IP2LOCATION-ISO3166-2.CSV',
-              filename_transformed: 'IP2LOCATION-ISO3166-2.CSV',
-              url: 'http://www.ip2location.com/downloads/Hi3sL9bnXfe/IP2LOCATION-ISO3166-2.ZIP'
+              name: 'iso3166',
+              url: 'https://en.wikipedia.org/wiki/ISO_3166-1'
             }
           ]
         end
@@ -31,35 +31,52 @@ module Jetel
 
       def extract(global_options, options, args)
         self.class.sources.pmap do |source|
-          unzip(source, global_options.merge(options))
+          downloaded_file = downloaded_file(source, global_options.merge(options))
+          dest_dir = extract_dir(source, global_options.merge(options))
+
+          puts "Extracting #{downloaded_file}"
+
+          FileUtils.mkdir_p(dest_dir)
+          FileUtils.cp(downloaded_file, dest_dir)
         end
       end
 
       def transform(global_options, options, args)
         self.class.sources.pmap do |source|
-          opts = global_options.merge(options)
+          extracted_file = extracted_file(source, global_options.merge(options))
+          dest_dir = transform_dir(source, global_options.merge(options))
+          FileUtils.mkdir_p(dest_dir)
+
+          puts "Transforming #{extracted_file}"
 
-          extracted_file = extracted_file(source, opts)
-          transformed_file = transformed_file(source, opts)
+          page = Nokogiri::HTML(open(extracted_file))
+          rows = page.css('table.wikitable.sortable tr')
 
-          FileUtils.mkdir_p(transform_dir(source, opts))
+          headers = [
+            'name',
+            'alpha2',
+            'alpha3',
+            'numeric'
+          ]
 
-          csv_opts = {
-            :headers => true
-          }
+          transformed_file = "#{extracted_file.gsub(extract_dir(source, global_options.merge(options)), dest_dir)}.csv"
+          CSV.open(transformed_file, 'w', :write_headers => true, :headers => headers, :quote_char => '"', :force_quotes => true) do |csv|
+            rows.each do |row|
+              td = row.css('td')
 
-          puts "Transforming #{extracted_file}"
-          CSV.open(extracted_file, 'r', csv_opts) do |csv|
-            headers = %w(
-              country_code
-              subdivision_name
-              code
-            )
-            CSV.open(transformed_file, 'w', :write_headers => true, :headers => headers) do |csv_out|
-              csv.each do |row|
-                next if row.length < 3
-                csv_out << row
-              end
+              next if td.length < 4
+
+              name = td[0].css('a').text
+              alpha2 = td[1].text
+              alpha3 = td[2].text
+              numeric = td[3].text
+
+              csv << [
+                 name,
+                 alpha2,
+                 alpha3,
+                 numeric
+              ]
             end
           end
         end

data/lib/jetel/version.rb

@@ -1,5 +1,5 @@
 # encoding: utf-8
 
 module Jetel
-  VERSION = '0.0.14'
+  VERSION = '0.0.15'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: jetel
 version: !ruby/object:Gem::Version
-  version: 0.0.14
+  version: 0.0.15
 platform: ruby
 authors:
 - Tomas Korcak
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-11-07 00:00:00.000000000 Z
+date: 2015-11-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -425,6 +425,7 @@ files:
 - lib/jetel/loaders/pg/sql/schema.sql.erb
 - lib/jetel/loaders/pg/sql/truncate_table.sql.erb
 - lib/jetel/modules/alexa/alexa.rb
+- lib/jetel/modules/gadm/gadm.rb
 - lib/jetel/modules/geolite/geolite.rb
 - lib/jetel/modules/ip/ip.rb
 - lib/jetel/modules/iso3166/iso3166.rb