jetel 0.0.15 → 0.0.16

data/lib/jetel/loaders/pg/sql/truncate_table.sql.erb

@@ -1,2 +1,22 @@
+-- Copyright (c) 2015, Tomas Korcak <korczis@gmail.com>
+
+-- Permission is hereby granted, free of charge, to any person obtaining a copy
+-- of this software and associated documentation files (the "Software"), to deal
+-- in the Software without restriction, including without limitation the rights
+-- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+-- copies of the Software, and to permit persons to whom the Software is
+-- furnished to do so, subject to the following conditions:
+--
+-- The above copyright notice and this permission notice shall be included in
+-- all copies or substantial portions of the Software.
+--
+-- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+-- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+-- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+-- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+-- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+-- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+-- THE SOFTWARE.
+
 TRUNCATE "<%= ctx[:table] %>";
 

data/lib/jetel/modules/alexa/alexa.rb

@@ -1,5 +1,24 @@
 # encoding: utf-8
 
+# Copyright (c) 2015, Tomas Korcak <korczis@gmail.com>
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
 require 'pmap'
 require 'pty'
 

data/lib/jetel/modules/gadm/gadm.rb

@@ -1,5 +1,7 @@
 # encoding: utf-8
 
+
+
 require 'pmap'
 
 require_relative '../../config/config'
@@ -24,7 +26,9 @@ module Jetel
             {
               name: name,
               url: "http://biogeo.ucdavis.edu/data/gadm2.8/shp/#{filename}",
-              filename_downloaded: filename
+              filename_downloaded: filename,
+              flat: true,
+              filename_transformed: "#{name}_adm?.topo.json"
             }
           end
 

data/lib/jetel/modules/geolite/geolite.rb

@@ -1,5 +1,24 @@
 # encoding: utf-8
 
+# Copyright (c) 2015, Tomas Korcak <korczis@gmail.com>
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
 require 'pmap'
 
 require_relative '../../config/config'

data/lib/jetel/modules/ip/ip.rb

@@ -1,5 +1,25 @@
 # encoding: utf-8
 
+# Copyright (c) 2015, Tomas Korcak <korczis@gmail.com>
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
 require 'pmap'
 require 'csv'
 

data/lib/jetel/modules/iso3166/iso3166.rb

@@ -1,5 +1,25 @@
 # encoding: utf-8
 
+# Copyright (c) 2015, Tomas Korcak <korczis@gmail.com>
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
 require 'nokogiri'
 require 'open-uri'
 require 'pmap'

data/lib/jetel/modules/module.rb

@@ -1,5 +1,27 @@
 # encoding: utf-8
 
+# Copyright (c) 2015, Tomas Korcak <korczis@gmail.com>
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+require 'multi_json'
+require 'pp'
 require 'zip'
 
 require_relative '../downloaders/downloader'
@@ -14,7 +36,7 @@ module Jetel
         def target_dir(modul, source, dir, *path)
           klass = modul.class.name.split('::').last
           source_name = Helper.sanitize(source[:name])
-          File.join(dir.kind_of?(String) ? dir : dir['download_dir'] || Config[:DATA_DIRECTORY], klass, source_name, path)
+          File.join(dir.kind_of?(Hash) ? dir['download_dir'] : dir || Config[:DATA_DIRECTORY], klass, source[:flat] ? '' : source_name, path)
         end
 
         def download_dir(modul, source, opts)
@@ -47,7 +69,11 @@ module Jetel
       end
 
       def download_source(source, opts)
-        downloader.download(source[:url], {:dir => download_dir(source, opts), :filename => source[:filename_downloaded]})
+        downloaded_file = downloaded_file(source, opts)
+
+        unless File.exists?(downloaded_file)
+          downloader.download(source[:url], {:dir => download_dir(source, opts), :filename => source[:filename_downloaded]})
+        end
       end
 
       def target_dir(source, opts, *path)
@@ -87,17 +113,33 @@ module Jetel
           end
         end
 
-        sources.pmap(8) do |source|
+        sources.map do |source|
           opts = global_options.merge(options)
 
           transformed_file = transformed_file(source, opts)
 
           loader = Helper.get_loader(opts['data_loader'])
 
-          loader.load(self, source, transformed_file, opts)
+          Dir.glob(transformed_file).each do |one_file|
+            puts "Loading file #{one_file}"
+            loader.load(self, source, one_file, opts)
+          end
         end
       end
 
+      def sources(global_options, options, _args)
+        opts = global_options.merge(options)
+
+        res = self.class.sources
+
+        if opts['format'] == 'json'
+          puts MultiJson.dump(res, :pretty => true)
+        else
+          pp res
+        end
+
+      end
+
       def unzip(source, options = {})
         downloaded_file = downloaded_file(source, options)
         dest_dir = extract_dir(source, options)

data/lib/jetel/modules/modules.rb

@@ -1,3 +1,25 @@
+# encoding: utf-8
+
+# Copyright (c) 2015, Tomas Korcak <korczis@gmail.com>
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
 require 'active_support/inflector'
 require_relative '../extensions/extensions'
 

data/lib/jetel/modules/nga/nga.rb

@@ -1,5 +1,25 @@
 # encoding: utf-8
 
+# Copyright (c) 2015, Tomas Korcak <korczis@gmail.com>
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
 require 'pmap'
 
 require_relative '../../modules/module'

data/lib/jetel/modules/sfpd/sfpd.rb

@@ -1,5 +1,25 @@
 # encoding: utf-8
 
+# Copyright (c) 2015, Tomas Korcak <korczis@gmail.com>
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
 require 'pmap'
 
 require_relative '../../config/config'

data/lib/jetel/modules/tiger/tiger.rb

@@ -0,0 +1,113 @@
+# encoding: utf-8
+
+# Copyright (c) 2015, Tomas Korcak <korczis@gmail.com>
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+require 'pmap'
+
+require_relative '../../config/config'
+require_relative '../../modules/module'
+
+require 'nokogiri'
+require 'open-uri'
+
+module Jetel
+  module Modules
+    class Tiger < Module
+      BASE_URL = 'http://www2.census.gov/geo/tiger/TIGER2015'
+      CSS_SELECTOR = '#innerPage > table > tr > td > a'
+
+      class << self
+        def sub_sources(base_url)
+          page = Nokogiri::HTML(open(base_url))
+
+          links = page.css(CSS_SELECTOR)
+
+          links[1..-1].map do |link|
+            next unless link
+            name = base_url.split('/').last # link.text.gsub('/', '')
+            href = link.attr('href')
+
+            {
+              name: name,
+              url: "#{base_url}#{href}".gsub('http://www2', 'ftp://ftp'),
+              filename_downloaded: href,
+              # flat: true,
+              # filename_transformed: "#{name}_adm?.topo.json"
+            }
+          end
+        end
+
+        def sources
+          page = Nokogiri::HTML(open(BASE_URL))
+
+          links = page.css(CSS_SELECTOR)
+
+          res = links[2..-1].pmap(8) do |link|
+            next unless link
+            href = link.attr('href')
+            url = "#{BASE_URL}/#{href}"
+
+            puts "Processing #{url}"
+            tmp = sub_sources(url)
+          end
+
+          res.flatten
+        end
+      end
+
+      def download(global_options, options, args)
+        self.class.sources.pmap(16) do |source|
+          download_source(source, global_options.merge(options))
+        end
+      end
+
+      def extract(global_options, options, args)
+        self.class.sources.map do |source|
+          unzip(source, global_options.merge(options))
+        end
+      end
+
+      def transform(global_options, options, args)
+        self.class.sources.pmap(8) do |source|
+          extracted_file = extracted_file(source, global_options.merge(options))
+          transformed_file = transformed_file(source, global_options.merge(options))
+          dest_dir = transform_dir(source, global_options.merge(options))
+          FileUtils.mkdir_p(dest_dir)
+
+          extracted_dir = extract_dir(source, global_options.merge(options))
+          Dir.glob("#{extracted_dir}/*.shp") do |shapefile|
+            puts "Transforming #{shapefile}"
+
+            # "topojson data/Gadm/AFG/extracted/AFG_adm0.shp -o data/Gadm/AFG/transformed/AFG_adm0.topo.json"
+            cmd = "topojson #{shapefile} -o #{shapefile.gsub(extracted_dir, dest_dir).gsub('.shp', '.topo.json')}"
+            puts cmd
+            PTY.spawn(cmd) do |stdout, stdin, pid|
+              begin
+                # Do stuff with the output here. Just printing to show it works
+                stdout.each { |line| print line }
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/jetel/modules/wifileaks/wifileaks.rb

@@ -1,5 +1,26 @@
 # encoding: utf-8
 
+# Copyright (c) 2015, Tomas Korcak <korczis@gmail.com>
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+require 'fileutils'
 require 'pmap'
 
 require_relative '../../config/config'
@@ -72,7 +93,6 @@ module Jetel
         res = super(global_options, options.merge({:delimiter => "\t"}), args)
         res
       end
-
     end
   end
 end