jetel 0.0.7 → 0.0.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 037ccf5c15decd72caae4f48d65c8d2525483cd2
-  data.tar.gz: 4bbf16791f9c6bbca7bfd52528dd2067e81aa56c
+  metadata.gz: f223ffcd2b8d33c2fe12cfb2a4f20472dadc5ae9
+  data.tar.gz: ac9d7188dbfce5b31be2a0adfa7c3327645a5f30
 SHA512:
-  metadata.gz: 58f2c194d83231c744c26fc28208812ad099e4accb635a84e4fc25f8091556c63043c862cdee054b88042427315994ea7a3ba5d62d81189055869d05d1c03358
-  data.tar.gz: e922b1cdea6f34d4abcd45fd20aa2a019c48e9a5ee9562ebfe7ceacbfa4a903c22e6a5a972342f3b82b3965cf4aba5167938e0048087d9e1d27e4c67665377be
+  metadata.gz: ffa892d6a6dfcfd26e19131f78cda6db99d625eff6f4cafe80f0035af387f4348b4497191a6d91a4b2e3b0b74d1534ce3113a8de5cb27fd1c88b35a8ac2da1f0
+  data.tar.gz: 9f51835cd7ffab18ac93d975ce6c017814026ec392a3057a93968fbf1941f0444f4dc42e4375476e8569f737808dadc16c3f3fbf1e0c8f2956f28bbbefae2a08

data/Gemfile.lock

@@ -1,9 +1,9 @@
 PATH
   remote: .
   specs:
-    jetel (0.0.6)
+    jetel (0.0.7)
       activesupport
-      csv2psql
+      csv2psql (~> 0.0.19)
       gli
       i18n
       json_pure
@@ -18,7 +18,7 @@ PATH
 PATH
   remote: /Users/tomaskorcak/dev/csv2psql
   specs:
-    csv2psql (0.0.15)
+    csv2psql (0.0.19)
       gli (~> 2.13, >= 2.13.2)
       json_pure (~> 1.8, >= 1.8.3)
       lru (~> 0.1, >= 0.1.0)

data/Rakefile

@@ -1 +1,4 @@
-require 'bundler/gem_tasks'
+namespace :gem do
+  require 'bundler/gem_tasks'
+end
+

data/jetel.gemspec

@@ -20,7 +20,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ['lib']
 
   spec.add_dependency 'activesupport'
-  spec.add_dependency 'csv2psql', '~> 0.0.16'
+  spec.add_dependency 'csv2psql', '~> 0.0.19'
   spec.add_dependency 'gli'
   spec.add_dependency 'i18n'
   spec.add_dependency 'json_pure'

data/lib/jetel/cli/cmd/modules_cmd.rb

@@ -11,12 +11,20 @@ MODULES_ACTIONS = {
   extract: nil,
   transform: nil,
   load: {
-    params: [{
-      desc: 'Column type',
-      default_value: nil,
-      arg_name: 'column-name=column-type',
-      flag: [:column_type]
-    }]
+    params: [
+      {
+        desc: 'Column type',
+        default_value: nil,
+        arg_name: 'column-name=column-type',
+        flag: [:column_type]
+      },
+      {
+        desc: 'Num Lines to analyze',
+        default_value: 1000,
+        arg_name: 'num',
+        flag: [:analyze_num_rows]
+      }
+    ]
   }
 }
 

data/lib/jetel/downloader/backends/aria.rb

@@ -14,7 +14,9 @@ module Jetel
 
         opts = BaseDownloader::OPTS_DOWNLOAD.merge(opts)
 
-        cmd = "aria2c -j 4 -t #{opts[:timeout]} -d \"#{opts[:dir]}\" #{url}"
+        filename = opts[:filename] || url.split('/').last
+
+        cmd = "aria2c -j 4 -t #{opts[:timeout]} -d \"#{opts[:dir]}\" -o \"#{filename}\" #{url}"
         puts(cmd)
 
         PTY.spawn(cmd) do |stdout, stdin, pid|

data/lib/jetel/loaders/pg/pg.rb

@@ -40,9 +40,10 @@ module Jetel
         super
 
         convert_opts = {
-          :l => 1_000,
+          :l => opts['analyze_num_rows'] && opts['analyze_num_rows'].to_i,
           :skip => 0,
-          :header => true
+          :header => true,
+          :delimiter => opts[:delimiter]
         }
 
         schema_list = Csv2Psql::Convert.generate_schema([file], convert_opts)
@@ -80,7 +81,8 @@ module Jetel
             :columns => schema[:columns],
             :source => source,
             :module => modul,
-            :file => File.absolute_path(modul.transformed_file(source, opts))
+            :file => File.absolute_path(modul.transformed_file(source, opts)),
+            :delimiter => opts[:delimiter]
           }
         }
 

data/lib/jetel/loaders/pg/sql/copy.sql.erb

@@ -1,6 +1,11 @@
 COPY "<%= ctx[:table] %>"
   FROM STDIN
-  WITH DELIMITER ','
+  <% if ctx[:delimiter] || ctx[:row_sep] %>
+    WITH DELIMITER '<%= ctx[:delimiter] || ctx[:row_sep] %>'
+  <% else %>
+    WITH DELIMITER ','
+  <% end %>
+
   CSV HEADER
 ;
 

data/lib/jetel/modules/geolite/geolite.rb

@@ -0,0 +1,50 @@
+# encoding: utf-8
+
+require 'pmap'
+
+require_relative '../../config/config'
+require_relative '../../modules/module'
+
+module Jetel
+  module Modules
+    class Geolite < Module
+      class << self
+        def sources
+          [
+            {
+              name: 'geolite',
+              # filename_downloaded: 'SFPD_Incidents_-_from_1_January_2003.csv',
+              filename_extracted: 'GeoLite2-City-Blocks-IPv4.csv',
+              filename_transformed: 'GeoLite2-City-Blocks-IPv4.csv',
+              url: 'http://geolite.maxmind.com/download/geoip/database/GeoLite2-City-CSV.zip'
+            }
+          ]
+        end
+      end
+
+      def download(global_options, options, args)
+        self.class.sources.pmap do |source|
+          download_source(source, global_options.merge(options))
+        end
+      end
+
+      def extract(global_options, options, args)
+        self.class.sources.pmap do |source|
+          unzip(source, global_options.merge(options))
+        end
+      end
+
+      def transform(global_options, options, args)
+        self.class.sources.pmap do |source|
+          extracted_file = extracted_file(source, global_options.merge(options))
+          dest_dir = transform_dir(source, global_options.merge(options))
+
+          puts "Transforming #{extracted_file}"
+
+          FileUtils.mkdir_p(dest_dir)
+          FileUtils.cp(extracted_file, dest_dir)
+        end
+      end
+    end
+  end
+end

data/lib/jetel/modules/iso3166/iso3166.rb

@@ -31,21 +31,7 @@ module Jetel
 
       def extract(global_options, options, args)
         self.class.sources.pmap do |source|
-          downloaded_file = downloaded_file(source, global_options.merge(options))
-          dest_dir = extract_dir(source, global_options.merge(options))
-
-          FileUtils.mkdir_p(dest_dir)
-
-          Zip::ZipFile.open(downloaded_file) do |zip_file|
-            # Handle entries one by one
-            zip_file.each do |entry|
-              # Extract to file/directory/symlink
-              puts "Extracting #{entry.name}"
-              dest_file = File.join(dest_dir, entry.name.split('/').last)
-              FileUtils.rm_rf(dest_file)
-              entry.extract(dest_file)
-            end
-          end
+          unzip(source, global_options.merge(options))
         end
       end
 

data/lib/jetel/modules/module.rb

@@ -1,5 +1,7 @@
 # encoding: utf-8
 
+require 'zip'
+
 require_relative '../downloader/downloader'
 require_relative '../helpers/helpers'
 
@@ -45,7 +47,7 @@ module Jetel
       end
 
       def download_source(source, opts)
-        downloader.download(source[:url], {:dir => download_dir(source, opts)})
+        downloader.download(source[:url], {:dir => download_dir(source, opts), :filename => source[:filename_downloaded]})
       end
 
       def target_dir(source, opts, *path)
@@ -95,6 +97,24 @@ module Jetel
           loader.load(self, source, transformed_file, opts)
         end
       end
+
+      def unzip(source, options = {})
+        downloaded_file = downloaded_file(source, options)
+        dest_dir = extract_dir(source, options)
+
+        FileUtils.mkdir_p(dest_dir)
+
+        Zip::ZipFile.open(downloaded_file) do |zip_file|
+          # Handle entries one by one
+          zip_file.each do |entry|
+            # Extract to file/directory/symlink
+            puts "Extracting #{entry.name}"
+            dest_file = File.join(dest_dir, entry.name.split('/').last)
+            FileUtils.rm_rf(dest_file)
+            entry.extract(dest_file)
+          end
+        end
+      end
     end
   end
 end

data/lib/jetel/modules/nga/nga.rb

@@ -1050,21 +1050,7 @@ module Jetel
 
       def extract(global_options, options, args)
         self.class.sources.pmap do |source|
-          downloaded_file = downloaded_file(source, global_options.merge(options))
-          dest_dir = extract_dir(source, global_options.merge(options))
-
-          FileUtils.mkdir_p(dest_dir)
-
-          Zip::ZipFile.open(downloaded_file) do |zip_file|
-            # Handle entries one by one
-            zip_file.each do |entry|
-              # Extract to file/directory/symlink
-              puts "Extracting #{entry.name}"
-              dest_file = File.join(dest_dir, entry.name.split('/').last)
-              FileUtils.rm_rf(dest_file)
-              entry.extract(dest_file)
-            end
-          end
+          unzip(source, global_options.merge(options))
         end
       end
 

data/lib/jetel/modules/wifileaks/wifileaks.rb

@@ -0,0 +1,78 @@
+# encoding: utf-8
+
+require 'pmap'
+
+require_relative '../../config/config'
+require_relative '../../modules/module'
+
+module Jetel
+  module Modules
+    class Wifileaks < Module
+      class << self
+        def sources
+          [
+            {
+              name: 'wifileaks',
+              url: 'http://download.wifileaks.cz/data/wifileaks_150709.tsv'
+            }
+          ]
+        end
+      end
+
+      def download(global_options, options, args)
+        self.class.sources.pmap do |source|
+          download_source(source, global_options.merge(options))
+        end
+      end
+
+      def extract(global_options, options, args)
+        self.class.sources.pmap do |source|
+          downloaded_file = downloaded_file(source, global_options.merge(options))
+          dest_dir = extract_dir(source, global_options.merge(options))
+
+          puts "Extracting #{downloaded_file}"
+
+          FileUtils.mkdir_p(dest_dir)
+          FileUtils.cp(downloaded_file, dest_dir)
+        end
+      end
+
+      def transform(global_options, options, args)
+        self.class.sources.pmap do |source|
+          opts = global_options.merge(options)
+
+          extracted_file = extracted_file(source, opts)
+          transformed_file = transformed_file(source, opts)
+
+          puts "Transforming #{extracted_file}"
+
+          headers = %w(
+            MAC
+            SSID
+            security
+            latitude
+            longitude
+            altitude
+            updated_at
+          )
+
+          File.open(extracted_file, 'r') do |file_in|
+            File.open(transformed_file, 'w') do |file_out|
+              file_out.puts(headers.join("\t"))
+              file_in.each do |line|
+                next if line == "\n"
+                file_out.puts(line.chomp)
+              end
+            end
+          end
+        end
+      end
+
+      def load(global_options, options, args)
+        res = super(global_options, options.merge({:delimiter => "\t"}), args)
+        res
+      end
+
+    end
+  end
+end

data/lib/jetel/version.rb

@@ -1,5 +1,5 @@
 # encoding: utf-8
 
 module Jetel
-  VERSION = '0.0.7'
+  VERSION = '0.0.8'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: jetel
 version: !ruby/object:Gem::Version
-  version: 0.0.7
+  version: 0.0.8
 platform: ruby
 authors:
 - Tomas Korcak
@@ -30,14 +30,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.0.16
+        version: 0.0.19
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.0.16
+        version: 0.0.19
 - !ruby/object:Gem::Dependency
   name: gli
   requirement: !ruby/object:Gem::Requirement
@@ -281,7 +281,6 @@ files:
 - README.md
 - Rakefile
 - bin/jetel
-- data/.gitkeepme
 - docks/postgresql/pull.sh
 - docks/postgresql/run.sh
 - jetel.gemspec
@@ -312,12 +311,14 @@ files:
 - lib/jetel/loaders/pg/sql/header.sql.erb
 - lib/jetel/loaders/pg/sql/schema.sql.erb
 - lib/jetel/loaders/pg/sql/truncate_table.sql.erb
+- lib/jetel/modules/geolite/geolite.rb
 - lib/jetel/modules/ip/ip.rb
 - lib/jetel/modules/iso3166/iso3166.rb
 - lib/jetel/modules/module.rb
 - lib/jetel/modules/modules.rb
 - lib/jetel/modules/nga/nga.rb
 - lib/jetel/modules/sfpd/sfpd.rb
+- lib/jetel/modules/wifileaks/wifileaks.rb
 - lib/jetel/version.rb
 - test/spec_helper.rb
 homepage: https://github.com/korczis/jetel

data/data/.gitkeepme

@@ -1 +0,0 @@
-# KEEP ME, PLEASE !!!