jetel 0.0.6 → 0.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 51f8e1d58f1480832113887664c59bc1fee6f142
-  data.tar.gz: c144f84c9664e281299bdc9016e81411a5f6c7b3
+  metadata.gz: 037ccf5c15decd72caae4f48d65c8d2525483cd2
+  data.tar.gz: 4bbf16791f9c6bbca7bfd52528dd2067e81aa56c
 SHA512:
-  metadata.gz: 8fde78020fdcadbf6b6eb6fe5a172607c3177f3a2099ee0de095748d7896d1811c0cf8c27ab9cef16fa2c67a244367c689aafa5b8cf681629949b320777b7f5e
-  data.tar.gz: fecde5a4930c26d987a4ff14b93755eb6401304f52944b3cf2ffb2e81fe82bdaf5d867c38f1ebd741def9c9b9fa7305bcd93541c80ac7ffe64ba541d58da698e
+  metadata.gz: 58f2c194d83231c744c26fc28208812ad099e4accb635a84e4fc25f8091556c63043c862cdee054b88042427315994ea7a3ba5d62d81189055869d05d1c03358
+  data.tar.gz: e922b1cdea6f34d4abcd45fd20aa2a019c48e9a5ee9562ebfe7ceacbfa4a903c22e6a5a972342f3b82b3965cf4aba5167938e0048087d9e1d27e4c67665377be

data/.idea/vcs.xml

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <project version="4">
   <component name="VcsDirectoryMappings">
-    <mapping directory="$PROJECT_DIR$" vcs="Git" />
+    <mapping directory="" vcs="Git" />
   </component>
 </project>

data/Gemfile

@@ -2,3 +2,5 @@ source 'https://rubygems.org'
 
 # Specify your gem's dependencies in untitled.gemspec
 gemspec
+
+# gem "csv2psql", :path => '/Users/tomaskorcak/dev/csv2psql'

data/Gemfile.lock

@@ -3,6 +3,7 @@ PATH
   specs:
     jetel (0.0.6)
       activesupport
+      csv2psql
       gli
       i18n
       json_pure
@@ -14,6 +15,17 @@ PATH
       terminal-table
       zip
 
+PATH
+  remote: /Users/tomaskorcak/dev/csv2psql
+  specs:
+    csv2psql (0.0.15)
+      gli (~> 2.13, >= 2.13.2)
+      json_pure (~> 1.8, >= 1.8.3)
+      lru (~> 0.1, >= 0.1.0)
+      multi_json (~> 1.11, >= 1.11.2)
+      rake (~> 10.4, >= 10.4.2)
+      terminal-table (~> 1.5, >= 1.5.2)
+
 GEM
   remote: https://rubygems.org/
   specs:
@@ -42,6 +54,7 @@ GEM
     i18n (0.7.0)
     json (1.8.3)
     json_pure (1.8.3)
+    lru (0.1.0)
     mime-types (2.6.2)
     mini_portile (0.6.2)
     minitest (5.8.2)
@@ -105,6 +118,7 @@ PLATFORMS
 DEPENDENCIES
   bundler (~> 1.5)
   coveralls
+  csv2psql!
   jetel!
   rake
   rspec

data/README.md

@@ -49,7 +49,15 @@ COMMANDS
 └── test
 ```
 
-## Executables
+## Examples
+
+**Plays nicely with [csv2psql](https://github.com/korczis/csv2psql)**
+
+```
+$ csv2psql convert -t --drop-table --create-table -t afrinic tmp/Ip/afrinic/transformed/delegated-afrinic-latest | psql -h 127.0.0.1 -U jetel
+
+$ csv2psql convert -t --drop-table --create-table -t apnic tmp/Ip/apnic/transformed/delegated-apnic-latest | psql -h 127.0.0.1 -U jetel
+```
 
 ### Rake
 

data/bin/jetel

@@ -1,5 +1,3 @@
 #! /usr/bin/env ruby
 
-require_relative '../lib/jetel'
-
 require_relative '../lib/jetel/cli/cli'

data/jetel.gemspec

@@ -20,6 +20,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ['lib']
 
   spec.add_dependency 'activesupport'
+  spec.add_dependency 'csv2psql', '~> 0.0.16'
   spec.add_dependency 'gli'
   spec.add_dependency 'i18n'
   spec.add_dependency 'json_pure'

data/lib/jetel/cli/cli.rb

@@ -1,5 +1,3 @@
 # encoding: UTF-8
 
-require 'pathname'
-
 require_relative 'app'

data/lib/jetel/cli/cmd/config_cmd.rb

@@ -2,10 +2,6 @@
 
 require 'pp'
 
-require_relative '../../version'
-
-require_relative '../shared'
-
 require_relative '../../config/config'
 
 desc 'Show config'

data/lib/jetel/cli/cmd/modules_cmd.rb

@@ -2,10 +2,6 @@
 
 require 'terminal-table'
 
-require_relative '../../version'
-
-require_relative '../shared'
-
 require_relative '../../modules/modules'
 
 MODULES = Jetel::Modules.modules
@@ -14,7 +10,14 @@ MODULES_ACTIONS = {
   download: nil,
   extract: nil,
   transform: nil,
-  load: nil
+  load: {
+    params: [{
+      desc: 'Column type',
+      default_value: nil,
+      arg_name: 'column-name=column-type',
+      flag: [:column_type]
+    }]
+  }
 }
 
 # Gets module name
@@ -32,9 +35,21 @@ end
 # @param action_command [String] Nested command name
 # @param action_desc [String] Nested command action description
 # @return [Object] Return value
-def register_module_action(c, _m, action_command, action_desc, &block)
-  c.desc(action_desc)
-  c.command(action_command) do |cmd|
+def register_module_action(c, modul, name, spec, &block)
+  module_name = modul[:name]
+  action_description = spec && spec[:description] || "#{name} #{module_name}"
+
+  params = spec && spec[:params] || []
+
+
+  c.desc(action_description)
+  c.command(name) do |cmd|
+    params.each do |param|
+      param.each do |name, val|
+        cmd.send name, val
+      end
+    end
+
     cmd.action(&block)
   end
 end
@@ -45,15 +60,12 @@ def register_module(m)
   desc "Module #{module_name}"
   command(m[:name], m[:class_name]) do |c|
     module_instance = m[:klass].new
-    module_name = m[:name]
 
-    MODULES_ACTIONS.each do |k, v|
-      next unless module_instance.respond_to?(k)
+    MODULES_ACTIONS.each do |name, spec|
+      next unless module_instance.respond_to?(name)
 
-      action_name = k
-      action_description = v || "#{action_name} #{module_name}"
-      register_module_action(c, m, action_name, action_description) do |global_options, options, args|
-        module_instance.send(k, global_options, options, args)
+      register_module_action(c, m, name, spec) do |global_options, options, args|
+        module_instance.send(name, global_options, options, args)
       end
     end
   end

data/lib/jetel/helpers/general_helper.rb

@@ -2,16 +2,19 @@
 
 require_relative '../config/config'
 
+require_relative '../helpers/helpers'
 require_relative '../loaders/loaders'
 
+require 'erb'
 require 'i18n'
+require 'ostruct'
 
 module Jetel
   module Helper
     class << self
       def target_dir(modul, dir, source)
-        klass = I18n.transliterate(modul.class.name.split('::').last).gsub(/[^0-9a-z_\-]/i, '_')
-        source_name = I18n.transliterate(source[:name]).gsub(/[^0-9a-z_\-]/i, '_')
+        klass = modul.class.name.split('::').last
+        source_name = Helper.sanitize(source[:name])
         File.join(dir || Config[:DATA_DIRECTORY], klass, source_name)
       end
 
@@ -24,6 +27,19 @@ module Jetel
 
         res[:klass].new(uri)
       end
+
+      def erb(template, vars)
+        ERB.new(template).result(OpenStruct.new(vars).instance_eval { binding })
+      end
+
+      def erb_template(file, vars)
+        template = File.open(file, 'r').read
+        ERB.new(template).result(OpenStruct.new(vars).instance_eval { binding })
+      end
+
+      def sanitize(str)
+        I18n.transliterate(str).gsub(/[^0-9a-z_\-]/i, '_')
+      end
     end
   end
 end

data/lib/jetel/loaders/pg/pg.rb

@@ -1,6 +1,10 @@
 require_relative '../loader'
 
+require_relative '../../helpers/helpers'
+
 require 'pg'
+require 'csv2psql/convert/convert'
+require 'csv2psql/analyzer/analyzer'
 
 module Jetel
   module Loaders
@@ -31,6 +35,83 @@ module Jetel
 
         @client = PG.connect(opts)
       end
+
+      def load(modul, source, file, opts)
+        super
+
+        convert_opts = {
+          :l => 1_000,
+          :skip => 0,
+          :header => true
+        }
+
+        schema_list = Csv2Psql::Convert.generate_schema([file], convert_opts)
+        _file_name, schema = schema_list.first
+
+        return nil if schema.nil?
+
+        analyzer = Csv2Psql::Analyzer.new
+        column_types = (opts['column_type'] && opts['column_type'].split(/[;,]/)) || []
+        column_types.each do |ct|
+          name, type = ct.split('=')
+
+          columns = schema[:columns] || []
+          column = columns.find do |k, v|
+            k.downcase == name
+          end
+
+          analyzer_type = analyzer.analyzers.find do |spec|
+            spec[:class].name.split('::').last.downcase == type.downcase
+          end
+
+          type_val = analyzer_type ? analyzer_type[:class].const_get(:TYPE) : type
+
+          if column
+            columns[column[0]] = {
+              type: type_val.to_sym,
+              null: true
+            }
+          end
+        end
+
+        ctx = {
+          :ctx => {
+            :table => Helper.sanitize(source[:name]).downcase,
+            :columns => schema[:columns],
+            :source => source,
+            :module => modul,
+            :file => File.absolute_path(modul.transformed_file(source, opts))
+          }
+        }
+
+        sql = Helper.erb_template(File.expand_path('../sql/schema.sql.erb', __FILE__), ctx)
+        sql.gsub!("\n\n", "\n")
+        puts sql
+        @client.exec(sql)
+
+        sql = Helper.erb_template(File.expand_path('../sql/copy.sql.erb', __FILE__), ctx)
+        sql.gsub!("\n\n", "\n")
+        puts sql
+        @client.exec(sql)
+
+        file = File.open(ctx[:ctx][:file], 'r')
+        while !file.eof?
+          # Add row to copy data
+          @client.put_copy_data(file.readline)
+        end
+
+        # We are done adding copy data
+        @client.put_copy_end
+
+        # Display any error messages
+        while res = @client.get_result
+          if e_message = res.error_message
+            p e_message
+          end
+        end
+
+        sql
+      end
     end
   end
 end

data/lib/jetel/loaders/pg/sql/copy.sql.erb

@@ -0,0 +1,6 @@
+COPY "<%= ctx[:table] %>"
+  FROM STDIN
+  WITH DELIMITER ','
+  CSV HEADER
+;
+

data/lib/jetel/loaders/pg/sql/create_table.sql.erb

@@ -0,0 +1,12 @@
+DROP TABLE IF EXISTS "<%= ctx[:table] %>";
+
+CREATE TABLE "<%= ctx[:table] %>"
+(
+<% ctx[:columns].each_with_index do |item, index| %>
+	<%= item %> TEXT<%= ", " if index < ctx[:columns].length - 1%>
+<% end %>
+)
+WITH (
+  OIDS=FALSE
+);
+

data/lib/jetel/loaders/pg/sql/drop_table.sql.erb

@@ -0,0 +1,2 @@
+DROP TABLE IF EXISTS "<%= ctx[:table] %>";
+

data/lib/jetel/loaders/pg/sql/header.sql.erb

@@ -0,0 +1,2 @@
+-- Table: "<%= ctx[:table] + "\n" %>"
+

data/lib/jetel/loaders/pg/sql/schema.sql.erb

@@ -0,0 +1,12 @@
+DROP TABLE IF EXISTS  "<%= ctx[:table] %>";
+
+CREATE TABLE "<%= ctx[:table] %>"
+(
+<% ctx[:columns].each_with_index do |item, index| %>
+  "<%= Csv2Psql::Generator.sanitize_header(item[0]) %>" <%= item[1][:type].upcase %><% if !item[1][:null] %> NOT NULL<% end %><%= ', ' if index < ctx[:columns].length - 1%>
+<% end %>
+)
+WITH (
+  OIDS=FALSE
+);
+

data/lib/jetel/loaders/pg/sql/truncate_table.sql.erb

@@ -0,0 +1,2 @@
+TRUNCATE "<%= ctx[:table] %>";
+

data/lib/jetel/modules/ip/ip.rb

@@ -10,41 +10,45 @@ require_relative '../../modules/module'
 module Jetel
   module Modules
     class Ip < Module
-      SOURCES = [
-        {
-          name: 'afrinic',
-          url: 'ftp://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-latest'
-        },
-        {
-          name: 'apnic',
-          url: 'ftp://ftp.apnic.net/pub/stats/apnic/delegated-apnic-latest'
-        },
-        {
-          name: 'arin',
-          url: 'ftp://ftp.arin.net/pub/stats/arin/delegated-arin-latest'
-        },
-        {
-          name: 'lacnic',
-          url: 'ftp://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-latest'
-        },
-        {
-          name: 'ripencc',
-          url: 'ftp://ftp.ripe.net/ripe/stats/delegated-ripencc-latest'
-        },
-        {
-          name: 'iana',
-          url: 'ftp://ftp.apnic.net/pub/stats/iana/delegated-iana-latest'
-        }
-      ]
+      class << self
+        def sources
+          [
+            {
+              name: 'afrinic',
+              url: 'ftp://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-latest'
+            },
+            {
+              name: 'apnic',
+              url: 'ftp://ftp.apnic.net/pub/stats/apnic/delegated-apnic-latest'
+            },
+            {
+              name: 'arin',
+              url: 'ftp://ftp.arin.net/pub/stats/arin/delegated-arin-latest'
+            },
+            {
+              name: 'lacnic',
+              url: 'ftp://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-latest'
+            },
+            {
+              name: 'ripencc',
+              url: 'ftp://ftp.ripe.net/ripe/stats/delegated-ripencc-latest'
+            },
+            {
+              name: 'iana',
+              url: 'ftp://ftp.apnic.net/pub/stats/iana/delegated-iana-latest'
+            }
+          ]
+        end
+      end
 
       def download(global_options, options, args)
-        SOURCES.pmap do |source|
+        self.class.sources.pmap do |source|
           download_source(source, global_options.merge(options))
         end
       end
 
       def extract(global_options, options, args)
-        SOURCES.pmap do |source|
+        self.class.sources.pmap do |source|
           downloaded_file = downloaded_file(source, global_options.merge(options))
           dest_dir = extract_dir(source, global_options.merge(options))
 
@@ -56,7 +60,7 @@ module Jetel
       end
 
       def transform(global_options, options, args)
-        SOURCES.pmap do |source|
+        self.class.sources.pmap do |source|
           opts = global_options.merge(options)
 
           extracted_file = extracted_file(source, opts)
@@ -89,18 +93,6 @@ module Jetel
           end
         end
       end
-
-      def load(global_options, options, args)
-        SOURCES.map do |source|
-          opts = global_options.merge(options)
-
-          transformed_file = transformed_file(source, opts)
-
-          loader = Helper.get_loader(opts['data_loader'])
-
-          loader.load(self, source, transformed_file, opts)
-        end
-      end
     end
   end
 end

data/lib/jetel/modules/iso3166/iso3166.rb

@@ -10,23 +10,27 @@ require_relative '../../modules/module'
 module Jetel
   module Modules
     class Iso3166 < Module
-      SOURCES = [
-        {
-          name: 'iso366',
-          filename_extracted: 'IP2LOCATION-ISO3166-2.CSV',
-          filename_transformed: 'IP2LOCATION-ISO3166-2.CSV',
-          url: 'http://www.ip2location.com/downloads/Hi3sL9bnXfe/IP2LOCATION-ISO3166-2.ZIP'
-        }
-      ]
+      class << self
+        def sources
+          [
+            {
+              name: 'iso366',
+              filename_extracted: 'IP2LOCATION-ISO3166-2.CSV',
+              filename_transformed: 'IP2LOCATION-ISO3166-2.CSV',
+              url: 'http://www.ip2location.com/downloads/Hi3sL9bnXfe/IP2LOCATION-ISO3166-2.ZIP'
+            }
+          ]
+        end
+      end
 
       def download(global_options, options, args)
-        SOURCES.pmap do |source|
+        self.class.sources.pmap do |source|
           download_source(source, global_options.merge(options))
         end
       end
 
       def extract(global_options, options, args)
-        SOURCES.pmap do |source|
+        self.class.sources.pmap do |source|
           downloaded_file = downloaded_file(source, global_options.merge(options))
           dest_dir = extract_dir(source, global_options.merge(options))
 
@@ -46,7 +50,7 @@ module Jetel
       end
 
       def transform(global_options, options, args)
-        SOURCES.pmap do |source|
+        self.class.sources.pmap do |source|
           opts = global_options.merge(options)
 
           extracted_file = extracted_file(source, opts)
@@ -74,9 +78,6 @@ module Jetel
           end
         end
       end
-
-      def load(global_options, options, args)
-      end
     end
   end
 end

data/lib/jetel/modules/module.rb

@@ -10,8 +10,8 @@ module Jetel
 
       class << self
         def target_dir(modul, source, dir, *path)
-          klass = I18n.transliterate(modul.class.name.split('::').last).gsub(/[^0-9a-z_\-]/i, '_')
-          source_name = I18n.transliterate(source[:name]).gsub(/[^0-9a-z_\-]/i, '_')
+          klass = modul.class.name.split('::').last
+          source_name = Helper.sanitize(source[:name])
           File.join(dir.kind_of?(String) ? dir : dir['download_dir'] || Config[:DATA_DIRECTORY], klass, source_name, path)
         end
 
@@ -75,6 +75,26 @@ module Jetel
       def transformed_file(source, opts)
         Module.transformed_file(self, source, opts)
       end
+
+      def load(global_options, options, args)
+        sources = self.class.sources
+        if args.length > 0
+          args = args.map(&:downcase)
+          sources = sources.select do |source|
+            args.index(source[:name].downcase)
+          end
+        end
+
+        sources.pmap(8) do |source|
+          opts = global_options.merge(options)
+
+          transformed_file = transformed_file(source, opts)
+
+          loader = Helper.get_loader(opts['data_loader'])
+
+          loader.load(self, source, transformed_file, opts)
+        end
+      end
     end
   end
 end