csv_fast_importer 1.0.0

data/lib/csv_fast_importer/database/mysql.rb

@@ -0,0 +1,28 @@
+require_relative './queryable'
+
+module CsvFastImporter
+  module Database
+    class Mysql < Queryable
+      identifier_quote_character '`'
+
+      def verify_compatibility(configuration)
+        raise 'Transactional not supported with MySQL database' if configuration.transactional_forced?
+      end
+
+      def bulk_import(file, table, columns, row_index_column: nil, column_separator:, encoding:)
+        columns_list_query = columns.map { |column| identify(column) }.join(',')
+        execute <<-SQL
+            LOAD DATA LOCAL INFILE '#{File.expand_path(file)}'
+            INTO TABLE #{identify(table)}
+            CHARACTER SET UTF8
+            FIELDS TERMINATED BY '#{column_separator}' OPTIONALLY ENCLOSED BY '"'
+            LINES TERMINATED BY '\\n'
+            IGNORE 1 LINES
+            (#{columns_list_query})
+            ;
+          SQL
+        query('SELECT ROW_COUNT()')
+      end
+    end
+  end
+end

data/lib/csv_fast_importer/database/postgres.rb

@@ -0,0 +1,36 @@
+require_relative './queryable'
+
+module CsvFastImporter
+  module Database
+    class Postgres < Queryable
+      identifier_quote_character '"'
+
+      def verify_compatibility(configuration)
+        #TODO verify postgresql version
+      end
+
+      def bulk_import(file, table, columns, row_index_column: nil, column_separator:, encoding:)
+        sql_columns = columns
+        sql_columns = [row_index_column] + sql_columns unless row_index_column.nil?
+        columns_list_query = sql_columns.map { |column| identify(column) }
+                                        .join(',')
+
+        row_index = 0
+        connection.copy_data <<-SQL do
+          COPY #{identify(table)} (#{columns_list_query})
+          FROM STDIN
+          DELIMITER '#{column_separator}'
+          CSV
+          ENCODING '#{encoding}';
+        SQL
+          while line = file.gets do
+            row_index += 1
+            line.prepend row_index.to_s << column_separator unless row_index_column.nil?
+            connection.put_copy_data line
+          end
+        end
+        row_index
+      end
+    end
+  end
+end

data/lib/csv_fast_importer/database/queryable.rb

@@ -0,0 +1,51 @@
+module CsvFastImporter
+  module Database
+
+    # Inherit from this class to create new custom database implementation
+    # Do not forget to call .identifier_quote_character
+    class Queryable
+
+      def initialize(connection)
+        @connection = connection
+      end
+
+      # Character used around identifiers (table or column name) to handle special characters
+      def self.identifier_quote_character(character)
+        define_method "identify" do |identifier|
+          character + identifier + character
+        end
+      end
+
+      def identify(table_or_column)
+        raise '#identify method not available. #identifier_quote_character is certainly missing'
+      end
+
+      def connection
+        @connection.raw_connection
+      end
+
+      def execute(query)
+        @connection.execute query
+      end
+
+      def query(query)
+        @connection.select_value query
+      end
+
+      def transaction
+        @connection.transaction do
+          yield
+        end
+      end
+
+      def delete_all(table)
+        execute "DELETE FROM #{identify(table)}"
+      end
+
+      def truncate(table)
+        execute "TRUNCATE TABLE #{identify(table)}"
+      end
+
+    end
+  end
+end

data/lib/csv_fast_importer/database_connection.rb

@@ -0,0 +1,19 @@
+require 'active_record'
+
+module CsvFastImporter
+
+  # Provide access to database driver/adapter
+  class DatabaseConnection
+
+    def self.adapter_name
+      @adapter_name ||= base_connection.adapter_name
+                                      .downcase
+                                      .to_sym
+    end
+
+    def self.base_connection
+      @base_connection ||= ActiveRecord::Base.connection
+    end
+
+  end
+end

data/lib/csv_fast_importer/database_factory.rb

@@ -0,0 +1,19 @@
+require_relative './database_connection'
+require_relative './database/mysql'
+require_relative './database/postgres'
+
+module CsvFastImporter
+
+  # Build corresponding CsvFastImporter database implementation based current adapter (fetched from DatabaseConnection)
+  module DatabaseFactory
+    DATABASES = { postgresql: CsvFastImporter::Database::Postgres,
+                  mysql2: CsvFastImporter::Database::Mysql
+                }
+
+    def self.build
+      adapter = CsvFastImporter::DatabaseConnection.adapter_name
+      return DATABASES[adapter].new(CsvFastImporter::DatabaseConnection.base_connection) if DATABASES.has_key?(adapter)
+      raise "Database adapter #{adapter} not supported by CsvFastImporter. Only #{DATABASES.keys.join(", ")} are supported"
+    end
+  end
+end

data/lib/csv_fast_importer/import.rb

@@ -0,0 +1,58 @@
+require_relative './database_factory'
+
+module CsvFastImporter
+
+  # Responsible for the main process
+  class Import
+
+    def initialize(configuration)
+      @configuration = configuration
+    end
+
+    def run
+      @db = CsvFastImporter::DatabaseFactory.build
+      @db.verify_compatibility @configuration
+
+      row_index = 0
+      within_transaction_if(@configuration.transactional?) do
+        table = @configuration.destination_table
+        columns = db_columns(@configuration)
+        if @configuration.deletion?
+          if @configuration.truncate?
+            @db.truncate table
+          else
+            @db.delete_all table
+          end
+        end
+        row_index = @db.bulk_import(@configuration.file,
+                                    table,
+                                    columns,
+                                    row_index_column: @configuration.row_index_column,
+                                    column_separator: @configuration.column_separator,
+                                    encoding:         @configuration.encoding)
+      end
+      row_index
+    end
+
+    def db_columns(configuration)
+      file_columns = configuration.file
+                                  .gets
+                                  .split(configuration.column_separator)
+                                  .map(&:strip)
+      db_columns = file_columns.map(&:downcase)
+                               .map { |column| configuration.mapping[column] || column }
+      db_columns
+    end
+
+    def within_transaction_if(transactional)
+      if transactional
+        @db.transaction do
+          yield
+        end
+      else
+        yield
+      end
+    end
+
+  end
+end

data/lib/csv_fast_importer/version.rb

@@ -0,0 +1,3 @@
+module CSVFastImporter
+  VERSION = "1.0.0"
+end

data/sample-app/.gitignore

@@ -0,0 +1,10 @@
+# Ignore bundler config.
+.bundle
+
+# Ignore all logfiles and tempfiles.
+log/*
+!/log/.keep
+tmp
+
+# Ignore cache
+vendor/cache

data/sample-app/Gemfile

@@ -0,0 +1,50 @@
+source 'https://rubygems.org'
+
+
+# Bundle edge Rails instead: gem 'rails', github: 'rails/rails'
+gem 'rails', '4.2.6'
+# Use sqlite3 as the database for Active Record
+#gem 'sqlite3'
+gem 'pg'
+# Use SCSS for stylesheets
+gem 'sass-rails', '~> 5.0'
+# Use Uglifier as compressor for JavaScript assets
+gem 'uglifier', '>= 1.3.0'
+# Use CoffeeScript for .coffee assets and views
+gem 'coffee-rails', '~> 4.1.0'
+# See https://github.com/rails/execjs#readme for more supported runtimes
+# gem 'therubyracer', platforms: :ruby
+
+# Use jquery as the JavaScript library
+gem 'jquery-rails'
+# Turbolinks makes following links in your web application faster. Read more: https://github.com/rails/turbolinks
+gem 'turbolinks'
+# Build JSON APIs with ease. Read more: https://github.com/rails/jbuilder
+gem 'jbuilder', '~> 2.0'
+# bundle exec rake doc:rails generates the API under doc/api.
+gem 'sdoc', '~> 0.4.0', group: :doc
+
+gem 'csv_fast_importer'
+
+# Use ActiveModel has_secure_password
+# gem 'bcrypt', '~> 3.1.7'
+
+# Use Unicorn as the app server
+# gem 'unicorn'
+
+# Use Capistrano for deployment
+# gem 'capistrano-rails', group: :development
+
+group :development, :test do
+  # Call 'byebug' anywhere in the code to stop execution and get a debugger console
+  gem 'byebug'
+end
+
+group :development do
+  # Access an IRB console on exception pages or by using <%= console %> in views
+  gem 'web-console', '~> 2.0'
+
+  # Spring speeds up development by keeping your application running in the background. Read more: https://github.com/rails/spring
+  gem 'spring'
+end
+

data/sample-app/Gemfile.lock

@@ -0,0 +1,172 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    actionmailer (4.2.6)
+      actionpack (= 4.2.6)
+      actionview (= 4.2.6)
+      activejob (= 4.2.6)
+      mail (~> 2.5, >= 2.5.4)
+      rails-dom-testing (~> 1.0, >= 1.0.5)
+    actionpack (4.2.6)
+      actionview (= 4.2.6)
+      activesupport (= 4.2.6)
+      rack (~> 1.6)
+      rack-test (~> 0.6.2)
+      rails-dom-testing (~> 1.0, >= 1.0.5)
+      rails-html-sanitizer (~> 1.0, >= 1.0.2)
+    actionview (4.2.6)
+      activesupport (= 4.2.6)
+      builder (~> 3.1)
+      erubis (~> 2.7.0)
+      rails-dom-testing (~> 1.0, >= 1.0.5)
+      rails-html-sanitizer (~> 1.0, >= 1.0.2)
+    activejob (4.2.6)
+      activesupport (= 4.2.6)
+      globalid (>= 0.3.0)
+    activemodel (4.2.6)
+      activesupport (= 4.2.6)
+      builder (~> 3.1)
+    activerecord (4.2.6)
+      activemodel (= 4.2.6)
+      activesupport (= 4.2.6)
+      arel (~> 6.0)
+    activesupport (4.2.6)
+      i18n (~> 0.7)
+      json (~> 1.7, >= 1.7.7)
+      minitest (~> 5.1)
+      thread_safe (~> 0.3, >= 0.3.4)
+      tzinfo (~> 1.1)
+    arel (6.0.4)
+    binding_of_caller (0.7.2)
+      debug_inspector (>= 0.0.1)
+    builder (3.2.3)
+    byebug (9.0.6)
+    coffee-rails (4.1.1)
+      coffee-script (>= 2.2.0)
+      railties (>= 4.0.0, < 5.1.x)
+    coffee-script (2.4.1)
+      coffee-script-source
+      execjs
+    coffee-script-source (1.12.2)
+    concurrent-ruby (1.0.5)
+    csv_fast_importer (1.0.0)
+      activerecord (>= 3.0)
+    debug_inspector (0.0.3)
+    erubis (2.7.0)
+    execjs (2.7.0)
+    ffi (1.9.18)
+    globalid (0.4.0)
+      activesupport (>= 4.2.0)
+    i18n (0.8.6)
+    jbuilder (2.7.0)
+      activesupport (>= 4.2.0)
+      multi_json (>= 1.2)
+    jquery-rails (4.3.1)
+      rails-dom-testing (>= 1, < 3)
+      railties (>= 4.2.0)
+      thor (>= 0.14, < 2.0)
+    json (1.8.6)
+    loofah (2.0.3)
+      nokogiri (>= 1.5.9)
+    mail (2.6.6)
+      mime-types (>= 1.16, < 4)
+    mime-types (3.1)
+      mime-types-data (~> 3.2015)
+    mime-types-data (3.2016.0521)
+    mini_portile2 (2.2.0)
+    minitest (5.10.3)
+    multi_json (1.12.1)
+    nokogiri (1.8.0)
+      mini_portile2 (~> 2.2.0)
+    pg (0.19.0)
+    rack (1.6.8)
+    rack-test (0.6.3)
+      rack (>= 1.0)
+    rails (4.2.6)
+      actionmailer (= 4.2.6)
+      actionpack (= 4.2.6)
+      actionview (= 4.2.6)
+      activejob (= 4.2.6)
+      activemodel (= 4.2.6)
+      activerecord (= 4.2.6)
+      activesupport (= 4.2.6)
+      bundler (>= 1.3.0, < 2.0)
+      railties (= 4.2.6)
+      sprockets-rails
+    rails-deprecated_sanitizer (1.0.3)
+      activesupport (>= 4.2.0.alpha)
+    rails-dom-testing (1.0.8)
+      activesupport (>= 4.2.0.beta, < 5.0)
+      nokogiri (~> 1.6)
+      rails-deprecated_sanitizer (>= 1.0.1)
+    rails-html-sanitizer (1.0.3)
+      loofah (~> 2.0)
+    railties (4.2.6)
+      actionpack (= 4.2.6)
+      activesupport (= 4.2.6)
+      rake (>= 0.8.7)
+      thor (>= 0.18.1, < 2.0)
+    rake (12.0.0)
+    rb-fsevent (0.10.2)
+    rb-inotify (0.9.10)
+      ffi (>= 0.5.0, < 2)
+    rdoc (4.3.0)
+    sass (3.5.1)
+      sass-listen (~> 4.0.0)
+    sass-listen (4.0.0)
+      rb-fsevent (~> 0.9, >= 0.9.4)
+      rb-inotify (~> 0.9, >= 0.9.7)
+    sass-rails (5.0.6)
+      railties (>= 4.0.0, < 6)
+      sass (~> 3.1)
+      sprockets (>= 2.8, < 4.0)
+      sprockets-rails (>= 2.0, < 4.0)
+      tilt (>= 1.1, < 3)
+    sdoc (0.4.2)
+      json (~> 1.7, >= 1.7.7)
+      rdoc (~> 4.0)
+    spring (2.0.2)
+      activesupport (>= 4.2)
+    sprockets (3.7.1)
+      concurrent-ruby (~> 1.0)
+      rack (> 1, < 3)
+    sprockets-rails (3.2.0)
+      actionpack (>= 4.0)
+      activesupport (>= 4.0)
+      sprockets (>= 3.0.0)
+    thor (0.20.0)
+    thread_safe (0.3.6)
+    tilt (2.0.8)
+    turbolinks (5.0.1)
+      turbolinks-source (~> 5)
+    turbolinks-source (5.0.3)
+    tzinfo (1.2.3)
+      thread_safe (~> 0.1)
+    uglifier (3.2.0)
+      execjs (>= 0.3.0, < 3)
+    web-console (2.3.0)
+      activemodel (>= 4.0)
+      binding_of_caller (>= 0.7.2)
+      railties (>= 4.0)
+      sprockets-rails (>= 2.0, < 4.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  byebug
+  coffee-rails (~> 4.1.0)
+  csv_fast_importer
+  jbuilder (~> 2.0)
+  jquery-rails
+  pg
+  rails (= 4.2.6)
+  sass-rails (~> 5.0)
+  sdoc (~> 0.4.0)
+  spring
+  turbolinks
+  uglifier (>= 1.3.0)
+  web-console (~> 2.0)
+
+BUNDLED WITH
+   1.13.2