bulk_importer 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 3a3981e26851ce310caa14a2fc257e9c651eccad
+  data.tar.gz: 27cbfe3409574d837f8af6fae3509c71def416bb
+SHA512:
+  metadata.gz: a25b6f749c6f54136edc8697d25dc3a5b270630f9f38a064fbb83a606038b4d45ac3a8b0f212831e2b4e5c85ebc461a6a19433181c7b3b8c1e1178c188347786
+  data.tar.gz: 7ff8e23b0af3ab71d61c8a8a18b972dc416f6bb9639985c54c8872deb23772326017c21faf7cd2a001f9ca8a8c5a7904140f037daa189f9756eb833b040d0c45

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/.travis.yml

@@ -0,0 +1,4 @@
+language: ruby
+rvm:
+  - 2.1.5
+before_install: gem install bundler -v 1.11.2

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in bulk_importer.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 Abel M. Osorio
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,77 @@
+# BulkImporter
+
+This gem allows you to import a big amount of data from files to your database as quick as possible.
+
+Supose you have a CSV file with 500k+ rows. Or even better: 2kk rows. Go, import it. Yes, I know, you can do it... right?. Now, do it in Rails... systematically :)
+This task can be very upset and slow. You need a _bulk import_ operation. Thats is it. Thanks to BulkImporter you only need to write a few sentences and you'll get your data imported _and_ updated.
+
+By now, we only support PostgreSQL 9.4+ databases thanks to the [COPY command](https://www.postgresql.org/docs/9.4/static/sql-copy.html).
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'bulk_importer'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install bulk_importer
+
+## Usage
+
+Suppose your CSV file _/tmp/names.csv_ (delimited by TABs) has the columns: _ID_ and _LOVELY_NAME_, and you need to import this information into your _names_ table, who has the fields _id_ and _name_.
+Also, you want to keep your prexistent data but updated it if necessary.
+
+```ruby
+csv_file = File.new '/tmp/names.csv'
+
+csv_columns = {
+  'ID'          => id,
+  'LOVELY_NAME' => name
+}
+
+# Suppose your table's key is the 'id' column.
+keys = csv_columns.first
+
+BulkImporter.import_from_csv(
+  'names',
+  csv_file,
+  csv_columns,
+  keys,
+  delimiter: '\t',
+  # UPDATE_MODE_UPDATE inserts the new data and update
+  # preexistent (it search by keys).
+  update_mode: BulkImportModule::UPDATE_MODE_UPDATE
+)
+```
+
+Thats all! The imported data will be inserted in your _names_ table in a few seconds.
+
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/abelosorio/bulk_importer.
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
+
+## TO-DO
+
+* Write more tests!
+* Decouple database queries. This gem should work with any database engine. Yes, even MySQL.
+* Since PostgreSQL 9.5 we could implement the UPDATE_MODE_UPDATE method by using [INSERT](https://www.postgresql.org/docs/9.5/static/sql-insert.html) command with the [ON CONFLICT](https://www.postgresql.org/docs/9.5/static/sql-insert.html#SQL-ON-CONFLICT) option.
+* Support different database schemas.

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList['test/**/*_test.rb']
+end
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "bulk_importer"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/bulk_importer.gemspec

@@ -0,0 +1,26 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'bulk_importer/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "bulk_importer"
+  spec.version       = BulkImporter::VERSION
+  spec.authors       = ["Abel M. Osorio"]
+  spec.email         = ["abel.m.osorio@gmail.com"]
+
+  spec.summary       = %q{Bulk importer for Ruby on Rails.}
+  spec.description   = %q{Import big amount of data into any table of your project.}
+  spec.homepage      = "https://github.com/abelosorio/bulk_importer"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.11"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "minitest", "~> 5.0"
+  spec.add_development_dependency "active_support"
+end

data/lib/bulk_importer/postgresql_module.rb

@@ -0,0 +1,137 @@
+# PostgreSQL module for Bulk importer.
+#
+# +see+ https://www.postgresql.org/docs/9.4/static/index.html
+#
+module PostgresqlModule
+  # Copy from file or standard input.
+  #
+  # +see+ https://www.postgresql.org/docs/9.4/static/sql-copy.html
+  #
+  # ==== Parameters
+  #
+  # * +from+   path_to_some_file|stdin.
+  # * +target+ Destination table name.
+  #
+  # ==== Options
+  #
+  # * +format+    File format. Defaults to 'csv'.
+  # * +delimiter+ Column separator character. Defaults ','.
+  # * +null+      String that represent null values. Defaults '' (empty).
+  # * +header+    File includes header? Defaults to True.
+  #
+  # ==== Return
+  #
+  # +integer+ Number of imported rows.
+  #
+  def self.copy_from(from, target, format: 'csv', delimiter: ',', null: '', header: true)
+    return -1 unless from == 'stdin' or from.is_a? File
+
+    result = ActiveRecord::Base.connection.execute self.make_import_sql(
+      from.is_a?(File) ? from.path : from,
+      target,
+      format:    format,
+      delimiter: delimiter,
+      null:      null,
+      header:    header
+    )
+
+    result.cmd_tuples
+  end
+
+  # Get column types.
+  #
+  # ==== Parameters
+  #
+  # * +table+
+  #
+  # ==== Return
+  #
+  # +array+
+  #
+  def self.get_column_types(table)
+    sql = self.make_get_column_types_sql(table)
+    types = {}
+
+    ActiveRecord::Base.connection.execute(sql).each do |row|
+      types[row['name']] = row['type']
+    end
+
+    types
+  end
+
+  # Make the SQL statement of COPY FROM sentence
+  #
+  # ==== Options
+  #
+  # * +format+    File format. Defaults to 'csv'.
+  # * +columns+   An optional array of columns to be copied. If no column list
+  #               is specified, all columns of the table will be copied.
+  # * +delimiter+ Column separator character. Defaults ','.
+  # * +null+      String that represent null values. Defaults '' (empty).
+  # * +header+    File includes header? Defaults to True.
+  #
+  # ==== Return
+  #
+  # +string+
+  #
+  def self.make_import_sql(from, target, format: 'csv', columns: [], delimiter: ',', null: '', header: true)
+    sql = []
+
+    sql << "COPY #{target}"
+    sql << "(#{columns.join(',')})" unless columns.empty?
+    sql << 'FROM'
+    sql << (from.downcase == 'stdin' ? 'STDIN' : "'#{from}'")
+    sql << self.make_import_options_sql(format, delimiter, null, header)
+
+    sql.join ' '
+  end
+
+  # Makes the SQL options to COPY command.
+  #
+  # * +format+    File format. Defaults to 'csv'.
+  # * +delimiter+ Column separator character. Defaults ','.
+  # * +null+      String that represent null values. Defaults '' (empty).
+  # * +header+    File includes header? Defaults to True.
+  #
+  # ==== Return
+  #
+  # +string+
+  #
+  def self.make_import_options_sql(format = 'csv', delimiter = ',', null = '', header = true)
+    return nil if [ format, delimiter, null, header ].all? &:blank?
+
+    options = []
+
+    options << 'WITH'
+    options << format.upcase unless format.blank?
+    options << "DELIMITER E'#{delimiter}'" unless delimiter.blank?
+    options << "NULL '#{null}'" unless null.blank?
+    options << 'HEADER' if header == true and format.downcase == 'csv'
+
+    options.join ' '
+  end
+
+  # Makes the SQL sentence to get column types. This should return two columns
+  # at least: name, type.
+  #
+  # +see+ https://www.postgresql.org/docs/9.4/static/infoschema-columns.html
+  #
+  # ==== Parameters
+  #
+  # * +table+
+  #
+  # ==== Return
+  #
+  # +PG::Result+
+  #
+  # TODO: Add support to different schemas.
+  #
+  def self.make_get_column_types_sql(table)
+    <<-eof
+      SELECT column_name as name,
+             udt_name as type
+        FROM information_schema.columns
+        WHERE table_name = '#{table}'
+    eof
+  end
+end

data/lib/bulk_importer/version.rb

@@ -0,0 +1,3 @@
+module BulkImporter
+  VERSION = "0.1.0"
+end

data/lib/bulk_importer.rb

@@ -0,0 +1,298 @@
+require "bulk_importer/version"
+require "active_support"
+require "active_support/core_ext"
+require "bulk_importer/postgresql_module.rb"
+
+module BulkImporter
+  # Update modes
+  UPDATE_MODE_APPEND  = 'append'
+  UPDATE_MODE_UPDATE  = 'update'
+  UPDATE_MODE_REPLACE = 'replace'
+
+  # Import data from a CSV file to an existing table
+  #
+  # ==== Parameters
+  #
+  # * +target+  Target table.
+  # * +file+    Source CSV file.
+  # * +columns+ Array of CSV columns.
+  # * +keys+    Primary keys of destination table.
+  #
+  # ==== Options
+  #
+  # * +delimiter+
+  # * +null+
+  # * +header+
+  # * +update_mode+ Update mode for imported data.
+  #
+  # ==== Updated modes
+  #
+  # * +self::UPDATE_MODE_APPEND+  Move only new data (default).
+  # * +self::UPDATE_MODE_UPDATE+  Insert new data and updated prexistent.
+  # * +self::UPDATE_MODE_REPLACE+ Truncate old data and insert the new one.
+  #
+  # ==== Return
+  #
+  # +integer+ Number of imported rows.
+  #
+  def self.import_from_csv(target, file, columns, keys, delimiter: ',', null: '', header: true, update_mode: UPDATE_MODE_APPEND)
+    return -1 unless file.is_a? File
+
+    conn = ActiveRecord::Base.connection
+    temp_name = target + '_' + Time.now.to_i.to_s + '_temporal'
+
+    begin
+      # Create temporary table (with all CSV fields)
+      Rails.logger.debug \
+        "[#{self.class}] Creating temporary table #{temp_name}(#{columns.keys})"
+      conn.execute self.make_create_temp_table_sql(temp_name, columns.keys)
+
+      # Import data
+      Rails.logger.debug \
+        "[#{self.class}] Importing data from #{file} to #{temp_name}"
+      PostgresqlModule.copy_from(
+        file,
+        temp_name,
+        format:    'csv',
+        delimiter: delimiter,
+        null:      null,
+        header:    header
+      )
+
+      # Move data from temporary table to target and return total imported rows
+      Rails.logger.debug \
+        "[#{self.class}] Moving new data to #{target} with mode #{update_mode}"
+      self.move_imported_data(temp_name, target, columns, keys, update_mode)
+    rescue Exception => e
+      Rails.logger.error e.message
+      Rails.logger.error e.backtrace
+      return -1
+    ensure
+      # Drop temporary table (if exists)
+      ActiveRecord::Base.connection.execute "DROP TABLE IF EXISTS #{temp_name}"
+    end
+  end
+
+  # Move imported data from origin (raw imported) to destination.
+  #
+  # ==== Parameters
+  #
+  # * +origin+      Origin (temporary table).
+  # * +destination+ Destination table.
+  # * +columns+     Array of CSV columns.
+  # * +keys+        Primary keys of destination table.
+  # * +update_mode+ Update mode.
+  #
+  # ==== Return
+  #
+  # +integer+ Number of imported rows.
+  #
+  def self.move_imported_data(origin, destination, columns, keys, update_mode)
+    unless self.is_update_mode_valid update_mode
+      raise "[#{self.class}] Unknown update mode: #{update_mode}"
+    end
+
+    queries = self.make_move_imported_data_sql(
+      origin,
+      destination,
+      columns,
+      keys,
+      update_mode
+    )
+
+    rows = 0
+
+    queries.each do |query|
+      Rails.logger.debug "[#{self.class}] Running query <<#{query}>>"
+      rows += ActiveRecord::Base.connection.execute(query).cmd_tuples
+    end
+
+    return rows
+  end
+
+  # Makes the SQL command to move the imported data.
+  #
+  # ==== Parameters
+  #
+  # * +origin+      Origin (temporary table).
+  # * +destination+ Destination table.
+  # * +columns+     Array of CSV columns.
+  # * +keys+        Primary keys of destination table.
+  # * +update_mode+ Update mode.
+  #
+  # ==== Return
+  #
+  # +array+ Array of queries to execute.
+  #
+  def self.make_move_imported_data_sql(origin, destination, columns, keys, update_mode)
+    case update_mode
+    when UPDATE_MODE_APPEND
+      # Insert new
+      self.make_update_mode_append_sql origin, destination, columns, keys
+    when UPDATE_MODE_UPDATE
+      # Insert new and Update prexistent
+      self.make_update_mode_update_sql origin, destination, columns, keys
+    when UPDATE_MODE_REPLACE
+      # Truncate destination and Insert new (all)
+      self.make_update_mode_replace_sql origin, destination, columns, keys
+    end
+  end
+
+  # Makes the SQL command to append new imported data.
+  #
+  # ==== Parameters
+  #
+  # * +origin+      Origin (temporary table).
+  # * +destination+ Destination table.
+  # * +columns+     Array of CSV columns.
+  # * +keys+        Primary keys of destination table.
+  #
+  # ==== Return
+  #
+  # +array+ Array of queries to execute
+  #
+  def self.make_update_mode_append_sql(origin, destination, columns, keys)
+    sql = []
+
+    columns = columns.delete_if { |item| columns[item].nil? }
+    types = {}
+
+    pg_types = PostgresqlModule.get_column_types destination
+    columns.values.each { |i| types[columns.invert[i]] = pg_types[i] }
+
+    sql << "INSERT INTO #{destination}"
+    sql << "(#{columns.values.join(',')})"
+    sql << "SELECT #{self.keys_to_list(columns.keys, 'o', types)}"
+    sql << "FROM #{origin} o"
+    sql << "WHERE (#{self.keys_to_list(keys.keys, 'o')}) NOT IN"
+    sql << "( SELECT (#{self.keys_to_list(keys.values, 'd')})"
+    sql << "FROM #{destination} d )"
+
+    [ sql.join(' ') ]
+  end
+
+  # Makes the SQL command to update prexistent data.
+  #
+  # ==== Parameters
+  #
+  # * +origin+      Origin (temporary table).
+  # * +destination+ Destination table.
+  # * +columns+     Array of CSV columns.
+  # * +keys+        Primary keys of destination table.
+  #
+  # ==== Return
+  #
+  # +array+ Array of queries to execute
+  #
+  def self.make_update_mode_update_sql(origin, destination, columns, keys)
+    q = self.make_update_mode_append_sql(origin, destination, columns, keys)
+
+    sql = []
+    types = {}
+
+    pg_types = PostgresqlModule.get_column_types destination
+    columns.values.each { |i| types[columns.invert[i]] = pg_types[i] }
+
+    sql << "UPDATE #{destination} d SET"
+    set = []
+    columns.delete_if { |item| columns[item].nil? }.keys.each do |column|
+      set << "#{columns[column]} = o.#{column}::#{types[column]}"
+    end
+    sql << set.join(',')
+    sql << "FROM #{origin} o"
+    sql << "WHERE (#{self.keys_to_list(keys.keys, 'o')})"
+    sql << "IN (#{self.keys_to_list(keys.values, 'd')})"
+
+    q << sql.join(' ')
+
+    return q
+  end
+
+  # Makes the SQL command to remove existing data and Insert new (all).
+  #
+  # ==== Parameters
+  #
+  # * +origin+      Origin (temporary table).
+  # * +destination+ Destination table.
+  # * +columns+     Array of CSV columns.
+  # * +keys+        Primary keys of destination table.
+  #
+  # ==== Return
+  #
+  # +array+ Array of queries to execute
+  #
+  def self.make_update_mode_replace_sql(origin, destination, columns, key)
+    q = []
+
+    q << "TRUNCATE TABLE #{destination}"
+    q.concat self.make_update_mode_append_sql origin, destination, columns, keys
+
+    return q
+  end
+
+  # Makes the SQL command to create a temporary table.
+  #
+  # ==== Parameters
+  #
+  # * +name+    Name of temporary table.
+  # * +columns+ Array of columns (just name).
+  #
+  # ==== Return
+  #
+  # +string+
+  #
+  def self.make_create_temp_table_sql(name, columns)
+    columns = columns.map { |i| i + ' text' }
+    "CREATE TEMPORARY TABLE #{name} (#{columns.join(',')})"
+  end
+
+  # Translate an array of keys in a list with an optional prefix.
+  #
+  # ==== Parameters
+  #
+  # * +keys+
+  # * +prefix+
+  # * +types+
+  #
+  # ==== Return
+  #
+  # * +string+
+  #
+  # TODO: Add doc
+  #
+  def self.keys_to_list(keys, prefix = nil, types = nil)
+    list = []
+
+    keys.each do |i|
+      col = i
+      col = [ prefix, i ].compact.join('.') unless prefix.nil?
+      col = [ col, types[i] ].compact.join('::') unless types.nil?
+
+      list << col
+    end
+
+    list.join ','
+  end
+
+  # Check if the update mode is valid.
+  #
+  # ==== Parameters
+  #
+  # * +update_mode+
+  #
+  # ==== Return
+  #
+  # +bool+
+  #
+  def self.is_update_mode_valid(update_mode)
+    valid_update_modes = []
+
+    self.constants.each do |constant|
+      if constant.to_s.start_with? 'UPDATE_MODE_'
+        valid_update_modes << self.const_get(constant)
+      end
+    end
+
+    valid_update_modes.include? update_mode.to_s
+  end
+end

metadata

@@ -0,0 +1,112 @@
+--- !ruby/object:Gem::Specification
+name: bulk_importer
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Abel M. Osorio
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2016-12-01 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: active_support
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Import big amount of data into any table of your project.
+email:
+- abel.m.osorio@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- bulk_importer.gemspec
+- lib/bulk_importer.rb
+- lib/bulk_importer/postgresql_module.rb
+- lib/bulk_importer/version.rb
+homepage: https://github.com/abelosorio/bulk_importer
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Bulk importer for Ruby on Rails.
+test_files: []