bulk_importer 0.2.1 → 0.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 55b8c71a91b7654fde766010d72fb4ce9870c322
-  data.tar.gz: e09f5235f7ebbbd06b3339f3f4c2bbd892e15335
+  metadata.gz: 5b35f73c6f0ad696286c181ec46e0b8cc00334b6
+  data.tar.gz: 5666a2358e00501877413c17783d6dc4de2d1246
 SHA512:
-  metadata.gz: 3994d8a548c294a2038b340b185f0645ede8671d000dacd389c233fff3aec6ae64d82a73b91536fda4a6c253fdc520cafa9bf366dd50a73ea712398a1251a770
-  data.tar.gz: d0a2c510e5cc5af410b3ad83b10b82ab0a5a3d0f090882d0bc56cfc518324c0036d23b88872d42df8a9f89def7087ff5795ec219e58f87894771c6634d659526
+  metadata.gz: c72a504a6946cff11319fb0ca0825ff345d6717c782daf6662a9717f399ca9ed5af9a231ae339713e2072eb6a73ca89197b687322529d119f73d2a42808ba0b2
+  data.tar.gz: 9e20be8acc80d4892237b6c57c6b8d373a852300ee42884ae8e9ae2139a66acb0340c15f41572812fdb772b7ee58753042bf3a82d1e4427481e58c4fd91b911f

data/README.md

@@ -1,4 +1,4 @@
-# BulkImporter [![Build Status](https://travis-ci.org/abelosorio/bulk_importer.svg?branch=master)](https://travis-ci.org/abelosorio/bulk_importer)
+# BulkImporter [![Build Status](https://travis-ci.org/abelosorio/bulk_importer.svg?branch=master)](https://travis-ci.org/abelosorio/bulk_importer) [![Gem Version](https://badge.fury.io/rb/bulk_importer.svg)](https://badge.fury.io/rb/bulk_importer)
 
 
 This gem allows you to import a big amount of data from files to your database as quick as possible.

data/lib/bulk_importer/postgresql_module.rb

@@ -224,4 +224,13 @@ module PostgresqlModule
 
     ActiveRecord::Base.connection.execute(query).first['rolsuper'] == 't'
   end
+
+  # Create index on table (column(s)).
+  #
+  # * +table+
+  # * +columns+
+  #
+  def self.create_index_on(table, columns)
+    ActiveRecord::Base.connection.add_index table, columns
+  end
 end

data/lib/bulk_importer/version.rb

@@ -1,3 +1,3 @@
 module BulkImporter
-  VERSION = "0.2.1"
+  VERSION = "0.2.2"
 end

data/lib/bulk_importer.rb

@@ -8,6 +8,9 @@ module BulkImporter
   UPDATE_MODE_UPDATE  = 'update'
   UPDATE_MODE_REPLACE = 'replace'
 
+  # Module name (used for loggin).
+  NAME = 'BulkImporter'
+
   # Import data from a CSV file to an existing table
   #
   # ==== Parameters
@@ -43,12 +46,12 @@ module BulkImporter
     begin
       # Create temporary table (with all CSV fields)
       Rails.logger.debug \
-        "[#{self.class}] Creating temporary table #{temp_name}(#{columns.keys})"
+        "[#{NAME}] Creating temporary table #{temp_name}(#{columns.keys})"
       conn.execute self.make_create_temp_table_sql(temp_name, columns.keys)
 
       # Import data
       Rails.logger.debug \
-        "[#{self.class}] Importing data from #{file} to #{temp_name}"
+        "[#{NAME}] Importing data from #{file} to #{temp_name}"
       PostgresqlModule.copy_from(
         file,
         temp_name,
@@ -60,7 +63,7 @@ module BulkImporter
 
       # Move data from temporary table to target and return total imported rows
       Rails.logger.debug \
-        "[#{self.class}] Moving new data to #{target} with mode #{update_mode}"
+        "[#{NAME}] Moving new data to #{target} with mode #{update_mode}"
       self.move_imported_data(temp_name, target, columns, keys, update_mode)
     rescue Exception => e
       Rails.logger.error e.message
@@ -68,7 +71,7 @@ module BulkImporter
       return -1
     ensure
       # Drop temporary table (if exists)
-      ActiveRecord::Base.connection.execute "DROP TABLE IF EXISTS #{temp_name}"
+      #ActiveRecord::Base.connection.execute "DROP TABLE IF EXISTS #{temp_name}"
     end
   end
 
@@ -88,9 +91,13 @@ module BulkImporter
   #
   def self.move_imported_data(origin, destination, columns, keys, update_mode)
     unless self.is_update_mode_valid update_mode
-      raise "[#{self.class}] Unknown update mode: #{update_mode}"
+      raise "[#{NAME}] Unknown update mode: #{update_mode}"
     end
 
+    # Create an index to improve move performance.
+    Rails.logger.debug "[#{NAME}] Creating index on #{origin}"
+    PostgresqlModule.create_index_on origin, keys.keys.map { |i| i.downcase }
+
     queries = self.make_move_imported_data_sql(
       origin,
       destination,
@@ -102,7 +109,7 @@ module BulkImporter
     rows = 0
 
     queries.each do |query|
-      Rails.logger.debug "[#{self.class}] Running query <<#{query}>>"
+      Rails.logger.debug "[#{NAME}] Running query <<#{query}>>"
       rows += ActiveRecord::Base.connection.execute(query).cmd_tuples
     end
 
@@ -243,7 +250,8 @@ module BulkImporter
   #
   def self.make_create_temp_table_sql(name, columns)
     columns = columns.map { |i| i + ' text' }
-    "CREATE TEMPORARY TABLE #{name} (#{columns.join(',')})"
+    #"CREATE TEMPORARY TABLE #{name} (#{columns.join(',')})"
+    "CREATE TABLE #{name} (#{columns.join(',')})"
   end
 
   # Translate an array of keys in a list with an optional prefix.

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bulk_importer
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.2.2
 platform: ruby
 authors:
 - Abel M. Osorio
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-12-03 00:00:00.000000000 Z
+date: 2016-12-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler