bulk_importer 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3a3981e26851ce310caa14a2fc257e9c651eccad
-  data.tar.gz: 27cbfe3409574d837f8af6fae3509c71def416bb
+  metadata.gz: fb82dcf1ba046594e065218c795dca1e874ad398
+  data.tar.gz: a44c73dfee9cf88acf5918e6928175c8b0575962
 SHA512:
-  metadata.gz: a25b6f749c6f54136edc8697d25dc3a5b270630f9f38a064fbb83a606038b4d45ac3a8b0f212831e2b4e5c85ebc461a6a19433181c7b3b8c1e1178c188347786
-  data.tar.gz: 7ff8e23b0af3ab71d61c8a8a18b972dc416f6bb9639985c54c8872deb23772326017c21faf7cd2a001f9ca8a8c5a7904140f037daa189f9756eb833b040d0c45
+  metadata.gz: 7524de391b276d92d69a1afb595deebb683e353f33f4c28d05cb1e1339c79a7ee07082a8cbbc464279c8e11e2acd799c0d8d4348ddbe77d3da07589265349b56
+  data.tar.gz: 078908245a45fb5c72ca07c403369c1a04b3d88ac42a51fcfa78128ee7082e5350e3ffa07fb0a1e695af4c159e7cc5123e8cae5247c6ae8ffb16b62bd27ea2cd

data/README.md

@@ -1,4 +1,5 @@
-# BulkImporter
+# BulkImporter [![Build Status](https://travis-ci.org/abelosorio/bulk_importer.svg?branch=master)](https://travis-ci.org/abelosorio/bulk_importer)
+
 
 This gem allows you to import a big amount of data from files to your database as quick as possible.
 

data/Rakefile

@@ -7,4 +7,4 @@ Rake::TestTask.new(:test) do |t|
   t.test_files = FileList['test/**/*_test.rb']
 end
 
-task :default => :spec
+task :default => :test

data/bulk_importer.gemspec

@@ -22,5 +22,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "bundler", "~> 1.11"
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "minitest", "~> 5.0"
-  spec.add_development_dependency "active_support"
+  spec.add_development_dependency "activesupport", "~> 4.2"
 end

data/lib/bulk_importer.rb

@@ -1,6 +1,5 @@
 require "bulk_importer/version"
-require "active_support"
-require "active_support/core_ext"
+require "active_support/all"
 require "bulk_importer/postgresql_module.rb"
 
 module BulkImporter

data/lib/bulk_importer/postgresql_module.rb

@@ -3,13 +3,13 @@
 # +see+ https://www.postgresql.org/docs/9.4/static/index.html
 #
 module PostgresqlModule
-  # Copy from file or standard input.
+  # Copy from file.
   #
   # +see+ https://www.postgresql.org/docs/9.4/static/sql-copy.html
   #
   # ==== Parameters
   #
-  # * +from+   path_to_some_file|stdin.
+  # * +from+   path_to_some_file.
   # * +target+ Destination table name.
   #
   # ==== Options
@@ -24,10 +24,63 @@ module PostgresqlModule
   # +integer+ Number of imported rows.
   #
   def self.copy_from(from, target, format: 'csv', delimiter: ',', null: '', header: true)
-    return -1 unless from == 'stdin' or from.is_a? File
+    return -1 unless from.is_a? File
 
-    result = ActiveRecord::Base.connection.execute self.make_import_sql(
-      from.is_a?(File) ? from.path : from,
+    if self.can_execute_copy
+      # Copy file directly.
+      result = ActiveRecord::Base.connection.execute self.make_import_sql(
+        from.path,
+        target,
+        format:    format,
+        delimiter: delimiter,
+        null:      null,
+        header:    header
+      )
+    else
+      # Copy file by sending it trough stdin.
+      result = self.copy_trough_stdin(
+        from.path,
+        target,
+        format:    format,
+        delimiter: delimiter,
+        null:      null,
+        header:    header
+      )
+    end
+
+    result.cmd_tuples
+  end
+
+  # Copy from file by sendind it trough stdin.
+  #
+  # You should not run this method directly, instead use copy_from.
+  #
+  # ==== Parameters
+  #
+  # * +from+   path_to_some_file.
+  # * +target+ Destination table name.
+  #
+  # ==== Options
+  #
+  # * +format+    File format. Defaults to 'csv'.
+  # * +delimiter+ Column separator character. Defaults ','.
+  # * +null+      String that represent null values. Defaults '' (empty).
+  # * +header+    File includes header? Defaults to True.
+  #
+  # ==== Return
+  #
+  # +integer+ Number of imported rows.
+  #
+  def self.copy_trough_stdin(from, target, format: 'csv', delimiter: ',', null: '', header: true)
+
+    # This solution was taken from:
+    # http://www.kadrmasconcepts.com/blog/2013/12/15/copy-millions-of-rows-to-postgresql-with-rails
+
+    conn = ActiveRecord::Base.connection
+    raw  = conn.raw_connection
+
+    raw.exec self.make_import_sql(
+      'stdin',
       target,
       format:    format,
       delimiter: delimiter,
@@ -35,7 +88,16 @@ module PostgresqlModule
       header:    header
     )
 
-    result.cmd_tuples
+    file = File.open from, 'r'
+
+    until file.eof?
+      raw.put_copy_data file.readline
+    end
+
+    raw.put_copy_end
+    file.close
+
+    raw.get_result
   end
 
   # Get column types.
@@ -134,4 +196,29 @@ module PostgresqlModule
         WHERE table_name = '#{table}'
     eof
   end
+
+  # Checks if the current db user can execute COPY.
+  #
+  # ==== Return
+  #
+  # +bool+
+  #
+  def self.can_execute_copy()
+    # Only superusers can execute COPY.
+    query = <<-eof
+      SELECT *
+        FROM pg_roles
+        WHERE rolsuper AND
+              rolname = CURRENT_USER
+    eof
+
+    begin
+      ActiveRecord::Base.connection.execute(query).cmd_tuples > 0
+    rescue Exception => e
+      Rails.logger.error e.message
+      Rails.logger.error e.backtrace
+
+      false
+    end
+  end
 end

data/lib/bulk_importer/version.rb

@@ -1,3 +1,3 @@
 module BulkImporter
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bulk_importer
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Abel M. Osorio
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-12-01 00:00:00.000000000 Z
+date: 2016-12-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -53,19 +53,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '5.0'
 - !ruby/object:Gem::Dependency
-  name: active_support
+  name: activesupport
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '4.2'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '4.2'
 description: Import big amount of data into any table of your project.
 email:
 - abel.m.osorio@gmail.com