beam 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4537b1cefd669c45e4db6e2feee8ba8644405fc7
-  data.tar.gz: aea8de9386ece2ccf66636201dd97c43ebf00830
+  metadata.gz: acd3a8c0a75cf6f919a96814829bbe6d77c05898
+  data.tar.gz: e84bbadfac36ad545ac651725681851d05b15cf6
 SHA512:
-  metadata.gz: 571b1f9e6e89fde10ace7122e58425248a3ba14e13414b7fdf19cee2a3aeb7e328a19f3b5cbf8d0eaf4a1edbffbd1f292fc7a52133948b25dfb04af771cac0f7
-  data.tar.gz: 25b4e17a6e0bf135e5343cf54bf84a29e8987f248afbfcb3623ea2f8b3f238c4a295dd015b630d9a0fb6c1e7d06c42d0ae880e210113eebfacce1e886a497374
+  metadata.gz: 3644af866a139d8737ac62a83718768240061849c0ba38d3239aa721e46e4cb3ff3d024f41991950310b65ecacdca1d4a9892fe7f1ce109d035c3113eddf5178
+  data.tar.gz: 4afcf1ad50f32f32eb170f9d466cbb4e163c8a7f1a94a4b2e98d6a206dbf955281250be5b816ca01edf104345092328de3fefb26a7063f67a8014c7c3b439817

data/CHANGELOG.md

@@ -1,2 +1,5 @@
+### 0.0.2
+- Added batch processing with activerecord-import
+
 ### 0.0.1
 - Basic version of csv uploader

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    beam (0.0.1)
+    beam (0.0.2)
 
 GEM
   remote: https://rubygems.org/

data/README.md

@@ -1,15 +1,32 @@
 # Beam
 
-A rubygem to simplifiy repetitive csv upload process for ActiveRecord models
+A rubygem to simplifiy repetitive csv upload process for ActiveRecord models. 
+Supports bulk upload with [activerecord-import](http://rubygems.org/gems/activerecord-import)
 
 ## Usage
 
-1. Add it to the model you want to import csv file
+1. Add it the application's Gemfile:
+    ```ruby
+    gem 'beam'
+    ```
+    
+    Add [activerecord-import gem](http://rubygems.org/gems/activerecord-import) to the application's Gemfile:
+    
+    ```ruby
+    gem 'activerecord-import', '0.4.1' # for rails-4.1 app
+    gem 'activerecord-import', '0.4.0' # for rails-4.0 app
+    gem 'activerecord-import', '0.3.1' # for rails-3.1+ app
+    ```
 
+2. Add it to the model you want to import csv file
+    
+    ```ruby
     extend Beam::Upload
+    ```
 
-2. Upload zipped csv file, e.g. users.csv.zip
-
+3. Upload zipped csv file, e.g. users.csv.zip
+  
+    ```ruby
     Model.upload_file(file_name, file_path)
     
     # where users.csv has headers and rows, e.g.:
@@ -18,13 +35,17 @@ A rubygem to simplifiy repetitive csv upload process for ActiveRecord models
     # Test2,test2@test.com
     # Test3,test3@test.com
     # Test4,test4@test.com
+    ```
 
-3. Get the output as:
-
-    - response hash, e.g. 
+4. Get the output as:
+  
+    ```ruby
+    # response hash, e.g. 
       {:errors=>1, :status=>200, :total_rows=>4, :error_rows=>[["Test1", nil, "is invalid"]]}
-    - error file, e.g.
+    # error file, e.g.
       for users.csv file, it creates errors_users.csv at the same path
+    # see records being saved in batch(by default) of 1_000 with activerecord-import gem
+    ```
 
 See beam/upload.rb for more details
 

data/beam.gemspec

@@ -20,4 +20,5 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency "bundler", "~> 1.3"
   spec.add_development_dependency "rake"
+
 end

data/lib/beam/upload.rb

@@ -6,13 +6,17 @@ module Beam
     #       file_path:        path/to/zip-file, e.g. Rails.root/tmp
     #       error_file_needed:true if you need error file to be created, default is true
     #       callback_method:  method which does the job to load records, default is parse method
-    def upload_file(file_name, file_path, error_file_needed=true, callback_method = "parse")
+    #       batch_process:    defailt is true, set to false if you do not use activerecord-import 
+    #                         for batch processing
+    def upload_file(file_name, file_path, error_file_needed=true, callback_method = "parse", batch_process=true)
       status = {}
-      @file_name = file_name
-      @file_path = file_path
+      @file_name          = file_name
+      @file_path          = file_path
       @error_file_needed  = error_file_needed
       @original_zip_file  = "#{@file_path}/#{@file_name}"
       @csv_file_name      = @file_name.gsub(/\.zip/,'')
+      @batch_process      = batch_process
+      @batch_size         = 1_1000
 
       begin
         delete_csv_if_present
@@ -59,11 +63,12 @@ module Beam
     end
 
     # parses the csv file, creates record for the model and returns response hash,e.g.
-    # {:errors=>1, :status=>200, :total_rows=>4, :error_rows=>[["Test1", nil, "is invalid"]]}
+    # \{:errors=>1, :status=>200, :total_rows=>4, :error_rows=>[["Test1", nil, "is invalid"]]\}
     # also, it creates error file to consume
     def parse
       response  = { errors: 0, status: 200, total_rows: 0, error_rows: []}
       index = 0
+      batch = []
 
       begin
         CSV.foreach("#{@file_path}/#{@csv_file_name}", :encoding => 'iso-8859-1:UTF-8', headers: true) do |row|
@@ -72,8 +77,10 @@ module Beam
           response[:total_rows] += 1
           begin
             record, response[:errors], error_row = validate_record(response[:errors], row_hash, index)
-            if error_row
-              response[:error_rows] << error_row
+            response[:error_rows] << error_row if error_row
+              
+            if @batch_process
+              batch = add_to_batch(batch, record)
             else
               record.save!
             end
@@ -82,6 +89,7 @@ module Beam
             response[:error_rows] << log_and_return_error(row_hash, e, index)
           end
         end
+        import_from_batch(batch)
       rescue Exception => e
         response[:errors] = 1
         log_and_return_error({}, e, '')
@@ -96,6 +104,19 @@ module Beam
       response
     end
 
+    def add_to_batch(batch, record)
+      batch << record if record
+      if batch.size >= @batch_size
+        import_from_batch(batch, true)
+        batch = []
+      end
+      batch
+    end
+
+    def import_from_batch(batch)
+      import(batch, :validate => false) if @batch_process
+    end
+
     def log_and_return_validation_error(record, row_hash, index)
       error_msg = record.errors.messages.values.join(', ')
       Rails.logger.error("Error on #{index}: \n #{row_hash.values + [error_msg]}")

data/lib/beam/version.rb

@@ -1,3 +1,3 @@
 module Beam
-  VERSION = "0.0.1"
+  VERSION = "0.0.2"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: beam
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - Gourav Tiwari
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-12-15 00:00:00.000000000 Z
+date: 2013-12-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler