rseed 1.0.7 → 1.0.8

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    OWFhM2Q5ZTEyYjhlZjhmM2Y5NzE2MzljODM5ZWQ4YzA3MWZlZTEzMg==
+    MGU2YjBiM2ExYWZhZjhiMGVmMjNkNjdkODFjZjMwODliZjA3NWUwYw==
   data.tar.gz: !binary |-
-    YTQzMGVjZjljMWUyZDI3NDZmYjBhM2RiNmI3ODZiMTg4NjgwZjljOQ==
+    YzRiNjZiOGNkMWI3YTg2NTM1NDFkYmU4M2Y3MmVkOWMwZjliMGQyMw==
 SHA512:
   metadata.gz: !binary |-
-    M2ZhZGNhMzY5MzQxM2NmNWUyMmFjYzAxY2FlMGNkMjJlYzliNDNhYzA5YjI5
-    NDk1NTc0YzdjMmE2MmVkYzU2ZTNjN2FmZjBkOTFmMmMyOTQxOTc2ZDZmYjgy
-    NGFiMmQzNDJiOTBlNGI5NmYyYWE1ZDJiYjdhMDQ4ZjUwNjQxZTk=
+    NmVhNjQ2MDNhYWY3ODliNDgyMzRiMDc3YmFiZWI4YWQxZDQ3MTcwMjg5MmIw
+    Y2Q4MWU2MTI2ZGE1MmE3YWU3NzBmMzY2MjM1N2Q0MTIwOTNkYjQ4ODI1MTAx
+    N2EzNTAwYmFmMTNmMWRmYzFkNGUzZDhhMDRlZDdhZWFmYWVmZjc=
   data.tar.gz: !binary |-
-    MjQ4ZWZjNjRjODQ5ZWY0OGEzMzJkMWIzYjc5YTkyZDk1NjE5NTcyOGI2ZDYz
-    N2RkODc4ZDllZmVlN2Q5ZDc4OTRiMjUzNmQyZTE0ZmRkODEwYmExY2RjMDM1
-    MmZiNDVlMmFjMjBmZmRkNjQ4NTIyOTcxNmQ1ODJiZjY1YTlkNDg=
+    ZjcyNjYxZDJmNjE5ZDQyNWZjZGRiMTM2ZmVkNzRmM2U4MDE3ZTA3MTM3NGI3
+    MjdhNTIzNDA2YzI2NWEzZjNkOTUyMDMzMzZkNWQxNDc4MjRkMzE1ZjYxMTlk
+    NjRiNWY5OTBjNjFkY2JhNDFlNDc5ZWU4MGNkMWM0ZDBmMWIyYzE=

data/README.rdoc

@@ -124,6 +124,13 @@ These rake tasks allow you to run seeds manually:
 
 In this case the file in db/rseed/users.csv would be run through the converter UserConverter. The options specified are available within the converter. In this case options["give_admin_access"] will evaluate to "true".
 
+== Processor Options
+
+* :within_transaction
+
+Setting this to true will wrap the entire deserialize in an transaction, avoiding calling a commit after each line.
+For large data sets, this should speed up insertion.
+
 == Seeding
 
 If you want to seed your Rails application using Rseed. The best method is to add lines like the following to db/seeds.rb

data/lib/rseed/helpers.rb

@@ -39,8 +39,9 @@ module Rseed
           progress_bar.format "#{"Complete".green} %t <%B> %C (%a)"
           progress_bar.finish if progress_bar.total
         when :error
-          processor.logger.error "Error processing record #{meta[:record_count]}"
+          processor.logger.error "Error during processing"
           processor.logger.error result[:message].to_s.red
+          processor.logger.error meta.to_s.cyan if meta
           processor.logger.error result[:error]
           processor.logger.error result[:backtrace].join('\n') if result[:backtrace]
       end

data/lib/rseed/processor.rb

@@ -22,6 +22,7 @@ module Rseed
         end
         converter.options = HashWithIndifferentAccess.new(converter_options)
       end
+      @within_transaction = options[:within_transaction]
       @adapter = adapter
       @converter = converter
     end
@@ -46,42 +47,45 @@ module Rseed
           if @converter.before_deserialize
             yield :processing
             start_time = Time.now
-            adapter.process do |values, meta|
-              result = {values: values}
-              meta ||= {}
-              begin
-                if @converter.deserialize_raw(values)
-                  result[:success] = true
-                else
+            wrap_inserts do
+              adapter.process do |values, meta|
+                result = {values: values}
+                meta ||= {}
+                begin
+                  if @converter.deserialize_raw(values)
+                    result[:success] = true
+                  else
+                    result[:success] = false
+                    result[:message] = "Failed to convert"
+                    result[:error] = @converter.error
+                  end
+                rescue Exception => e
                   result[:success] = false
-                  result[:message] = "Failed to convert"
-                  result[:error] = @converter.error
+                  result[:message] = "Exception during deserialize"
+                  result[:error] = e.message
+                  result[:backtrace] = e.backtrace
                 end
-              rescue Exception => e
-                result[:success] = false
-                result[:message] = "Exception during deserialize"
-                result[:error] = e.message
-                result[:backtrace] = e.backtrace
-              end
 
-              total_records = meta[:total_records] unless meta[:total_records].nil?
-              record_count = meta[:record_count] unless meta[:record_count].nil?
-              # Calculate the ETA
-              if record_count and total_records
-                remaining = total_records - record_count
-                tpr = (Time.now - start_time)/record_count
-                meta[:eta] = remaining * tpr
-              end
+                total_records = meta[:total_records] unless meta[:total_records].nil?
+                record_count = meta[:record_count] unless meta[:record_count].nil?
+                # Calculate the ETA
+                if record_count and total_records
+                  remaining = total_records - record_count
+                  tpr = (Time.now - start_time)/record_count
+                  meta[:eta] = remaining * tpr
+                end
 
-               # Log any errors
-              unless result[:success]
-                logger.error result[:message].to_s.red
-                logger.error result[:error].to_s.red
-                logger.error result[:backtrace].to_s unless result[:backtrace].to_s.blank?
+                # Log any errors
+                unless result[:success]
+                  logger.error result[:message].to_s.red
+                  logger.error result[:error].to_s.red
+                  logger.error meta.to_s.cyan
+                  logger.error result[:backtrace].to_s unless result[:backtrace].to_s.blank?
+                end
+                yield :processing, result, meta
               end
-              yield :processing, result, meta
+              @converter.after_deserialize
             end
-            @converter.after_deserialize
           else
             yield :error, {success: false, message: 'Before deserialize failed', error: @converter.error}
           end
@@ -91,7 +95,19 @@ module Rseed
       rescue Exception => e
         yield :error, {success: false, message: 'Exception during Processing', error: e.message, backtrace: e.backtrace}
       end
-      yield :complete, {success: true }, {total_records: total_records, record_count: record_count}
+      yield :complete, {success: true}, {total_records: total_records, record_count: record_count}
+    end
+
+    protected
+
+    def wrap_inserts &block
+      if @within_transaction
+        ActiveRecord::Base.transaction do
+          yield
+        end
+      else
+        yield
+      end
     end
   end
 end

data/lib/rseed/version.rb

@@ -1,3 +1,3 @@
 module Rseed
-  VERSION = "1.0.7"
+  VERSION = "1.0.8"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rseed
 version: !ruby/object:Gem::Version
-  version: 1.0.7
+  version: 1.0.8
 platform: ruby
 authors:
 - David Monagle
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-11-13 00:00:00.000000000 Z
+date: 2013-11-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: colorize