activerecord-import 1.0.8 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b32b91e70dc3ec25461a37092b774709300798912b930aeb9e7ae57dba41230d
-  data.tar.gz: 5b64d6bc14cae05785300a72cd830e552cd5b21d0dbb304e4e527f69de9865c5
+  metadata.gz: 4c61743fafaad0de04ccf2c6bff4439fe66aed03e951e5c42e4f7926cf8dac39
+  data.tar.gz: c58d8992957e546b73bb7f7b90eaf2a741f007d2109a16db0e3b3d71b376e1da
 SHA512:
-  metadata.gz: 04a16537b5cd4ef535e7ed1c49397b1313d2717f5b60ed43cdee42f220d3a0094d0b4cce6241470c892a184b09d74b28ece67d28506508964f962c9282aa763c
-  data.tar.gz: d84f9ffea3a04a2ca3b2bb3dfc05307a90876d5c7e9f8e155605dbc7a6bbcec8cd3cfefc7aac82a7e14de0ae5334cdeba3ac6467c00206ccd15e016aebe8854e
+  metadata.gz: 98638b63235eae1c16f27d3be932b8ce4e8f236b24350766792a4fc563d199cb1d048d25efa5d22abf3a21017fa07b9474c3168a4be6c1ba5aaa6e0843d584a3
+  data.tar.gz: 5c62bed2684f1b6d60d7e26b8ad13c6b7173a64835eeca9474082122425ab1f340f9063868224eda2539878210887701022e1ff6e6a37593872682bf90768d3a

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## Changes in 1.1.0
+
+### New Features
+
+*  Add batch progress reporting. Thanks to @gee-forr via \##729.
+
 ## Changes in 1.0.8
 
 ### Fixes
@@ -6,6 +12,8 @@
 
 ## Changes in 1.0.7
 
+### New Features
+
 * Use @@max_allowed_packet session variable instead of querying SHOW VARIABLES. Thanks to @diclophis via \#706.
 * Add option :track_validation_failures. When this is set to true, failed_instances will be an array of arrays, with each inner array having the form [:index_in_dataset, :object_with_errors]. Thanks to @rorymckinley  via \#684.
 

data/README.markdown

@@ -231,6 +231,18 @@ columns = [ :title ]
 Book.import columns, books, batch_size: 2
 ```
 
+If your import is particularly large or slow (possibly due to [callbacks](#callbacks)) whilst batch importing, you might want a way to report back on progress. This is supported by passing a callable as the `batch_progress` option. e.g:
+
+```ruby
+my_proc = ->(rows_size, num_batches, current_batch_number, batch_duration_in_secs) {
+  # Using the arguments provided to the callable, you can
+  # send an email, post to a websocket,
+  # update slack, alert if import is taking too long, etc.
+}
+
+Book.import columns, books, batch_size: 2, batch_progress: my_proc
+```
+
 #### Recursive
 
 NOTE: This only works with PostgreSQL and ActiveRecord objects. This won't work with

data/lib/activerecord-import/import.rb

@@ -805,17 +805,29 @@ class ActiveRecord::Base
       if supports_import?
         # generate the sql
         post_sql_statements = connection.post_sql_statements( quoted_table_name, options )
+        import_size = values_sql.size
+
+        batch_size = options[:batch_size] || import_size
+        run_proc = options[:batch_size].to_i.positive? && options[:batch_progress].respond_to?( :call )
+        progress_proc = options[:batch_progress]
+        current_batch = 0
+        batches = (import_size / batch_size.to_f).ceil
 
-        batch_size = options[:batch_size] || values_sql.size
         values_sql.each_slice(batch_size) do |batch_values|
+          batch_started_at = Time.now.to_i
+
           # perform the inserts
           result = connection.insert_many( [insert_sql, post_sql_statements].flatten,
             batch_values,
             options,
             "#{model_name} Create Many" )
+
           number_inserted += result.num_inserts
           ids += result.ids
           results += result.results
+          current_batch += 1
+
+          progress_proc.call(import_size, batches, current_batch, Time.now.to_i - batch_started_at) if run_proc
         end
       else
         transaction(requires_new: true) do

data/lib/activerecord-import/version.rb

@@ -1,5 +1,5 @@
 module ActiveRecord
   module Import
-    VERSION = "1.0.8".freeze
+    VERSION = "1.1.0".freeze
   end
 end

data/test/import_test.rb

@@ -405,6 +405,15 @@ describe "#import" do
         assert_equal 3, result.num_inserts if Topic.supports_import?
       end
     end
+
+    it "should accept and call an optional callable to run after each batch" do
+      lambda_called = 0
+
+      my_proc = ->(_row_count, _batches, _batch, _duration) { lambda_called += 1 }
+      Topic.import Build(10, :topics), batch_size: 4, batch_progress: my_proc
+
+      assert_equal 3, lambda_called
+    end
   end
 
   context "with :synchronize option" do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: activerecord-import
 version: !ruby/object:Gem::Version
-  version: 1.0.8
+  version: 1.1.0
 platform: ruby
 authors:
 - Zach Dennis
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-02-20 00:00:00.000000000 Z
+date: 2021-05-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord