batches_task_processor 0.3.2 → 0.3.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9e3ca93ca15247aafcea0afc6873b11bc26d8519e7b5ef477ceefd9174552888
-  data.tar.gz: e2669c5ed9c3f8afdefa0d5e59ee7750437560d95b78792dfa5c43da2edbbf1e
+  metadata.gz: 69f248153b65e49d24ebbebaa064cb3f5e03aec503cb2d2b9add636c96e8aad4
+  data.tar.gz: d020b38e7e8fea12d7a7df841ca976881ab86b7dee326f3befd4ccc94d205033
 SHA512:
-  metadata.gz: 140fa98d409be395a422f2687d64d0690b1b65ae1ded1cd7e00e2b0d3217680c65b8629b7aea8da5b38dd6b62fb79acd8631ae250bfe262121b1107466b65f16
-  data.tar.gz: e594672fb7e890d0e475f7480f6b503c34d64fc4525c50c6bfe47909b1c4b573ab6af62db13140d847cd6a5ce49332e42d7b3ea8db75d5931d27b284dea41b99
+  metadata.gz: 0bfdfae31beed3d6bb7ea8c3004f842bd5936a94d9f188ebfaeb573c44f38207aeef2ea343bc9db848ce66c50e1ff27d9da4c5a261f98b783c95551e7a3f7059
+  data.tar.gz: 4219b4010280239086ee4a9dabe50876a495f599f2d4450227cd8c8dac105ae9e798aa7e2348838b52b8e9c7f78a61aded822e334f9674cca5c1edb92cd8784a

data/README.md

@@ -18,8 +18,12 @@ And then execute: `bundle install && bundle exec rake db:migrate`
     key: 'my_process',
     data: Article.all.limit(200000).pluck(:id),
     qty_jobs: 10,
+    queue_name: 'default',
     preload_job_items: 'Article.where(id: items)',
-    process_item: 'puts "my article ID: #{item.id}"'
+    process_item: %{
+      puts "Processing article #{item.id}..."
+      HugeArticleProcessor.new(item).call
+    }
   )
   task.start!
   ```

data/lib/batches_task_processor/model.rb

@@ -4,11 +4,11 @@ require 'csv'
 module BatchesTaskProcessor
   class Model < ActiveRecord::Base
     self.table_name = 'batches_task_processors'
-    has_many :items, class_name: 'BatchesTaskProcessor::ModelItem', dependent: :destroy, foreign_key: :batches_task_processors_id
+    has_many :items, class_name: 'BatchesTaskProcessor::ModelItem', dependent: :delete_all, foreign_key: :batches_task_processors_id
     validates :process_item, presence: true
     validates :key, presence: true
-    before_create :apply_data_uniqueness
-    before_create :check_qty_jobs
+    before_save :apply_data_uniqueness
+    before_save :check_qty_jobs
     # state: :pending, :processing, :finished, :canceled
 
     def qty_items_job
@@ -38,11 +38,16 @@ module BatchesTaskProcessor
     end
 
     def status
-      Rails.logger.info "Process status: #{task_model.items.count}/#{task_model.data.count}"
+      Rails.logger.info "Process status: #{items.count}/#{data.count}"
+    end
+
+    def retry_failures
+      start!
     end
 
     def export
-      path = Rails.root.join('tmp/batches_task_processor_result.csv')
+      filename = (key || 'batches_task_processor_result').try(:parameterize)
+      path = Rails.root.join("tmp/#{filename}.csv")
       data = items.pluck(:key, :result, :error_details)
       data = [['Key', 'Result', 'Error details']] + data
       File.write(path, data.map(&:to_csv).join)

data/lib/batches_task_processor/processor.rb

@@ -52,10 +52,10 @@ module BatchesTaskProcessor
     def run_job(job)
       log "Running ##{job} job..."
       items = job_items(job)
-      (items.try(:find_each) || items.each).with_index do |item, index|
+      (items.try(:find_each) || items.each).with_index(1) do |item, index|
         key = item.try(:id) || item
         break log('Process cancelled') if process_cancelled?
-        next log("Skipping #{key}...") if already_processed?(key)
+        next if already_processed?(key)
 
         start_process_item(item, job, key, index)
       end
@@ -72,14 +72,14 @@ module BatchesTaskProcessor
     def start_process_item(item, job, key, index)
       log "Processing key: #{key}, job: #{job}, counter: #{index}/#{task_model.qty_items_job}"
       result = process_item(item)
-      task_model.items.create!(key: key, result: result.to_s[0..255])
+      task_model.items.where(key: key).first_or_initialize.update!(result: result, error_details: nil)
     rescue => e
-      task_model.items.create!(key: key, error_details: e.message)
+      task_model.items.where(key: key).first_or_initialize.update!(result: nil, error_details: e.message)
       log "Process failed #{job}/#{key}: #{e.message}"
     end
 
     def already_processed?(key)
-      task_model.items.where(key: key).exists?
+      task_model.items.where(key: key, error_details: nil).exists?
     end
 
     def process_cancelled?

data/lib/batches_task_processor/version.rb

@@ -1,3 +1,3 @@
 module BatchesTaskProcessor
-  VERSION = "0.3.2"
+  VERSION = "0.3.5"
 end

data/lib/db/migrate/20220727101904_add_batches_task_processor.rb

@@ -6,7 +6,7 @@ class AddBatchesTaskProcessor < ActiveRecord::Migration[5.0]
       t.string :key
       t.string :state, default: :pending
       t.json :data, default: [] if support_json?
-      t.text :data unless support_json?
+      t.text :data, limit: 999999 unless support_json?
       t.integer :qty_jobs, default: 10
       t.datetime :finished_at
       t.text :preload_job_items
@@ -18,7 +18,7 @@ class AddBatchesTaskProcessor < ActiveRecord::Migration[5.0]
     create_table :batches_task_processor_items do |t|
       t.belongs_to :batches_task_processors, foreign_key: true, index: { name: 'index_batches_task_processors_parent_id' }
       t.string :key
-      t.text :result
+      t.text :result, limit: 999999
       t.text :error_details
       t.timestamps
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: batches_task_processor
 version: !ruby/object:Gem::Version
-  version: 0.3.2
+  version: 0.3.5
 platform: ruby
 authors:
 - Owen Peredo
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-08-02 00:00:00.000000000 Z
+date: 2022-08-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails