batches_task_processor 0.2.0 → 0.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a9292ab75aea73468e3c48bf668388c3767e6e55aa36872adcd044014b20c7b1
-  data.tar.gz: 6955b56074ea63f120010518807a03a58f053b23f57b7dc3064e9c8705b84d14
+  metadata.gz: 9e3ca93ca15247aafcea0afc6873b11bc26d8519e7b5ef477ceefd9174552888
+  data.tar.gz: e2669c5ed9c3f8afdefa0d5e59ee7750437560d95b78792dfa5c43da2edbbf1e
 SHA512:
-  metadata.gz: f88cecfa026896d758f24c260cc9b2f6a2516a83e80df80bd8862b44fe650ceedd07cb867fdbb6a723bf0235b79cdc612090c8b0cf361a5af3471af2572c4358
-  data.tar.gz: 7fc18b9188e50f2ee84c61a67bbd4b4e6769c39aa2658c0a0de8b92ff44542a3c2d1b2560468f2ae6e829d415cc03af387dd4b85f07d1199ac90bb3ae0a8dd88
+  metadata.gz: 140fa98d409be395a422f2687d64d0690b1b65ae1ded1cd7e00e2b0d3217680c65b8629b7aea8da5b38dd6b62fb79acd8631ae250bfe262121b1107466b65f16
+  data.tar.gz: e594672fb7e890d0e475f7480f6b503c34d64fc4525c50c6bfe47909b1c4b573ab6af62db13140d847cd6a5ce49332e42d7b3ea8db75d5931d27b284dea41b99

data/README.md

@@ -1,5 +1,6 @@
 # BatchesTaskProcessor
-Gem that allows to process huge amount of any kind of tasks in parallel using batches.
+Ruby Gem that allows to process huge amount of any kind of tasks in parallel using batches with the ability to cancel at any time and rerun later (excludes the already processed ones when rerunning) which reduces the process time dramatically. 
+The jobs created can be processed in background (via background jobs) or in the foreground (inline).
 
 ## Installation
 Add this line to your application's Gemfile:
@@ -7,36 +8,31 @@ Add this line to your application's Gemfile:
 ```ruby
 gem "batches_task_processor"
 ```
-And then execute: `bundle install`
-
+And then execute: `bundle install && bundle exec rake db:migrate`
 
 ## Usage 
-- Register a new task: 
+- Register a new task:     
+  The following will process 200k items with 10 jobs parallelly each one in charge of 20k items (recommended `preload_job_items` for performance reasons):
   ```ruby
   task = BatchesTaskProcessor::Model.create!(
     key: 'my_process',
-    data: [1, 2, 3],
-    qty_jobs: 10,
-    process_item: 'puts "my item: #{item}"'
-  )
-  ```
-  Activerecord sample (recommended `preload_job_items` for performance reasons):
-  ```ruby
-  task = BatchesTaskProcessor::Model.create!(
-    key: 'my_process',
-    data: Article.all.pluck(:id),
+    data: Article.all.limit(200000).pluck(:id),
     qty_jobs: 10,
     preload_job_items: 'Article.where(id: items)',
-    process_item: 'puts "my article: #{item.id}"'
+    process_item: 'puts "my article ID: #{item.id}"'
   )
+  task.start!
   ```
-  
-- Run the corresponding rake task:     
-  Copy the `task.id` from step one and use it in the following code:    
-  `RUNNER_MODEL_ID=<id-here> rake batches_task_processor:call`
-  
 ![Photo](./img.png)
 
+## Task api  
+  - `task.start!` starts the task (initializes the jobs)
+  - `task.cancel` cancels the task at any time and stops processing the items
+  - `task.export` exports the items that were processed in a csv file
+  - `task.status` prints the current status of the task
+  - `task.items` returns the items that were processed so far       
+    Each item includes the following attributes: `# { key: 'value from items', result: "value returned from the process_item callback", error_details: "error message from the process_message callback if failed" }`
+
 ## TODO
 - update tests
 
@@ -44,15 +40,11 @@ And then execute: `bundle install`
 Settings:    
 - `data` (Array<Integer|String>) Array of whole items to be processed.
 - `key` (Mandatory) key to be used to identify the task.
-- `qty_jobs` (Optional) number of jobs to be created. Default: `10`
+- `queue_name` (String, default `default`) name of the background queue to be used (If `nil`, will run the process inline).
+- `qty_jobs` (Optional) number of jobs to be created (all `data` items will be distributed across this qty of jobs). Default: `10`
 - `process_item` (Mandatory) callback to be called to perform each item where `item` variable holds the current item value. Sample: `'Article.find(item).update_column(:title, "changed")'`
 - `preload_job_items` (Optional) callback that allows to preload items list and/or associations where `items` variable holds the current chunk of items to be processed (by default returns the same list). Sample: `Article.where(id: items)`
 
-Tasks (requires `RUNNER_MODEL_ID` env variable):    
-- `rake batches_task_processor:call` Starts the processing of jobs (Skips already processed ones when rerunning after cancel).
-- `rake batches_task_processor:status` Prints the process status.
-- `rake batches_task_processor:cancel` Marks as cancelled the process and stops processing jobs (Change into `pending` to rerun again).
-
 ## Contributing
 Contribution directions go here.
 

data/lib/batches_task_processor/model.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require 'csv'
 module BatchesTaskProcessor
   class Model < ActiveRecord::Base
     self.table_name = 'batches_task_processors'
@@ -7,6 +8,7 @@ module BatchesTaskProcessor
     validates :process_item, presence: true
     validates :key, presence: true
     before_create :apply_data_uniqueness
+    before_create :check_qty_jobs
     # state: :pending, :processing, :finished, :canceled
 
     def qty_items_job
@@ -17,18 +19,46 @@ module BatchesTaskProcessor
       update!(state: :finished, finished_at: Time.current)
     end
 
-    def cancel!
+    def all_processed?
+      items.count == data.count
+    end
+
+    # Text data columns support (Mysql only)
+    def data
+      self[:data].is_a?(String) ? JSON.parse(self[:data] || '[]') : self[:data]
+    end
+
+    # ********* user methods
+    def start!
+      Processor.new(id).call
+    end
+
+    def cancel
       update!(state: :canceled)
     end
 
-    def all_processed?
-      items.count == data.count
+    def status
+      Rails.logger.info "Process status: #{task_model.items.count}/#{task_model.data.count}"
     end
 
+    def export
+      path = Rails.root.join('tmp/batches_task_processor_result.csv')
+      data = items.pluck(:key, :result, :error_details)
+      data = [['Key', 'Result', 'Error details']] + data
+      File.write(path, data.map(&:to_csv).join)
+      Rails.logger.info "Exported to #{path}"
+    end
+    # ********* end user methods
+
     private
 
     def apply_data_uniqueness
       self.data = data.uniq
     end
+
+    # Fix: at least 1 item per job
+    def check_qty_jobs
+      self.qty_jobs = data.count if data.count < qty_jobs
+    end
   end
 end

data/lib/batches_task_processor/processor.rb

@@ -3,11 +3,10 @@
 require 'active_support/all'
 module BatchesTaskProcessor
   class Processor
-    RUNNER_JOB_KEY = 'RUNNER_JOB_KEY'
-    attr_reader :model_id
+    attr_reader :task_id
 
-    def initialize(model_id = nil)
-      @model_id = model_id || ENV['RUNNER_MODEL_ID']
+    def initialize(task_id = nil)
+      @task_id = task_id || ENV['RUNNER_TASK_ID']
     end
 
     def call
@@ -18,37 +17,38 @@ module BatchesTaskProcessor
       run_job(job_no.to_i)
     end
 
-    def status
-      log "Process status: #{process_model.items.count}/#{process_model.data.count}"
-    end
-
-    def cancel
-      process_model.cancel!
-    end
-
     private
 
     # @example item.perform_my_action
     def process_item(item)
-      instance_eval(process_model.process_item)
+      instance_eval(task_model.process_item)
     end
 
     # @example Article.where(no: items)
     def preload_job_items(items)
-      instance_eval(process_model.preload_job_items || 'items')
+      instance_eval(task_model.preload_job_items || 'items')
     end
 
     def init_jobs
-      jobs = process_model.qty_jobs
+      jobs = task_model.qty_jobs
       log "Initializing #{jobs} jobs..."
       jobs.times.each do |index|
-        log "Starting ##{index} job..."
-        env_vars = "RUNNER_JOB_NO=#{index} RUNNER_MODEL_ID=#{model_id}"
-        pid = Process.spawn("#{env_vars} rake batches_task_processor:process_job &")
-        Process.detach(pid)
+        if task_model.queue_name
+          log "Scheduling ##{index} job..."
+          BatchesTaskProcessor::ProcessorJob.set(queue: task_model.queue_name).perform_later(task_id, index)
+        else
+          start_inline_job(index)
+        end
       end
     end
 
+    def start_inline_job(job_no)
+      log "Starting ##{job_no} job..."
+      env_vars = "RUNNER_JOB_NO=#{job_no} RUNNER_TASK_ID=#{task_id}"
+      pid = Process.spawn("#{env_vars} rake batches_task_processor:process_job &")
+      Process.detach(pid)
+    end
+
     def run_job(job)
       log "Running ##{job} job..."
       items = job_items(job)
@@ -61,38 +61,38 @@ module BatchesTaskProcessor
       end
 
       log "Finished #{job} job..."
-      process_model.finish! if process_model.all_processed?
+      task_model.finish! if task_model.all_processed?
     end
 
     def job_items(job)
-      res = process_model.data.each_slice(process_model.qty_items_job).to_a[job]
+      res = task_model.data.each_slice(task_model.qty_items_job).to_a[job]
       preload_job_items(res)
     end
 
     def start_process_item(item, job, key, index)
-      log "Processing #{job}/#{key}: #{index}/#{process_model.qty_items_job}"
+      log "Processing key: #{key}, job: #{job}, counter: #{index}/#{task_model.qty_items_job}"
       result = process_item(item)
-      process_model.items.create!(key: key, result: result.to_s[0..255])
+      task_model.items.create!(key: key, result: result.to_s[0..255])
     rescue => e
-      process_model.items.create!(key: key, error_details: e.message)
+      task_model.items.create!(key: key, error_details: e.message)
       log "Process failed #{job}/#{key}: #{e.message}"
     end
 
     def already_processed?(key)
-      process_model.items.where(key: key).exists?
+      task_model.items.where(key: key).exists?
     end
 
     def process_cancelled?
-      process_model.state == 'cancelled'
+      task_model.state == 'cancelled'
     end
 
     def log(msg)
       puts "BatchesTaskProcessor => #{msg}"
     end
 
-    def process_model
+    def task_model
       klass = BatchesTaskProcessor::Model.all
-      model_id ? klass.find(model_id) : klass.last
+      task_id ? klass.find(task_id) : klass.last
     end
   end
 end

data/lib/batches_task_processor/processor_job.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module BatchesTaskProcessor
+  class ProcessorJob < ActiveJob::Base
+    queue_as :default
+
+    def perform(task_id, job_no)
+      Processor.new(task_id).process_job(job_no)
+    end
+  end
+end

data/lib/batches_task_processor/version.rb

@@ -1,3 +1,3 @@
 module BatchesTaskProcessor
-  VERSION = "0.2.0"
+  VERSION = "0.3.2"
 end

data/lib/batches_task_processor.rb

@@ -3,6 +3,7 @@
 require "batches_task_processor/version"
 require "batches_task_processor/railtie"
 require "batches_task_processor/processor"
+require "batches_task_processor/processor_job"
 require "batches_task_processor/model"
 require "batches_task_processor/model_item"
 

data/lib/db/migrate/20220727101904_add_batches_task_processor.rb

@@ -5,11 +5,13 @@ class AddBatchesTaskProcessor < ActiveRecord::Migration[5.0]
     create_table :batches_task_processors do |t|
       t.string :key
       t.string :state, default: :pending
-      t.json :data, default: []
+      t.json :data, default: [] if support_json?
+      t.text :data unless support_json?
       t.integer :qty_jobs, default: 10
       t.datetime :finished_at
       t.text :preload_job_items
       t.text :process_item, null: false
+      t.string :queue_name, default: :default
       t.timestamps
     end
 
@@ -21,4 +23,10 @@ class AddBatchesTaskProcessor < ActiveRecord::Migration[5.0]
       t.timestamps
     end
   end
+
+  def support_json?
+    connector_name = ActiveRecord::Base.connection.adapter_name.downcase
+    no_json = connector_name.include?('mysql') || connector_name.include?('sqlite')
+    !no_json
+  end
 end

data/lib/tasks/batches_task_processor_tasks.rake

@@ -3,22 +3,11 @@
 namespace :batches_task_processor do
   desc 'Starts the Batches Task Processor'
   task call: :environment do
-    BatchesTaskProcessor::Processor.new(ENV['RUNNER_MODEL_ID']).call
+    BatchesTaskProcessor::Processor.new(ENV['RUNNER_TASK_ID']).call
   end
 
   desc 'Starts the Batches Task Processor'
   task process_job: :environment do
-    BatchesTaskProcessor::Processor.new(ENV['RUNNER_MODEL_ID']).process_job(ENV['RUNNER_JOB_NO'])
-  end
-
-
-  desc 'Prints the status of the Task Processor'
-  task status: :environment do
-    BatchesTaskProcessor::Processor.new(ENV['RUNNER_MODEL_ID']).status
-  end
-
-  desc 'Cancels the Batches Task Processor'
-  task cancel: :environment do
-    BatchesTaskProcessor::Processor.new(ENV['RUNNER_MODEL_ID']).cancel
+    BatchesTaskProcessor::Processor.new(ENV['RUNNER_TASK_ID']).process_job(ENV['RUNNER_JOB_NO'])
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: batches_task_processor
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.2
 platform: ruby
 authors:
 - Owen Peredo
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-07-31 00:00:00.000000000 Z
+date: 2022-08-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -37,6 +37,7 @@ files:
 - lib/batches_task_processor/model.rb
 - lib/batches_task_processor/model_item.rb
 - lib/batches_task_processor/processor.rb
+- lib/batches_task_processor/processor_job.rb
 - lib/batches_task_processor/railtie.rb
 - lib/batches_task_processor/version.rb
 - lib/db/migrate/20220727101904_add_batches_task_processor.rb