batches_task_processor 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a9292ab75aea73468e3c48bf668388c3767e6e55aa36872adcd044014b20c7b1
-  data.tar.gz: 6955b56074ea63f120010518807a03a58f053b23f57b7dc3064e9c8705b84d14
+  metadata.gz: 66cd035ce5d4863e28c7052efc51b79d2cfcc3cae0b0ab84e14a8a3921da9dee
+  data.tar.gz: b38a41a2489cbc422e3d18c108fd61c00f46304e17b179e6ec109b2b29e956a0
 SHA512:
-  metadata.gz: f88cecfa026896d758f24c260cc9b2f6a2516a83e80df80bd8862b44fe650ceedd07cb867fdbb6a723bf0235b79cdc612090c8b0cf361a5af3471af2572c4358
-  data.tar.gz: 7fc18b9188e50f2ee84c61a67bbd4b4e6769c39aa2658c0a0de8b92ff44542a3c2d1b2560468f2ae6e829d415cc03af387dd4b85f07d1199ac90bb3ae0a8dd88
+  metadata.gz: 1618b2a8460a30828df870b559a299d75283ab960d9d74c51743da5f1ab3992c61f9e7ed0f1eefc6af3a89e18a5a719387003497c55def0fdd668e3a5902689b
+  data.tar.gz: 3f822048a4ee3efa3244416348e680ba38cfd779d5e3de8cff2d66add02c9cd465272e78d12c5a127d7d6caaf738f8ca9af6b2d6b9892db759d48cd7ca830529

data/README.md

@@ -1,5 +1,6 @@
 # BatchesTaskProcessor
-Gem that allows to process huge amount of any kind of tasks in parallel using batches.
+Ruby Gem that allows to process huge amount of any kind of tasks in parallel using batches with the ability to cancel at any time. 
+The jobs created can be processed in background or in the foreground (inline) with the ability to rerun/retry later (excludes the already processed ones).
 
 ## Installation
 Add this line to your application's Gemfile:
@@ -7,36 +8,30 @@ Add this line to your application's Gemfile:
 ```ruby
 gem "batches_task_processor"
 ```
-And then execute: `bundle install`
-
+And then execute: `bundle install && bundle exec rake db:migrate`
 
 ## Usage 
-- Register a new task: 
+- Register a new task:     
+  The following will process 200 items with 10 jobs parallelly each one in charge of 20 items (recommended `preload_job_items` for performance reasons):
   ```ruby
   task = BatchesTaskProcessor::Model.create!(
     key: 'my_process',
-    data: [1, 2, 3],
-    qty_jobs: 10,
-    process_item: 'puts "my item: #{item}"'
-  )
-  ```
-  Activerecord sample (recommended `preload_job_items` for performance reasons):
-  ```ruby
-  task = BatchesTaskProcessor::Model.create!(
-    key: 'my_process',
-    data: Article.all.pluck(:id),
+    data: Article.all.limit(200).pluck(:id),
     qty_jobs: 10,
     preload_job_items: 'Article.where(id: items)',
     process_item: 'puts "my article: #{item.id}"'
   )
+  task.start!
   ```
-  
-- Run the corresponding rake task:     
-  Copy the `task.id` from step one and use it in the following code:    
-  `RUNNER_MODEL_ID=<id-here> rake batches_task_processor:call`
-  
 ![Photo](./img.png)
 
+## Task api  
+  - `task.start!` starts the task (initializes the jobs)
+  - `task.cancel` cancels the task and stops processing the items
+  - `task.export` exports the items that were processed in a csv file
+  - `task.items` returns the items that were processed    
+    Each item includes the following attributes: `# { result: "value returned from the process_item callback", error_details: "error message from the process_message callback if failed" }`
+
 ## TODO
 - update tests
 
@@ -44,15 +39,11 @@ And then execute: `bundle install`
 Settings:    
 - `data` (Array<Integer|String>) Array of whole items to be processed.
 - `key` (Mandatory) key to be used to identify the task.
-- `qty_jobs` (Optional) number of jobs to be created. Default: `10`
+- `queue_name` (String, default `default`) name of the background queue to be used (If `nil`, will run the process inline).
+- `qty_jobs` (Optional) number of jobs to be created (all `data` items will be distributed across this qty of jobs). Default: `10`
 - `process_item` (Mandatory) callback to be called to perform each item where `item` variable holds the current item value. Sample: `'Article.find(item).update_column(:title, "changed")'`
 - `preload_job_items` (Optional) callback that allows to preload items list and/or associations where `items` variable holds the current chunk of items to be processed (by default returns the same list). Sample: `Article.where(id: items)`
 
-Tasks (requires `RUNNER_MODEL_ID` env variable):    
-- `rake batches_task_processor:call` Starts the processing of jobs (Skips already processed ones when rerunning after cancel).
-- `rake batches_task_processor:status` Prints the process status.
-- `rake batches_task_processor:cancel` Marks as cancelled the process and stops processing jobs (Change into `pending` to rerun again).
-
 ## Contributing
 Contribution directions go here.
 

data/lib/batches_task_processor/model.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require 'csv'
 module BatchesTaskProcessor
   class Model < ActiveRecord::Base
     self.table_name = 'batches_task_processors'
@@ -17,13 +18,31 @@ module BatchesTaskProcessor
       update!(state: :finished, finished_at: Time.current)
     end
 
-    def cancel!
+    def all_processed?
+      items.count == data.count
+    end
+
+    # ********* user methods
+    def start!
+      Processor.new(id).call
+    end
+
+    def cancel
       update!(state: :canceled)
     end
 
-    def all_processed?
-      items.count == data.count
+    def status
+      Rails.logger.info "Process status: #{task_model.items.count}/#{task_model.data.count}"
+    end
+
+    def export
+      path = Rails.root.join('tmp/batches_task_processor_result.csv')
+      data = items.pluck(:key, :result, :error_details)
+      data = [['Key', 'Result', 'Error details']] + data
+      File.write(path, data.map(&:to_csv).join)
+      Rails.logger.info "Exported to #{path}"
     end
+    # ********* end user methods
 
     private
 

data/lib/batches_task_processor/processor.rb

@@ -3,11 +3,10 @@
 require 'active_support/all'
 module BatchesTaskProcessor
   class Processor
-    RUNNER_JOB_KEY = 'RUNNER_JOB_KEY'
-    attr_reader :model_id
+    attr_reader :task_id
 
-    def initialize(model_id = nil)
-      @model_id = model_id || ENV['RUNNER_MODEL_ID']
+    def initialize(task_id = nil)
+      @task_id = task_id || ENV['RUNNER_TASK_ID']
     end
 
     def call
@@ -18,37 +17,38 @@ module BatchesTaskProcessor
       run_job(job_no.to_i)
     end
 
-    def status
-      log "Process status: #{process_model.items.count}/#{process_model.data.count}"
-    end
-
-    def cancel
-      process_model.cancel!
-    end
-
     private
 
     # @example item.perform_my_action
     def process_item(item)
-      instance_eval(process_model.process_item)
+      instance_eval(task_model.process_item)
     end
 
     # @example Article.where(no: items)
     def preload_job_items(items)
-      instance_eval(process_model.preload_job_items || 'items')
+      instance_eval(task_model.preload_job_items || 'items')
     end
 
     def init_jobs
-      jobs = process_model.qty_jobs
+      jobs = task_model.qty_jobs
       log "Initializing #{jobs} jobs..."
       jobs.times.each do |index|
-        log "Starting ##{index} job..."
-        env_vars = "RUNNER_JOB_NO=#{index} RUNNER_MODEL_ID=#{model_id}"
-        pid = Process.spawn("#{env_vars} rake batches_task_processor:process_job &")
-        Process.detach(pid)
+        if task_model.queue_name
+          log "Scheduling ##{index} job..."
+          BatchesTaskProcessor::ProcessorJob.set(queue: task_model.queue_name).perform_later(task_id, index)
+        else
+          start_inline_job(index)
+        end
       end
     end
 
+    def start_inline_job(job_no)
+      log "Starting ##{job_no} job..."
+      env_vars = "RUNNER_JOB_NO=#{job_no} RUNNER_TASK_ID=#{task_id}"
+      pid = Process.spawn("#{env_vars} rake batches_task_processor:process_job &")
+      Process.detach(pid)
+    end
+
     def run_job(job)
       log "Running ##{job} job..."
       items = job_items(job)
@@ -61,38 +61,38 @@ module BatchesTaskProcessor
       end
 
       log "Finished #{job} job..."
-      process_model.finish! if process_model.all_processed?
+      task_model.finish! if task_model.all_processed?
     end
 
     def job_items(job)
-      res = process_model.data.each_slice(process_model.qty_items_job).to_a[job]
+      res = task_model.data.each_slice(task_model.qty_items_job).to_a[job]
       preload_job_items(res)
     end
 
     def start_process_item(item, job, key, index)
-      log "Processing #{job}/#{key}: #{index}/#{process_model.qty_items_job}"
+      log "Processing #{job}/#{key}: #{index}/#{task_model.qty_items_job}"
       result = process_item(item)
-      process_model.items.create!(key: key, result: result.to_s[0..255])
+      task_model.items.create!(key: key, result: result.to_s[0..255])
     rescue => e
-      process_model.items.create!(key: key, error_details: e.message)
+      task_model.items.create!(key: key, error_details: e.message)
       log "Process failed #{job}/#{key}: #{e.message}"
     end
 
     def already_processed?(key)
-      process_model.items.where(key: key).exists?
+      task_model.items.where(key: key).exists?
     end
 
     def process_cancelled?
-      process_model.state == 'cancelled'
+      task_model.state == 'cancelled'
     end
 
     def log(msg)
       puts "BatchesTaskProcessor => #{msg}"
     end
 
-    def process_model
+    def task_model
       klass = BatchesTaskProcessor::Model.all
-      model_id ? klass.find(model_id) : klass.last
+      task_id ? klass.find(task_id) : klass.last
     end
   end
 end

data/lib/batches_task_processor/processor_job.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module BatchesTaskProcessor
+  class ProcessorJob < ActiveJob::Base
+    queue_as :default
+
+    def perform(task_id, job_no)
+      Processor.new(task_id).process_job(job_no)
+    end
+  end
+end

data/lib/batches_task_processor/version.rb

@@ -1,3 +1,3 @@
 module BatchesTaskProcessor
-  VERSION = "0.2.0"
+  VERSION = "0.3.0"
 end

data/lib/batches_task_processor.rb

@@ -3,6 +3,7 @@
 require "batches_task_processor/version"
 require "batches_task_processor/railtie"
 require "batches_task_processor/processor"
+require "batches_task_processor/processor_job"
 require "batches_task_processor/model"
 require "batches_task_processor/model_item"
 

data/lib/db/migrate/20220727101904_add_batches_task_processor.rb

@@ -10,6 +10,7 @@ class AddBatchesTaskProcessor < ActiveRecord::Migration[5.0]
       t.datetime :finished_at
       t.text :preload_job_items
       t.text :process_item, null: false
+      t.string :queue_name, default: :default
       t.timestamps
     end
 

data/lib/tasks/batches_task_processor_tasks.rake

@@ -3,22 +3,11 @@
 namespace :batches_task_processor do
   desc 'Starts the Batches Task Processor'
   task call: :environment do
-    BatchesTaskProcessor::Processor.new(ENV['RUNNER_MODEL_ID']).call
+    BatchesTaskProcessor::Processor.new(ENV['RUNNER_TASK_ID']).call
   end
 
   desc 'Starts the Batches Task Processor'
   task process_job: :environment do
-    BatchesTaskProcessor::Processor.new(ENV['RUNNER_MODEL_ID']).process_job(ENV['RUNNER_JOB_NO'])
-  end
-
-
-  desc 'Prints the status of the Task Processor'
-  task status: :environment do
-    BatchesTaskProcessor::Processor.new(ENV['RUNNER_MODEL_ID']).status
-  end
-
-  desc 'Cancels the Batches Task Processor'
-  task cancel: :environment do
-    BatchesTaskProcessor::Processor.new(ENV['RUNNER_MODEL_ID']).cancel
+    BatchesTaskProcessor::Processor.new(ENV['RUNNER_TASK_ID']).process_job(ENV['RUNNER_JOB_NO'])
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: batches_task_processor
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Owen Peredo
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-07-31 00:00:00.000000000 Z
+date: 2022-08-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -37,6 +37,7 @@ files:
 - lib/batches_task_processor/model.rb
 - lib/batches_task_processor/model_item.rb
 - lib/batches_task_processor/processor.rb
+- lib/batches_task_processor/processor_job.rb
 - lib/batches_task_processor/railtie.rb
 - lib/batches_task_processor/version.rb
 - lib/db/migrate/20220727101904_add_batches_task_processor.rb