batches_task_processor 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0d65fcf4a0f70a2c325afe4ac8e61142c810d8fa884ef5eb6caac62c43f6461a
-  data.tar.gz: 974ef55a01812679696cecc85fbf4d6cd3f9ba3d8849a258a3f5b63ecab9b3e2
+  metadata.gz: a9292ab75aea73468e3c48bf668388c3767e6e55aa36872adcd044014b20c7b1
+  data.tar.gz: 6955b56074ea63f120010518807a03a58f053b23f57b7dc3064e9c8705b84d14
 SHA512:
-  metadata.gz: d1fb278e8d5ca7b07011e635ccef6de8b60f8acb3584b2bb877a7aef1985b0a6f0d2506fa80e0c920d69ae09aecda8a930caf9544f2ed98ba61c6f8cccb751b5
-  data.tar.gz: 6f4b98fca6315f6c088d32c5d7df7c43c56d66a7304308c9049b76d6762472001fd2347f44f444baf059fbd55f8c439cac45aa484b655ba753b9853952626c37
+  metadata.gz: f88cecfa026896d758f24c260cc9b2f6a2516a83e80df80bd8862b44fe650ceedd07cb867fdbb6a723bf0235b79cdc612090c8b0cf361a5af3471af2572c4358
+  data.tar.gz: 7fc18b9188e50f2ee84c61a67bbd4b4e6769c39aa2658c0a0de8b92ff44542a3c2d1b2560468f2ae6e829d415cc03af387dd4b85f07d1199ac90bb3ae0a8dd88

data/README.md

@@ -1,6 +1,5 @@
 # BatchesTaskProcessor
-Gem that allows to process huge amount of tasks in parallel using batches (Supports for array or activerecord collections).    
-This gem depends on `Rails.cache` to save results of processing (In the future: use a database table instead).
+Gem that allows to process huge amount of any kind of tasks in parallel using batches.
 
 ## Installation
 Add this line to your application's Gemfile:
@@ -11,42 +10,48 @@ gem "batches_task_processor"
 And then execute: `bundle install`
 
 
-## Usage
-- Create an initializer file for your application:    
-  Sample Array:    
+## Usage 
+- Register a new task: 
   ```ruby
-    # config/initializers/batches_task_processor.rb
-    BatchesTaskProcessor::Config.configure do |config|
-      config.per_page = 100
-      config.calculate_items = -> { [1,2,3,5] }
-      config.preload_job_items = -> { |items| Article.where(id: items) }
-      config.process_item = -> { |item| MyService.new.process(item) }
-    end
+  task = BatchesTaskProcessor::Model.create!(
+    key: 'my_process',
+    data: [1, 2, 3],
+    qty_jobs: 10,
+    process_item: 'puts "my item: #{item}"'
+  )
   ```
-  Sample ActiveRecord Collection:
-    ```ruby
-      # config/initializers/batches_task_processor.rb
-      BatchesTaskProcessor::Config.configure do |config|
-        config.per_page = 100
-        config.calculate_items = -> { Article.where(created_at: 10.days.ago..Time.current) }
-        config.process_item = -> { |item| MyService.new.process(item) }
-      end
-    ```
+  Activerecord sample (recommended `preload_job_items` for performance reasons):
+  ```ruby
+  task = BatchesTaskProcessor::Model.create!(
+    key: 'my_process',
+    data: Article.all.pluck(:id),
+    qty_jobs: 10,
+    preload_job_items: 'Article.where(id: items)',
+    process_item: 'puts "my article: #{item.id}"'
+  )
+  ```
+  
+- Run the corresponding rake task:     
+  Copy the `task.id` from step one and use it in the following code:    
+  `RUNNER_MODEL_ID=<id-here> rake batches_task_processor:call`
+  
+![Photo](./img.png)
+
+## TODO
+- update tests
 
 ## Api
 Settings:    
-- `config.calculate_items` (Mandatory) method called to calculate the whole list of items to process
-- `config.process_item(item)` (Mandatory) method called to process each item
-- `config.per_page` (Optional) number of items in one batch
-- `config.preload_job_items(items)` (Optional) Allows to preload associations or load objects list. Provides `items` which is a chunk of items to process.
-Tasks:    
-- `rake batches_task_processor:call` Starts the processing of jobs.
-- `rake batches_task_processor:process_job` (Only for internal usage). 
-- `rake batches_task_processor:retry` Retries the processing of all jobs (ignores already processed).
+- `data` (Array<Integer|String>) Array of whole items to be processed.
+- `key` (Mandatory) key to be used to identify the task.
+- `qty_jobs` (Optional) number of jobs to be created. Default: `10`
+- `process_item` (Mandatory) callback to be called to perform each item where `item` variable holds the current item value. Sample: `'Article.find(item).update_column(:title, "changed")'`
+- `preload_job_items` (Optional) callback that allows to preload items list and/or associations where `items` variable holds the current chunk of items to be processed (by default returns the same list). Sample: `Article.where(id: items)`
+
+Tasks (requires `RUNNER_MODEL_ID` env variable):    
+- `rake batches_task_processor:call` Starts the processing of jobs (Skips already processed ones when rerunning after cancel).
 - `rake batches_task_processor:status` Prints the process status.
-- `rake batches_task_processor:cancel` Marks as cancelled the process and stops processing jobs.
-- `rake batches_task_processor:clear` Removes all process logs or tmp data.
-
+- `rake batches_task_processor:cancel` Marks as cancelled the process and stops processing jobs (Change into `pending` to rerun again).
 
 ## Contributing
 Contribution directions go here.

data/lib/batches_task_processor/model.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module BatchesTaskProcessor
+  class Model < ActiveRecord::Base
+    self.table_name = 'batches_task_processors'
+    has_many :items, class_name: 'BatchesTaskProcessor::ModelItem', dependent: :destroy, foreign_key: :batches_task_processors_id
+    validates :process_item, presence: true
+    validates :key, presence: true
+    before_create :apply_data_uniqueness
+    # state: :pending, :processing, :finished, :canceled
+
+    def qty_items_job
+      @qty_items_job ||= (data.count.to_f / qty_jobs).ceil
+    end
+
+    def finish!
+      update!(state: :finished, finished_at: Time.current)
+    end
+
+    def cancel!
+      update!(state: :canceled)
+    end
+
+    def all_processed?
+      items.count == data.count
+    end
+
+    private
+
+    def apply_data_uniqueness
+      self.data = data.uniq
+    end
+  end
+end

data/lib/batches_task_processor/model_item.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module BatchesTaskProcessor
+  class ModelItem < ActiveRecord::Base
+    self.table_name = 'batches_task_processor_items'
+    belongs_to :parent, class_name: 'BatchesTaskProcessor::Model'
+  end
+end

data/lib/batches_task_processor/processor.rb

@@ -4,141 +4,95 @@ require 'active_support/all'
 module BatchesTaskProcessor
   class Processor
     RUNNER_JOB_KEY = 'RUNNER_JOB_KEY'
+    attr_reader :model_id
+
+    def initialize(model_id = nil)
+      @model_id = model_id || ENV['RUNNER_MODEL_ID']
+    end
 
     def call
-      init_cache
       init_jobs
     end
 
     def process_job(job_no)
-      run_job(job_no.to_i, calculate_items)
-    end
-
-    def retry
-      init_jobs
+      run_job(job_no.to_i)
     end
 
     def status
-      res = Rails.cache.read(RUNNER_JOB_KEY)
-      res[:jobs] = res[:jobs].times.map { |i| job_registry(i)[:items].count }
-      puts "Process status: #{res.inspect}"
+      log "Process status: #{process_model.items.count}/#{process_model.data.count}"
     end
 
     def cancel
-      data = Rails.cache.read(RUNNER_JOB_KEY)
-      data[:cancelled] = true
-      Rails.cache.write(RUNNER_JOB_KEY, data)
-    end
-
-    def clear
-      res = Rails.cache.read(RUNNER_JOB_KEY)
-      res[:jobs].times.each { |i| job_registry(i, :delete) }
-      Rails.cache.delete(RUNNER_JOB_KEY)
+      process_model.cancel!
     end
 
     private
 
-    # ****** customizations
-    # @example ['article_id1', 'article_id2', 'article_id3']
-    # @example Article.where(created_at: 1.month_ago..Time.current)
-    def calculate_items
-      instance_exec(&BatchesTaskProcessor::Config.calculate_items)
-    end
-
     # @example item.perform_my_action
     def process_item(item)
-      instance_exec(item, &BatchesTaskProcessor::Config.process_item)
-    end
-
-    def per_page
-      BatchesTaskProcessor::Config.per_page
+      instance_eval(process_model.process_item)
     end
 
     # @example Article.where(no: items)
     def preload_job_items(items)
-      instance_exec(items, &BatchesTaskProcessor::Config.preload_job_items)
-    end
-    # ****** end customizations
-
-    def init_cache
-      items = calculate_items
-      jobs = (items.count.to_f / per_page).ceil
-      data = { jobs: jobs, count: items.count, date: Time.current, finished_jobs: [], cancelled: false }
-      main_registry(data)
+      instance_eval(process_model.preload_job_items || 'items')
     end
 
     def init_jobs
-      jobs = main_registry[:jobs]
+      jobs = process_model.qty_jobs
       log "Initializing #{jobs} jobs..."
       jobs.times.each do |index|
         log "Starting ##{index} job..."
-        pid = Process.spawn("RUNNER_JOB_NO=#{index} rake batches_task_processor:process_job &")
+        env_vars = "RUNNER_JOB_NO=#{index} RUNNER_MODEL_ID=#{model_id}"
+        pid = Process.spawn("#{env_vars} rake batches_task_processor:process_job &")
         Process.detach(pid)
       end
     end
 
-    def run_job(job, items)
+    def run_job(job)
       log "Running ##{job} job..."
-      preload_job_items(job_items(items, job)).each_with_index do |item, index|
+      items = job_items(job)
+      (items.try(:find_each) || items.each).with_index do |item, index|
         key = item.try(:id) || item
         break log('Process cancelled') if process_cancelled?
-        next log("Skipping #{key}...") if already_processed?(job, key)
+        next log("Skipping #{key}...") if already_processed?(key)
 
         start_process_item(item, job, key, index)
       end
 
-      mark_finished_job(job)
       log "Finished #{job} job..."
+      process_model.finish! if process_model.all_processed?
     end
 
-    def job_items(items, job)
-      items.is_a?(Array) ? items.each_slice(per_page).to_a[job] : items.offset(job * per_page).limit(per_page)
+    def job_items(job)
+      res = process_model.data.each_slice(process_model.qty_items_job).to_a[job]
+      preload_job_items(res)
     end
 
     def start_process_item(item, job, key, index)
-      log "Processing #{job}/#{key}: #{index}/#{per_page}"
-      process_item(item)
-      update_job_cache(job, key)
+      log "Processing #{job}/#{key}: #{index}/#{process_model.qty_items_job}"
+      result = process_item(item)
+      process_model.items.create!(key: key, result: result.to_s[0..255])
     rescue => e
-      update_job_cache(job, key, e.message)
+      process_model.items.create!(key: key, error_details: e.message)
       log "Process failed #{job}/#{key}: #{e.message}"
     end
 
-    def main_registry(new_data = nil)
-      Rails.cache.write(RUNNER_JOB_KEY, new_data, expires_in: 1.week) if new_data
-      new_data || Rails.cache.read(RUNNER_JOB_KEY)
-    end
-
-    def mark_finished_job(job)
-      main_registry(main_registry.merge(finished_jobs: main_registry[:finished_jobs] + [job]))
-    end
-
-    def job_registry(job, new_data = nil)
-      key = "#{RUNNER_JOB_KEY}/#{job}"
-      default_data = { items: [], errors: [] }
-      Rails.cache.write(key, default_data, expires_in: 1.week) unless Rails.cache.read(key)
-      Rails.cache.write(key, new_data, expires_in: 1.week) if new_data
-      Rails.cache.delete(key) if new_data == :delete
-      new_data || Rails.cache.read(key)
-    end
-
-    def update_job_cache(job, value, error = nil)
-      data = job_registry(job)
-      data[:items] << value
-      data[:errors] << [value, error] if error
-      job_registry(job, data)
-    end
-
-    def already_processed?(job, value)
-      job_registry(job)[:items].include?(value)
+    def already_processed?(key)
+      process_model.items.where(key: key).exists?
     end
 
     def process_cancelled?
-      Rails.cache.read(RUNNER_JOB_KEY)[:cancelled]
+      process_model.state == 'cancelled'
     end
 
     def log(msg)
       puts "BatchesTaskProcessor => #{msg}"
     end
+
+    def process_model
+      klass = BatchesTaskProcessor::Model.all
+      model_id ? klass.find(model_id) : klass.last
+    end
   end
 end

data/lib/batches_task_processor/railtie.rb

@@ -6,5 +6,9 @@ module BatchesTaskProcessor
     rake_tasks do
       load 'tasks/batches_task_processor_tasks.rake'
     end
+    initializer :append_migrations do |app|
+      path = File.join(File.expand_path('../../', __FILE__), 'db/migrate')
+      app.config.paths["db/migrate"] << path
+    end
   end
 end

data/lib/batches_task_processor/version.rb

@@ -1,3 +1,3 @@
 module BatchesTaskProcessor
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

data/lib/batches_task_processor.rb

@@ -1,18 +1,10 @@
+# frozen_string_literal: true
+
 require "batches_task_processor/version"
 require "batches_task_processor/railtie"
 require "batches_task_processor/processor"
-
+require "batches_task_processor/model"
+require "batches_task_processor/model_item"
 
 module BatchesTaskProcessor
-  class Config
-    cattr_accessor(:per_page) { 5000 }
-    cattr_accessor(:calculate_items) { -> { raise('Implement calculate_items method') } }
-    cattr_accessor(:process_item) { -> (_item) { raise('Implement calculate_items method') } }
-    cattr_accessor(:preload_job_items) { -> (items) { items } }
-
-
-    def self.configure
-      yield self
-    end
-  end
 end

data/lib/db/migrate/20220727101904_add_batches_task_processor.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+class AddBatchesTaskProcessor < ActiveRecord::Migration[5.0]
+  def change
+    create_table :batches_task_processors do |t|
+      t.string :key
+      t.string :state, default: :pending
+      t.json :data, default: []
+      t.integer :qty_jobs, default: 10
+      t.datetime :finished_at
+      t.text :preload_job_items
+      t.text :process_item, null: false
+      t.timestamps
+    end
+
+    create_table :batches_task_processor_items do |t|
+      t.belongs_to :batches_task_processors, foreign_key: true, index: { name: 'index_batches_task_processors_parent_id' }
+      t.string :key
+      t.text :result
+      t.text :error_details
+      t.timestamps
+    end
+  end
+end

data/lib/tasks/batches_task_processor_tasks.rake

@@ -3,31 +3,22 @@
 namespace :batches_task_processor do
   desc 'Starts the Batches Task Processor'
   task call: :environment do
-    BatchesTaskProcessor::Processor.new.call
+    BatchesTaskProcessor::Processor.new(ENV['RUNNER_MODEL_ID']).call
   end
 
   desc 'Starts the Batches Task Processor'
   task process_job: :environment do
-    BatchesTaskProcessor::Processor.new.process_job(ENV['RUNNER_JOB_NO'])
+    BatchesTaskProcessor::Processor.new(ENV['RUNNER_MODEL_ID']).process_job(ENV['RUNNER_JOB_NO'])
   end
 
-  desc 'Retries the Batches Task Processor'
-  task retry: :environment do
-    BatchesTaskProcessor::Processor.new.retry
-  end
 
   desc 'Prints the status of the Task Processor'
   task status: :environment do
-    BatchesTaskProcessor::Processor.new.status
+    BatchesTaskProcessor::Processor.new(ENV['RUNNER_MODEL_ID']).status
   end
 
   desc 'Cancels the Batches Task Processor'
   task cancel: :environment do
-    BatchesTaskProcessor::Processor.new.cancel
-  end
-
-  desc 'Clears the Batches Task Processor cache'
-  task clear: :environment do
-    BatchesTaskProcessor::Processor.new.clear
+    BatchesTaskProcessor::Processor.new(ENV['RUNNER_MODEL_ID']).cancel
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: batches_task_processor
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Owen Peredo
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-07-29 00:00:00.000000000 Z
+date: 2022-07-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -34,9 +34,12 @@ files:
 - README.md
 - Rakefile
 - lib/batches_task_processor.rb
+- lib/batches_task_processor/model.rb
+- lib/batches_task_processor/model_item.rb
 - lib/batches_task_processor/processor.rb
 - lib/batches_task_processor/railtie.rb
 - lib/batches_task_processor/version.rb
+- lib/db/migrate/20220727101904_add_batches_task_processor.rb
 - lib/tasks/batches_task_processor_tasks.rake
 homepage: https://github.com/owen2345/batches-task-processor
 licenses: []