batches_task_processor 0.1.0 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0d65fcf4a0f70a2c325afe4ac8e61142c810d8fa884ef5eb6caac62c43f6461a
-  data.tar.gz: 974ef55a01812679696cecc85fbf4d6cd3f9ba3d8849a258a3f5b63ecab9b3e2
+  metadata.gz: 6b83bdfebcd837e7ee632a01502d5796ad29ad096e63486cd4d9513dd5882908
+  data.tar.gz: a8e2fbfafe6c76ca0ddab0810b24797dbc5ce3b6eceb84f15c2b587078a8d558
 SHA512:
-  metadata.gz: d1fb278e8d5ca7b07011e635ccef6de8b60f8acb3584b2bb877a7aef1985b0a6f0d2506fa80e0c920d69ae09aecda8a930caf9544f2ed98ba61c6f8cccb751b5
-  data.tar.gz: 6f4b98fca6315f6c088d32c5d7df7c43c56d66a7304308c9049b76d6762472001fd2347f44f444baf059fbd55f8c439cac45aa484b655ba753b9853952626c37
+  metadata.gz: a2aca1fb9e37d1c3c7e7af5e51cc593bcbbf1d83f37654bb81f9ef0cc45740a5d33467f3ead66a20f93fedf25f5ac1b81568a49019673ee0f47bf61bf3fd2629
+  data.tar.gz: 43d671b910781c69ac56bee5c363213c7590c162e5f17cf48882e31ea5dcc20ae7311186b30c5d9b9fdbc037a3e342097543b46d084112477c794057ccc89612

data/README.md

@@ -1,6 +1,6 @@
 # BatchesTaskProcessor
-Gem that allows to process huge amount of tasks in parallel using batches (Supports for array or activerecord collections).    
-This gem depends on `Rails.cache` to save results of processing (In the future: use a database table instead).
+Ruby Gem that allows to process huge amount of any kind of tasks in parallel using batches with the ability to cancel at any time. 
+The jobs created can be processed in background or in the foreground (inline) with the ability to rerun/retry later (excludes the already processed ones).
 
 ## Installation
 Add this line to your application's Gemfile:
@@ -8,45 +8,42 @@ Add this line to your application's Gemfile:
 ```ruby
 gem "batches_task_processor"
 ```
-And then execute: `bundle install`
+And then execute: `bundle install && bundle exec rake db:migrate`
 
-
-## Usage
-- Create an initializer file for your application:    
-  Sample Array:    
+## Usage 
+- Register a new task:     
+  The following will process 200k items with 10 jobs parallelly each one in charge of 20k items (recommended `preload_job_items` for performance reasons):
   ```ruby
-    # config/initializers/batches_task_processor.rb
-    BatchesTaskProcessor::Config.configure do |config|
-      config.per_page = 100
-      config.calculate_items = -> { [1,2,3,5] }
-      config.preload_job_items = -> { |items| Article.where(id: items) }
-      config.process_item = -> { |item| MyService.new.process(item) }
-    end
+  task = BatchesTaskProcessor::Model.create!(
+    key: 'my_process',
+    data: Article.all.limit(200000).pluck(:id),
+    qty_jobs: 10,
+    preload_job_items: 'Article.where(id: items)',
+    process_item: 'puts "my article ID: #{item.id}"'
+  )
+  task.start!
   ```
-  Sample ActiveRecord Collection:
-    ```ruby
-      # config/initializers/batches_task_processor.rb
-      BatchesTaskProcessor::Config.configure do |config|
-        config.per_page = 100
-        config.calculate_items = -> { Article.where(created_at: 10.days.ago..Time.current) }
-        config.process_item = -> { |item| MyService.new.process(item) }
-      end
-    ```
+![Photo](./img.png)
+
+## Task api  
+  - `task.start!` starts the task (initializes the jobs)
+  - `task.cancel` cancels the task at any time and stops processing the items
+  - `task.export` exports the items that were processed in a csv file
+  - `task.status` prints the current status of the task
+  - `task.items` returns the items that were processed so far       
+    Each item includes the following attributes: `# { key: 'value from items', result: "value returned from the process_item callback", error_details: "error message from the process_message callback if failed" }`
+
+## TODO
+- update tests
 
 ## Api
 Settings:    
-- `config.calculate_items` (Mandatory) method called to calculate the whole list of items to process
-- `config.process_item(item)` (Mandatory) method called to process each item
-- `config.per_page` (Optional) number of items in one batch
-- `config.preload_job_items(items)` (Optional) Allows to preload associations or load objects list. Provides `items` which is a chunk of items to process.
-Tasks:    
-- `rake batches_task_processor:call` Starts the processing of jobs.
-- `rake batches_task_processor:process_job` (Only for internal usage). 
-- `rake batches_task_processor:retry` Retries the processing of all jobs (ignores already processed).
-- `rake batches_task_processor:status` Prints the process status.
-- `rake batches_task_processor:cancel` Marks as cancelled the process and stops processing jobs.
-- `rake batches_task_processor:clear` Removes all process logs or tmp data.
-
+- `data` (Array<Integer|String>) Array of whole items to be processed.
+- `key` (Mandatory) key to be used to identify the task.
+- `queue_name` (String, default `default`) name of the background queue to be used (If `nil`, will run the process inline).
+- `qty_jobs` (Optional) number of jobs to be created (all `data` items will be distributed across this qty of jobs). Default: `10`
+- `process_item` (Mandatory) callback to be called to perform each item where `item` variable holds the current item value. Sample: `'Article.find(item).update_column(:title, "changed")'`
+- `preload_job_items` (Optional) callback that allows to preload items list and/or associations where `items` variable holds the current chunk of items to be processed (by default returns the same list). Sample: `Article.where(id: items)`
 
 ## Contributing
 Contribution directions go here.

data/lib/batches_task_processor/model.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+require 'csv'
+module BatchesTaskProcessor
+  class Model < ActiveRecord::Base
+    self.table_name = 'batches_task_processors'
+    has_many :items, class_name: 'BatchesTaskProcessor::ModelItem', dependent: :destroy, foreign_key: :batches_task_processors_id
+    validates :process_item, presence: true
+    validates :key, presence: true
+    before_create :apply_data_uniqueness
+    # state: :pending, :processing, :finished, :canceled
+
+    def qty_items_job
+      @qty_items_job ||= (data.count.to_f / qty_jobs).ceil
+    end
+
+    def finish!
+      update!(state: :finished, finished_at: Time.current)
+    end
+
+    def all_processed?
+      items.count == data.count
+    end
+
+    # ********* user methods
+    def start!
+      Processor.new(id).call
+    end
+
+    def cancel
+      update!(state: :canceled)
+    end
+
+    def status
+      Rails.logger.info "Process status: #{task_model.items.count}/#{task_model.data.count}"
+    end
+
+    def export
+      path = Rails.root.join('tmp/batches_task_processor_result.csv')
+      data = items.pluck(:key, :result, :error_details)
+      data = [['Key', 'Result', 'Error details']] + data
+      File.write(path, data.map(&:to_csv).join)
+      Rails.logger.info "Exported to #{path}"
+    end
+    # ********* end user methods
+
+    private
+
+    def apply_data_uniqueness
+      self.data = data.uniq
+    end
+  end
+end

data/lib/batches_task_processor/model_item.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module BatchesTaskProcessor
+  class ModelItem < ActiveRecord::Base
+    self.table_name = 'batches_task_processor_items'
+    belongs_to :parent, class_name: 'BatchesTaskProcessor::Model'
+  end
+end

data/lib/batches_task_processor/processor.rb

@@ -3,142 +3,96 @@
 require 'active_support/all'
 module BatchesTaskProcessor
   class Processor
-    RUNNER_JOB_KEY = 'RUNNER_JOB_KEY'
+    attr_reader :task_id
 
-    def call
-      init_cache
-      init_jobs
-    end
-
-    def process_job(job_no)
-      run_job(job_no.to_i, calculate_items)
+    def initialize(task_id = nil)
+      @task_id = task_id || ENV['RUNNER_TASK_ID']
     end
 
-    def retry
+    def call
       init_jobs
     end
 
-    def status
-      res = Rails.cache.read(RUNNER_JOB_KEY)
-      res[:jobs] = res[:jobs].times.map { |i| job_registry(i)[:items].count }
-      puts "Process status: #{res.inspect}"
-    end
-
-    def cancel
-      data = Rails.cache.read(RUNNER_JOB_KEY)
-      data[:cancelled] = true
-      Rails.cache.write(RUNNER_JOB_KEY, data)
-    end
-
-    def clear
-      res = Rails.cache.read(RUNNER_JOB_KEY)
-      res[:jobs].times.each { |i| job_registry(i, :delete) }
-      Rails.cache.delete(RUNNER_JOB_KEY)
+    def process_job(job_no)
+      run_job(job_no.to_i)
     end
 
     private
 
-    # ****** customizations
-    # @example ['article_id1', 'article_id2', 'article_id3']
-    # @example Article.where(created_at: 1.month_ago..Time.current)
-    def calculate_items
-      instance_exec(&BatchesTaskProcessor::Config.calculate_items)
-    end
-
     # @example item.perform_my_action
     def process_item(item)
-      instance_exec(item, &BatchesTaskProcessor::Config.process_item)
-    end
-
-    def per_page
-      BatchesTaskProcessor::Config.per_page
+      instance_eval(task_model.process_item)
     end
 
     # @example Article.where(no: items)
     def preload_job_items(items)
-      instance_exec(items, &BatchesTaskProcessor::Config.preload_job_items)
-    end
-    # ****** end customizations
-
-    def init_cache
-      items = calculate_items
-      jobs = (items.count.to_f / per_page).ceil
-      data = { jobs: jobs, count: items.count, date: Time.current, finished_jobs: [], cancelled: false }
-      main_registry(data)
+      instance_eval(task_model.preload_job_items || 'items')
     end
 
     def init_jobs
-      jobs = main_registry[:jobs]
+      jobs = task_model.qty_jobs
       log "Initializing #{jobs} jobs..."
       jobs.times.each do |index|
-        log "Starting ##{index} job..."
-        pid = Process.spawn("RUNNER_JOB_NO=#{index} rake batches_task_processor:process_job &")
-        Process.detach(pid)
+        if task_model.queue_name
+          log "Scheduling ##{index} job..."
+          BatchesTaskProcessor::ProcessorJob.set(queue: task_model.queue_name).perform_later(task_id, index)
+        else
+          start_inline_job(index)
+        end
       end
     end
 
-    def run_job(job, items)
+    def start_inline_job(job_no)
+      log "Starting ##{job_no} job..."
+      env_vars = "RUNNER_JOB_NO=#{job_no} RUNNER_TASK_ID=#{task_id}"
+      pid = Process.spawn("#{env_vars} rake batches_task_processor:process_job &")
+      Process.detach(pid)
+    end
+
+    def run_job(job)
       log "Running ##{job} job..."
-      preload_job_items(job_items(items, job)).each_with_index do |item, index|
+      items = job_items(job)
+      (items.try(:find_each) || items.each).with_index do |item, index|
         key = item.try(:id) || item
         break log('Process cancelled') if process_cancelled?
-        next log("Skipping #{key}...") if already_processed?(job, key)
+        next log("Skipping #{key}...") if already_processed?(key)
 
         start_process_item(item, job, key, index)
       end
 
-      mark_finished_job(job)
       log "Finished #{job} job..."
+      task_model.finish! if task_model.all_processed?
     end
 
-    def job_items(items, job)
-      items.is_a?(Array) ? items.each_slice(per_page).to_a[job] : items.offset(job * per_page).limit(per_page)
+    def job_items(job)
+      res = task_model.data.each_slice(task_model.qty_items_job).to_a[job]
+      preload_job_items(res)
     end
 
     def start_process_item(item, job, key, index)
-      log "Processing #{job}/#{key}: #{index}/#{per_page}"
-      process_item(item)
-      update_job_cache(job, key)
+      log "Processing key: #{key}, job: #{job}, counter: #{index}/#{task_model.qty_items_job}"
+      result = process_item(item)
+      task_model.items.create!(key: key, result: result.to_s[0..255])
     rescue => e
-      update_job_cache(job, key, e.message)
+      task_model.items.create!(key: key, error_details: e.message)
       log "Process failed #{job}/#{key}: #{e.message}"
     end
 
-    def main_registry(new_data = nil)
-      Rails.cache.write(RUNNER_JOB_KEY, new_data, expires_in: 1.week) if new_data
-      new_data || Rails.cache.read(RUNNER_JOB_KEY)
-    end
-
-    def mark_finished_job(job)
-      main_registry(main_registry.merge(finished_jobs: main_registry[:finished_jobs] + [job]))
-    end
-
-    def job_registry(job, new_data = nil)
-      key = "#{RUNNER_JOB_KEY}/#{job}"
-      default_data = { items: [], errors: [] }
-      Rails.cache.write(key, default_data, expires_in: 1.week) unless Rails.cache.read(key)
-      Rails.cache.write(key, new_data, expires_in: 1.week) if new_data
-      Rails.cache.delete(key) if new_data == :delete
-      new_data || Rails.cache.read(key)
-    end
-
-    def update_job_cache(job, value, error = nil)
-      data = job_registry(job)
-      data[:items] << value
-      data[:errors] << [value, error] if error
-      job_registry(job, data)
-    end
-
-    def already_processed?(job, value)
-      job_registry(job)[:items].include?(value)
+    def already_processed?(key)
+      task_model.items.where(key: key).exists?
     end
 
     def process_cancelled?
-      Rails.cache.read(RUNNER_JOB_KEY)[:cancelled]
+      task_model.state == 'cancelled'
     end
 
     def log(msg)
       puts "BatchesTaskProcessor => #{msg}"
     end
+
+    def task_model
+      klass = BatchesTaskProcessor::Model.all
+      task_id ? klass.find(task_id) : klass.last
+    end
   end
 end

data/lib/batches_task_processor/processor_job.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module BatchesTaskProcessor
+  class ProcessorJob < ActiveJob::Base
+    queue_as :default
+
+    def perform(task_id, job_no)
+      Processor.new(task_id).process_job(job_no)
+    end
+  end
+end

data/lib/batches_task_processor/railtie.rb

@@ -6,5 +6,9 @@ module BatchesTaskProcessor
     rake_tasks do
       load 'tasks/batches_task_processor_tasks.rake'
     end
+    initializer :append_migrations do |app|
+      path = File.join(File.expand_path('../../', __FILE__), 'db/migrate')
+      app.config.paths["db/migrate"] << path
+    end
   end
 end

data/lib/batches_task_processor/version.rb

@@ -1,3 +1,3 @@
 module BatchesTaskProcessor
-  VERSION = "0.1.0"
+  VERSION = "0.3.1"
 end

data/lib/batches_task_processor.rb

@@ -1,18 +1,11 @@
+# frozen_string_literal: true
+
 require "batches_task_processor/version"
 require "batches_task_processor/railtie"
 require "batches_task_processor/processor"
-
+require "batches_task_processor/processor_job"
+require "batches_task_processor/model"
+require "batches_task_processor/model_item"
 
 module BatchesTaskProcessor
-  class Config
-    cattr_accessor(:per_page) { 5000 }
-    cattr_accessor(:calculate_items) { -> { raise('Implement calculate_items method') } }
-    cattr_accessor(:process_item) { -> (_item) { raise('Implement calculate_items method') } }
-    cattr_accessor(:preload_job_items) { -> (items) { items } }
-
-
-    def self.configure
-      yield self
-    end
-  end
 end

data/lib/db/migrate/20220727101904_add_batches_task_processor.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+class AddBatchesTaskProcessor < ActiveRecord::Migration[5.0]
+  def change
+    create_table :batches_task_processors do |t|
+      t.string :key
+      t.string :state, default: :pending
+      t.json :data, default: []
+      t.integer :qty_jobs, default: 10
+      t.datetime :finished_at
+      t.text :preload_job_items
+      t.text :process_item, null: false
+      t.string :queue_name, default: :default
+      t.timestamps
+    end
+
+    create_table :batches_task_processor_items do |t|
+      t.belongs_to :batches_task_processors, foreign_key: true, index: { name: 'index_batches_task_processors_parent_id' }
+      t.string :key
+      t.text :result
+      t.text :error_details
+      t.timestamps
+    end
+  end
+end

data/lib/tasks/batches_task_processor_tasks.rake

@@ -3,31 +3,11 @@
 namespace :batches_task_processor do
   desc 'Starts the Batches Task Processor'
   task call: :environment do
-    BatchesTaskProcessor::Processor.new.call
+    BatchesTaskProcessor::Processor.new(ENV['RUNNER_TASK_ID']).call
   end
 
   desc 'Starts the Batches Task Processor'
   task process_job: :environment do
-    BatchesTaskProcessor::Processor.new.process_job(ENV['RUNNER_JOB_NO'])
-  end
-
-  desc 'Retries the Batches Task Processor'
-  task retry: :environment do
-    BatchesTaskProcessor::Processor.new.retry
-  end
-
-  desc 'Prints the status of the Task Processor'
-  task status: :environment do
-    BatchesTaskProcessor::Processor.new.status
-  end
-
-  desc 'Cancels the Batches Task Processor'
-  task cancel: :environment do
-    BatchesTaskProcessor::Processor.new.cancel
-  end
-
-  desc 'Clears the Batches Task Processor cache'
-  task clear: :environment do
-    BatchesTaskProcessor::Processor.new.clear
+    BatchesTaskProcessor::Processor.new(ENV['RUNNER_TASK_ID']).process_job(ENV['RUNNER_JOB_NO'])
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: batches_task_processor
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.3.1
 platform: ruby
 authors:
 - Owen Peredo
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-07-29 00:00:00.000000000 Z
+date: 2022-08-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -34,9 +34,13 @@ files:
 - README.md
 - Rakefile
 - lib/batches_task_processor.rb
+- lib/batches_task_processor/model.rb
+- lib/batches_task_processor/model_item.rb
 - lib/batches_task_processor/processor.rb
+- lib/batches_task_processor/processor_job.rb
 - lib/batches_task_processor/railtie.rb
 - lib/batches_task_processor/version.rb
+- lib/db/migrate/20220727101904_add_batches_task_processor.rb
 - lib/tasks/batches_task_processor_tasks.rake
 homepage: https://github.com/owen2345/batches-task-processor
 licenses: []