massive 0.1.0

data/lib/massive/notifiers/pusher.rb

@@ -0,0 +1,17 @@
+module Massive
+  module Notifiers
+    class Pusher < Base
+      protected
+
+      def send_notification(message, data, &block)
+        data = block.call if block_given?
+
+        client.trigger(id, message, data)
+      end
+
+      def client
+        @client ||= options[:client] || ::Pusher
+      end
+    end
+  end
+end

data/lib/massive/process.rb

@@ -0,0 +1,69 @@
+module Massive
+  class Process
+    include Mongoid::Document
+    include Mongoid::Timestamps
+
+    field :cancelled_at, type: Time
+
+    embeds_many :steps, class_name: 'Massive::Step'
+
+    def self.find_step(process_id, step_id)
+      find(process_id).steps.find(step_id)
+    end
+
+    def self.find_job(process_id, step_id, job_id)
+      find_step(process_id, step_id).jobs.find(job_id)
+    end
+
+    def enqueue_next
+      next_step.try(:enqueue)
+    end
+
+    def next_step
+      step = steps.not_completed.not_started.first
+      step.try(:enqueued?) ? nil : step
+    end
+
+    def processed_percentage
+      total_weight > 0 ? total_steps_processed_percentage.to_f / total_weight : 0
+    end
+
+    def completed?
+      steps.not_completed.none?
+    end
+
+    def cancelled?
+      cancelled_at? || redis.exists(cancelled_key)
+    end
+
+    def cancel
+      self.cancelled_at = Time.now
+      redis.setex(cancelled_key, 1.day, true)
+      save
+    end
+
+    def active_model_serializer
+      super || Massive::ProcessSerializer
+    end
+
+    protected
+      def redis
+        Massive.redis
+      end
+
+      def cancelled_key
+        "#{self.class.name.underscore}:#{id}:cancelled"
+      end
+
+    private
+      def total_weight
+        steps.map(&:weight).sum
+      end
+
+      def total_steps_processed_percentage
+        steps.inject(0) do |result, step|
+          result += step.processed_percentage * step.weight
+        end
+      end
+  end
+end

data/lib/massive/process_serializer.rb

@@ -0,0 +1,12 @@
+module Massive
+  class ProcessSerializer < ActiveModel::Serializer
+    attributes :id, :created_at, :updated_at, :processed_percentage
+    attribute :completed?, key: :completed
+
+    has_many :steps
+
+    def id
+      object.id.to_s
+    end
+  end
+end

data/lib/massive/retry.rb

@@ -0,0 +1,49 @@
+module Massive
+  module Retry
+    extend ActiveSupport::Concern
+
+    included do
+      retry_interval 2
+      maximum_retries 10
+
+      def self.inherited(base)
+        super
+
+        base.retry_interval retry_interval
+        base.maximum_retries maximum_retries
+      end
+    end
+
+    def retrying(&block)
+      self.retries = 0
+
+      begin
+        block.call
+      rescue Massive::Cancelled, SignalException
+        # re-raise cancelled and signal exceptions since they are not an actual error
+        raise
+      rescue StandardError => e
+        self.retries += 1
+
+        if self.retries < self.class.maximum_retries
+          Kernel.sleep self.class.retry_interval
+          retry
+        else
+          raise e
+        end
+      end
+    end
+
+    module ClassMethods
+      def retry_interval(value=nil)
+        @retry_interval = value if value
+        @retry_interval
+      end
+
+      def maximum_retries(value=nil)
+        @maximum_retries = value if value
+        @maximum_retries
+      end
+    end
+  end
+end

data/lib/massive/status.rb

@@ -0,0 +1,59 @@
+module Massive
+  module Status
+    extend ActiveSupport::Concern
+
+    included do
+      field :started_at,   type: Time
+      field :finished_at,  type: Time
+      field :failed_at,    type: Time
+      field :cancelled_at, type: Time
+
+      field :last_error,   type: String
+      field :retries,      type: Integer, default: 0
+
+      scope :started,       ne(started_at: nil)
+      scope :not_started,   where(started_at: nil)
+      scope :completed,     ne(finished_at: nil)
+      scope :not_completed, where(finished_at: nil)
+      scope :failed,        ne(failed_at: nil)
+      scope :cancelled,     ne(cancelled_at: nil)
+    end
+
+    def start!
+      update_attributes(attributes_to_reset)
+    end
+
+    def started?
+      !failed? && started_at?
+    end
+
+    def completed?
+      !failed? && finished_at?
+    end
+
+    def failed?
+      failed_at?
+    end
+
+    def enqueued?
+      item = Resque.peek(self.class.queue)
+      item.present? && (item["class"] == self.class.name) && (item["args"] == args_for_resque)
+    end
+
+    protected
+
+    def attributes_to_reset
+      {
+        started_at: Time.now,
+        finished_at: nil,
+        failed_at: nil,
+        cancelled_at: nil,
+        retries: 0,
+        last_error: nil
+      }
+    end
+
+    def args_for_resque
+    end
+  end
+end

data/lib/massive/step.rb

@@ -0,0 +1,143 @@
+module Massive
+  class Step
+    include Mongoid::Document
+    include Mongoid::Timestamps
+
+    include Massive::Status
+    include Massive::MemoryConsumption
+    include Massive::TimingSupport
+    include Massive::Locking
+    include Massive::Notifications
+
+    embedded_in :process, class_name: 'Massive::Process'
+    embeds_many :jobs,    class_name: 'Massive::Job'
+
+    field :total_count,  type: Integer
+    field :weight,       type: Integer, default: 1
+    field :job_class,    type: String,  default: -> { self.class.job_class }
+    field :execute_next, type: Boolean, default: false
+
+    define_model_callbacks :work
+    define_model_callbacks :complete
+
+    def self.perform(process_id, step_id)
+      Massive::Process.find_step(process_id, step_id).work
+    end
+
+    def self.queue
+      :massive_step
+    end
+
+    def self.calculates_total_count_with(&block)
+      define_method(:calculate_total_count, &block)
+    end
+
+    def self.limit_ratio(value=nil)
+      @limit_ratio = value if value
+      @limit_ratio
+    end
+
+    def self.job_class(value=nil)
+      @job_class = value if value
+      @job_class
+    end
+
+    def self.inherited(child)
+      super
+
+      child.job_class   self.job_class
+      child.limit_ratio self.limit_ratio
+    end
+
+    limit_ratio 3000 => 1000, 0 => 100
+    job_class 'Massive::Job'
+
+    def enqueue
+      Resque.enqueue(self.class, process.id.to_s, id.to_s)
+    end
+
+    def start!
+      super
+      notify(:start)
+    end
+
+    def work
+      start!
+
+      run_callbacks :work do
+        process_step
+      end
+
+      complete
+    end
+
+    def process_step
+      self.jobs = number_of_jobs.times.map do |index|
+        job_class.constantize.new(job_params(index))
+      end
+    end
+
+    def complete
+      if completed_all_jobs? && !locked?(:complete)
+        run_callbacks :complete do
+          update_attributes finished_at: Time.now, failed_at: nil, memory_consumption: current_memory_consumption
+          notify(:complete)
+        end
+
+        process.enqueue_next if execute_next?
+      end
+    end
+
+    def completed_all_jobs?
+      reload if persisted?
+
+      jobs.all?(&:completed?)
+    end
+
+    def processed
+      jobs.map(&:processed).sum
+    end
+
+    def processed_percentage
+      total_count && total_count > 0 ? processed.to_f / total_count : 0
+    end
+
+    def processing_time
+      jobs.map(&:elapsed_time).sum
+    end
+
+    def limit
+      @limit ||= self.class.limit_ratio.find { |count, l| total_count >= count }.last
+    end
+
+    def calculate_total_count
+      0
+    end
+
+    def active_model_serializer
+      super || Massive::StepSerializer
+    end
+
+    protected
+
+    def job_params(index)
+      {
+        offset: index * limit,
+        limit: limit,
+        step: self
+      }
+    end
+
+    def number_of_jobs
+      (total_count.to_f / limit).ceil
+    end
+
+    def attributes_to_reset
+      super.merge(total_count: total_count || calculate_total_count)
+    end
+
+    def args_for_resque
+      [process.id.to_s, id.to_s]
+    end
+  end
+end

data/lib/massive/step_serializer.rb

@@ -0,0 +1,12 @@
+module Massive
+  class StepSerializer < ActiveModel::Serializer
+    attributes :id, :created_at, :updated_at, :started_at, :finished_at, :failed_at,
+               :last_error, :retries, :memory_consumption, :total_count,
+               :processed, :processed_percentage, :processing_time, :elapsed_time,
+               :notifier_id
+
+    def id
+      object.id.to_s
+    end
+  end
+end

data/lib/massive/timing_support.rb

@@ -0,0 +1,10 @@
+module Massive
+  module TimingSupport
+    def elapsed_time
+      start  = started_at || 0
+      finish = finished_at || Time.now
+
+      started_at? ? finish - start : 0
+    end
+  end
+end

data/lib/massive/version.rb

@@ -0,0 +1,3 @@
+module Massive
+  VERSION = "0.1.0"
+end

data/massive.gemspec

@@ -0,0 +1,23 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'massive/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = "massive"
+  gem.version       = Massive::VERSION
+  gem.authors       = ["Vicente Mundim"]
+  gem.email         = ["vicente.mundim@gmail.com"]
+  gem.description   = %q{Parallelize processing of large files and/or data using Resque, Redis and MongoDB}
+  gem.summary       = %q{Parallelize processing of large files and/or data using Resque, Redis and MongoDB}
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+
+  gem.add_dependency "resque"
+  gem.add_dependency "mongoid", "~> 3.1.x"
+  gem.add_dependency "file_processor", "0.2.0"
+  gem.add_dependency "active_model_serializers"
+end

data/spec/fixtures/custom_job.rb

@@ -0,0 +1,4 @@
+class CustomJob < Massive::Job
+  retry_interval 5
+  maximum_retries 20
+end

data/spec/fixtures/custom_step.rb

@@ -0,0 +1,19 @@
+class CustomStep < Massive::Step
+  calculates_total_count_with { 100 }
+  job_class 'CustomJob'
+
+  limit_ratio 3000 => 1500, 2000 => 1000, 0 => 100
+
+  protected
+
+  def job_params(index)
+    {
+      offset: index * limit,
+      limit: limit,
+      custom_param: "some_param"
+    }
+  end
+end
+
+class InheritedStep < Massive::Step
+end

data/spec/models/massive/cancelling_spec.rb

@@ -0,0 +1,83 @@
+require "spec_helper"
+
+class Cancellable
+  include Massive::Cancelling
+
+  attr_accessor :cancelled, :work_count, :work_done_count, :cancelled_exception
+
+  def initialize(work_count)
+    self.work_count = work_count
+  end
+
+  def cancelled?
+    cancelled == true
+  end
+
+  def work(&block)
+    self.work_done_count = 0
+
+    work_count.times do |iteration|
+      cancelling do
+        block.call(self, iteration)
+        self.work_done_count += 1
+      end
+    end
+  rescue Massive::Cancelled => e
+    self.cancelled_exception = e
+  end
+end
+
+describe Massive::Cancelling do
+  let(:work_count) { 3 }
+  subject(:cancellable) { Cancellable.new(work_count) }
+
+  context "when it is never cancelled" do
+    it "does not cancel the work" do
+      cancellable.work { |cancellable| }
+      cancellable.work_done_count.should eq(cancellable.work_count)
+    end
+
+    it "does not raises a cancelled exception" do
+      cancellable.work { |cancellable| }
+      cancellable.cancelled_exception.should be_nil
+    end
+  end
+
+  context "when it is cancelled before actually performing any work" do
+    before { cancellable.cancelled = true }
+
+    it "cancels the work before the first iteration" do
+      cancellable.work { |cancellable|  }
+      cancellable.work_done_count.should eq(0)
+    end
+
+    it "raises a cancelled exception" do
+      cancellable.work { |cancellable| }
+      cancellable.cancelled_exception.should be_present
+    end
+  end
+
+  context "when it is cancelled while performing some work" do
+    it "cancels the work before performing the iteration" do
+      cancellable.work { |cancellable, iteration| cancellable.cancelled = (iteration == work_count - 2) }
+      cancellable.work_done_count.should eq(2)
+    end
+
+    it "raises a cancelled exception" do
+      cancellable.work { |cancellable, iteration| cancellable.cancelled = (iteration == work_count - 2) }
+      cancellable.cancelled_exception.should be_present
+    end
+  end
+
+  context "when it is cancelled while performing the last iteration" do
+    it "performs all the work" do
+      cancellable.work { |cancellable, iteration| cancellable.cancelled = (iteration == work_count - 1) }
+      cancellable.work_done_count.should eq(work_count)
+    end
+
+    it "does not raise a cancelled exception" do
+      cancellable.work { |cancellable, iteration| cancellable.cancelled = (iteration == work_count - 1) }
+      cancellable.cancelled_exception.should be_nil
+    end
+  end
+end