massive 0.1.0

data/Rakefile

@@ -0,0 +1,8 @@
+require "bundler/gem_tasks"
+
+require "rspec/core/rake_task"
+
+desc "Run all examples"
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/lib/massive.rb

@@ -0,0 +1,63 @@
+require "massive/version"
+
+module Massive
+  autoload :MemoryConsumption, 'massive/memory_consumption'
+  autoload :TimingSupport,     'massive/timing_support'
+  autoload :Status,            'massive/status'
+  autoload :Locking,           'massive/locking'
+  autoload :Retry,             'massive/retry'
+  autoload :Cancelling,        'massive/cancelling'
+
+  autoload :Process,           'massive/process'
+  autoload :Step,              'massive/step'
+  autoload :Job,               'massive/job'
+
+  autoload :File,              'massive/file'
+  autoload :FileProcess,       'massive/file_process'
+  autoload :FileStep,          'massive/file_step'
+  autoload :FileJob,           'massive/file_job'
+
+  autoload :Notifications,     'massive/notifications'
+  autoload :Notifiers,         'massive/notifiers'
+
+  autoload :ProcessSerializer, 'massive/process_serializer'
+  autoload :StepSerializer,    'massive/step_serializer'
+
+  class Cancelled < StandardError; end
+
+  def self.redis
+    @redis ||= Resque.redis
+  end
+
+  def self.fog_credentials
+    @fog_credentials
+  end
+
+  def self.fog_credentials=(values=nil)
+    @fog_credentials = values
+  end
+
+  def self.fog_authenticated_url_expiration
+    @fog_authenticated_url_expiration
+  end
+
+  def self.fog_authenticated_url_expiration=(value=nil)
+    @fog_authenticated_url_expiration = value
+  end
+
+  def self.fog_directory
+    @fog_directory
+  end
+
+  def self.fog_directory=(directory=nil)
+    @fog_directory = directory
+  end
+
+  self.fog_directory = 'massive'
+  self.fog_authenticated_url_expiration = 1 * 60 * 60
+end
+
+require "resque"
+require "mongoid"
+require "active_model_serializers"
+require "file_processor"

data/lib/massive/cancelling.rb

@@ -0,0 +1,20 @@
+module Massive
+  module Cancelling
+    extend ActiveSupport::Concern
+
+    # Override this to provide logic for whether it should be cancelled or not
+    def cancelled?
+    end
+
+    def cancelling(&block)
+      raise Massive::Cancelled.new(cancelled_exception_message) if cancelled?
+      block.call
+    end
+
+    private
+
+    def cancelled_exception_message
+      "Cancelled #{self.class.name} - #{self.id if respond_to?(:id)}"
+    end
+  end
+end

data/lib/massive/file.rb

@@ -0,0 +1,80 @@
+module Massive
+  class File
+    include Mongoid::Document
+    include Mongoid::Timestamps
+
+    embedded_in :process
+
+    field :url,         type: String
+    field :filename,    type: String
+
+    field :encoding,    type: String
+    field :col_sep,     type: String
+    field :total_count, type: Integer
+    field :use_headers, type: Boolean, default: true
+
+    field :headers,     type: Array, default: -> { [] }
+    field :sample_data, type: Array, default: -> { [] }
+
+    def processor
+      @processor ||= FileProcessor::CSV.new(url, processor_options)
+    end
+
+    def gather_info!
+      clear_info
+
+      self.encoding    = processor.detected_encoding
+      self.col_sep     = processor.col_sep
+      self.total_count = processor.total_count
+      self.headers     = processor.shift && processor.headers if use_headers?
+
+      processor.process_range(limit: 3) do |row|
+        self.sample_data << (use_headers? ? row.fields : row)
+      end
+
+      save
+    end
+
+    def url
+      read_attribute(:url).presence || authenticated_url
+    end
+
+    private
+
+    def clear_info
+      [:encoding, :col_sep, :total_count, :headers].each { |attr| self[attr] = nil }
+
+      sample_data.clear
+    end
+
+    def processor_options
+      {
+        headers:  use_headers?,
+        encoding: encoding,
+        col_sep:  col_sep
+      }
+    end
+
+    def authenticated_url
+      if can_use_fog?
+        fog_file.respond_to?(:url) ? fog_file.url(Time.current.to_i + Massive.fog_authenticated_url_expiration) : fog_file.public_url
+      end
+    end
+
+    def can_use_fog?
+      filename && Massive.fog_credentials.present?
+    end
+
+    def fog_connection
+      @fog_connection ||= Fog::Storage.new(Massive.fog_credentials)
+    end
+
+    def fog_directory
+      @fog_directory ||= fog_connection.directories.get(Massive.fog_directory)
+    end
+
+    def fog_file
+      @fog_file ||= fog_directory.files.get(filename)
+    end
+  end
+end

data/lib/massive/file_job.rb

@@ -0,0 +1,9 @@
+module Massive
+  class FileJob < Job
+    delegate :file, to: :step
+
+    def each_item(&block)
+      file.processor.process_range(offset: offset + processed, limit: limit - processed, &block)
+    end
+  end
+end

data/lib/massive/file_process.rb

@@ -0,0 +1,7 @@
+module Massive
+  class FileProcess < Massive::Process
+    embeds_one :file,  class_name: 'Massive::File', autobuild: true
+
+    accepts_nested_attributes_for :file
+  end
+end

data/lib/massive/file_step.rb

@@ -0,0 +1,7 @@
+module Massive
+  class FileStep < Step
+    calculates_total_count_with { file.total_count }
+
+    delegate :file, to: :process
+  end
+end

data/lib/massive/job.rb

@@ -0,0 +1,115 @@
+module Massive
+  class Job
+    include Mongoid::Document
+    include Mongoid::Timestamps
+
+    include Massive::Status
+    include Massive::MemoryConsumption
+    include Massive::TimingSupport
+    include Massive::Retry
+    include Massive::Cancelling
+
+    embedded_in :step, class_name: 'Massive::Step'
+
+    field :processed,   type: Integer, default: 0
+    field :offset,      type: Integer, default: 0
+    field :limit,       type: Integer, default: -1
+
+    delegate :process, :notify, to: :step
+
+    define_model_callbacks :work
+
+    after_create :enqueue
+
+    def self.perform(process_id, step_id, job_id)
+      Massive::Process.find_job(process_id, step_id, job_id).work
+    end
+
+    def self.queue
+      :massive_job
+    end
+
+    def enqueue
+      Resque.enqueue(self.class, process.id.to_s, step.id.to_s, id.to_s)
+    end
+
+    def work
+      handle_errors do
+        cancelling do
+          start!
+
+          run_callbacks :work do
+            each_item do |item, index|
+              retrying do
+                cancelling do
+                  process_each(item, index)
+                  increment_processed
+                  notify(:progress)
+                end
+              end
+            end
+          end
+
+          finish!
+        end
+      end
+    end
+
+    def finish!
+      update_attributes(finished_at: Time.now, memory_consumption: current_memory_consumption)
+
+      step.complete
+    end
+
+    def each_item(&block)
+      # iterate through each item within offset/limit range
+    end
+
+    def process_each(item, index)
+      # process an item
+    end
+
+    protected
+
+    def attributes_to_reset
+      super.merge(processed: 0)
+    end
+
+    def cancelled?
+      process.cancelled?
+    end
+
+    private
+
+    def handle_errors(&block)
+      block.call
+    rescue Massive::Cancelled => e
+      assign_attributes(cancelled_at: Time.now)
+      step.update_attributes(cancelled_at: Time.now)
+
+      notify(:cancelled)
+    rescue StandardError, SignalException => e
+      step.failed_at = Time.now
+
+      assign_attributes(
+        last_error: e.message,
+        failed_at: Time.now,
+        processed: 0,
+        retries: retries
+      )
+
+      step.save
+      notify(:failed)
+
+      raise e
+    end
+
+    def increment_processed
+      inc(:processed, 1)
+    end
+
+    def args_for_resque
+      [process.id.to_s, step.id.to_s, id.to_s]
+    end
+  end
+end

data/lib/massive/locking.rb

@@ -0,0 +1,27 @@
+module Massive
+  module Locking
+    def locked?(key, expire_in=60 * 1000)
+      lock_key = lock_key_for(key)
+
+      !redis.setnx(lock_key, Time.now.to_i + (expire_in)/1000).tap do |result|
+        expire(lock_key, expire_in) if result
+      end
+    end
+
+    protected
+
+    def lock_key_for(key)
+      "#{self.class.name.underscore}:#{id}:#{key}"
+    end
+
+    def expire(lock_key, expire_in)
+      redis.pexpire(lock_key, expire_in)
+    rescue Redis::CommandError
+      redis.expire(lock_key, (expire_in/1000).to_i)
+    end
+
+    def redis
+      @redis ||= Massive.redis
+    end
+  end
+end

data/lib/massive/memory_consumption.rb

@@ -0,0 +1,15 @@
+module Massive
+  module MemoryConsumption
+    extend ActiveSupport::Concern
+
+    included do
+      field :memory_consumption, type: Integer, default: 0
+    end
+
+    def current_memory_consumption
+      IO.popen("ps -o rss= -p #{::Process.pid}") { |io| io.gets.to_i }
+    rescue StandardError
+      0
+    end
+  end
+end

data/lib/massive/notifications.rb

@@ -0,0 +1,40 @@
+module Massive
+  module Notifications
+    extend ActiveSupport::Concern
+
+    included do
+      notifier :base
+    end
+
+    def notify(message)
+      if active_model_serializer
+        notifier.notify(message) do
+          active_model_serializer.new(reload)
+        end
+      end
+    end
+
+    def notifier
+      @notifier ||= self.class.notifier_class.new(notifier_id, self.class.notifier_options)
+    end
+
+    def notifier_id
+      "#{self.class.name.underscore.gsub('/', '-')}-#{id}"
+    end
+
+    module ClassMethods
+      def notifier(name, options={})
+        @notifier_class = name.is_a?(Class) ? name : "massive/notifiers/#{name}".camelize.constantize
+        @notifier_options = options
+      end
+
+      def notifier_class
+        @notifier_class || Massive::Notifiers::Base
+      end
+
+      def notifier_options
+        @notifier_options
+      end
+    end
+  end
+end

data/lib/massive/notifiers.rb

@@ -0,0 +1,6 @@
+module Massive
+  module Notifiers
+    autoload :Base,   'massive/notifiers/base'
+    autoload :Pusher, 'massive/notifiers/pusher'
+  end
+end

data/lib/massive/notifiers/base.rb

@@ -0,0 +1,32 @@
+module Massive
+  module Notifiers
+    class Base
+      include Massive::Locking
+
+      attr_accessor :id, :last, :options
+
+      def initialize(id, options={})
+        self.id   = id
+        self.last = {}
+
+        self.options = options || {}
+      end
+
+      def notify(message, data=nil, &block)
+        send_notification(message, data, &block) unless locked?(message, expiration)
+      end
+
+      protected
+
+      def send_notification(message, data, &block)
+        data = block.call if block_given?
+
+        self.last = { message: message, data: data }
+      end
+
+      def expiration
+        options[:expiration] || 1000  # 1 second between each notification
+      end
+    end
+  end
+end