jobba 1.0.0

data/lib/jobba/query.rb

@@ -0,0 +1,84 @@
+require 'jobba/clause'
+require 'jobba/clause_factory'
+
+class Jobba::Query
+
+  include Jobba::Common
+
+  def where(options)
+    options.each do |kk,vv|
+      clauses.push(Jobba::ClauseFactory.new_clause(kk,vv))
+    end
+
+    self
+  end
+
+  def count
+    run(&COUNT_STATUSES)
+  end
+
+  def empty?
+    count == 0
+  end
+
+  # At the end of a chain of `where`s, the user will call methods that expect
+  # to run on the result of the executed `where`s.  So if we don't know what
+  # the method is, execute the `where`s and pass the method to its output.
+
+  def method_missing(method_name, *args)
+    if Jobba::Statuses.instance_methods.include?(method_name)
+      run(&GET_STATUSES).send(method_name, *args)
+    else
+      super
+    end
+  end
+
+  def respond_to?(method_name)
+    Jobba::Statuses.instance_methods.include?(method_name) || super
+  end
+
+  protected
+
+  attr_accessor :clauses
+
+  def initialize
+    @clauses = []
+  end
+
+  GET_STATUSES = ->(working_set) {
+    ids = Jobba.redis.zrange(working_set, 0, -1)
+    Jobba::Statuses.new(ids)
+  }
+
+  COUNT_STATUSES = ->(working_set) {
+    Jobba.redis.zcard(working_set)
+  }
+
+  def run(&working_set_block)
+
+    # TODO PUT IN MULTI BLOCKS WHERE WE CAN!
+
+    load_default_clause if clauses.empty?
+    working_set = nil
+
+    clauses.each_with_index do |clause, ii|
+      clause_set = clause.to_new_set
+
+      if working_set.nil?
+        working_set = clause_set
+      else
+        redis.zinterstore(working_set, [working_set, clause_set], weights: [0, 0])
+        redis.del(clause_set)
+      end
+    end
+
+    working_set_block.call(working_set).tap do
+      redis.del(working_set)
+    end
+  end
+
+  def load_default_clause
+    where(state: Jobba::State::ALL.collect(&:name))
+  end
+
+end

data/lib/jobba/state.rb

@@ -0,0 +1,59 @@
+class Jobba::State
+
+  attr_reader :name, :timestamp_name
+
+  def initialize(name, timestamp_name)
+    @name = name
+    @timestamp_name = timestamp_name
+  end
+
+  def self.from_name(state_name)
+    ALL.select{|state| state.name == state_name}.first
+  end
+
+  UNQUEUED        = new('unqueued', 'recorded_at')
+  QUEUED          = new('queued', 'queued_at')
+  WORKING         = new('working', 'started_at')
+  SUCCEEDED       = new('succeeded', 'succeeded_at')
+  FAILED          = new('failed', 'failed_at')
+  KILLED          = new('killed', 'killed_at')
+  UNKNOWN         = new('unknown', 'recorded_at')
+
+  ALL = [
+    UNQUEUED,
+    QUEUED,
+    WORKING,
+    SUCCEEDED,
+    FAILED,
+    KILLED,
+    UNKNOWN
+  ].freeze
+
+  COMPLETED = [
+    SUCCEEDED,
+    FAILED
+  ].freeze
+
+  INCOMPLETE = [
+    UNQUEUED,
+    QUEUED,
+    WORKING,
+    KILLED
+  ].freeze
+
+  ENTERABLE = [
+    UNQUEUED,
+    QUEUED,
+    WORKING,
+    SUCCEEDED,
+    FAILED,
+    KILLED,
+    UNKNOWN
+  ].freeze
+
+end
+
+
+
+
+

data/lib/jobba/status.rb

@@ -0,0 +1,304 @@
+require 'json'
+require 'ostruct'
+
+module Jobba
+  class Status
+
+    include Jobba::Common
+
+    def self.create!
+      create(state: State::UNQUEUED)
+    end
+
+    # Finds the job with the specified ID and returns it.  If no such ID
+    # exists in the store, returns a job with 'unknown' state and sets it
+    # in the store
+    def self.find!(id)
+      find(id) || create(id: id)
+    end
+
+    # Finds the job with the specified ID and returns it.  If no such ID
+    # exists in the store, returns nil.
+    def self.find(id)
+      if (hash = raw_redis_hash(id))
+        new(raw: hash)
+      else
+        nil
+      end
+    end
+
+    def self.local_attrs
+      %w(id state progress errors data kill_requested_at job_name job_args) +
+      State::ALL.collect(&:timestamp_name)
+    end
+
+    def reload!
+      @json_encoded_attrs = self.class.raw_redis_hash(id)
+      self.class.local_attrs.each{|aa| send("#{aa}=",nil)}
+      self
+    end
+
+    # If the attributes are nil, the attribute accessors lazily parse their values
+    # from the JSON retrieved from redis.  That way there's no parsing that isn't used.
+    # As an extra step, convert state names into State objects.
+
+    local_attrs.each do |attribute|
+      class_eval <<-eoruby
+        def #{attribute}
+          @#{attribute} ||= load_from_json_encoded_attrs('#{attribute}')
+        end
+
+        protected
+
+        attr_writer :#{attribute}
+      eoruby
+    end
+
+    State::ENTERABLE.each do |state|
+      define_method("#{state.name}!") do
+        return self if state == self.state
+
+        redis.multi do
+          leave_current_state!
+          enter_state!(state)
+        end
+
+        self
+      end
+    end
+
+    State::ALL.each do |state|
+      define_method("#{state.name}?") do
+        state == self.state
+      end
+    end
+
+    def completed?
+      failed? || succeeded?
+    end
+
+    def incomplete?
+      !completed?
+    end
+
+    def request_kill!
+      time, usec_int = now
+      if redis.hsetnx(job_key, :kill_requested_at, usec_int)
+        @kill_requested_at = time
+      end
+    end
+
+    def kill_requested?
+      !kill_requested_at.nil?
+    end
+
+    def set_progress(at, out_of = nil)
+      progress = compute_fractional_progress(at, out_of)
+      set(progress: progress)
+    end
+
+    def set_job_name(job_name)
+      raise ArgumentError, "`job_name` must not be blank" if job_name.nil? || job_name.empty?
+      raise StandardError, "`job_name` can only be set once" if !self.job_name.nil?
+
+      redis.multi do
+        set(job_name: job_name)
+        redis.sadd(job_name_key, id)
+      end
+    end
+
+    def add_job_arg(arg_name, arg)
+      raise ArgumentError, "`arg_name` must not be blank" if arg_name.nil? || arg_name.empty?
+      raise ArgumentError, "`arg` must not be blank" if arg.nil? || arg.empty?
+
+      redis.multi do
+        self.job_args[arg_name.to_sym] = arg
+        redis.hset(job_args_key, arg_name, arg)
+        redis.sadd(job_arg_key(arg), id)
+      end
+    end
+
+    # def add_error(error, options = { })
+    #   options = { is_fatal: false }.merge(options)
+    #   @errors << { is_fatal: options[:is_fatal],
+    #                code: error.code,
+    #                message: error.message,
+    #                data: error.data }
+    #   set(errors: @errors)
+    # end
+
+    def save(data)
+      set(data: data)
+    end
+
+    def delete
+      completed? ?
+        delete! :
+        raise(NotCompletedError, "This status cannot be deleted because it " \
+                                 "isn't complete.  Use `delete!` if you want to " \
+                                 "delete anyway.")
+    end
+
+    def delete!
+      redis.multi do
+        redis.del(job_key)
+
+        State::ALL.each do |state|
+          redis.srem(state.name, id)
+          redis.zrem(state.timestamp_name, id)
+        end
+
+        redis.srem(job_name_key, id)
+
+        redis.del(job_args_key)
+        job_args.marshal_dump.values.each do |arg|
+          redis.srem(job_arg_key(arg), id)
+        end
+      end
+    end
+
+    protected
+
+    def self.create(attrs)
+      new(attrs.merge!(persist: true))
+    end
+
+    def self.raw_redis_hash(id)
+      main_hash, job_args_hash = redis.multi do
+        redis.hgetall(job_key(id))
+        redis.hgetall(job_args_key(id))
+      end
+
+      return nil if main_hash.empty?
+
+      main_hash['job_args'] = job_args_hash.to_json if !job_args_hash.nil?
+      main_hash
+    end
+
+    def leave_current_state!
+      redis.srem(state.name, id)
+    end
+
+    def enter_state!(state)
+      time, usec_int = now
+      set(state: state.name, state.timestamp_name => usec_int)
+      self.state = state
+      self.send("#{state.timestamp_name}=",time)
+      redis.zadd(state.timestamp_name, usec_int, id)
+      redis.sadd(state.name, id)
+    end
+
+    def initialize(attrs = {})
+      # If we get a raw hash, don't parse the attributes until they are requested
+
+      @json_encoded_attrs = attrs[:raw]
+
+      if !@json_encoded_attrs.nil? && !@json_encoded_attrs.empty?
+        @json_encoded_attrs['data'] ||= "{}"
+        @json_encoded_attrs['job_args'] ||= "{}"
+      else
+        @id       = attrs[:id]       || attrs['id']       || SecureRandom.uuid
+        @state    = attrs[:state]    || attrs['state']    || State::UNKNOWN
+        @progress = attrs[:progress] || attrs['progress'] || 0
+        @errors   = attrs[:errors]   || attrs['errors']   || []
+        @data     = attrs[:data]     || attrs['data']     || {}
+        @job_args = OpenStruct.new # TODO need this and the above job args init?
+
+        if attrs[:persist]
+          redis.multi do
+            set({
+              id: id,
+              progress: progress,
+              errors: errors
+            })
+            enter_state!(state)
+          end
+        end
+      end
+    end
+
+    def load_from_json_encoded_attrs(attribute_name)
+      json = (@json_encoded_attrs || {})[attribute_name]
+      attribute = json.nil? ? nil : JSON.parse(json, quirks_mode: true)
+
+      case attribute_name
+      when 'state'
+        State.from_name(attribute)
+      when /.*_at/
+        attribute.nil? ? nil : Jobba::Utils.time_from_usec_int(attribute.to_i)
+      when 'job_args'
+        OpenStruct.new(attribute)
+      else
+        attribute
+      end
+    end
+
+    def set(incoming_hash)
+      apply_consistency_rules!(incoming_hash)
+      set_hash_locally(incoming_hash)
+      set_hash_in_redis(incoming_hash)
+    end
+
+    def apply_consistency_rules!(hash)
+      hash[:progress] = 1.0 if hash[:state] == State::SUCCEEDED
+    end
+
+    def set_hash_locally(hash)
+      hash.each{ |key, value| self.send("#{key}=", value) }
+    end
+
+    def set_hash_in_redis(hash)
+      redis_key_value_array =
+        hash.to_a
+            .collect{|kv_array| [kv_array[0], kv_array[1].to_json]}
+            .flatten(1)
+
+      Jobba.redis.hmset(job_key, *redis_key_value_array)
+    end
+
+    def job_name_key
+      "job_name:#{job_name}"
+    end
+
+    def job_key
+      self.class.job_key(id)
+    end
+
+    def self.job_key(id)
+      raise(ArgumentError, "`id` cannot be nil") if id.nil?
+      "id:#{id}"
+    end
+
+    def job_args_key
+      self.class.job_args_key(id)
+    end
+
+    def self.job_args_key(id)
+      raise(ArgumentError, "`id` cannot be nil") if id.nil?
+      "job_args:#{id}"
+    end
+
+    def job_arg_key(arg)
+      "job_arg:#{arg}"
+    end
+
+    def compute_fractional_progress(at, out_of)
+      if at.nil?
+        raise ArgumentError, "Must specify at least `at` argument to `progress` call"
+      elsif at < 0
+        raise ArgumentError, "progress cannot be negative (at=#{at})"
+      elsif out_of && out_of < at
+        raise ArgumentError, "`out_of` must be greater than `at` in `progress` calls"
+      elsif out_of.nil? && (at < 0 || at > 1)
+        raise ArgumentError, "If `out_of` not specified, `at` must be in the range [0.0, 1.0]"
+      end
+
+      at.to_f / (out_of || 1).to_f
+    end
+
+    def now
+      [time = Jobba::Time.now, Utils.time_to_usec_int(time)]
+    end
+
+  end
+end

data/lib/jobba/statuses.rb

@@ -0,0 +1,74 @@
+class Jobba::Statuses
+
+  include Jobba::Common
+  extend Forwardable
+
+  attr_reader :ids
+
+  def all
+    load
+  end
+
+  def_delegator :@ids, :empty?
+  def_delegators :all, :first, :any?, :none?, :all?, :each, :each_with_index,
+                       :map, :collect, :select, :count
+
+  def delete
+    if any?(&:incomplete?)
+      raise(Jobba::NotCompletedError,
+            "This status cannot be deleted because it isn't complete.  Use " \
+            "`delete!` if you want to delete anyway.")
+    end
+
+    delete!
+  end
+
+  def delete!
+    load
+    redis.multi do
+      @cache.each(&:delete!)
+    end
+    @cache = []
+    @ids = []
+  end
+
+  def request_kill!
+    load
+    redis.multi do
+      @cache.each(&:request_kill!)
+    end
+  end
+
+  def multi(&block)
+    load
+    redis.multi do
+      @cache.each{|status| block.call(status, redis)}
+    end
+  end
+
+  protected
+
+  def load
+    @cache ||= get_all!
+  end
+
+  def get_all!
+    id_keys = @ids.collect{|id| "id:#{id}"}
+
+    raw_statuses = redis.pipelined do
+      id_keys.each do |key|
+        redis.hgetall(key)
+      end
+    end
+
+    raw_statuses.collect do |raw_status|
+      Jobba::Status.new(raw: raw_status)
+    end
+  end
+
+  def initialize(*ids)
+    @ids = [ids].flatten.compact
+    @cache = nil
+  end
+
+end

data/lib/jobba/time.rb

@@ -0,0 +1,21 @@
+class Jobba::Time
+
+  # We can only accurately record times in redis up to microseconds.  Some
+  # platforms, e.g. Mac OS, give Time up to microseconds while others, e.g.
+  # Linux, give it up to nanoseconds.  To make our specs happy and to gel
+  # with what redis is giving us, Jobba uses this Time class to enforce
+  # rounding away precision beyond microseconds.
+
+  def self.new(*args)
+    Time.new(*args).round(6)
+  end
+
+  def self.now
+    Time.new.round(6)
+  end
+
+  def self.at(*args)
+    Time.at(*args).round(6)
+  end
+
+end

data/lib/jobba/utils.rb

@@ -0,0 +1,25 @@
+module Jobba::Utils
+
+  # Represent time as an integer number of us since epoch
+  # (helps avoid redis precision issues)
+  def self.time_to_usec_int(time)
+    case time
+    when ::Time
+      time.strftime("%s%6N").to_i
+    when Float
+      # assuming that time is the number of seconds since epoch
+      # to avoid precision issues, convert to a string, remove
+      # the decimal, and convert back to an integer
+      sprintf("%0.6f", time.to_f).gsub(/\./,'').to_i
+    when Integer
+      time
+    when String
+      time.to_i
+    end
+  end
+
+  def self.time_from_usec_int(int)
+    Jobba::Time.at(int / 1000000, int % 1000000)
+  end
+
+end

data/lib/jobba/version.rb

@@ -0,0 +1,3 @@
+module Jobba
+  VERSION = "1.0.0"
+end

data/lib/jobba.rb

@@ -0,0 +1,44 @@
+require "redis"
+require "redis-namespace"
+
+require "jobba/version"
+require "jobba/exceptions"
+require "jobba/time"
+require "jobba/utils"
+require "jobba/configuration"
+require "jobba/common"
+require "jobba/state"
+require "jobba/status"
+require "jobba/statuses"
+require "jobba/query"
+
+module Jobba
+
+  def self.where(*args)
+    Query.new.where(*args)
+  end
+
+  def self.all
+    Query.new.all
+  end
+
+  def self.count
+    Query.new.count
+  end
+
+  def self.configure
+    yield configuration
+  end
+
+  def self.configuration
+    @configuration ||= Configuration.new
+  end
+
+  def self.redis
+    @redis ||= Redis::Namespace.new(
+      configuration.namespace,
+      redis: Redis.new(configuration.redis_options || {})
+    )
+  end
+
+end