jobba 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 5188a37939c0f9b306f0d98e01dfb9abf6ef3a31
+  data.tar.gz: 9cc2a58ff46d55c13aa25bec9e98b395916fd13b
+SHA512:
+  metadata.gz: 8f6d7780ac4361ba25a5f73f4a1df1746c6954e947497fcb6e1f97d6b9d5c360d891835fd12cb2f09f52c557d9abc304a881b5fd3e40131c3ad7521a812d1a6a
+  data.tar.gz: e7fec3fbfd1e73496816907efbc28723daa1d148180b1cda7e71ec47ec176a5cbf4280be40491b751ed59a86a7fe35d7736a146b902dfe3a9a85851c90fcf8b9

data/.gitignore

@@ -0,0 +1,10 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+.byebug*

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.ruby-gemset

@@ -0,0 +1 @@
+jobba

data/.ruby-version

@@ -0,0 +1 @@
+2.2.3

data/.travis.yml

@@ -0,0 +1,15 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.2.3
+cache: bundler
+bundler_args: --retry=6
+script:
+  - bundle exec rake
+services:
+  - redis-server
+notifications:
+  email: false
+env:
+  - USE_REAL_REDIS=false
+  - USE_REAL_REDIS=true

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in jobba.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 Rice University
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,377 @@
+# jobba
+
+[![Build Status](https://travis-ci.org/openstax/jobba.svg?branch=master)](https://travis-ci.org/openstax/jobba)
+[![Code Climate](https://codeclimate.com/github/openstax/jobba/badges/gpa.svg)](https://codeclimate.com/github/openstax/jobba)
+
+Redis-based background job status tracking.
+
+## Installation
+
+```ruby
+# Gemfile
+gem 'jobba'
+```
+
+or
+
+```
+$> gem install jobba
+```
+
+## Configuration
+
+To configure Jobba, put the following code in your applications
+initialization logic (eg. in the config/initializers in a Rails app):
+
+```ruby
+Jobba.configure do |config|
+  # Whatever options should be passed to `Redis.new` (see https://github.com/redis/redis-rb)
+  config.redis_options = { url: "redis://:p4ssw0rd@10.0.1.1:6380/15" }
+  # top-level redis prefix
+  config.namespace = "jobba"
+end
+```
+
+## Getting status objects
+
+If you know you need a new `Status`, call `create!`:
+
+```ruby
+Jobba::Status.create!
+```
+
+If you are looking for a status:
+
+```ruby
+Jobba::Status.find(id)
+```
+
+which will return `nil` if no such `Status` is found. If you always want a `Status` object back,
+call:
+
+```ruby
+Jobba::Status.find!(id)
+```
+
+The results of `find!` will always start in an `unknown` state.
+
+## Basic Use with ActiveJob
+
+```ruby
+class MyJob < ::ActiveJob::Base
+  def self.perform_later(an_arg:, another_arg:)
+    status = Jobba::Status.create!
+    args.push(status.id)
+
+    # In theory we'd mark as queued right after the call to super, but this messes
+    # up when the activejob adapter runs the job right away
+    status.queued!
+    super(*args, &block)
+
+    # return the Status ID in case it needs to be noted elsewhere
+    status.id
+  end
+
+  def perform(*args, &block)
+    # Pop the ID argument added by perform_later and get a Status
+    status = Jobba::Status.find!(args.pop)
+    status.working!
+
+    # ... do stuff ...
+
+    status.succeeded!
+  end
+end
+```
+
+## Change States
+
+One of the main functions of Jobba is to let a job advance its status through a series of states:
+
+* `unqueued`
+* `queued`
+* `working`
+* `succeeded`
+* `failed`
+* `killed`
+* `unknown`
+
+Put a `Status` into one of these states by calling `that_state!`, e.g.
+
+```ruby
+my_state.working!
+```
+
+The `unqueued` state is entered when a `Status` is first created.  The `unknown` state is entered when `find!(id)` is called but the `id` is not known.  You can re-enter these states with the `!` methods, but note that the `recorded_at` timestamp will not be updated.
+
+The **first time a state is entered**, a timestamp is recorded for that state.  Not all timestamp names match the state names:
+
+| State | Timestamp |
+|-------|-----------|
+|unqueued  | recorded_at |
+|queued    | queued_at   |
+|working   | started_at  |
+|succeeded | succeeded_at |
+|failed    | failed_at    |
+|killed    | killed_at    |
+|unknown   | recorded_at  |
+
+There is also a special timestamp for when a kill is requested, `kill_requested_at`.  More about this later.
+
+The order of states is not enforced, and you do not have to use all states.  However, note that you'll only be able to query for states you use (Jobba doesn't automatically travel through states you skip) and if you're using an unusual order your time-based queries will have to reflect that order.
+
+## Mark Progress
+
+If you want to have a way to track the progress of a job, you can call:
+
+```ruby
+my_status.set_progress(0.7)   # 70% complete
+my_status.set_progress(7,10)  # 70% complete
+my_status.set_progress(14,20) # 70% complete
+```
+
+This is useful if you need to show a progress bar on your client, for example.
+
+## Recording Job Errors
+
+...
+
+## Saving Job-specific Data
+
+Jobba provides a `data` field in all `Status` objects that you can use for storing job-specific data.  Note that the data must be in a format that can be serialized to JSON.  Recommend sticking with basic data types, arrays, primitives, hashes, etc.
+
+```ruby
+my_status.save({a: 'blah', b: [1,2,3]})
+my_status.save("some string")
+```
+
+## Setting Job Name and Arguments
+
+If you want to be able to query for all statuses for a certain kind of job, you can set the job's name in the status:
+
+```ruby
+my_status.set_job_name("MySpecialJob")
+```
+
+If you want to be able to query for all statuses that take a certain argument as input, you can add job arguments to a status:
+
+```ruby
+my_status.add_job_arg(arg_name, arg)
+```
+
+where `arg_name` is what the argument is called in your job (e.g. `"input_1"`) and `arg` is a way to identify the argument (e.g. `"gid://app/Person/72").
+
+You probably will only want to track complex arguments, e.g. models in your application.  E.g. you could have a `Book` model and a `PublishBook` background job and you may want to see all of the `PublishBook` jobs that have status for the `Book` with ID `53`.
+
+## Killing Jobs
+
+While Jobba can't really kill jobs (it doesn't control your job-running library), it has a facility for marking that you'd like a job to be killed.
+
+```ruby
+a_status.request_kill!
+```
+
+Then a job itself can occassionally come up for air and check
+
+```ruby
+my_status.kill_requested?
+```
+
+and if that returns `true`, it can attempt to gracefully terminate itself.
+
+Note that when a kill is requested, the job will continue to be in some other state (e.g. `working`) until it is in fact killed, at which point the job should call:
+
+```ruby
+my_status.killed!
+```
+
+to change the state to `killed`.
+
+## Status Objects
+
+When you get hold of a `Status`, via `create!`, `find`, `find!`, or as the result of a query, it will have the following attributes (some of which may be nil):
+
+| Attribute | Description |
+|-----------|-------------|
+| `id` | A Jobba-created UUID |
+| `state` | one of the states above |
+| `progress` | a float between 0.0 and 1.0 |
+| `errors` | TBD |
+| `data` | job-specific data |
+| `job_name` | The name of the job |
+| `job_args` | An hash of job arguments, {arg_name: arg, ...} |
+| `recorded_at` | Ruby `Time` timestamp |
+| `queued_at` | Ruby `Time` timestamp |
+| `started_at` | Ruby `Time` timestamp |
+| `succeeded_at` | Ruby `Time` timestamp |
+| `failed_at` | Ruby `Time` timestamp |
+| `killed_at` | Ruby `Time` timestamp |
+| `recorded_at` | Ruby `Time` timestamp |
+| `kill_requested_at` | Ruby `Time` timestamp |
+
+A `Status` object also methods to check if it is in certain states:
+
+* `reload!`
+* `unqueued?`
+* `queued?`
+* `working?`
+* `succeeded?`
+* `failed?`
+* `killed?`
+* `unknown?`
+
+And two conveience methods for checking groups of states:
+
+* `completed?`
+* `incomplete?`
+
+You can also call `reload!` on a `Status` to have it reset its state to what is stored in redis.
+
+## Deleting Job Statuses
+
+Once jobs are completed or otherwise no longer interesting, it'd be nice to clear them out of redis.  You can do this with:
+
+```ruby
+my_status.delete    # freaks out if `my_status` isn't complete
+my_status.delete!   # always deletes
+```
+
+## Querying for Statuses
+
+Jobba has an activerecord-like query interface for finding Status objects.
+
+### Basic Query Examples
+
+**State**
+
+```ruby
+Jobba.where(state: :unqueued)
+Jobba.where(state: :queued)
+Jobba.where(state: :working)
+Jobba.where(state: :succeeded)
+Jobba.where(state: :failed)
+Jobba.where(state: :killed)
+Jobba.where(state: :unknown)
+```
+
+Two convenience "state" queries have been added:
+
+```ruby
+Jobba.where(state: :completed)   # includes succeeded, failed
+Jobba.where(state: :incomplete)  # includes unqueued, queued, working, killed
+```
+
+You can query combinations of states too:
+
+```ruby
+Jobba.where(state: [:queued, :working])
+```
+
+**State Timestamp**
+
+```ruby
+Jobba.where(recorded_at: {after: time_1})
+Jobba.where(queued_at: [time_1, nil])
+Jobba.where(started_at: {before: time_2})
+Jobba.where(started_at: [nil, time_2])
+Jobba.where(succeeded_at: {after: time_1, before: time_2})
+Jobba.where(failed_at: [time_1, time_2])
+```
+
+Note that you cannot query on `kill_requested_at`.  The time arguments can be Ruby `Time` objects or a number of microseconds since the epoch represented as a float, integer, or string.
+
+Note that, in operations having to do with time, this gem ignores anything beyond microseconds.
+
+**Job Name**
+
+(requires having called the optional `set_job_name` method)
+
+```ruby
+Jobba.where(job_name: "MySpecialBackgroundJob")
+Jobba.where(job_name: ["MySpecialBackgroundJob", "MyOtherJob"])
+```
+
+**Job Arguments**
+
+(requires having called the optional `add_job_arg` method)
+
+```ruby
+Jobba.where(job_arg: "gid://app/MyModel/42")
+Jobba.where(job_arg: "gid://app/Person/86")
+```
+
+### Query Chaining
+
+Queries can be chained! (intersects the results of each `where` clause)
+
+```ruby
+Jobba.where(state: :queued).where(recorded_at: {after: some_time})
+Jobba.where(job_name: "MyTroublesomeJob").where(state: :failed)
+```
+
+### Operations on Queries
+
+When you have a query you can run the following methods on it.  These act like what you'd expect for a Ruby array.
+
+* `first`
+* `any?`
+* `none?`
+* `all?`
+* `each`
+* `each_with_index`
+* `map`
+* `collect`
+* `select`
+* `empty?`
+* `count`
+
+`empty?` and `count` are performed in redis without bringing back all query results to Ruby.
+
+You can also call two special methods directly on `Jobba`:
+
+```ruby
+Jobba.all     # returns all statuses
+Jobba.count   # returns count of all statuses
+```
+
+## Statuses Object
+
+Calling `all` on a query returns a `Statuses` object, which is just a collection of `Status` objects.  It has a few methods for doing bulk operations on all `Status` objects in it.
+
+* `delete`
+* `delete!`
+* `request_kill!`
+
+These work like describe above for individual `Status` objects.
+
+There is also a not-very-tested `multi` operation that takes a block and executes the block inside a Redis `multi` call.
+
+```ruby
+my_statuses.multi do |status, redis|
+  # do stuff on `status` using the `redis` connection
+end
+```
+
+## Notes
+
+### Times
+
+Note that, in operations having to do with time, this gem ignores anything beyond microseconds.
+
+### Efficiency
+
+Jobba strives to do all of its operations as efficiently as possible using built-in redis operations.  If you find a place where the efficiency can be improved, please submit an issue or a pull request.
+
+## TODO
+
+1. Provide job min, max, and average durations.
+2. Implement `add_error`.
+8. Specs that test scale.
+9. Make sure we're calling `multi` or `pipelined` everywhere we can.
+11. Make sure we're consistent on completed/complete incompleted/incomplete.
+12. Should more `Statuses` operations return `Statuses`, e.g. `each`, so that they can be chained?
+
+
+
+
+

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "jobba"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/jobba.gemspec

@@ -0,0 +1,30 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'jobba/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "jobba"
+  spec.version       = Jobba::VERSION
+  spec.authors       = ["JP Slavinsky"]
+  spec.email         = ["jps@kindlinglabs.com"]
+
+  spec.summary       = %q{Redis-based background job status tracking.}
+  spec.description   = %q{Redis-based background job status tracking.}
+  spec.homepage      = "https://github.com/openstax/jobba"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency "redis", "~> 3.2"
+  spec.add_runtime_dependency "redis-namespace"
+
+  spec.add_development_dependency "bundler", "~> 1.10"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "byebug"
+  spec.add_development_dependency "fakeredis"
+end

data/lib/jobba/clause.rb

@@ -0,0 +1,44 @@
+class Jobba::Clause
+  attr_reader :keys, :min, :max
+
+  include Jobba::Common
+
+  # if `keys` or `suffixes` is an array, all entries will be included in the resulting set
+  def initialize(prefix: nil, suffixes: nil, keys: nil, min: nil, max: nil)
+    if keys.nil? && prefix.nil? && suffixes.nil?
+      raise ArgumentError, "Either `keys` or both `prefix` and `suffix` must be specified."
+    end
+
+    if (prefix.nil? && !suffixes.nil?) || (!prefix.nil? && suffixes.nil?)
+      raise ArgumentError, "When `prefix` is given, so must `suffix` be, and vice versa."
+    end
+
+    if keys
+      @keys = [keys].flatten
+    else
+      prefix = "#{prefix}:" unless prefix[-1] == ":"
+      @keys = [suffixes].flatten.collect{|suffix| prefix + suffix}
+    end
+
+    @min = min
+    @max = max
+  end
+
+  def to_new_set
+    new_key = "temp:#{SecureRandom.hex(10)}"
+
+    # Make a copy of the data into new_key then filter values if indicated
+    # (always making a copy gets normal sets into a sorted set key OR if
+    # already sorted gives us a safe place to filter out values without
+    # perturbing the original sorted set).
+
+    if !keys.empty?
+      redis.zunionstore(new_key, keys)
+      redis.zremrangebyscore(new_key, '-inf', "(#{min}") unless min.nil?
+      redis.zremrangebyscore(new_key, "(#{max}", '+inf') unless max.nil?
+    end
+
+    new_key
+  end
+
+end

data/lib/jobba/clause_factory.rb

@@ -0,0 +1,82 @@
+require 'jobba/clause'
+
+class Jobba::ClauseFactory
+
+  def self.new_clause(key, value)
+    if value.nil?
+      raise ArgumentError, "Nil search criteria are not currently " \
+                           "accepted in a Jobba `where` call"
+    end
+
+    case key.to_sym
+    when :state
+      state_clause(value)
+    when :job_name
+      Jobba::Clause.new(prefix: "job_name", suffixes: value)
+    when :job_arg
+      Jobba::Clause.new(prefix: "job_arg", suffixes: value)
+    when /.*_at/
+      timestamp_clause(key, value)
+    else
+      raise ArgumentError, "#{key} is not a valid key in a Jobba `where` call"
+    end
+  end
+
+  protected
+
+  def self.timestamp_clause(timestamp_name, options)
+    validate_timestamp_name!(timestamp_name)
+
+    min, max =
+      case options
+      when Array
+        if options.length != 2
+          raise ArgumentError, "Wrong number of array entries for '#{timestamp_name}'."
+        end
+
+        [options[0], options[1]]
+      when Hash
+        [options[:after], options[:before]]
+      else
+        raise ArgumentError,
+              "#{option_value} is not a valid value for a " +
+              "#{option_key} key in a Jobba `where` call"
+      end
+
+    min = Jobba::Utils.time_to_usec_int(min)
+    max = Jobba::Utils.time_to_usec_int(max)
+
+    Jobba::Clause.new(keys: timestamp_name, min: min, max: max)
+  end
+
+  def self.state_clause(state)
+    state = [state].flatten.collect { |ss|
+      case ss
+      when :completed
+        Jobba::State::COMPLETED.collect(&:name)
+      when :incomplete
+        Jobba::State::INCOMPLETE.collect(&:name)
+      else
+        ss
+      end
+    }.uniq
+
+    validate_state_name!(state)
+    Jobba::Clause.new(keys: state)
+  end
+
+  def self.validate_state_name!(state_name)
+    [state_name].flatten.each do |name|
+      if Jobba::State::ALL.none?{|state| state.name == name.to_s}
+        raise ArgumentError, "'#{name}' is not a valid state name."
+      end
+    end
+  end
+
+  def self.validate_timestamp_name!(timestamp_name)
+    if Jobba::State::ALL.none?{|state| state.timestamp_name == timestamp_name.to_s}
+      raise ArgumentError, "'#{timestamp_name}' is not a valid timestamp."
+    end
+  end
+
+end

data/lib/jobba/common.rb

@@ -0,0 +1,17 @@
+module Jobba::Common
+
+  def redis
+    Jobba.redis
+  end
+
+  module ClassMethods
+    def redis
+      Jobba.redis
+    end
+  end
+
+  def self.included(base)
+    base.extend(ClassMethods)
+  end
+
+end

data/lib/jobba/configuration.rb

@@ -0,0 +1,10 @@
+class Jobba::Configuration
+
+  attr_accessor :redis_options
+  attr_accessor :namespace
+
+  def initialize
+    @redis_options = {}
+    @namespace = "jobba"
+  end
+end

data/lib/jobba/exceptions.rb

@@ -0,0 +1,5 @@
+module Jobba
+
+  class NotCompletedError < StandardError; end
+
+end