ruby_job 0.1.1

data/COPYING.LESSER

@@ -0,0 +1,165 @@
+                  GNU LESSER GENERAL PUBLIC LICENSE
+                       Version 3, 29 June 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+
+  This version of the GNU Lesser General Public License incorporates
+the terms and conditions of version 3 of the GNU General Public
+License, supplemented by the additional permissions listed below.
+
+  0. Additional Definitions.
+
+  As used herein, "this License" refers to version 3 of the GNU Lesser
+General Public License, and the "GNU GPL" refers to version 3 of the GNU
+General Public License.
+
+  "The Library" refers to a covered work governed by this License,
+other than an Application or a Combined Work as defined below.
+
+  An "Application" is any work that makes use of an interface provided
+by the Library, but which is not otherwise based on the Library.
+Defining a subclass of a class defined by the Library is deemed a mode
+of using an interface provided by the Library.
+
+  A "Combined Work" is a work produced by combining or linking an
+Application with the Library.  The particular version of the Library
+with which the Combined Work was made is also called the "Linked
+Version".
+
+  The "Minimal Corresponding Source" for a Combined Work means the
+Corresponding Source for the Combined Work, excluding any source code
+for portions of the Combined Work that, considered in isolation, are
+based on the Application, and not on the Linked Version.
+
+  The "Corresponding Application Code" for a Combined Work means the
+object code and/or source code for the Application, including any data
+and utility programs needed for reproducing the Combined Work from the
+Application, but excluding the System Libraries of the Combined Work.
+
+  1. Exception to Section 3 of the GNU GPL.
+
+  You may convey a covered work under sections 3 and 4 of this License
+without being bound by section 3 of the GNU GPL.
+
+  2. Conveying Modified Versions.
+
+  If you modify a copy of the Library, and, in your modifications, a
+facility refers to a function or data to be supplied by an Application
+that uses the facility (other than as an argument passed when the
+facility is invoked), then you may convey a copy of the modified
+version:
+
+   a) under this License, provided that you make a good faith effort to
+   ensure that, in the event an Application does not supply the
+   function or data, the facility still operates, and performs
+   whatever part of its purpose remains meaningful, or
+
+   b) under the GNU GPL, with none of the additional permissions of
+   this License applicable to that copy.
+
+  3. Object Code Incorporating Material from Library Header Files.
+
+  The object code form of an Application may incorporate material from
+a header file that is part of the Library.  You may convey such object
+code under terms of your choice, provided that, if the incorporated
+material is not limited to numerical parameters, data structure
+layouts and accessors, or small macros, inline functions and templates
+(ten or fewer lines in length), you do both of the following:
+
+   a) Give prominent notice with each copy of the object code that the
+   Library is used in it and that the Library and its use are
+   covered by this License.
+
+   b) Accompany the object code with a copy of the GNU GPL and this license
+   document.
+
+  4. Combined Works.
+
+  You may convey a Combined Work under terms of your choice that,
+taken together, effectively do not restrict modification of the
+portions of the Library contained in the Combined Work and reverse
+engineering for debugging such modifications, if you also do each of
+the following:
+
+   a) Give prominent notice with each copy of the Combined Work that
+   the Library is used in it and that the Library and its use are
+   covered by this License.
+
+   b) Accompany the Combined Work with a copy of the GNU GPL and this license
+   document.
+
+   c) For a Combined Work that displays copyright notices during
+   execution, include the copyright notice for the Library among
+   these notices, as well as a reference directing the user to the
+   copies of the GNU GPL and this license document.
+
+   d) Do one of the following:
+
+       0) Convey the Minimal Corresponding Source under the terms of this
+       License, and the Corresponding Application Code in a form
+       suitable for, and under terms that permit, the user to
+       recombine or relink the Application with a modified version of
+       the Linked Version to produce a modified Combined Work, in the
+       manner specified by section 6 of the GNU GPL for conveying
+       Corresponding Source.
+
+       1) Use a suitable shared library mechanism for linking with the
+       Library.  A suitable mechanism is one that (a) uses at run time
+       a copy of the Library already present on the user's computer
+       system, and (b) will operate properly with a modified version
+       of the Library that is interface-compatible with the Linked
+       Version.
+
+   e) Provide Installation Information, but only if you would otherwise
+   be required to provide such information under section 6 of the
+   GNU GPL, and only to the extent that such information is
+   necessary to install and execute a modified version of the
+   Combined Work produced by recombining or relinking the
+   Application with a modified version of the Linked Version. (If
+   you use option 4d0, the Installation Information must accompany
+   the Minimal Corresponding Source and Corresponding Application
+   Code. If you use option 4d1, you must provide the Installation
+   Information in the manner specified by section 6 of the GNU GPL
+   for conveying Corresponding Source.)
+
+  5. Combined Libraries.
+
+  You may place library facilities that are a work based on the
+Library side by side in a single library together with other library
+facilities that are not Applications and are not covered by this
+License, and convey such a combined library under terms of your
+choice, if you do both of the following:
+
+   a) Accompany the combined library with a copy of the same work based
+   on the Library, uncombined with any other library facilities,
+   conveyed under the terms of this License.
+
+   b) Give prominent notice with the combined library that part of it
+   is a work based on the Library, and explaining where to find the
+   accompanying uncombined form of the same work.
+
+  6. Revised Versions of the GNU Lesser General Public License.
+
+  The Free Software Foundation may publish revised and/or new versions
+of the GNU Lesser General Public License from time to time. Such new
+versions will be similar in spirit to the present version, but may
+differ in detail to address new problems or concerns.
+
+  Each version is given a distinguishing version number. If the
+Library as you received it specifies that a certain numbered version
+of the GNU Lesser General Public License "or any later version"
+applies to it, you have the option of following the terms and
+conditions either of that published version or of any later version
+published by the Free Software Foundation. If the Library as you
+received it does not specify a version number of the GNU Lesser
+General Public License, you may choose any version of the GNU Lesser
+General Public License ever published by the Free Software Foundation.
+
+  If the Library as you received it specifies that a proxy can decide
+whether future versions of the GNU Lesser General Public License shall
+apply, that proxy's public statement of acceptance of any version is
+permanent authorization for you to choose that version for the
+Library.

data/Gemfile

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
+
+# Specify your gem's dependencies in ruby_job.gemspec
+gemspec

data/Guardfile

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+# A sample Guardfile
+# More info at https://github.com/guard/guard#readme
+
+## Uncomment and set this to only include directories you want to watch
+# directories %w(app lib config test spec features) \
+#  .select{|d| Dir.exist?(d) ? d : UI.warning("Directory #{d} does not exist")}
+
+## Note: if you are using the `directories` clause above and you are not
+## watching the project directory ('.'), then you will want to move
+## the Guardfile to a watched dir and symlink it back, e.g.
+#
+#  $ mkdir config
+#  $ mv Guardfile config/
+#  $ ln -s config/Guardfile .
+#
+# and, you'll have to watch "config/Guardfile" instead of "Guardfile"
+
+# Note: The cmd option is now required due to the increasing number of ways
+#       rspec may be run, below are examples of the most common uses.
+#  * bundler: 'bundle exec rspec'
+#  * bundler binstubs: 'bin/rspec'
+#  * spring: 'bin/rspec' (This will use spring if running and you have
+#                          installed the spring binstubs per the docs)
+#  * zeus: 'zeus rspec' (requires the server to be started separately)
+#  * 'just' rspec: 'rspec'
+
+guard :rspec, cmd: 'bundle exec rspec' do
+  require 'guard/rspec/dsl'
+  dsl = Guard::RSpec::Dsl.new(self)
+
+  # Feel free to open issues for suggestions and improvements
+
+  # RSpec files
+  rspec = dsl.rspec
+  watch(rspec.spec_helper) { rspec.spec_dir }
+  watch(rspec.spec_support) { rspec.spec_dir }
+  watch(rspec.spec_files)
+
+  # Ruby files
+  ruby = dsl.ruby
+  dsl.watch_spec_files_for(ruby.lib_files)
+
+  # Turnip features and steps
+  watch(%r{^spec/acceptance/(.+)\.feature$})
+  watch(%r{^spec/acceptance/steps/(.+)_steps\.rb$}) do |m|
+    Dir[File.join("**/#{m[1]}.feature")][0] || 'spec/acceptance'
+  end
+end

data/LICENSE

@@ -0,0 +1,5 @@
+Copyright (c) 2019 Marco Imperatore
+
+RubyJob is an Open Source project licensed under the terms of
+the LGPLv3 license.  Please see COPYING.LESSER or
+<http://www.gnu.org/licenses/lgpl-3.0.html> for license text.

data/LICENSE.txt

@@ -0,0 +1,5 @@
+Copyright (c) 2019 Marco Imperatore
+
+RubyJob is an Open Source project licensed under the terms of
+the LGPLv3 license.  Please see COPYING.LESSER or
+<http://www.gnu.org/licenses/lgpl-3.0.html> for license text.

data/README.md

@@ -0,0 +1,226 @@
+[![Code Climate](https://codeclimate.com/github/mimperatore/ruby_job.svg)](https://codeclimate.com/github/mimperatore/ruby_job)
+[![Test Coverage](https://codeclimate.com/github/mimperatore/ruby_job/badges/coverage.svg)](https://codeclimate.com/github/mimperator/ruby_job/coverage)
+[![Build Status](https://travis-ci.com/mimperatore/ruby_job.svg?branch=master)](https://codecov.io/gh/mimperatore/ruby_job/branch/master)
+
+# RubyJob
+
+RubyJob is a framework for running jobs.
+
+The current version behaves much like [Sucker Punch](https://github.com/brandonhilkert/sucker_punch), in that it
+only supports an [In-Memory Job Store](https://github.com/mimperatore/ruby_job/blob/master/lib/ruby_job/in_memory_job_store.rb)
+implemented through a fast [Fibonacci Heap](https://github.com/mudge/fibonacci_heap).
+
+The initial version, which supports only a single queue, runs **200% faster than Sucker Punch**, capable of processing **1,000,000** simple jobs in **28 seconds**
+vs. Sucker Punch's 59 seconds (measured on on a MacBook Pro 2.3GHz with 16GB of RAM).
+
+Additional features are in the works, including:
+- Support for multiple queues & queue priorities
+- Persistent Job Stores for:
+  - Redis
+  - Cassandra
+- Batches & Job nesting
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'ruby_job'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install ruby_job
+
+## Usage
+
+### A simple example
+
+#### Define your worker class
+```ruby
+class MyWorker
+  include RubyJob::Worker
+
+  def perform
+    #job code goes here
+  end
+end
+```
+
+#### Setup your JobStore
+```ruby
+MyWorker.jobstore = RubyJob::InMemoryJobStore.new
+MyWorker.perform_async
+```
+
+#### Run your server
+```ruby
+server = RubyJob::ThreadedServer.new(num_threads: 10, jobstore: MyWorker.jobstore)
+server_thread = server.start
+server_thread.join
+```
+
+### Setting up the default JobStore
+Jobs are enqueued to the default JobStore of the worker class:
+
+```ruby
+MyWorker.jobstore = RubyJob::InMemoryJobStore.new # attach the JobStore to the MyWorker class
+```
+
+If the worker class doesn't have a JobStore attached to it, jobs will be enqueued to `Worker.jobstore`.
+
+```ruby
+Worker.jobstore = RubyJob::InMemoryJobStore.new # jobs will be queued here, if MyWorker doesn't have `jobstore` set.
+```
+
+### Enqueuing jobs
+There are 2 ways you can enqueue your jobs:
+
+#### Using <i>#perform_*</i> (recommended approach)
+```ruby
+MyWorker.jobstore = RubyJob::InMemoryJobStore.new
+MyWorker.perform_async # will enqueue on `MyWorker.jobstore`, or `Worker.jobstore` if the former isn't set.
+```
+
+**Note:** you must ensure either `MyWorker.jobstore` or `Worker.jobstore` is set to a valid JobStore.
+
+#### Using Job#enqueue
+```ruby
+MyWorker.jobstore = RubyJob::InMemoryJobStore.new
+job = Job.new(worker_class_name: 'MyWorker', args: [], start_at: Time.now)
+job.enqueue
+```
+
+### Dequeuing jobs
+In some situations, it's important to remove a previously enqueued job from the queue, so that it does not run in the future.
+To do so:
+```ruby
+job.dequeue
+```
+
+### Schedule a Job for execution (asynchronously)
+
+**Note:** Jobs are scheduled to nearest **millisecond** of the specified start time.
+
+#### Immediately (ASAP)
+```ruby
+MyWorker.perform_async # schedule to run asynchonously, asap
+```
+
+#### Delayed
+```ruby
+MyWorker.perform_in(5.5) # schedule to run asynchonously, in 5.5 seconds
+```
+
+#### At a specific time
+```ruby
+MyWorker.perform_at(a_particular_time) # schedule to run asynchonously, at the specified time
+```
+
+### Executing a Job immediately (synchronously)
+```ruby
+MyWorker.perform # run the job synchronously now
+```
+
+### Threaded Server (the job processor)
+A threaded server is provided to process the queued jobs.  It is instantiated by specifying the number of workers (threads) to spawn,
+and the JobStore it will be processing.
+```ruby
+server = RubyJob::ThreadedServer.new(num_threads: 10, jobstore: MyWorker.jobstore)
+```
+
+#### Server options
+```ruby
+server.set(wait: true)
+```
+
+- `wait`[boolean]: determines whether the server should exit when there aren't any processable jobs in the queue.  Defaults to `true`.
+- `wait_delay`[float]: number of seconds to wait (sleep).  Defaults to `0.5`.
+
+
+#### Starting the server
+Queued jobs will only run when a Server, attached to the JobStore the jobs have been enqueued to, has been started.
+
+```ruby
+server_thread = server.start
+server_thread.join # if needed, depending on your use case
+```
+
+#### Halting the server
+A running server can be halted as follows:
+```ruby
+server.halt_at(Time.now + 5)
+```
+
+```ruby
+server.halt # equivalent to halt_at(Time.now)
+```
+
+`Halting` causes the server to stop processing jobs scheduled to start after the specified halt time. Once the halt time has been
+reached, the server waits if the `wait` option is `true`, or exits otherwise.
+
+Halting the server can be useful in production, when you want to temporarily pause job processing.
+
+#### Resuming the server
+A halted server can be resumed with:
+```ruby
+server.resume
+```
+
+```ruby
+server.resume_until(Time.now + 5) # equivalent to: resume && halt_at(Time.now + 5)
+```
+
+With `resume`, the server picks up jobs from where it left off and keeps processing them as if it never stopped.  Note that a server
+that's been halted for a significant amount of time will pick up old jobs that may have been intended to start significantly in the past, so
+ensure you take that into account in your job processing code if you care about this situation.
+
+### Retries
+Jobs will be not be retried by default.  To have jobs retry, the worker class must define a `retry?` method that
+returns a tuple indicating whether the job should be retried, and how long the retry delay should be: [do_retry, retry_delay]
+```ruby
+  MAX_RETRIES = 5
+  INITIAL_RETRY_DELAY = 0.5
+
+  def retry?(attempt:, error:)
+    # determine whether a retry is required, based on the attempt number and error passed in
+    do_retry = error.is_a?(RetriableError) && (attempt < MAX_RETRIES)
+
+    [do_retry, INITIAL_RETRY_DELAY * 2**(attempt-1)] # exponential backoff
+  end
+```
+
+`attempt` starts at `1` and `error` is the exception that was raised by the last attempt.
+
+**Note:** the current implementation uses `sleep` to implement the retry delay.  This isn't ideal, as it prevents the thread
+processing the job from servicing another job that's ready to run.  In the future, this will be changed such that the job
+is put back onto the job queue to start at a later time.  Feel free to put together a PR if you're interested in seeing this
+change sooner rather than later.
+
+**Note:** the retry delay is the time between the end of the last attempt and the start of the new attempt
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an
+interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and
+then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/mimperatore/ruby_job. This project is intended to be a safe, welcoming space
+for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
+
+## License
+
+The gem is available as open source under the terms of the [GNU Lesser General Public License Version 3 (LGPLv3)](https://www.gnu.org/licenses/lgpl-3.0.html).
+
+## Code of Conduct
+
+Everyone interacting in the RubyJob project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow
+the [code of conduct](https://github.com/mimperatore/ruby_job/blob/master/CODE_OF_CONDUCT.md).

data/Rakefile

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'

data/_config.yml

@@ -0,0 +1 @@
+theme: jekyll-theme-cayman

data/bin/console

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'ruby_job'
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require 'irb'
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/codecov.yml

@@ -0,0 +1,28 @@
+ignore:
+  - spec/**
+
+codecov:
+  require_ci_to_pass: yes
+
+coverage:
+  precision: 2
+  round: down
+  range: "99...100"
+
+  status:
+    project: yes
+    patch: yes
+    changes: no
+
+parsers:
+  gcov:
+    branch_detection:
+      conditional: yes
+      loop: yes
+      method: yes
+      macro: no
+
+comment:
+  layout: "reach,diff,flags,tree"
+  behavior: default
+  require_changes: no

data/lib/ruby_job/in_memory_job_store.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+require 'forwardable'
+require 'fibonacci_heap'
+require 'digest/sha1'
+
+module RubyJob
+  class InMemoryJobStore < JobStore
+    attr_reader :pause_starting_at
+
+    def initialize
+      super
+      @semaphore = Mutex.new
+      @next_uuid = 0
+      @pause_starting_at = nil
+    end
+
+    def enqueue(job)
+      raise 'job does not have an assigned uuid' unless job.uuid
+
+      @semaphore.synchronize { queue.push(job) }
+    end
+
+    def dequeue(job)
+      @semaphore.synchronize { queue.delete(job) }
+    end
+
+    def pause_at(time)
+      @pause_starting_at = time
+    end
+
+    def fetch
+      @options[:wait] ? fetch_next_or_wait : fetch_next
+    end
+
+    def size
+      queue.size
+    end
+
+    def next_uuid
+      @semaphore.synchronize { @next_uuid += 1 }
+    end
+
+    private
+
+    def queue
+      @queue ||= JobPriorityQueue.new
+    end
+
+    def paused_before?(time)
+      @pause_starting_at && @pause_starting_at <= time
+    end
+
+    def fetch_next
+      @semaphore.synchronize do
+        queue.pop if (top = queue.top) && top.start_at <= [Time.now, @pause_starting_at].compact.min
+      end
+    end
+
+    def fetch_next_or_wait
+      job = nil
+      loop do
+        job = fetch_next
+        break if job || (paused_before?(Time.now) && !@options[:wait])
+
+        sleep(@options[:wait_delay])
+      end
+      job
+    end
+
+    class JobPriorityQueue
+      extend Forwardable
+
+      def_delegators :@pqueue, :size
+
+      def initialize
+        @pqueue = FibonacciHeap::Heap.new
+      end
+
+      def push(job)
+        @pqueue.insert(job, key_for(job))
+      end
+
+      def pop
+        @pqueue.pop
+      end
+
+      def top
+        @pqueue.min
+      end
+
+      def delete(job)
+        @pqueue.delete(job)
+      end
+
+      private
+
+      def key_for(job)
+        job.start_at.to_f.round(3) + job.uuid.to_f / 1_000
+      end
+    end
+  end
+end