pg_jobs 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 2ce76759a48458e681b0c2b619dae3848f83d776615fff1a3890d0743748070c
+  data.tar.gz: 886badb6799a69f7eb0d257f070d052a978ad715616e26560701d21be19bc2df
+SHA512:
+  metadata.gz: a1b93e2389c7283e54d4c0d28d70441f9fe79c667beaf9c088b776d379ebec58c9d3d70cdfd8863a54cbed1e286508568c8b90fb843c8e310d4fecfa1bb67e74
+  data.tar.gz: 7003bed8eb73bacc539466ddc770acfba1e3d8b52e7eb9567a808f47fedf3f656d6d827083272cf04c9e5d352fd808ed8461631c4d0d7ecf984a4cb939ec6fb1

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright 2017 Moritz Breit
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,83 @@
+[![Gem](https://img.shields.io/gem/v/pg_jobs.svg)](https://rubygems.org/gems/pg_jobs)
+[![license MIT](https://img.shields.io/github/license/mbreit/pg_jobs.svg)](https://github.com/mbreit/pg_jobs/blob/codeclimate/MIT-LICENSE)
+[![build](https://img.shields.io/travis/com/mbreit/pg_jobs/master.svg)](https://travis-ci.com/mbreit/pg_jobs)
+[![maintainability](https://img.shields.io/codeclimate/maintainability/mbreit/pg_jobs.svg)](https://codeclimate.com/github/mbreit/pg_jobs)
+[![coverage](https://img.shields.io/codeclimate/coverage/mbreit/pg_jobs.svg)](https://codeclimate.com/github/mbreit/pg_jobs)
+[![docs](https://inch-ci.org/github/mbreit/pg_jobs.svg?branch=master)](https://inch-ci.org/github/mbreit/pg_jobs)
+
+# PgJobs
+
+Simple Active Job worker for PostgreSQL using LISTEN/NOTIFY and
+SKIP LOCKED.
+
+Supports most Active Job features like multiple queues, priorities
+and wait times.
+
+## Dependencies
+
+* PostgreSQL >= 9.5 to use SKIP LOCKED
+* Ruby >= 2.3
+* Rails >= 5.1
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'pg_jobs'
+```
+
+And then execute:
+
+```bash
+bundle
+```
+
+Then copy the migrations and migrate your database:
+
+```bash
+bin/rails pg_jobs_engine:install:migrations
+bin/rails db:migrate
+```
+
+To configure the Active Job adapter add this to your environment
+configuration (config/environments/production.rb):
+
+```ruby
+config.active_job.queue_adapter = :pg_jobs
+```
+
+If you want to run all your jobs in one queue, we recommend to configure
+ActionMailer to use the `default` queue:
+
+```ruby
+config.action_mailer.deliver_later_queue_name = 'default'
+```
+
+## Usage
+
+Just schedule your work with Active Job, then run one or multiple
+workers for the default queue with
+
+```bash
+bin/rails runner PgJobs.work
+```
+
+or for other queues with
+
+```bash
+bin/rails runner "PgJobs.work(:my_queue)"
+```
+
+For more documentation about Active Job and how to use different queues,
+scheduled jobs, priorities and error handling, see the
+[Active Job Rails Guide](https://guides.rubyonrails.org/active_job_basics.html).
+
+## Contributing
+
+Use Github issues and pull requests.
+
+## License
+
+The gem is available as open source under the terms of the
+[MIT License](http://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,47 @@
+begin
+  require 'bundler/setup'
+rescue LoadError
+  puts 'You must `gem install bundler` and `bundle install` to run rake tasks'
+end
+
+require 'rdoc/task'
+
+RDoc::Task.new(:rdoc) do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title    = 'PgJobs'
+  rdoc.options << '--line-numbers'
+  rdoc.rdoc_files.include('README.md')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+
+require 'bundler/gem_tasks'
+
+namespace :db do
+  task :init do
+    require 'zlib'
+    require 'active_record'
+
+    ActiveRecord::Base.configurations = YAML.load_file('test/database.yml')
+    ActiveRecord::Tasks::DatabaseTasks.migrations_paths = [File.expand_path('db/migrate', __dir__)]
+  end
+
+  desc 'Create test database'
+  task create: :init do
+    ActiveRecord::Tasks::DatabaseTasks.create_current('test')
+  end
+
+  desc 'Migrate test database'
+  task migrate: :init do
+    ActiveRecord::Tasks::DatabaseTasks.migrate
+  end
+end
+
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << 'test'
+  t.libs << 'app/models'
+  t.test_files = FileList['test/**/*_test.rb']
+end
+
+task default: ['db:create', 'db:migrate', :test]

data/app/models/pg_job.rb

@@ -0,0 +1,66 @@
+# ActiveRecord model for jobs
+#
+# Schema:
+#
+# | column        | type      | default   | null  |
+# |---------------|-----------|-----------|-------|
+# | job_data      | jsonb     |           | false |
+# | priority      | integer   | 100       | false |
+# | queue_name    | string    | 'default' | false |
+# | created_at    | timestamp |           | false |
+# | scheduled_for | timestamp |           | true  |
+class PgJob < ActiveRecord::Base
+  scope :due, -> { where('scheduled_for IS NULL OR scheduled_for <= ?', Time.current) }
+  scope :queue, ->(name) { where(queue_name: name).order(:priority, :created_at) }
+
+  validates :queue_name, format: { with: /\A[a-zA-Z0-9_]+\z/ }
+
+  after_create :notify_workers
+
+  # Yields a single job from the given queue that is scheduled for
+  # execution now. Does not block.
+  #
+  # Uses row locking with `SELECT ... FOR UPDATE SKIP LOCKED`
+  # to prevent race conditions.
+  #
+  # Returns false if no job has been found.
+  #
+  # @param queue_name [String] Name of the queue to look for a due job
+  def self.yield_job(queue_name)
+    transaction do
+      job = queue(queue_name).due.lock('FOR UPDATE SKIP LOCKED').first
+      return false unless job
+
+      yield job
+
+      job.destroy!
+    end
+  end
+
+  # Yields jobs when they are schedules to be executed.
+  # If the job queue is empty, it uses PostgreSQL LISTEN/NOTIFY support
+  # to block and wait for new jobs.
+  #
+  # @param queue_name [String] The name of the queue to work on
+  # @param timeout [integer] Interval to check for due jobs
+  def self.yield_jobs(queue_name, timeout, &block)
+    connection.execute "LISTEN pg_jobs_#{queue_name}"
+    loop do
+      # Consume all pending NOTIFY events
+      while connection.raw_connection.notifies; end
+      # Work jobs as long as there are pending jobs in the queue
+      while yield_job(queue_name, &block); end
+      # Wait for next NOTIFY event
+      logger.debug "[pg_jobs] [#{queue_name}] No jobs found, calling wait_for_notify(#{timeout})"
+      connection.raw_connection.wait_for_notify(timeout)
+    end
+  ensure
+    connection.execute "UNLISTEN pg_jobs_#{queue_name}"
+  end
+
+  # Notifies job workers that a new job is present using
+  # PostgreSQL NOTIFIY.
+  def notify_workers
+    PgJob.connection.execute "NOTIFY pg_jobs_#{queue_name}"
+  end
+end

data/db/migrate/20170317150614_create_jobs.rb

@@ -0,0 +1,14 @@
+class CreateJobs < ActiveRecord::Migration[5.0]
+  def change
+    create_table :pg_jobs do |t|
+      t.jsonb :job_data, null: false
+      t.integer :priority, default: 100, null: false
+      t.string :queue_name, null: false, default: 'default'
+
+      t.timestamp :created_at, null: false
+      t.timestamp :scheduled_for
+    end
+    add_index :pg_jobs, %i[queue_name scheduled_for priority created_at],
+              name: 'index_pg_jobs_worker'
+  end
+end

data/lib/active_job/queue_adapters/pg_jobs_adapter.rb

@@ -0,0 +1,17 @@
+module ActiveJob
+  module QueueAdapters
+    # Adapter for ActiveJob to run jobs with pg_jobs
+    #
+    # This lives in ActiveJob::QueueAdapters module so it can be used with
+    # config.active_job.queue_adapter = :pg_jobs
+    class PgJobsAdapter
+      def enqueue(job)
+        PgJobs.enqueue(job)
+      end
+
+      def enqueue_at(job, timestamp)
+        PgJobs.enqueue(job, timestamp)
+      end
+    end
+  end
+end

data/lib/pg_jobs.rb

@@ -0,0 +1,88 @@
+require 'pg_jobs/engine'
+require 'active_job/queue_adapters/pg_jobs_adapter'
+
+# Simple ActiveJob worker for PostgreSQL using LISTEN/NOTIFY and
+# SKIP LOCKED.
+#
+# Supports most ActiveJob features like multiple queues, priorities
+# and wait times.
+#
+# To use this as your Rails job queue, add this to your environment
+# configuration (config/environments/production.rb):
+#
+#   config.active_job.queue_adapter = :pg_jobs
+#
+# Then run one or multiple workers for the default queue with
+#
+#   bin/rails runner PgJobs.work
+#
+# or for other queues with
+#
+#   bin/rails runner "PgJobs.work(:my_queue)"
+#
+# Needs PostgreSQL 9.5 to use SKIP LOCKED.
+module PgJobs
+  # Run a worker process for a given queue name.
+  # Will run all scheduled jobs in the queue ordered by their
+  # priorities (lowest first) and then wait for PostgreSQL LISTEN
+  # events to run new jobs. For jobs that are scheduled for a later
+  # time, it wakes up in an interval given by the timeout parameter
+  # to check for jobs that became due in the meantime.
+  #
+  # Handles SIGTERM for graceful shutdown. This signal will interrupt
+  # neither the execution of a job nor waiting for a new job,
+  # so a shorter timeout means a faster shutdown on SIGTERM.
+  #
+  # @param queue_name [String] The name of the queue to work on
+  # @param timeout [integer] Interval to check for due jobs
+  # @param exit_signals [Array<String>] Array of signal names for graceful exit
+  def self.work(queue_name = 'default', timeout: 10, exit_signals: %w[INT TERM])
+    exit_signal = false
+    job_running = false
+
+    exit_signals.each do |signal|
+      Signal.trap(signal) do
+        raise SignalException, signal unless job_running
+
+        # Put this message to STDERR because the logger cannot be used in a trap context
+        STDERR.puts "Received signal #{signal}, waiting for current job to finish"
+        exit_signal = true
+      end
+    end
+
+    Rails.logger.info do
+      "[pg_jobs] [#{queue_name}] " \
+      "Starting pg_jobs worker for queue '#{queue_name}' with wait timeout #{timeout} seconds"
+    end
+
+    PgJob.yield_jobs(queue_name, timeout) do |pg_job|
+      job_running = true
+      execute_job(pg_job)
+      job_running = false
+
+      break if exit_signal
+    end
+  end
+
+  # Enqueue a new job to run at a given time or immediately
+  #
+  # @param job [ActiveJob::Base] The ActiveJob job object to schedule
+  # @param scheduled_for [Integer,Time] Timestamp when the job should be
+  #   executed. Use nil if the job should be run immediately.
+  def self.enqueue(job, scheduled_for = nil)
+    PgJob.create!(job_data: job.serialize,
+                  scheduled_for: scheduled_for && Time.at(scheduled_for),
+                  priority: job.priority || 100,
+                  queue_name: job.queue_name || 'default')
+  end
+
+  # Execute a PgJob instance. Calls `ActiveJob::Base.execute`.
+  def self.execute_job(pg_job)
+    ActiveJob::Base.execute(pg_job.job_data)
+  rescue => e
+    Rails.logger.error do
+      "[pg_jobs] [#{pg_job.queue_name}] [#{pg_job.job_data['job_id']}] " \
+      "Error while executing job: #{e}\n" + e.backtrace.join("\n")
+    end
+  end
+end

data/lib/pg_jobs/engine.rb

@@ -0,0 +1,4 @@
+module PgJobs
+  class Engine < ::Rails::Engine
+  end
+end

data/lib/pg_jobs/version.rb

@@ -0,0 +1,3 @@
+module PgJobs
+  VERSION = '0.1.0'.freeze
+end

metadata

@@ -0,0 +1,135 @@
+--- !ruby/object:Gem::Specification
+name: pg_jobs
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Moritz Breit
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2018-10-14 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.18'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.18'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '5.1'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '6.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '5.1'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '6.0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 5.9.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 5.9.0
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.59.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.59.1
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.16.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.16.1
+description: Simple ActiveJob queue for PostgreSQL using LISTEN/NOTIFY
+email:
+- mail@moritz-breit.de
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- MIT-LICENSE
+- README.md
+- Rakefile
+- app/models/pg_job.rb
+- db/migrate/20170317150614_create_jobs.rb
+- lib/active_job/queue_adapters/pg_jobs_adapter.rb
+- lib/pg_jobs.rb
+- lib/pg_jobs/engine.rb
+- lib/pg_jobs/version.rb
+homepage: https://github.com/mbreit/pg_jobs/
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '2.3'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.6
+signing_key: 
+specification_version: 4
+summary: Simple ActiveJob queue for PostgreSQL using LISTEN/NOTIFY
+test_files: []