gouda 0.1.0 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f36a7c7b361cb8008f34b4a178c6bf1eb2a358e95502d0550fc14fef78ef5ed7
-  data.tar.gz: 6d2bc5778d6284f212189f4311e575393c5d9c1da13e4618f0b9a5d6ace9e954
+  metadata.gz: 0fa853c78222eb23897ccb31ed465fc231aa5894641fe0d1991ade90a5e3fc8d
+  data.tar.gz: e9680b441d3fe9c3da7fadf50272c033db712296104870d016b278da2c1d92bd
 SHA512:
-  metadata.gz: 71a324a3bae3ee17c2ed547915b1cc841442059c30f0c85138f1b220a7c9dc2b307e6d0363fba12069be0b893ce74e1cefe81f5d3fc1ceec427b56a7331be256
-  data.tar.gz: 8e9d521dc92ccd14175611e21535ed9e5b71610e74c22fd4038e11a16c104b9fc73d93431e1e9aae38bb37e739d91447f3739ddd8099083e4d6e821db1426ea5
+  metadata.gz: 9a64544cd45d14400ab949a848e0325ee5d5305d648f7f38239279f93e1f8d2d32dac368708317aafbf470b48e16f88a0ffe4bad6890798ef53adea0566da5f6
+  data.tar.gz: e2140d4da50c4afe8edadd51bb3049b60935e4c0273d235dcb1988efa2900362ea81451a58df04ba3f4db58209380ea9a58ccedd42972806bd5be2cd9f19d7d4

data/.github/workflows/ci.yml

@@ -9,28 +9,34 @@ env:
 jobs:
   test:
     name: Tests
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
+    if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository
+    strategy:
+      matrix:
+        ruby:
+          - '2.7'
+          - '3.3'
     services:
       postgres:
-        image: postgres:15-alpine
+        image: postgres
         env:
           POSTGRES_PASSWORD: postgres
+        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
         ports:
           - 5432:5432
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 100ms
-          --health-timeout 1s
-          --health-retries 100
 
-    if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository
     steps:
       - name: Checkout
         uses: actions/checkout@v4
       - name: Setup Ruby
         uses: ruby/setup-ruby@v1
         with:
-          ruby-version: '3.2'
+          ruby-version: ${{ matrix.ruby }}
           bundler-cache: true
       - name: "Tests and Lint"
         run: bundle exec rake
+        env:
+          PGHOST: localhost
+          PGUSER: postgres
+          PGPASSWORD: postgres
+          TESTOPTS: "--fail-fast"

data/.gitignore

@@ -5,5 +5,6 @@
 /doc/
 /pkg/
 /spec/reports/
-/tmp/
+/tmp/*
+!/tmp/.keep
 Gemfile.lock

data/.ruby-version

@@ -1 +1 @@
-3.2.2
+3.3.1

data/CHANGELOG.md

@@ -1,6 +1,13 @@
 ## [Unreleased]
 
-## [0.1.0] - 2023-05-27
+## [0.1.0] - 2023-06-10
 
 - Initial release
 
+## [0.1.1] - 2023-06-10
+
+- Fix support for older ruby versions until 2.7
+
+## [0.1.2] - 2023-06-11
+
+- Updated readme and method renaming in Scheduler

data/README.md

@@ -11,7 +11,96 @@ $ bundle install
 $ bin/rails g gouda:install
 ```
 
-## Usage
+Gouda is build as a lightweight alternative to [good_job](https://github.com/bensheldon/good_job) and has been created before [solid_queue.](https://github.com/rails/solid_queue/)
+It is _smaller_ than solid_queue though.
+
+It was designed to enable job processing using `SELECT ... FOR UPDATE SKIP LOCKED` on Postgres so that we could use pg_bouncer in our system setup. 
+
+
+## Key concepts in Gouda: Workload
+
+Gouda is built around the concept of a **Workload.** A workload is not the same as an ActiveJob. A workload is a single execution of a task - the task may be an entire ActiveJob, or a retry of an ActiveJob, or a part of a sequence of ActiveJobs initiated using [job-iteration](https://github.com/shopify/job-iteration)
+
+You can easily have multiple `Workloads` stored in your queue which reference the same job. However, when you are using Gouda it is important to always keep the distinction between the two in mind.
+
+When an ActiveJob gets first initialised, it receives a randomly-generated ActiveJob ID, which is normally a UUID. This UUID will be reused when a job gets retried, or when job-iteration is in use - but it will exist across multiple Gouda workloads.
+
+A `Workload` can only be in one of the three states: `enqueued`, `executing` and `finished`. It does not matter whether the workload has raised an exception, or was manually canceled before it started performing, or succeeded - its terminal state is always going to be `finished`, regardless. This is done on purpose: Gouda uses a number of partial indexes in Postgres which allows it to maintain uniqueness, but only among jobs which are either waiting to start or already running. Additionally, _only the transitions between those states_ are guarded by `BEGIN...COMMIT` and it is the selection on those states that is supplemented by `SELECT ... FOR UPDATE SKIP LOCKED`. The only time locks are placed on a particular `gouda_workloads` row is when this update is about to take place (`SELECT` then `UPDATE`). This makes Gouda a good fit for use with pg_bouncer in transaction mode.
+
+Understanding workload identity is key for making good use of Gouda. For example, an ActiveJob that gets retried can take the following shape in Gouda:
+
+```
+ ____________________________         _______________________________________________
+| ActiveJob(id="0abc-...34") | ----> |  Workload(id="f67b-...123",state="finished")  |
+ ‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾        ‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾
+ ____________________________         _______________________________________________
+| ActiveJob(id="0abc-...34") | ----> |  Workload(id="5e52-...456",state="finished")  |
+ ‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾        ‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾
+ ____________________________         _______________________________________________
+| ActiveJob(id="0abc-...34") | ----> |  Workload(id="8a41-...789",state="enqueued")  |
+ ‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾        ‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾
+```
+
+This would happen if, for example, the ActiveJob raises an exception inside `perform` and is configured to `retry_on` after this exception. Same for job-iteration:
+
+```
+ _______________________________________         _______________________________________________
+| ActiveJob(id="0abc-...34",cursor=nil) | ----> |  Workload(id="f67b-...123",state="finished")  |
+ ‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾        ‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾
+ _______________________________________         _______________________________________________
+| ActiveJob(id="0abc-...34",cursor=123) | ----> |  Workload(id="5e52-...456",state="finished")  |
+ ‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾        ‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾
+ _______________________________________         _______________________________________________
+| ActiveJob(id="0abc-...34",cursor=456) | ----> |  Workload(id="8a41-...789",state="executing") |
+ ‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾        ‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾
+```
+
+A key thing to remember when reading the Gouda source code is that **workloads and jobs are not the same thing.** A single job **may span multiple workloads.**
+
+## Key concepts in Gouda: concurrency keys
+
+Gouda has a few indexes on the `gouda_workloads` table which will:
+
+* Forbid inserting another `enqueued` workload with the same `enqueue_concurrency_key` value. Uniqueness is on that column only.
+* Forbid a workload from transition into `executing` when another workload with the same `execution_concurrency_key` is already running.
+
+These are compatible with good_job concurrency keys, with one major distinction: we use unique indices and not counters, so these keys can be used
+to **prevent concurrent executions** but not to **limit the load on the system**, and the limit of 1 is always enforced.
+
+## Key concepts in Gouda: `executing_on`
+
+A `Workload` is executing on a particular `executing_on` entity - usually a worker thread. That entity gets a pseudorandom ID . The `executing_on` value can be used to see, for example, whether a particular worker thread has hung. If multiple jobs have a far-behind `updated_at` and are all `executing`, this likely means that the worker has crashed or hung. The value can also be used to build a table of currently running workers.
+
+## Usage tips: bulkify your enqueues
+
+When possible, Gouda uses `enqueue_all` to `INSERT` as many jobs at once as possible. With modern servers this allows for very rapid insertion of very large
+batches of jobs. It is supplemented by a module which will make all `perform_later` calls buffered and submitted to the queue in bulk:
+
+```ruby
+Gouda.in_bulk do
+  User.joined_recently.find_each do |user|
+    WelcomeMailer.with(user:).welcome_email.deliver_later
+  end
+end
+```
+
+If there are multiple ActiveJob adapters configured and you bulk-enqueue a job which uses an adapter different than Gouda, `in_bulk` will try to use `enqueue_all` on that
+adapter as well.
+
+## Usage tips: co-commit
+
+Gouda is designed to `COMMIT` the workload together with your business data. It does not need `after_commit` unless you so choose. In fact,
+the main advantage of DB-based job queues such as Gouda is that you can always rely on the fact that the workload will be enqueued only
+once the data it needs to operate on is already available for reading. This is guaranteed to work:
+
+```ruby
+User.transaction do
+  freshly_joined_user = User.create!(user_params)
+  WelcomeMailer.with(user: freshly_joined_user).welcome_email.deliver_later
+end
+```
+
+## Web UI
 
 At the moment the Gouda UI is proprietary, so this gem only provides a "headless" implementation. We expect this to change in the future.
 

data/gouda.gemspec

@@ -4,12 +4,12 @@ Gem::Specification.new do |spec|
   spec.name = "gouda"
   spec.version = Gouda::VERSION
   spec.summary = "Job Scheduler"
-  spec.description = "Job Scheduler for Rails"
+  spec.description = "Job Scheduler for Rails and PostgreSQL"
   spec.authors = ["Sebastian van Hesteren", "Julik Tarkhanov"]
   spec.email = ["sebastian@cheddar.me", "me@julik.nl"]
   spec.homepage = "https://rubygems.org/gems/gouda"
   spec.license = "MIT"
-  spec.required_ruby_version = Gem::Requirement.new(">= 2.4.0")
+  spec.required_ruby_version = Gem::Requirement.new(">= 2.7.0")
   spec.require_paths = ["lib"]
 
   spec.metadata["homepage_uri"] = spec.homepage

data/lib/gouda/active_job_extensions/concurrency.rb

@@ -21,7 +21,7 @@ module Gouda
             enqueue_limit = total_limit
           end
 
-          self.gouda_concurrency_config = {perform_limit:, enqueue_limit:, key:}
+          self.gouda_concurrency_config = {perform_limit: perform_limit, enqueue_limit: enqueue_limit, key: key}
         end
       end
 

data/lib/gouda/railtie.rb

@@ -34,8 +34,6 @@ module Gouda
     # The `to_prepare` block which is executed once in production
     # and before each request in development.
     config.to_prepare do
-      Gouda::Scheduler.update_schedule_from_config!
-
       if defined?(Rails) && Rails.respond_to?(:application)
         config_from_rails = Rails.application.config.try(:gouda)
         if config_from_rails
@@ -52,6 +50,9 @@ module Gouda
         Gouda.config.polling_sleep_interval_seconds = 0.2
         Gouda.config.logger.level = Gouda.config.log_level
       end
+
+      Gouda::Scheduler.build_scheduler_entries_list!
+      Gouda::Scheduler.upsert_workloads_from_entries_list!
     end
   end
 end

data/lib/gouda/scheduler.rb

@@ -53,7 +53,33 @@ module Gouda::Scheduler
     end
   end
 
-  def self.update_schedule_from_config!(cron_table_hash = nil)
+  # Takes in a Hash formatted with cron entries in the format similar
+  # to good_job, and builds a table of scheduler entries. A scheduler
+  # entry references a particular job class name, the set of arguments to
+  # be passed to the job when performing it, and either the interval
+  # to repeat the job after or a cron pattern. This method does not
+  # insert the actual Workloads into the database but just builds the
+  # table of the entries. That table gets consulted when workloads finish
+  # to determine whether the workload that just ran was scheduled or ad-hoc,
+  # and whether the subsequent workload has to be enqueued.
+  #
+  # If no table is given the method will attempt to read the table from
+  # Rails application config from `[:gouda][:cron]`.
+  #
+  # The table is a Hash of entries, and the keys are the names of the workload
+  # to be enqueued - those keys are also used to ensure scheduled workloads
+  # only get scheduled once.
+  #
+  # @param cron_table_hash[Hash] a hash of the following shape:
+  #     {
+  #       download_invoices_every_minute: {
+  #         cron: "* * * * *",
+  #         class: "DownloadInvoicesJob",
+  #         args: ["immediate"]
+  #       }
+  #     }
+  # @return Array[Entry]
+  def self.build_scheduler_entries_list!(cron_table_hash = nil)
     Gouda.logger.info "Updating scheduled workload entries..."
     if cron_table_hash.blank?
       config_from_rails = Rails.application.config.try(:gouda)
@@ -72,10 +98,16 @@ module Gouda::Scheduler
       # `class` is a reserved keyword and a method that exists on every Ruby object so...
       cron_entry_params[:job_class] ||= cron_entry_params.delete(:class)
       params_with_defaults = defaults.merge(cron_entry_params)
-      Entry.new(name:, **params_with_defaults)
+      Entry.new(name: name, **params_with_defaults)
     end
   end
 
+  # Once a workload has finished (doesn't matter whether it raised an exception
+  # or completed successfully), it is going to be passed to this method to enqueue
+  # the next scheduled workload
+  #
+  # @param finished_workload[Gouda::Workload]
+  # @return void
   def self.enqueue_next_scheduled_workload_for(finished_workload)
     return unless finished_workload.scheduler_key
 
@@ -86,11 +118,23 @@ module Gouda::Scheduler
     Gouda.enqueue_jobs_via_their_adapters([timer_entry.build_active_job])
   end
 
+  # Returns the list of entries of the scheduler which are currently known. Normally the
+  # scheduler will hold the list of entries loaded from the Rails config.
+  #
+  # @return Array[Entry]
   def self.entries
     @cron_table || []
   end
 
-  def self.update_scheduled_workloads!
+  # Will upsert (`INSERT ... ON CONFLICT UPDATE`) workloads for all entries which are in the scheduler entries
+  # table (the table needs to be read or hydrated first using `build_scheduler_entries_list!`). This is done
+  # in a transaction. Any workloads which have been previously inserted from the scheduled entries, but no
+  # longer have a corresponding scheduler entry, will be deleted from the database. If there already are workloads
+  # with the corresponding scheduler key they will not be touched and will be performed with their previously-defined
+  # arguments.
+  #
+  # @return void
+  def self.upsert_workloads_from_entries_list!
     table_entries = @cron_table || []
 
     # Remove any cron keyed workloads which no longer match config-wise

data/lib/gouda/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Gouda
-  VERSION = "0.1.0"
+  VERSION = "0.1.2"
 end

data/lib/gouda/worker.rb

@@ -135,7 +135,7 @@ module Gouda
           break if check_shutdown.call
 
           did_process = Gouda.config.app_executor.wrap do
-            Gouda::Workload.checkout_and_perform_one(executing_on: worker_id_and_thread_id, queue_constraint:, in_progress: executing_workload_ids)
+            Gouda::Workload.checkout_and_perform_one(executing_on: worker_id_and_thread_id, queue_constraint: queue_constraint, in_progress: executing_workload_ids)
           end
 
           # If no job was retrieved the queue is likely empty. Relax the polling then and ease off.

data/lib/gouda/workload.rb

@@ -66,7 +66,7 @@ class Gouda::Workload < ActiveRecord::Base
             # Appsignal.increment_counter("gouda_workloads_revived", 1, job_class: workload.active_job_class_name)
 
             interrupted_at = workload.last_execution_heartbeat_at
-            workload.update!(state: "finished", interrupted_at:, last_execution_heartbeat_at: Time.now.utc, execution_finished_at: Time.now.utc)
+            workload.update!(state: "finished", interrupted_at: interrupted_at, last_execution_heartbeat_at: Time.now.utc, execution_finished_at: Time.now.utc)
             revived_job = ActiveJob::Base.deserialize(workload.active_job_data)
             # Save the interrupted_at timestamp so that upon execution the new job will raise a Gouda::Interrpupted exception.
             # The exception can then be handled like any other ActiveJob exception (using rescue_from or similar).
@@ -116,7 +116,7 @@ class Gouda::Workload < ActiveRecord::Base
       uncached do # Necessary because we SELECT with a clock_timestamp() which otherwise gets cached by ActiveRecord query cache
         transaction do
           jobs.first.tap do |job|
-            job&.update!(state: "executing", executing_on:, last_execution_heartbeat_at: Time.now.utc, execution_started_at: Time.now.utc)
+            job&.update!(state: "executing", executing_on: executing_on, last_execution_heartbeat_at: Time.now.utc, execution_started_at: Time.now.utc)
           end
         rescue ActiveRecord::RecordNotUnique
           # It can happen that due to a race the `execution_concurrency_key NOT IN` does not capture
@@ -133,7 +133,7 @@ class Gouda::Workload < ActiveRecord::Base
   # @param in_progress[#add,#delete] Used for tracking work in progress for heartbeats
   def self.checkout_and_perform_one(executing_on:, queue_constraint: Gouda::AnyQueue, in_progress: Set.new)
     # Select a job and mark it as "executing" which will make it unavailable to any other
-    workload = checkout_and_lock_one(executing_on:, queue_constraint:)
+    workload = checkout_and_lock_one(executing_on: executing_on, queue_constraint: queue_constraint)
     if workload
       in_progress.add(workload.id)
       workload.perform_and_update_state!

data/lib/gouda.rb

@@ -46,7 +46,7 @@ module Gouda
   end
 
   def self.start
-    Gouda::Scheduler.update_scheduled_workloads!
+    Gouda::Scheduler.upsert_workloads_from_entries_list!
 
     queue_constraint = if ENV["GOUDA_QUEUES"]
       Gouda.parse_queue_constraint(ENV["GOUDA_QUEUES"])
@@ -57,7 +57,7 @@ module Gouda
     Gouda.logger.info("Gouda version: #{Gouda::VERSION}")
     Gouda.logger.info("Worker threads: #{Gouda.config.worker_thread_count}")
 
-    Gouda.worker_loop(n_threads: Gouda.config.worker_thread_count, queue_constraint:)
+    Gouda.worker_loop(n_threads: Gouda.config.worker_thread_count, queue_constraint: queue_constraint)
   end
 
   def self.config

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gouda
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.2
 platform: ruby
 authors:
 - Sebastian van Hesteren
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-06-10 00:00:00.000000000 Z
+date: 2024-06-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -123,7 +123,7 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: Job Scheduler for Rails
+description: Job Scheduler for Rails and PostgreSQL
 email:
 - sebastian@cheddar.me
 - me@julik.nl
@@ -157,6 +157,7 @@ files:
 - lib/gouda/version.rb
 - lib/gouda/worker.rb
 - lib/gouda/workload.rb
+- tmp/.keep
 homepage: https://rubygems.org/gems/gouda
 licenses:
 - MIT
@@ -172,14 +173,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.4.0
+      version: 2.7.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.10
+rubygems_version: 3.5.9
 signing_key:
 specification_version: 4
 summary: Job Scheduler