gouda 0.1.4 → 0.1.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cbebec9ae881152e66d8a8e763d74e2314c6694cc46a1b547323bb4fe01fd505
-  data.tar.gz: f9836ba9594cf2473485efbe6be8dd6b001e0e514315cb84170a62d525adbaf3
+  metadata.gz: ec67da4c5d0dc236c02b65c770e87fe4a0a967eef8df50f0cb4ae452077f3bdb
+  data.tar.gz: 6089a03c99a944fcfedbe623810c72b513c52e2fe9afd698904ccab1f7cc3aa3
 SHA512:
-  metadata.gz: 2bf05ef7dcfe8cc682f54da18f8ad8a415f5db3f103928188ad04ab714f4a2799b4998b25c5d53432776d4f913e22fe0a761bd1fe11ea2eb17763f89201cb809
-  data.tar.gz: 636a2aa356b4fed48dece844dec2f26d46a430d6e4a0d956c56d5c5121d30f8d3fb8f87174b82e4d7ab6a58f3e0bcd99d3e07c18c631390cc6233b8cac2d48ba
+  metadata.gz: bb445a3c45a28d47f29124775f8f2f8c2cc9be8c6fb1069c8c3f1d00eca8269ca53d897c2cb069962bec90ee6b8b9916af133558b0692a3a64d46e3666fbb6f4
+  data.tar.gz: 3e82a95cf5f6107a8f9f7874a93d4bbbc58ac9a6badb2f684edca870cf6fbd3512be37299b192dcbb7a68a4be9fb638ea89af88625ae70459f1334a03b541407

data/CHANGELOG.md

@@ -23,3 +23,16 @@
 - Reduce logging in local test runs.
 - Bump local ruby version to 3.3.3
 
+## [0.1.5] - 2023-06-18
+
+- Update documentation
+- Don't pass on scheduler keys to retries
+
+## [0.1.6] - 2023-06-18
+
+- Fix: don't upsert workloads twice when starting Gouda.
+- Add back in Appsignal calls
+
+## [0.1.7] - 2023-06-21
+
+- Separate all instrumentation to use ActiveSupport::Notification

data/README.md

@@ -1,7 +1,10 @@
 Gouda is an ActiveJob adapter used at Cheddar. It requires PostgreSQL and a recent version of Rails.
 
-⚠️ At the moment Gouda is only used internally at Cheddar. We do not provide support for it, nor do we accept
-issues or feature requests. This is likely to change in the future.
+> [!CAUTION]
+> At the moment Gouda is only used internally at Cheddar. Any support to external parties is on best-effort
+> basis. While we are happy to see issues and pull requests, we can't guarantee that those will be addressed
+> quickly. The library does receive rapid updates which may break your application if you come to depend on
+> the library. That is to be expected.
 
 ## Installation
 
@@ -11,10 +14,10 @@ $ bundle install
 $ bin/rails g gouda:install
 ```
 
-Gouda is build as a lightweight alternative to [good_job](https://github.com/bensheldon/good_job) and has been created before [solid_queue.](https://github.com/rails/solid_queue/)
-It is _smaller_ than solid_queue though.
-
-It was designed to enable job processing using `SELECT ... FOR UPDATE SKIP LOCKED` on Postgres so that we could use pg_bouncer in our system setup. 
+Gouda is a lightweight alternative to [good_job](https://github.com/bensheldon/good_job) and [solid_queue.](https://github.com/rails/solid_queue/) - while
+more similar to the latter. It has been created prior to solid_queue and is smaller. It was designed to enable job processing using `SELECT ... FOR UPDATE SKIP LOCKED`
+on Postgres so that we could use pg_bouncer in our system setup. We have also observed that `SKIP LOCKED` causes less load on our database than advisory locking,
+especially as queue depths would grow. 
 
 
 ## Key concepts in Gouda: Workload

data/gouda.gemspec

@@ -7,13 +7,13 @@ Gem::Specification.new do |spec|
   spec.description = "Job Scheduler for Rails and PostgreSQL"
   spec.authors = ["Sebastian van Hesteren", "Julik Tarkhanov"]
   spec.email = ["sebastian@cheddar.me", "me@julik.nl"]
-  spec.homepage = "https://rubygems.org/gems/gouda"
+  spec.homepage = "https://github.com/cheddar-me/gouda"
   spec.license = "MIT"
   spec.required_ruby_version = Gem::Requirement.new(">= 2.7.0")
   spec.require_paths = ["lib"]
 
-  spec.metadata["homepage_uri"] = spec.homepage
-  spec.metadata["source_code_uri"] = "https://github.com/cheddar-me/gouda"
+  spec.metadata["homepage_uri"] = 
+  spec.metadata["source_code_uri"] = spec.homepage
   spec.metadata["changelog_uri"] = "https://github.com/cheddar-me/gouda/CHANGELOG.md"
 
   spec.files = Dir.chdir(File.expand_path(__dir__)) do

data/lib/active_job/queue_adapters/gouda_adapter.rb

@@ -1,8 +1,10 @@
 # frozen_string_literal: true
 
-module ActiveJob # :nodoc:
-  module QueueAdapters # :nodoc:
-    class GoudaAdapter < Gouda::Adapter
-    end
-  end
+# The sole purpose of this class is so that you can do
+# `config.active_job.queue_adapter = :gouda` in your Rails
+# config, as Rails insists on resolving the adapter module
+# name from the symbol automatically. If Rails ever allows
+# us to "register" an adapter to a symbol this module can
+# be removed later.
+class ActiveJob::QueueAdapters::GoudaAdapter < Gouda::Adapter
 end

data/lib/gouda/adapter.rb

@@ -57,10 +57,11 @@ class Gouda::Adapter
       # We can't tell Postgres to ignore conflicts on _both_ the scheduler key and the enqueue concurrency key but not on
       # the ID - it is either "all indexes" or "just one", but never "this index and that index". MERGE https://www.postgresql.org/docs/current/sql-merge.html
       # is in theory capable of solving this but let's not complicate things all to hastily, the hour is getting late
+      scheduler_key = active_job.try(:executions) == 0 ? active_job.scheduler_key : nil # only enforce scheduler key on first workload
       {
         active_job_id: active_job.job_id, # Multiple jobs can have the same ID due to retries, job-iteration etc.
         scheduled_at: active_job.scheduled_at || t_now,
-        scheduler_key: active_job.scheduler_key, # So that the scheduler_key gets retained between retries
+        scheduler_key: scheduler_key,
         priority: active_job.priority,
         execution_concurrency_key: extract_execution_concurrency_key(active_job),
         enqueue_concurrency_key: extract_enqueue_concurrency_key(active_job),
@@ -82,7 +83,7 @@ class Gouda::Adapter
     # Use batches of 500 so that we do not exceed the maximum statement size or do not create a transaction for the
     # insert which times out
     inserted_ids_and_positions = bulk_insert_attributes.each_slice(500).flat_map do |chunk|
-      ActiveSupport::Notifications.instrument("insert_all.gouda", {n_rows: chunk.size}) do |payload|
+      Gouda.instrument(:insert_all, n_rows: chunk.size) do |payload|
         rows = Gouda::Workload.insert_all(chunk, returning: [:id, :position_in_bulk])
         payload[:inserted_jobs] = rows.length
         payload[:rejected_jobs] = chunk.size - rows.length

data/lib/gouda/bulk.rb

@@ -1,6 +1,22 @@
 # frozen_string_literal: true
 
 module Gouda
+  # Inside this call, all `perform_later` calls on ActiveJob subclasses
+  # (including mailers) will be buffered. The call is reentrant, so you
+  # can have multiple `in_bulk` calls with arbitrary nesting. At the end
+  # of the block, the buffered jobs will be enqueued using their respective
+  # adapters. If an adapter supports `enqueue_all` (Sidekiq does in recent
+  # releases of Rails, for example), this functionality will be used. This
+  # method is especially useful when doing things such as mass-emails, or
+  # maintenance tasks where a large swath of jobs gets enqueued at once.
+  #
+  # @example
+  #    Gouda.in_bulk do
+  #      User.recently_joined.find_each do |recently_joined_user|
+  #        GreetingJob.perform_later(recently_joined_user)
+  #      end
+  #    end
+  # @return [Object] the return value of the block
   def self.in_bulk(&blk)
     if Thread.current[:gouda_bulk_buffer].nil?
       Thread.current[:gouda_bulk_buffer] = []

data/lib/gouda/queue_constraints.rb

@@ -1,44 +1,48 @@
 # frozen_string_literal: true
 
 module Gouda
+  # A queue constraint supports just one method, `to_sql`, which returns
+  # a condition based on the `queue_name` value of the `gouda_workloads`
+  # table. The minimal constraint is just a no-op - it allows execution
+  # of jobs from all queues in the system.
   module AnyQueue
     def self.to_sql
       "1=1"
     end
   end
 
+  # Allows execution of jobs only from specified queues
+  # For example, if you have a queue named "gpu", and you run
+  # jobs requiring a GPU on this queue, on your worker script
+  # running on GPU-equipped machines you could use
+  # `OnlyQueuesConstraint.new([:gpu])`
   class OnlyQueuesConstraint < Struct.new(:queue_names)
     def to_sql
       placeholders = (["?"] * queue_names.length).join(",")
-      ActiveRecord::Base.sanitize_sql_array([<<~SQL, *queue_names])
+      Gouda::Workload.sanitize_sql_array([<<~SQL, *queue_names])
         queue_name IN (#{placeholders})
       SQL
     end
   end
 
+  # Allows execution of jobs from queues except the given ones
+  # For example, if you have a queue named "emails" which is time-critical,
+  # on all other machines your worker script can specify
+  # `ExceptQueueConstraint.new([:emails])`
   class ExceptQueueConstraint < Struct.new(:queue_names)
     def to_sql
       placeholders = (["?"] * queue_names.length).join(",")
-      ActiveRecord::Base.sanitize_sql_array([<<~SQL, *queue_names])
+      Gouda::Workload.sanitize_sql_array([<<~SQL, *queue_names])
         queue_name NOT IN (#{placeholders})
       SQL
     end
   end
 
-  def self.parse_queue_constraint(constraint_str_from_envvar)
-    parsed = queue_parser(constraint_str_from_envvar)
-    if parsed[:include]
-      OnlyQueuesConstraint.new(parsed[:include])
-    elsif parsed[:exclude]
-      ExceptQueueConstraint.new(parsed[:exclude])
-    else
-      AnyQueue
-    end
-  end
-
-  # Parse a string representing a group of queues into a more readable data
-  # structure.
-  # @param string [String] Queue string
+  # Parse a string representing a group of queues into a queue constraint
+  # Note that this works similar to good_job. For example, the
+  # constraints do not necessarily compose all that well.
+  #
+  # @param queue_constraint_str[String] Queue string
   # @return [Hash]
   #   How to match a given queue. It can have the following keys and values:
   #   - +{ all: true }+ indicates that all queues match.
@@ -49,8 +53,8 @@ module Gouda
   # @example
   #   Gouda::QueueConstraints.queue_parser('-queue1,queue2')
   #   => { exclude: [ 'queue1', 'queue2' ] }
-  def self.queue_parser(string)
-    string = string.presence || "*"
+  def self.parse_queue_constraint(queue_constraint_str)
+    string = queue_constraint_str.presence || "*"
 
     case string.first
     when "-"
@@ -63,11 +67,11 @@ module Gouda
     queues = string.split(",").map(&:strip)
 
     if queues.include?("*")
-      {all: true}
+      AnyQueue
     elsif exclude_queues
-      {exclude: queues}
+      ExceptQueueConstraint.new([queues])
     else
-      {include: queues}
+      OnlyQueuesConstraint.new([queues])
     end
   end
 end

data/lib/gouda/railtie.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 module Gouda
+  UNINITIALISED_DATABASE_EXCEPTIONS = [ActiveRecord::NoDatabaseError, ActiveRecord::StatementInvalid, ActiveRecord::ConnectionNotEstablished]
+
   class Railtie < Rails::Railtie
     rake_tasks do
       task preload: :setup do
@@ -54,7 +56,7 @@ module Gouda
       Gouda::Scheduler.build_scheduler_entries_list!
       begin
         Gouda::Scheduler.upsert_workloads_from_entries_list!
-      rescue ActiveRecord::NoDatabaseError
+      rescue *Gouda::UNINITIALISED_DATABASE_EXCEPTIONS
         # Do nothing. On a freshly checked-out Rails app, running even unrelated Rails tasks
         # (such as asset compilation) - or, more importantly, initial db:create -
         # will cause a NoDatabaseError, as this is a chicken-and-egg problem. That error

data/lib/gouda/scheduler.rb

@@ -11,6 +11,9 @@ module Gouda::Scheduler
       [name, interval_seconds, cron, job_class].compact.join("_")
     end
 
+    # Tells when this particular task should next run
+    #
+    # @return [Time]
     def next_at
       if interval_seconds
         first_existing = Gouda::Workload.where(scheduler_key: scheduler_key).where("scheduled_at > NOW()").order("scheduled_at DESC").pluck(:scheduled_at).first
@@ -22,6 +25,9 @@ module Gouda::Scheduler
       end
     end
 
+    # Builds the ActiveJob which can be enqueued for this entry
+    #
+    # @return [ActiveJob::Base]
     def build_active_job
       next_at = self.next_at
       return unless next_at

data/lib/gouda/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Gouda
-  VERSION = "0.1.4"
+  VERSION = "0.1.7"
 end

data/lib/gouda/worker.rb

@@ -163,7 +163,8 @@ module Gouda
         # Find jobs which just hung and clean them up (mark them as "finished" and enqueue replacement workloads if possible)
         Gouda::Workload.reap_zombie_workloads
       rescue => e
-        # Appsignal.add_exception(e)
+        Gouda.instrument(:exception, exception: e)
+
         warn "Uncaught exception during housekeeping (#{e.class} - #{e}"
       end
 

data/lib/gouda/workload.rb

@@ -63,7 +63,7 @@ class Gouda::Workload < ActiveRecord::Base
           workload.with_lock("FOR UPDATE SKIP LOCKED") do
             Gouda.logger.info { "Reviving (re-enqueueing) Gouda workload #{workload.id} after interruption" }
 
-            # Appsignal.increment_counter("gouda_workloads_revived", 1, job_class: workload.active_job_class_name)
+            Gouda.instrument(:workloads_revived_counter, size: 1, job_class: workload.active_job_class_name)
 
             interrupted_at = workload.last_execution_heartbeat_at
             workload.update!(state: "finished", interrupted_at: interrupted_at, last_execution_heartbeat_at: Time.now.utc, execution_finished_at: Time.now.utc)
@@ -110,7 +110,7 @@ class Gouda::Workload < ActiveRecord::Base
       .lock("FOR UPDATE SKIP LOCKED")
       .limit(1)
 
-    _first_available_workload = ActiveSupport::Notifications.instrument("checkout_and_lock_one.gouda", {queue_constraint: queue_constraint.to_sql}) do |payload|
+    _first_available_workload = Gouda.instrument(:checkout_and_lock_one, queue_constraint: queue_constraint.to_sql) do |payload|
       payload[:condition_sql] = jobs.to_sql
       payload[:retried_checkouts_due_to_concurrent_exec] = 0
       uncached do # Necessary because we SELECT with a clock_timestamp() which otherwise gets cached by ActiveRecord query cache

data/lib/gouda.rb

@@ -46,8 +46,6 @@ module Gouda
   end
 
   def self.start
-    Gouda::Scheduler.upsert_workloads_from_entries_list!
-
     queue_constraint = if ENV["GOUDA_QUEUES"]
       Gouda.parse_queue_constraint(ENV["GOUDA_QUEUES"])
     else
@@ -72,6 +70,11 @@ module Gouda
     Gouda.config.logger
   end
 
+  def self.instrument(channel, **options, &block)
+    ActiveSupport::Notifications.instrument("#{channel}.gouda", **options, &block)
+  end
+
+
   def self.create_tables(active_record_schema)
     active_record_schema.create_enum :gouda_workload_state, %w[enqueued executing finished]
     active_record_schema.create_table :gouda_workloads, id: :uuid do |t|

data/test/gouda/gouda_test.rb

@@ -645,6 +645,15 @@ class GoudaTest < ActiveSupport::TestCase
     assert_equal ["did-run"], Thread.current[:gouda_test_side_effects]
   end
 
+  test "instrumentation" do
+    payload = subscribed_notification_for("workloads_revived_counter.gouda") do
+      Gouda.instrument(:workloads_revived_counter, size: 1, job_class: "test_class")
+    end
+
+    assert_equal "test_class", payload[:job_class]
+    assert_equal 1, payload[:size]
+  end
+
   def sample_description(sample)
     values = sample.map(&:to_f).sort
 

data/test/gouda/scheduler_test.rb

@@ -24,8 +24,6 @@ class GoudaSchedulerTest < ActiveSupport::TestCase
     class MegaError < StandardError
     end
 
-    gouda_control_concurrency_with(enqueue_limit: 1, key: -> { self.class.to_s })
-
     retry_on StandardError, wait: :polynomially_longer, attempts: 5
     retry_on Gouda::InterruptError, wait: 0, attempts: 5
     retry_on MegaError, attempts: 3, wait: 0
@@ -55,6 +53,29 @@ class GoudaSchedulerTest < ActiveSupport::TestCase
     assert Gouda::Workload.count > 3
   end
 
+  test "retries do not have a scheduler_key" do
+    tab = {
+      second_minutely: {
+        cron: "*/1 * * * * *", # every second
+        class: "GoudaSchedulerTest::FailingJob"
+      }
+    }
+
+    assert_nothing_raised do
+      Gouda::Scheduler.build_scheduler_entries_list!(tab)
+      Gouda::Scheduler.upsert_workloads_from_entries_list!
+    end
+
+    assert_equal 1, Gouda::Workload.enqueued.count
+    assert_equal "second_minutely_*/1 * * * * *_GoudaSchedulerTest::FailingJob", Gouda::Workload.enqueued.first.scheduler_key
+    sleep(2)
+    Gouda::Workload.checkout_and_perform_one(executing_on: "Unit test")
+
+    assert_equal 1, Gouda::Workload.retried.reload.count
+    assert_nil Gouda::Workload.retried.first.scheduler_key
+    assert_equal "enqueued", Gouda::Workload.retried.first.state
+  end
+
   test "re-inserts the next subsequent job after executing the queued one" do
     tab = {
       second_minutely: {

data/test/gouda/test_helper.rb

@@ -67,4 +67,17 @@ class ActiveSupport::TestCase
       end
     end
   end
+
+  def subscribed_notification_for(notification)
+    payload = nil
+    subscription = ActiveSupport::Notifications.subscribe notification do |name, start, finish, id, _payload|
+      payload = _payload
+    end
+
+    yield
+
+    ActiveSupport::Notifications.unsubscribe(subscription)
+
+    return payload
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gouda
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.1.7
 platform: ruby
 authors:
 - Sebastian van Hesteren
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-06-14 00:00:00.000000000 Z
+date: 2024-06-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -166,12 +166,12 @@ files:
 - test/gouda/workload_test.rb
 - test/support/assert_helper.rb
 - tmp/.keep
-homepage: https://rubygems.org/gems/gouda
+homepage: https://github.com/cheddar-me/gouda
 licenses:
 - MIT
 metadata:
-  homepage_uri: https://rubygems.org/gems/gouda
   source_code_uri: https://github.com/cheddar-me/gouda
+  homepage_uri: https://github.com/cheddar-me/gouda
   changelog_uri: https://github.com/cheddar-me/gouda/CHANGELOG.md
 post_install_message:
 rdoc_options: []
@@ -188,7 +188,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.11
+rubygems_version: 3.5.13
 signing_key:
 specification_version: 4
 summary: Job Scheduler