pgbus 0.14.2 → 0.15.0

data/lib/pgbus/fair_share.rb

@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+
+module Pgbus
+  # Fair share scheduling across tenants (issue #426).
+  #
+  # At enqueue time the configured callable (`config.fair_share`) resolves a
+  # key — typically a tenant id — and an optional weight for the job. Both
+  # ride inside the job payload hash (same pattern as Concurrency::METADATA_KEY
+  # and Batch::METADATA_KEY), so they survive every path that re-sends a
+  # payload: blocked-execution promotion, DLQ retry, dashboard retry, bulk
+  # enqueue. On the read side Client#read_batch_fair interleaves across keys
+  # proportionally to weight (see Client::FairRead).
+  #
+  # Events (issue #427) use the same keys and the same read primitive: the
+  # configured `config.event_fair_share` callable receives the Pgbus::Event at
+  # publish time and the key rides in the event *envelope* (a sibling of
+  # event_id / payload / published_at — never inside the user payload), so the
+  # same `message->>'pgbus_fair_key'` expression, index and read SQL serve both
+  # jobs and events, and the outbox (which stores the envelope) carries it.
+  module FairShare
+    METADATA_KEY = "pgbus_fair_key"
+    WEIGHT_KEY = "pgbus_fair_weight"
+    DEFAULT_WEIGHT = 1
+
+    class << self
+      def enabled?(config = Pgbus.configuration)
+        !config.fair_share.nil?
+      end
+
+      def event_enabled?(config = Pgbus.configuration)
+        !config.event_fair_share.nil?
+      end
+
+      # Returns the payload with fair-share metadata merged in, or the payload
+      # itself (same object) when fair share is off or the job is unkeyed.
+      # Exceptions raised by the callable propagate — a key resolver that
+      # cannot run is a programmer error, not something to swallow at enqueue.
+      def inject_metadata(active_job, payload_hash, config = Pgbus.configuration)
+        return payload_hash unless enabled?(config)
+
+        tag(payload_hash, resolve(active_job, config))
+      end
+
+      # Event twin of inject_metadata: returns the event envelope (the hash
+      # Publisher.build_event_data produced) with fair-share metadata merged
+      # in, or the envelope itself (same object) when event fair share is off
+      # or the callable declines to key the event.
+      def inject_event_metadata(event, event_data, config = Pgbus.configuration)
+        return event_data unless event_enabled?(config)
+
+        tag(event_data, resolve_event(event, config))
+      end
+
+      # [key, weight] for the job, or nil when the callable declines to key it.
+      def resolve(active_job, config = Pgbus.configuration)
+        normalize(config.fair_share.call(active_job))
+      end
+
+      # [key, weight] for the event, or nil when the callable declines.
+      def resolve_event(event, config = Pgbus.configuration)
+        normalize(config.event_fair_share.call(event))
+      end
+
+      def extract_key(payload)
+        payload[METADATA_KEY]
+      end
+
+      def extract_weight(payload)
+        payload[WEIGHT_KEY] || DEFAULT_WEIGHT
+      end
+
+      private
+
+      def tag(hash, resolved)
+        return hash unless resolved
+
+        key, weight = resolved
+        tagged = hash.merge(METADATA_KEY => key)
+        tagged[WEIGHT_KEY] = weight unless weight == DEFAULT_WEIGHT
+        tagged
+      end
+
+      def normalize(result)
+        return nil if result.nil?
+
+        raw_key, raw_weight = result.is_a?(Array) ? result : [result, nil]
+        [normalize_key(raw_key), normalize_weight(raw_weight)]
+      end
+
+      def normalize_key(raw)
+        key = case raw
+              when String, Symbol, Integer then raw.to_s
+              else
+                raise ArgumentError,
+                      "fair_share key must be a String, Symbol, or Integer (got #{raw.class})"
+              end
+        raise ArgumentError, "fair_share key must not be empty" if key.empty?
+
+        key
+      end
+
+      def normalize_weight(raw)
+        return DEFAULT_WEIGHT if raw.nil?
+        return raw if raw.is_a?(Numeric) && raw.positive?
+
+        raise ArgumentError, "fair_share weight must be a positive Numeric (got #{raw.inspect})"
+      end
+    end
+  end
+end

data/lib/pgbus/outbox.rb

@@ -18,7 +18,14 @@ module Pgbus
 
     def publish_event(routing_key, payload, headers: nil)
       Instrumentation.instrument("pgbus.outbox.publish", routing_key: routing_key, kind: :event) do
-        event_data = EventBus::Publisher.build_event_data(payload)
+        # routing_key: in the envelope so the consumer can dispatch the relayed
+        # event to handlers — pgmq.send_topic matches queues at relay time but
+        # stamps nothing on the message, and Consumer#handle_message reads the
+        # routing key from the envelope (fixed alongside issue #431; previously
+        # relayed outbox events matched zero handlers and were archived unrun).
+        event_data = EventBus::Publisher.build_event_data(payload, routing_key: routing_key)
+        event_data = EventBus::Publisher.tag_fair_share(event_data, payload, routing_key: routing_key, headers: headers)
+        event_data = EventBus::Publisher.tag_current(event_data)
         OutboxEntry.create!(
           routing_key: routing_key,
           payload: event_data,

data/lib/pgbus/process/consumer.rb

@@ -116,6 +116,7 @@ module Pgbus
         setup_signals
         start_heartbeat
         setup_subscriptions
+        ensure_fair_indexes
         start_wake_source
         Pgbus.logger.info do
           "[Pgbus] Consumer started: topics=#{topics.join(",")} threads=#{threads} " \
@@ -188,7 +189,9 @@ module Pgbus
         active_queues = @queue_names.reject { |q| @circuit_breaker.paused?(q) }
         return [] if active_queues.empty?
 
-        if active_queues.size == 1
+        if fair_share_enabled?
+          fetch_fair(active_queues, qty)
+        elsif active_queues.size == 1
           queue = active_queues.first
           (Pgbus.client.read_batch(queue, qty: qty) || []).map { |m| [queue, m] }
         else
@@ -282,6 +285,40 @@ module Pgbus
         end
       end
 
+      # Fair share for consumers (issue #427): one weighted-interleave read per
+      # subscriber queue, in list order with the remaining capacity — strict
+      # order across queues, fair across tenants WITHIN each queue. Same loop
+      # shape as Worker#fetch_fair; the consumer has no priority levels or
+      # group mode, so this is the whole fair branch.
+      def fetch_fair(active_queues, qty)
+        remaining = qty
+        results = []
+
+        active_queues.each do |queue|
+          break if remaining <= 0
+
+          messages = Pgbus.client.read_batch_fair(queue, qty: remaining) || []
+          messages.each { |m| results << [queue, m] }
+          remaining -= messages.size
+        end
+
+        results
+      end
+
+      def fair_share_enabled?
+        FairShare.event_enabled?(config)
+      end
+
+      # Build the fair index on every subscriber queue this consumer reads
+      # (idempotent, memoized in the client, CONCURRENTLY so a populated queue
+      # is never write-locked). Subscriber queues created later get theirs from
+      # Subscriber#setup!.
+      def ensure_fair_indexes
+        return unless fair_share_enabled?
+
+        @queue_names.each { |q| Pgbus.client.ensure_fair_index(q) }
+      end
+
       # Signal the loop to exit cleanly once a recycle limit is hit. The clean
       # exit gets an immediate supervisor restart (supervisor.rb:305-307), so a
       # fresh fork replaces this one before its memory grows unbounded — the

data/lib/pgbus/process/worker.rb

@@ -54,6 +54,11 @@ module Pgbus
           raise ArgumentError,
                 "Invalid group_mode: #{@group_mode.inspect}. Must be nil, :fifo, or :round_robin"
         end
+        if FairShare.enabled?(config) && (@group_mode || config.group_mode)
+          raise Pgbus::ConfigurationError,
+                "fair_share and group_mode are mutually exclusive — this worker has group_mode " \
+                "#{(@group_mode || config.group_mode).inspect} while config.fair_share is set"
+        end
         @single_active_consumer = single_active_consumer
         @consumer_priority = consumer_priority
         @lifecycle = Lifecycle.new
@@ -165,6 +170,7 @@ module Pgbus
         setup_signals
         start_heartbeat
         resolve_wildcard_queues
+        ensure_fair_indexes
         start_wake_source
         @lifecycle.transition_to!(:running)
         Pgbus.logger.info do
@@ -284,6 +290,8 @@ module Pgbus
             fetch_prioritized(active_queues, qty)
           elsif @group_mode
             fetch_grouped(active_queues, qty)
+          elsif fair_share_enabled?
+            fetch_fair(active_queues, qty)
           elsif active_queues.size == 1
             queue = active_queues.first
             messages = Pgbus.client.read_batch(queue, qty: qty) || []
@@ -391,6 +399,38 @@ module Pgbus
         results
       end
 
+      # Fair share (issue #426): one weighted-interleave read per queue, in
+      # list order with the remaining capacity — strict priority ACROSS queues
+      # (the capsule DSL contract), fair WITHIN each queue. Same loop shape as
+      # fetch_prioritized; under priority_levels the client composes both.
+      def fetch_fair(active_queues, qty)
+        remaining = qty
+        results = []
+
+        active_queues.each do |queue|
+          break if remaining <= 0
+
+          messages = Pgbus.client.read_batch_fair(queue, qty: remaining) || []
+          messages.each { |m| results << [queue, m] }
+          remaining -= messages.size
+        end
+
+        results
+      end
+
+      def fair_share_enabled?
+        FairShare.enabled?(config)
+      end
+
+      # Build the fair index on every queue this worker serves (idempotent,
+      # memoized in the client, CONCURRENTLY so a populated queue is never
+      # write-locked). Queues created later get theirs at creation time.
+      def ensure_fair_indexes
+        return unless fair_share_enabled?
+
+        queues.each { |q| Pgbus.client.ensure_fair_index(q) }
+      end
+
       def priority_enabled?
         config.priority_levels && config.priority_levels > 1
       end
@@ -448,6 +488,7 @@ module Pgbus
         return if @last_wildcard_resolve && (monotonic_now - @last_wildcard_resolve) < WILDCARD_REFRESH_INTERVAL
 
         resolve_wildcard_queues
+        ensure_fair_indexes
       end
 
       # When a "relation does not exist" error occurs, the queue was deleted.

data/lib/pgbus/testing.rb

@@ -54,7 +54,8 @@ module Pgbus
           break unless event
 
           Pgbus::EventBus::Registry.instance.handlers_for(event.routing_key).each do |subscriber|
-            subscriber.handler_class.new.handle(event)
+            # Restore the publisher's Current (issue #431) like the consumer does.
+            Pgbus::CurrentAttributes.restore(event.context) { subscriber.handler_class.new.handle(event) }
           end
 
           @mutex.synchronize { @events.shift }

data/lib/pgbus/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Pgbus
-  VERSION = "0.14.2"
+  VERSION = "0.15.0"
 end

data/lib/pgbus/web/job_context.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Pgbus
+  module Web
+    # Presents the Current attributes persisted in a job payload
+    # (Pgbus::CurrentAttributes::METADATA_KEY) for the dashboard's Context
+    # card: { "Current" => { "tenant" => "gid://app/Tenant/42", ... } }.
+    #
+    # Pure presentation: ActiveJob argument encodings are unwrapped to display
+    # strings (a GlobalID shows its gid, a serialized Symbol/Time its value) —
+    # nothing is constantized or located. Callers pass the payload through
+    # PayloadFilter first (ApplicationHelper#pgbus_parse_message does), so
+    # sensitive values arrive already redacted.
+    module JobContext
+      AJ_GLOBALID = "_aj_globalid"
+      AJ_SERIALIZED = "_aj_serialized"
+      AJ_META_PREFIX = "_aj_"
+
+      module_function
+
+      def from_payload(payload)
+        data = parse(payload)
+        stored = data && data[Pgbus::CurrentAttributes::METADATA_KEY]
+        return nil unless stored.is_a?(Hash) && stored.any?
+
+        stored.to_h do |klass_name, attrs|
+          [klass_name.to_s, present_attrs(attrs)]
+        end
+      end
+
+      def present_attrs(attrs)
+        return {} unless attrs.is_a?(Hash)
+
+        attrs.each_with_object({}) do |(name, value), out|
+          next if name.to_s.start_with?(AJ_META_PREFIX)
+
+          out[name.to_s] = display(value)
+        end
+      end
+
+      # String form of a value for the card. Hashes/arrays that are ActiveJob
+      # encodings collapse to their payload; anything else renders as JSON
+      # (stable across Ruby versions — Hash#inspect changed in 3.4).
+      def display(value)
+        unwrapped = unwrap(value)
+        unwrapped.is_a?(String) ? unwrapped : JSON.generate(unwrapped)
+      end
+
+      def unwrap(value)
+        case value
+        when Hash
+          return value[AJ_GLOBALID] if value.key?(AJ_GLOBALID)
+          return unwrap(value["value"]) if value.key?(AJ_SERIALIZED) && value.key?("value")
+
+          value.each_with_object({}) do |(k, v), out|
+            next if k.to_s.start_with?(AJ_META_PREFIX)
+
+            out[k] = unwrap(v)
+          end
+        when Array
+          value.map { |v| unwrap(v) }
+        else
+          value
+        end
+      end
+
+      def parse(payload)
+        case payload
+        when Hash then payload
+        when String then JSON.parse(payload)
+        end
+      rescue JSON::ParserError
+        nil
+      end
+      private_class_method :parse
+    end
+  end
+end

data/lib/pgbus.rb

@@ -49,6 +49,8 @@ module Pgbus
   # comes back with a msg_id count that doesn't match the number of jobs sent —
   # a data-integrity signal that some jobs may not have been persisted.
   class EnqueueError < Error; end
+  # A Current attribute could not be serialized for job persistence (issue #430).
+  class CurrentAttributesError < Error; end
 
   # Raised by the execution pools when work can't be accepted: the pool is
   # shutting down, or it's momentarily at capacity. Consumer-reachable during

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pgbus
 version: !ruby/object:Gem::Version
-  version: 0.14.2
+  version: 0.15.0
 platform: ruby
 authors:
 - Mikael Henriksson
@@ -257,6 +257,7 @@ files:
 - lib/pgbus.rb
 - lib/pgbus/active_job/adapter.rb
 - lib/pgbus/active_job/batch_id.rb
+- lib/pgbus/active_job/current_attributes.rb
 - lib/pgbus/active_job/executor.rb
 - lib/pgbus/autovacuum_tuning.rb
 - lib/pgbus/batch.rb
@@ -269,6 +270,7 @@ files:
 - lib/pgbus/client.rb
 - lib/pgbus/client/connection_health.rb
 - lib/pgbus/client/ensure_stream_queue.rb
+- lib/pgbus/client/fair_read.rb
 - lib/pgbus/client/notify_stream.rb
 - lib/pgbus/client/read_after.rb
 - lib/pgbus/client/resizable_pool.rb
@@ -277,6 +279,7 @@ files:
 - lib/pgbus/concurrency/semaphore.rb
 - lib/pgbus/configuration.rb
 - lib/pgbus/configuration/capsule_dsl.rb
+- lib/pgbus/current_attributes.rb
 - lib/pgbus/database_tasks_guard.rb
 - lib/pgbus/dedicated_connection.rb
 - lib/pgbus/dedup_cache.rb
@@ -292,6 +295,7 @@ files:
 - lib/pgbus/execution_pools/async_pool.rb
 - lib/pgbus/execution_pools/thread_pool.rb
 - lib/pgbus/failed_event_recorder.rb
+- lib/pgbus/fair_share.rb
 - lib/pgbus/generators/database_target_detector.rb
 - lib/pgbus/generators/migration_detector.rb
 - lib/pgbus/health_probe.rb
@@ -394,6 +398,7 @@ files:
 - lib/pgbus/web/data_source.rb
 - lib/pgbus/web/health_app.rb
 - lib/pgbus/web/health_server.rb
+- lib/pgbus/web/job_context.rb
 - lib/pgbus/web/metrics_serializer.rb
 - lib/pgbus/web/payload_filter.rb
 - lib/pgbus/web/stream_app.rb