pgbus 0.17.1 → 0.17.2

data/lib/pgbus/web/data_source/list_counts.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+module Pgbus
+  module Web
+    class DataSource
+      # Row totals for the dashboard's paged lists (issue #496). list_count
+      # stops at COUNT_CAP + 1 rows, so a table with millions of rows costs no
+      # more than one with ten thousand; past the cap the pager shows "10,000+".
+      # The same bound JobList uses for its state tabs. batches_count and
+      # recurring_tasks_count are exact (summary_stats' badge reads the latter);
+      # they predate the pagers and live here to keep data_source.rb small.
+      module ListCounts
+        COUNT_CAP = 10_000
+
+        Count = Data.define(:total, :capped) do
+          def capped? = capped
+        end
+
+        EMPTY_COUNT = Count.new(total: 0, capped: false)
+
+        # Semaphores and parked jobs, FULL OUTER JOINed: a key can have parked
+        # rows and no semaphore (its holder died and the sweep removed the row)
+        # or a semaphore and nothing parked. Shared with concurrency_keys.
+        CONCURRENCY_KEYS_FROM = <<~SQL
+          FROM pgbus_semaphores s
+          FULL OUTER JOIN (
+            SELECT concurrency_key, COUNT(*) AS parked_count, MIN(created_at) AS oldest_parked_at
+            FROM pgbus_blocked_executions
+            GROUP BY concurrency_key
+          ) b ON s.key = b.concurrency_key
+        SQL
+
+        LIST_MODELS = {
+          batches: -> { BatchEntry },
+          job_locks: -> { UniquenessKey },
+          outbox: -> { OutboxEntry },
+          processed_events: -> { ProcessedEvent },
+          recurring_tasks: -> { RecurringTask }
+        }.freeze
+
+        # list – :batches, :job_locks, :concurrency_keys, :outbox, :processed_events or :recurring_tasks.
+        def list_count(list)
+          raise ArgumentError, "unknown list: #{list.inspect}" unless list == :concurrency_keys || LIST_MODELS.key?(list)
+
+          bounded_count(list)
+        end
+
+        def batches_count
+          BatchEntry.count
+        rescue StandardError => e
+          Pgbus.logger.debug { "[Pgbus::Web] Error counting batches: #{e.message}" }
+          0
+        end
+
+        def recurring_tasks_count
+          RecurringTask.count
+        rescue StandardError => e
+          Pgbus.logger.debug { "[Pgbus::Web] Error counting recurring tasks: #{e.message}" }
+          0
+        end
+
+        private
+
+        def bounded_count(list)
+          rows = list == :concurrency_keys ? concurrency_keys_probe : LIST_MODELS.fetch(list).call.limit(COUNT_CAP + 1).count
+          Count.new(total: [rows.to_i, COUNT_CAP].min, capped: rows.to_i > COUNT_CAP)
+        rescue StandardError => e
+          Pgbus.logger.debug { "[Pgbus::Web] Error counting #{list}: #{e.message}" }
+          EMPTY_COUNT
+        end
+
+        def concurrency_keys_probe
+          connection.select_value(<<~SQL, "Pgbus Concurrency Keys Count")
+            SELECT COUNT(*) FROM (SELECT 1 #{CONCURRENCY_KEYS_FROM} LIMIT #{COUNT_CAP + 1}) c
+          SQL
+        end
+      end
+    end
+  end
+end

data/lib/pgbus/web/data_source/process_list.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module Pgbus
+  module Web
+    class DataSource
+      # The Processes page's rows and the context ProcessState needs to put
+      # them in words (issue #493). Every status in one result set is derived
+      # against one ProcessState::Thresholds, so one now.
+      module ProcessList
+        # thresholds: pass the ones the caller puts the rows in words with, so
+        # badge and sentence share one now.
+        def processes(thresholds: process_thresholds)
+          process_rows.map { |r| format_process(r, thresholds) }
+        rescue StandardError => e
+          Pgbus.logger.debug { "[Pgbus::Web] Error fetching processes: #{e.message}" }
+          []
+        end
+
+        # The dashboard host's config: the stall limits the page states are
+        # this host's stall_threshold, as the badge always was.
+        def process_thresholds(now: Time.now)
+          config = Pgbus.configuration
+          ProcessState::Thresholds.new(
+            now: now, alive_threshold: Process::Heartbeat::ALIVE_THRESHOLD, stall_threshold: config.stall_threshold,
+            dispatch_interval: config.dispatch_interval, recurring_schedule_interval: config.recurring_schedule_interval,
+            polling_interval: config.polling_interval, notify_fallback_poll: Process::Consumer::NOTIFY_FALLBACK_POLL_SECONDS,
+            notify_wakeup: config.worker_notify_wakeup?, reap_interval: Process::Dispatcher::REAP_INTERVAL
+          )
+        end
+
+        def process_context(rows:, thresholds: process_thresholds)
+          ProcessState.context_for(rows, thresholds)
+        end
+
+        private
+
+        # The raw rows, memoized per instance like queues_with_metrics (cleared
+        # by reset_cache!): the home page reads them for the stats, the signals,
+        # the health verdict, the queue lines and the panel (issue #492). Only
+        # the rows: each caller derives statuses against its own thresholds.
+        def process_rows
+          @process_rows ||= connection.select_all("SELECT * FROM pgbus_processes ORDER BY kind, created_at").to_a
+        end
+
+        def format_process(row, thresholds)
+          heartbeat = JobState.time(row["last_heartbeat_at"])
+          metadata = row["metadata"].is_a?(String) ? JSON.parse(row["metadata"]) : row["metadata"]
+          status = ProcessState.derive(kind: row["kind"], metadata: metadata, heartbeat_at: heartbeat,
+                                       thresholds: thresholds)
+          started_at = metadata.is_a?(Hash) && metadata["started_at"]
+
+          {
+            id: row["id"].to_i,
+            kind: row["kind"],
+            hostname: row["hostname"],
+            pid: row["pid"].to_i,
+            metadata: metadata,
+            last_heartbeat_at: heartbeat,
+            healthy: status == :healthy,
+            status: status,
+            created_at: row["created_at"],
+            supervisor_pid: metadata.is_a?(Hash) ? metadata["supervisor_pid"] : nil,
+            started_at: started_at ? Time.at(started_at.to_f) : nil
+          }
+        end
+      end
+    end
+  end
+end

data/lib/pgbus/web/data_source/queue_summary.rb

@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+
+module Pgbus
+  module Web
+    class DataSource
+      # The facts behind a queue page's summary in words (issue #491), turned
+      # into sentences by Pgbus::Web::QueueSummary. Both methods take the
+      # physical queue name the page is for.
+      module QueueSummary
+        RUNNING = { paused: false, reason: nil, paused_at: nil, resumes_at: nil, trip_count: 0 }.freeze
+
+        # Pause state is kept per logical queue, so every priority level and
+        # the bare table share it.
+        def queue_pause_state(name)
+          pause_state_of(QueueState.find_by(queue_name: logical_queue_name(name)))
+        rescue StandardError => e
+          Pgbus.logger.error { "[Pgbus::Web] Error reading pause state for #{name}: #{e.class}: #{e.message}" }
+          RUNNING
+        end
+
+        # { physical name => pause state } for many queues in one query (the
+        # home page's queues table, issue #492). Raises: the caller degrades.
+        def queue_pause_states(names)
+          states = QueueState.where(queue_name: names.map { |n| logical_queue_name(n) }.uniq).index_by(&:queue_name)
+          names.to_h { |name| [name, pause_state_of(states[logical_queue_name(name)])] }
+        end
+
+        # healthy: and streams: let a caller describing many queues load the
+        # process rows and the stream registry once (issue #492).
+        def queue_drainers(name, healthy: processes.select { |p| p[:healthy] }, streams: stream_queue_names)
+          table = name.to_s.delete_suffix(Pgbus::DEAD_LETTER_SUFFIX)
+          level = priority_level_of(table)
+          logical = level ? logical_queue_name(table) : table.delete_prefix("#{Pgbus.configuration.queue_prefix}_")
+          capsules, wildcard = draining_capsules(name)
+          {
+            logical: logical, capsules: capsules, wildcard: wildcard,
+            handler: handler_queue_physical_names.include?(name), stream: streams.include?(name),
+            live_workers: healthy.count { |p| p[:kind].to_s == "worker" && worker_drains?(p, logical) },
+            live_consumers: healthy.count { |p| p[:kind].to_s == "consumer" && consumer_drains?(p, name) },
+            siblings: level ? priority_siblings(name, logical) : [], priority_level: level
+          }
+        end
+
+        private
+
+        def pause_state_of(state)
+          return RUNNING unless state&.paused
+
+          { paused: true, reason: state.paused_reason, paused_at: state.paused_at,
+            resumes_at: state.circuit_breaker_resume_at, trip_count: state.circuit_breaker_trip_count.to_i }
+        end
+
+        # The level of a priority sub-table, or nil. `_pN` is legal in an
+        # ordinary queue name, so only a table the queue strategy creates for
+        # the stripped logical name counts.
+        def priority_level_of(table)
+          level = table[Pgbus::Web::QueueSummary::PRIORITY_LEVEL, 1]
+          return unless level && @client.physical_queue_names(logical_queue_name(table)).include?(table)
+
+          level.to_i
+        rescue StandardError => e
+          Pgbus.logger.debug { "[Pgbus::Web] Error resolving priority level of #{table}: #{e.class}: #{e.message}" }
+          nil
+        end
+
+        # [names of the capsules whose queues expand to this table, wildcard?].
+        # An anonymous capsule goes by its first queue. Fails open to a
+        # wildcard, like drained_queue_names: never a false "nothing drains it".
+        def draining_capsules(name)
+          capsules = Array(Pgbus.configuration.workers)
+          wildcard = capsules.any? { |c| capsule_queue_list(c).include?("*") }
+          names = capsules.filter_map do |c|
+            queues = capsule_queue_list(c) - ["*"]
+            next unless queues.any? { |q| @client.physical_queue_names(q).include?(name) }
+
+            (c[:name] || c["name"] || queues.first).to_s
+          end
+          [names, wildcard]
+        rescue StandardError => e
+          Pgbus.logger.error { "[Pgbus::Web] Error resolving capsules for #{name}: #{e.class}: #{e.message}" }
+          [[], true]
+        end
+
+        def capsule_queue_list(capsule)
+          Array(capsule[:queues] || capsule["queues"]).map(&:to_s)
+        end
+
+        # A heartbeat lists queue names as configured (Array or comma-separated
+        # String); tables are named after their normalized form ("bulk-imports"
+        # → pgbus_bulk_imports), so compare that.
+        def worker_drains?(process, logical)
+          queues = Array((process[:metadata] || {})["queues"]).flat_map { |q| q.to_s.split(",") }.map(&:strip)
+          queues.include?("*") || queues.any? { |q| normalized_queue(q) == logical }
+        end
+
+        # A consumer drains a handler queue when one of its topic filters
+        # overlaps a subscription routed to it (the rule the consumer itself
+        # uses to pick queues). A heartbeat without topics counts.
+        def consumer_drains?(process, name)
+          topics = Array((process[:metadata] || {})["topics"])
+          return true if topics.empty?
+
+          patterns = registered_subscribers.select { |s| s[:physical_queue_name] == name }.map { |s| s[:pattern] }
+          registry = EventBus::Registry.instance
+          topics.any? { |t| patterns.any? { |pattern| registry.pattern_overlaps?(t.to_s, pattern) } }
+        end
+
+        def normalized_queue(queue)
+          Pgbus.configuration.queue_name(queue).delete_prefix("#{Pgbus.configuration.queue_prefix}_")
+        rescue ArgumentError
+          queue
+        end
+
+        def priority_siblings(name, logical)
+          queues_with_metrics.map { |q| q[:name] }.select do |n|
+            n != name && n.match?(Pgbus::Web::QueueSummary::PRIORITY_LEVEL) && logical_queue_name(n) == logical
+          end
+        end
+      end
+    end
+  end
+end