roundhouse_ui 0.9.1 → 0.11.0

data/lib/roundhouse_ui/backends/sidekiq.rb

@@ -6,10 +6,9 @@ module RoundhouseUi
     # backend (Solid Queue) can supply duck-typed equivalents without touching
     # controllers or views.
     #
-    # Deliberately thin for now: each method returns the same Sidekiq object the
-    # controllers used before, so this refactor changes no behavior. The contract
-    # (what a "set" / "entry" must respond to) will be firmed up as the Solid
-    # Queue backend is built against it.
+    # Mostly thin — most methods return the Sidekiq object the controllers used before
+    # the port was extracted. The exceptions are the batched reads (queue_summaries)
+    # and the capability list, which the Solid Queue backend firmed up.
     class Sidekiq
       def name = "Sidekiq"
 
@@ -18,7 +17,14 @@ module RoundhouseUi
       # fetcher), so :native_pause is withheld and the "not enforced" warning
       # applies — but Sidekiq Pro ships its own enforced pause, so there it is
       # advertised and the warning drops away.
-      CAPABILITIES = %i[retries dead scheduled busy workers redis capsules].freeze
+      CAPABILITIES = %i[retries dead scheduled busy workers redis capsules cancel history
+                        retry_job enqueue enqueue_now snapshots].freeze
+
+      # Process-local, deliberately: this is a cheap staleness gate, not shared
+      # state. Every process reaches its own conclusion within CONCURRENCY_TTL.
+      class << self
+        attr_accessor :concurrency_cache
+      end
 
       def supports?(capability)
         return RoundhouseUi::Pause.native? if capability == :native_pause
@@ -29,6 +35,80 @@ module RoundhouseUi
       def stats        = ::Sidekiq::Stats.new
       def queues       = ::Sidekiq::Queue.all
       def queue(name)  = ::Sidekiq::Queue.new(name)
+
+      # The jobs waiting on one queue. Sidekiq::Queue is Enumerable and pages
+      # through Redis in chunks, so callers that stop early (browse fills one
+      # page and breaks) never load a large queue into memory.
+      def queue_jobs(name) = ::Sidekiq::Queue.new(name)
+
+      # Sidekiq keeps a per-day counter whether anyone reads it or not, so this
+      # is free history — no storage of ours, no migration, works everywhere.
+      # Oldest first: a chart should not have to reverse it.
+      def history(days)
+        h = ::Sidekiq::Stats::History.new(days)
+        processed = h.processed
+        failed = h.failed
+        processed.keys.sort.map do |date|
+          RoundhouseUi::Day.new(date: date, processed: processed[date].to_i, failed: failed[date].to_i)
+        end
+      end
+
+      # Total worker threads across every reporting process, for the capacity
+      # figure (#36). Deliberately not Stats#workers_size, which counts threads
+      # *busy this instant* — that goes to zero on an idle fleet, and dividing a
+      # required rate by it would claim the fleet has infinite headroom.
+      #
+      # Two things this call has to get right, because it runs on the poll
+      # endpoint that every open tab hits every few seconds:
+      #
+      #   * `ProcessSet.new(false)` skips Sidekiq's dead-process cleanup, which
+      #     issues a SET and can issue an SREM. Roundhouse is an observer; a
+      #     read endpoint must not write to Redis, least of all on a read-only
+      #     install. Sidekiq's own workers prune the set.
+      #   * The result is cached briefly. Thread counts change on deploy or
+      #     scale, not between polls, so re-reading the process set on every
+      #     poll spends round-trips on an answer that has not moved.
+      CONCURRENCY_TTL = 15.0 # seconds
+
+      def concurrency
+        now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+        cached = self.class.concurrency_cache
+        return cached[:value] if cached && now - cached[:at] < CONCURRENCY_TTL
+
+        value = ::Sidekiq::ProcessSet.new(false).sum { |p| p["concurrency"].to_i }
+        self.class.concurrency_cache = { value: value, at: now }
+        value
+      rescue StandardError
+        nil
+      end
+
+      # Depth and latency for every queue in two Redis round-trips, regardless
+      # of how many queues there are.
+      #
+      # Sidekiq::Queue#size and #latency each issue their own command, so the
+      # obvious `queues.map { ... }` costs one SSCAN plus a LLEN and an LRANGE
+      # per queue — on an app with sixty queues, a couple of hundred round-trips
+      # to render one page. Sidekiq's own Queues#lengths pipelines the LLENs for
+      # the same reason; this pipelines the LRANGEs alongside them.
+      #
+      # Queues with no ready work are kept, at size 0: an operator has to be able
+      # to pause or snapshot a queue that happens to be empty right now.
+      def queue_summaries
+        # Sidekiq 8 ships an equivalent (Stats#queue_summaries) but 6.5 and 7 do
+        # not, and its Data objects carry a `paused` field ours cannot. Branching
+        # on it would hand callers a different shape depending on which Sidekiq
+        # the host runs, so this stays one implementation for every supported
+        # version — the reason the gem is portable in the first place.
+        names = ::Sidekiq.redis { |c| c.call("SMEMBERS", "queues") }.sort
+        return [] if names.empty?
+
+        sizes, oldest = batch_read(names).each_slice(names.size).to_a
+        names.each_with_index.map do |name, i|
+          RoundhouseUi::QueueSummary.new(name: name, size: sizes[i].to_i,
+                                        latency: latency_from(oldest[i]))
+        end
+      end
+
       def retry_set    = ::Sidekiq::RetrySet.new
       def dead_set     = ::Sidekiq::DeadSet.new
       def scheduled_set = ::Sidekiq::ScheduledSet.new
@@ -60,6 +140,45 @@ module RoundhouseUi
         end
       end
 
+      private
+
+      # Everything goes through conn.call, whose splat signature is identical on
+      # redis-rb (Sidekiq 6.x) and redis-client (7+) — the same reason the rest of
+      # this gem avoids the high-level command methods. Pipelining is
+      # feature-detected the way DurationCollector does it.
+      def batch_read(names)
+        commands = names.map { |n| [ "LLEN", "queue:#{n}" ] } +
+                   names.map { |n| [ "LRANGE", "queue:#{n}", -1, -1 ] }
+        ::Sidekiq.redis do |conn|
+          if conn.respond_to?(:pipelined)
+            conn.pipelined { |pipe| commands.map { |c| pipe.call(*c) } }
+          else
+            commands.map { |c| conn.call(*c) }
+          end
+        end
+      end
+
+      # The tail entry of a list is its oldest.
+      #
+      # Two timestamp formats exist and the difference is three orders of
+      # magnitude: Sidekiq 8 writes enqueued_at as integer epoch milliseconds,
+      # 6.5 and 7 write float epoch seconds. Treating one as the other does not
+      # produce a slightly wrong latency, it produces nonsense. This mirrors the
+      # dispatch in Sidekiq 8's own ApiUtils#calculate_latency.
+      #
+      # A payload that will not parse must not take down the page.
+      def latency_from(entry)
+        payload = entry.is_a?(Array) ? entry.first : entry
+        return 0.0 unless payload
+
+        at = RoundhouseUi.enqueued_at(::Sidekiq.load_json(payload))
+        at ? (Time.now - at).to_f : 0.0
+      rescue StandardError
+        0.0
+      end
+
+      public
+
       # --- pause (via the opt-in Roundhouse fetcher; not native) ---
       def paused_queues     = RoundhouseUi::Pause.paused_set
       def pause(name)       = RoundhouseUi::Pause.pause!(name)

data/lib/roundhouse_ui/backends/solid_queue.rb

@@ -14,7 +14,10 @@ module RoundhouseUi
 
       # No retries/redis/capsules/workers (deferred); pause is native (no fetcher
       # warning). dead/scheduled/busy are supported.
-      CAPABILITIES = %i[dead scheduled busy native_pause].freeze
+      # retry_job: SolidQueue::FailedExecution#retry exists, so dead-set retry works.
+      # Absent on purpose: :enqueue and :enqueue_now (push and add_to_queue are not
+      # implemented), :snapshots and :redis (both need Redis).
+      CAPABILITIES = %i[dead scheduled busy native_pause retry_job].freeze
       def supports?(capability) = CAPABILITIES.include?(capability)
 
       def stats
@@ -32,6 +35,44 @@ module RoundhouseUi
       def queues = ::SolidQueue::Queue.all
       def queue(name) = ::SolidQueue::Queue.new(name)
 
+      # The jobs waiting on one queue, in the same JobSet shape the other sets
+      # use — so the queue detail page reuses the shared browse/search path
+      # rather than growing a second one.
+      # Solid Queue's processes table records threads per worker the same way.
+      def concurrency
+        sq(:Process).where(kind: "Worker").sum { |p| p.metadata&.dig("thread_pool_size").to_i }
+      rescue StandardError
+        nil
+      end
+
+      def queue_jobs(name) = JobSet.new(sq(:ReadyExecution).queued_as(name))
+
+      # Depth and latency for every queue in two queries, not two per queue.
+      #
+      # SolidQueue::Queue#size and #latency each run their own COUNT and MIN, so
+      # rendering the Queues page over sixty queues meant well over a hundred
+      # round-trips to the queue database. One GROUP BY answers both for every
+      # queue at once.
+      #
+      # Queue.all is still consulted, because it lists queues by distinct job
+      # name — including ones with nothing ready right now — and an operator has
+      # to be able to pause or clear a queue that happens to be empty.
+      def queue_summaries
+        counts = sq(:ReadyExecution).group(:queue_name)
+                                    .pluck(Arel.sql("queue_name, COUNT(*), MIN(created_at)"))
+                                    .to_h { |name, size, oldest| [ name, [ size, oldest ] ] }
+
+        names = (::SolidQueue::Queue.all.map(&:name) | counts.keys).sort
+        now = Time.current
+        names.map do |name|
+          size, oldest = counts[name]
+          RoundhouseUi::QueueSummary.new(
+            name: name, size: size.to_i,
+            latency: oldest ? (now - oldest).to_i : 0
+          )
+        end
+      end
+
       # No retry set in Solid Queue — surfaced via supports?(:retries) => false.
       def retry_set = EMPTY_SET
 
@@ -105,8 +146,18 @@ module RoundhouseUi
           }
         end
 
-        def retry = @execution.retry if @execution.respond_to?(:retry)
-        def delete = @execution.destroy
+        # Guarded at CALL time. As a trailing `if` on an endless def this was evaluated
+        # when the class body loaded, against a nil class-level ivar — so the method
+        # was never defined and every Retry raised NoMethodError.
+        def retry
+          raise NotImplementedError, "this entry cannot be retried" unless @execution.respond_to?(:retry)
+
+          @execution.retry
+        end
+        # discard removes the job row along with the execution and takes the
+        # row lock while it does; destroy drops only the execution, leaving an
+        # orphaned solid_queue_jobs row that nothing will ever pick up again.
+        def delete = @execution.respond_to?(:discard) ? @execution.discard : @execution.destroy
 
         private
 

data/lib/roundhouse_ui/demo.rb

@@ -0,0 +1,130 @@
+# frozen_string_literal: true
+
+require "digest"
+require "securerandom"
+
+module RoundhouseUi
+  # Workers that do real work and really fail, so a console has something to show.
+  #
+  # NOT loaded by `require "roundhouse_ui"`. Ask for it explicitly, in an
+  # initializer you would not ship:
+  #
+  #   require "roundhouse_ui/demo" if Rails.env.development?
+  #
+  # Then drive it with `rake roundhouse_ui:demo:load[10]`.
+  #
+  # Each class refuses to run outside development or test. A worker that sleeps
+  # for twelve seconds and then raises is a fine thing to have in a demo and a
+  # bad thing to have reachable in production, and "nobody will enqueue it" is
+  # not a control.
+  module Demo
+    class Base
+      include Sidekiq::Job
+
+      # Three attempts, so a failing job reaches the dead set inside a short
+      # demo instead of backing off for hours.
+      sidekiq_options retry: 3
+
+      # Subclasses override. Milliseconds of work, and the chance of failing.
+      DURATION = (100..400)
+      FAILURE_RATE = 0.0
+      ERRORS = [ [ RuntimeError, "something went wrong" ] ].freeze
+
+      def perform(payload = {})
+        unless %w[development test].include?(ENV["RAILS_ENV"] || Rails.env.to_s)
+          raise "RoundhouseUi::Demo workers do not run outside development"
+        end
+
+        work_for(self.class::DURATION)
+        maybe_fail(payload)
+      end
+
+      private
+
+      # Real elapsed time, and real CPU — a job that only sleeps shows up on the
+      # Busy page but never in a flame graph, and the point is to look like work.
+      def work_for(range)
+        budget = rand(range) / 1000.0
+        deadline = now + budget
+        digest = SecureRandom.hex(8)
+        while now < deadline
+          200.times { digest = Digest::SHA256.hexdigest(digest) }
+          sleep 0.01
+        end
+        digest
+      end
+
+      def maybe_fail(payload)
+        return if rand >= self.class::FAILURE_RATE
+
+        klass, message = self.class::ERRORS.sample
+        raise klass, "#{message} (account #{payload['account_id'] || '—'})"
+      end
+
+      def now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    end
+
+    # A payment path: quick, and fails often enough to group.
+    class CheckoutWorker < Base
+      OWNER = :core
+      QUEUE = "stripe"
+      DURATION = (200..1_200)
+      FAILURE_RATE = 0.18
+      ERRORS = [
+        [ Timeout::Error, "Stripe API timed out after 8s" ],
+        [ ArgumentError, "no such price" ]
+      ].freeze
+    end
+
+    # Mail: fast and mostly reliable, so the throughput line has volume.
+    class DigestMailerWorker < Base
+      OWNER = :growth
+      QUEUE = "invite_mailer"
+      DURATION = (60..300)
+      FAILURE_RATE = 0.04
+      ERRORS = [ [ IOError, "421 4.7.0 Try again later" ] ].freeze
+    end
+
+    # Search reindex: slow enough to sit on the Busy page while you look at it.
+    class ReindexWorker < Base
+      OWNER = :platform
+      QUEUE = "default"
+      DURATION = (1_500..5_000)
+      FAILURE_RATE = 0.09
+      ERRORS = [ [ IOError, "connection reset by peer" ] ].freeze
+    end
+
+    # The long one: guaranteed to be mid-flight whenever you open Busy.
+    class TranscodeWorker < Base
+      OWNER = :platform
+      QUEUE = "uploaded_files"
+      DURATION = (6_000..20_000)
+      FAILURE_RATE = 0.07
+      ERRORS = [ [ RuntimeError, "ffmpeg exited with status 1" ] ].freeze
+    end
+
+    # The flaky one: the group that dominates the Errors page.
+    class EmbeddingWorker < Base
+      OWNER = :ai
+      QUEUE = "ai"
+      DURATION = (400..2_500)
+      FAILURE_RATE = 0.24
+      ERRORS = [
+        [ Timeout::Error, "embedding endpoint timed out" ],
+        [ RuntimeError, "429 rate limited" ],
+        [ KeyError, "key not found: :embedding" ]
+      ].freeze
+    end
+
+    class GamificationWorker < Base
+      OWNER = :training
+      QUEUE = "gamification"
+      DURATION = (100..600)
+      FAILURE_RATE = 0.06
+      ERRORS = [ [ RuntimeError, "badge already awarded" ] ].freeze
+    end
+
+    ALL = [ CheckoutWorker, DigestMailerWorker, ReindexWorker,
+            TranscodeWorker, EmbeddingWorker, GamificationWorker ].freeze
+  end
+end

data/lib/roundhouse_ui/error_groups.rb

@@ -14,6 +14,7 @@ module RoundhouseUi
     def initialize(query: nil, limit: DEFAULT_SCAN_LIMIT)
       @query = query.to_s.strip
       @limit = limit
+      @tag_cache = {}
     end
 
     def call
@@ -34,12 +35,22 @@ module RoundhouseUi
       end
 
       list = groups.values.sort_by { |g| -g[:count] }
-      list = list.select { |g| "#{g[:klass]} #{g[:error]}".downcase.include?(@query.downcase) } if @query.present?
+      list = list.select { |g| matches?(g) } if @query.present?
       Result.new(groups: list, scanned: scanned, truncated: truncated)
     end
 
     private
 
+    # Tag values are part of the haystack here for the same reason they are on
+    # the job sets: typing a squad name should find that squad's failures. A
+    # group's entries all share a class, so the tag is constant for the group
+    # and resolves once per group rather than per scanned entry.
+    def matches?(group)
+      haystack = [ group[:klass], group[:error] ]
+      haystack.concat(Tags.for(klass: group[:klass], item: {}, cache: @tag_cache).values) if RoundhouseUi.job_tags
+      haystack.join(" ").downcase.include?(@query.downcase)
+    end
+
     # Sidekiq's native sets, plus the sidekiq-failures `failed` set when opted in
     # and loaded. Its FailureSet is a Sidekiq::JobSet, so it iterates like the rest.
     def sources
@@ -51,9 +62,19 @@ module RoundhouseUi
     end
 
     def record(groups, source, entry)
-      error = entry.item["error_class"] || "UnknownError"
-      group = (groups["#{entry.klass}|#{error}"] ||= {
-        klass: entry.klass, error: error, count: 0, last_at: nil, queues: [], sources: []
+      # The bucket key and the stored label must unwrap together — split them and
+      # rows merge under one name while a different one renders. Everything
+      # downstream reads group[:klass]: the label, the search haystack, the tag
+      # lookup, the Datadog link, and the dashboard's errors_path(q:) href.
+      #
+      # item is bound once because Solid Queue's Entry#item builds a fresh Hash
+      # and touches the execution on every call, and this runs up to
+      # DEFAULT_SCAN_LIMIT times per render.
+      item  = entry.item
+      klass = RoundhouseUi.unwrapped_class(entry.klass, item)
+      error = item["error_class"] || "UnknownError"
+      group = (groups["#{klass}|#{error}"] ||= {
+        klass: klass, error: error, count: 0, last_at: nil, queues: [], sources: []
       })
       group[:count]  += 1
       group[:queues] |= [ entry.queue ]