deployangel 0.1.28 → 0.1.29

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fde8310f8129827b4b759369146c2346542a776ab13ac91fe7001617e93a7ebb
-  data.tar.gz: 15a216435704ea45fce03e0a1433d615dc28f6c07fdd6e33c90bebed23361779
+  metadata.gz: e62d94a1e10efd0431f1cfad4f7aad29f954481919a609f8112e5f517291eeb1
+  data.tar.gz: 408bcc12ae37d4f93e74e5e31ad6a8b921b30481cedf81f84055c07c62677af0
 SHA512:
-  metadata.gz: e6e7ae24a04e3999af5865e2cf5be14054cdfc92b5644b1a4f5d8bd56c407faf874b122db23508bbcb3b268e461a89d68a16a3a235e166cf457e10e063a4c3f9
-  data.tar.gz: 3e2bb9e11c4d8b35134fdb9088beabcaf0ba5b99b484c9e4b0e2dfc3b710aa544507b55f6a3a222c0446d4b6edbec0f935f0049f03e2039cc19e253767560530
+  metadata.gz: e26a0379ee22236add3b1742a3ef174ebd3f39def909d3e230b4fadce49b1f2a3f62daa41c2aa02e05e5c12e59a78b9d6c94b7436bca9b986c56b20053f49783
+  data.tar.gz: 9d7f93900e16f63f1ba593063a31b0d03eb3f218877c65ea6c5a9b13153bd9a3520c8e5555370f8d94e53d89741f0a709143483b8ffa196e0e2e23f9a42a4b31

data/CHANGELOG.md

@@ -1,5 +1,18 @@
 # Changelog
 
+## 0.1.29 (2026-10-16)
+
+- Reports stuck work, so DeployAngel can fail a release in which a job hangs
+  or the queue keeps growing while every job still succeeds. Each minute's
+  payload says how many jobs are still running and how long the oldest of
+  each class has been running (ActiveJob on any adapter, and native Sidekiq
+  jobs). Sidekiq and Solid Queue worker processes also read the queue backlog,
+  the jobs ready to run and waiting, once a minute from the reporter thread:
+  one pipelined Redis call for Sidekiq, one `COUNT` of
+  `solid_queue_ready_executions` for Solid Queue. Web processes don't read
+  it. Needs a DeployAngel cloud that understands the new fields; older ones
+  ignore them.
+
 ## 0.1.28 (2026-10-12)
 
 - Clearer setup for coding agents, from a first-day rehearsal: `setup` says

data/README.md

@@ -88,6 +88,10 @@ the dashboard says so.
   queue latency, per job class. Works with every ActiveJob adapter (Solid
   Queue, Sidekiq, GoodJob, ...) and with native Sidekiq jobs through a server
   middleware. Failures that `retry_on` or `discard_on` handle still count.
+- Jobs still running at the end of each minute, and how long the oldest of
+  each class has been running, so a job that hangs is noticed. Sidekiq and
+  Solid Queue worker processes also send the queue backlog (how many jobs are
+  waiting), read once a minute off the request path.
 - Release identity, runtime versions, and a per-process instance ID.
 
 - Exceptions: a stable fingerprint, the exception class, the first line of the

data/lib/deployangel/agent.rb

@@ -86,6 +86,25 @@ module DeployAngel
       warn_once(:record_job, "DeployAngel failed to record a job: #{e.class}: #{e.message}")
     end
 
+    # Marks a job as running until #job_finished, so one that hangs is still
+    # reported. Returns the token to pass to it.
+    def job_started(job_class)
+      return unless @active
+
+      after_fork! if Process.pid != @pid
+      start_reporter
+      @running_jobs.start(job_class)
+    rescue StandardError => e
+      warn_once(:job_started, "DeployAngel failed to record a running job: #{e.class}: #{e.message}")
+      nil
+    end
+
+    def job_finished(token)
+      @running_jobs.finish(token) if token
+    rescue StandardError => e
+      warn_once(:job_finished, "DeployAngel failed to record a finished job: #{e.class}: #{e.message}")
+    end
+
     def record_discard(job_class:)
       return unless @active
 
@@ -135,7 +154,9 @@ module DeployAngel
     def flush(include_current: false)
       return 0 unless @active
 
-      @aggregator.drain(include_current: include_current, max_periods: config.max_queued_payloads).each do |period|
+      periods = @aggregator.drain(include_current: include_current, max_periods: config.max_queued_payloads)
+      add_stuck_work(periods.last, include_current) if periods.any?
+      periods.each do |period|
         @buffer.push(Transport.encode(Protocol.telemetry(period, instance: @instance, release: release,
           runtime: @runtime, capabilities: capabilities)))
       end
@@ -198,10 +219,24 @@ module DeployAngel
     end
 
     private
+      # What's still running, on the latest period only: as of its end, or
+      # now for the in-progress one at shutdown. The backlog is read here, in
+      # the reporter thread, but not at shutdown, which has seconds to send.
+      def add_stuck_work(period, include_current)
+        ended_at = [ period.started_at + Aggregator::PERIOD_SECONDS, @clock.call ].min
+        @running_jobs.snapshot(ended_at).each do |job_class, (count, oldest_ms)|
+          period.record_running(job_class, count, oldest_ms, config.max_routes)
+        end
+        period.queue_backlog = QueueBacklog.read unless include_current
+      rescue StandardError => e
+        warn_once(:stuck_work, "DeployAngel failed to read running jobs: #{e.class}: #{e.message}")
+      end
+
       def reset_process_state
         @pid = Process.pid
         @instance = Instance.new(env: @env, now: @clock.call)
         @aggregator = Aggregator.new(max_routes: config.max_routes, clock: @clock)
+        @running_jobs = RunningJobs.new(clock: @clock)
         @buffer = Buffer.new(config.max_queued_payloads)
         @paused_until = nil
         @metadata_sent = false

data/lib/deployangel/core/aggregator.rb

@@ -19,9 +19,16 @@ module DeployAngel
       end
     end
 
-    JobStats = Struct.new(:processed, :failed, :discarded, :duration, :queue_latency) do
+    # running and oldest_running_ms: jobs still running as the period is
+    # sent, and how long the oldest had been running at its end.
+    JobStats = Struct.new(:processed, :failed, :discarded, :duration, :queue_latency, :running, :oldest_running_ms) do
       def self.empty
-        new(0, 0, 0, Histogram.new, Histogram.new)
+        new(0, 0, 0, Histogram.new, Histogram.new, 0, nil)
+      end
+
+      def add_running(count, oldest_ms)
+        self.running += count
+        self.oldest_running_ms = [ oldest_running_ms, oldest_ms ].compact.max
       end
 
       def record(failed, discarded, duration_ms, queue_latency_ms)
@@ -54,6 +61,8 @@ module DeployAngel
     class Period
       attr_reader :started_at, :requests, :status_counts, :unhandled_exceptions, :histogram, :routes,
         :jobs, :job_classes, :exceptions, :exceptions_truncated, :checkpoints
+      # Jobs waiting in the job backend's queues, read as the period is sent.
+      attr_accessor :queue_backlog
 
       def initialize(started_at)
         @started_at = started_at
@@ -97,6 +106,14 @@ module DeployAngel
         (@job_classes[key] ||= JobStats.empty).record(failed, discarded, duration_ms, queue_latency_ms)
       end
 
+      # A class with jobs running and none finished is listed too, so a job
+      # that hangs is reported.
+      def record_running(job_class, count, oldest_ms, max_classes)
+        @jobs.add_running(count, oldest_ms)
+        key = @job_classes.key?(job_class) || @job_classes.size < max_classes - 1 ? job_class : OTHER_ROUTE
+        (@job_classes[key] ||= JobStats.empty).add_running(count, oldest_ms)
+      end
+
       def record_discard(job_class, max_classes)
         @jobs.record_discard
         key = @job_classes.key?(job_class) || @job_classes.size < max_classes - 1 ? job_class : OTHER_ROUTE

data/lib/deployangel/core/protocol.rb

@@ -32,7 +32,7 @@ module DeployAngel
         end,
         "exceptions" => period.exceptions.values.map { |e| e.merge("sources" => e["sources"].to_h).compact },
         "exceptions_truncated" => period.exceptions_truncated,
-        "jobs" => job_stats(period.jobs),
+        "jobs" => job_stats(period.jobs).merge({ "queue_backlog" => period.queue_backlog }.compact),
         "job_classes" => period.job_classes.map { |key, stats| { "key" => key }.merge(job_stats(stats)) },
         # http and job are always sent, 0 when none, so the cloud can tell
         # this agent from one that doesn't say where checkpoints happened.
@@ -42,14 +42,18 @@ module DeployAngel
       }
     end
 
+    # running and oldest_running_ms only when jobs were running.
     def job_stats(stats)
-      {
+      fields = {
         "processed" => stats.processed,
         "failed" => stats.failed,
         "discarded" => stats.discarded,
         "duration_histogram" => stats.duration.to_protocol,
         "queue_latency_histogram" => stats.queue_latency.to_protocol
       }
+      return fields unless stats.running.positive?
+
+      fields.merge("running" => stats.running, "oldest_running_ms" => stats.oldest_running_ms)
     end
 
     def runtime(framework: nil, framework_version: nil)

data/lib/deployangel/core/running_jobs.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module DeployAngel
+  # The jobs this process is running now, so each payload can say how long
+  # the oldest of each class has been running: a job that hangs never
+  # finishes, so it never shows up in the counts. Starting and finishing is
+  # a hash update under a mutex.
+  class RunningJobs
+    # More than any process runs at once; past it, new jobs go untracked
+    # rather than growing memory.
+    MAX = 1_000
+
+    def initialize(clock:)
+      @clock = clock
+      @jobs = {}.compare_by_identity
+      @mutex = Mutex.new
+    end
+
+    # Returns the token to pass to #finish.
+    def start(job_class)
+      token = Object.new
+      @mutex.synchronize { @jobs[token] = [ job_class.to_s, @clock.call ] if @jobs.size < MAX }
+      token
+    end
+
+    def finish(token)
+      @mutex.synchronize { @jobs.delete(token) }
+    end
+
+    # { job_class => [running, oldest_running_ms] } for jobs that started by
+    # at and are still running.
+    def snapshot(at)
+      @mutex.synchronize do
+        @jobs.each_value.with_object({}) do |(job_class, started_at), running|
+          next if started_at > at
+
+          age_ms = ((at - started_at) * 1000).round
+          count, oldest = running[job_class]
+          running[job_class] = [ count.to_i + 1, [ oldest.to_i, age_ms ].max ]
+        end
+      end
+    end
+  end
+end

data/lib/deployangel/queue_backlog.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module DeployAngel
+  # Reads how many jobs are ready to run and waiting in the job backend's
+  # queues. Only job worker processes read it, once a minute from the
+  # reporter thread as they send telemetry, never in a request or job. Every
+  # worker sees the same queues, and the cloud keeps the largest reading.
+  module QueueBacklog
+    module_function
+
+    # Solid Queue runs each worker in its own process, forked by the
+    # supervisor, so it's marked as each one starts.
+    def install
+      return if @installed || !defined?(::SolidQueue) || !::SolidQueue.respond_to?(:on_worker_start)
+
+      @installed = true
+      ::SolidQueue.on_worker_start { worker!(:solid_queue) }
+    rescue StandardError
+      nil
+    end
+
+    # Called where the backend says this process works its queues: Sidekiq's
+    # server configuration, or a Solid Queue worker starting.
+    def worker!(backend)
+      @backend = backend
+    end
+
+    def backend
+      @backend
+    end
+
+    # Jobs waiting, or nil when this process doesn't work a queue it can read.
+    def read
+      case @backend
+      when :sidekiq then sidekiq
+      when :solid_queue then solid_queue
+      end
+    rescue StandardError, LoadError
+      nil
+    end
+
+    # One pipelined call for every queue's length.
+    def sidekiq
+      require "sidekiq/api"
+      ::Sidekiq::Stats.new.enqueued
+    end
+
+    # Ready executions only: scheduled and blocked jobs aren't waiting on a
+    # worker. The connection goes straight back to the pool.
+    def solid_queue
+      model = ::SolidQueue::ReadyExecution
+      model.connection_pool.with_connection { model.count }
+    end
+  end
+end

data/lib/deployangel/rails/active_job.rb

@@ -12,12 +12,14 @@ module DeployAngel
     module ActiveJob
       STATE = :@__deployangel_job_state
       PREVIOUS_CONTEXT = :@__deployangel_previous_context
+      RUNNING = :@__deployangel_running
 
       # Marks the fiber as running a job for as long as perform.active_job
-      # lasts, so checkpoints recorded by the job say so. ActiveSupport calls
-      # finish from an ensure, so the previous context (an HTTP request, for
-      # a job performed inline) comes back even when the job raises. It is
-      # kept on the job, so only a perform that started here restores.
+      # lasts, so checkpoints recorded by the job say so, and the job as
+      # running, so one that hangs is reported. ActiveSupport calls finish
+      # from an ensure, so the previous context (an HTTP request, for a job
+      # performed inline) comes back even when the job raises. Both are kept
+      # on the job, so only a perform that started here finishes them.
       module ContextListener
         module_function
 
@@ -26,6 +28,7 @@ module DeployAngel
           return unless job && DeployAngel.recording?
 
           job.instance_variable_set(PREVIOUS_CONTEXT, ExecutionContext.enter(ExecutionContext::JOB))
+          job.instance_variable_set(RUNNING, DeployAngel.job_started(job.class.name))
         rescue StandardError
           nil
         end
@@ -35,6 +38,7 @@ module DeployAngel
           return unless job&.instance_variable_defined?(PREVIOUS_CONTEXT)
 
           ExecutionContext.restore(job.remove_instance_variable(PREVIOUS_CONTEXT))
+          DeployAngel.job_finished(job.remove_instance_variable(RUNNING)) if job.instance_variable_defined?(RUNNING)
         rescue StandardError
           nil
         end

data/lib/deployangel/rails/railtie.rb

@@ -12,6 +12,7 @@ module DeployAngel
       initializer "deployangel.jobs" do
         ActiveSupport.on_load(:active_job) { DeployAngel::Rails::ActiveJob.install }
         DeployAngel::Sidekiq.install
+        DeployAngel::QueueBacklog.install
       end
 
       # Handled reports (Rails.error.handle / Rails.error.report) are shown for

data/lib/deployangel/sidekiq.rb

@@ -17,6 +17,7 @@ module DeployAngel
         started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
         latency = DeployAngel::Sidekiq.queue_latency_ms(job)
         previous_context = ExecutionContext.enter(ExecutionContext::JOB)
+        running = DeployAngel.job_started(job["class"])
         begin
           yield
         rescue Exception => e # rubocop:disable Lint/RescueException -- recorded, then re-raised untouched
@@ -25,6 +26,7 @@ module DeployAngel
           raise
         ensure
           ExecutionContext.restore(previous_context)
+          DeployAngel.job_finished(running)
         end
         record(job, started, latency, failed: false)
       end
@@ -46,6 +48,7 @@ module DeployAngel
       @installed = true
       DeployAngel.add_capability("jobs")
       ::Sidekiq.configure_server do |config|
+        QueueBacklog.worker!(:sidekiq)
         config.server_middleware { |chain| chain.add(ServerMiddleware) }
         # Jobs that exhausted their retries are discarded.
         config.death_handlers << ->(job, _error) { record_death(job) } if config.respond_to?(:death_handlers)

data/lib/deployangel/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module DeployAngel
-  VERSION = "0.1.28"
+  VERSION = "0.1.29"
 end

data/lib/deployangel.rb

@@ -8,6 +8,7 @@ require_relative "deployangel/core/fingerprint"
 require_relative "deployangel/core/release"
 require_relative "deployangel/core/instance"
 require_relative "deployangel/core/aggregator"
+require_relative "deployangel/core/running_jobs"
 require_relative "deployangel/core/protocol"
 require_relative "deployangel/core/execution_context"
 require_relative "deployangel/core/buffer"
@@ -57,6 +58,14 @@ module DeployAngel
       agent&.record_job(**attributes)
     end
 
+    def job_started(job_class)
+      agent&.job_started(job_class)
+    end
+
+    def job_finished(token)
+      agent&.job_finished(token)
+    end
+
     def record_exception(exception, **options)
       agent&.record_exception(exception, **options)
     end
@@ -128,6 +137,7 @@ require_relative "deployangel/rails/error_subscriber"
 require_relative "deployangel/tasks"
 require_relative "deployangel/metadata"
 require_relative "deployangel/rails/metadata"
+require_relative "deployangel/queue_backlog"
 require_relative "deployangel/sidekiq"
 require_relative "deployangel/apartment"
 require_relative "deployangel/rails/railtie" if defined?(::Rails::Railtie)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: deployangel
 version: !ruby/object:Gem::Version
-  version: 0.1.28
+  version: 0.1.29
 platform: ruby
 authors:
 - Jordan Owens
@@ -45,6 +45,7 @@ files:
 - lib/deployangel/core/protocol.rb
 - lib/deployangel/core/redaction.rb
 - lib/deployangel/core/release.rb
+- lib/deployangel/core/running_jobs.rb
 - lib/deployangel/core/transport.rb
 - lib/deployangel/credentials.rb
 - lib/deployangel/doctor.rb
@@ -52,6 +53,7 @@ files:
 - lib/deployangel/fork_hook.rb
 - lib/deployangel/mcp.rb
 - lib/deployangel/metadata.rb
+- lib/deployangel/queue_backlog.rb
 - lib/deployangel/rack/http.rb
 - lib/deployangel/rails/active_job.rb
 - lib/deployangel/rails/error_subscriber.rb