deployangel 0.1.27 → 0.1.29

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 480a4d0a0e8ff4206659a86d40ea0fbfb2c5cc20f55e205aa5a724914add1acf
-  data.tar.gz: bdd81c91faac1642d5f78633ab48bd2efb80e15d6a8b3022d4a0d8e18c967f4f
+  metadata.gz: e62d94a1e10efd0431f1cfad4f7aad29f954481919a609f8112e5f517291eeb1
+  data.tar.gz: 408bcc12ae37d4f93e74e5e31ad6a8b921b30481cedf81f84055c07c62677af0
 SHA512:
-  metadata.gz: e669f7ad136b1328c7f672120dc00b7f5afb20a09a6ca7b27a37c25e26709f678aec6c5fb24dcc1de68c2ffd04a73e1687d2132f09c94ea95cabf72d4731c597
-  data.tar.gz: 88e02d9c250287ac051b233bada6a7469507860aa62cc03390dee8b566770eba38c79431b0e882778b9351a9ca59d03c68b26c8d9cd188c390ee65ad0a4093e8
+  metadata.gz: e26a0379ee22236add3b1742a3ef174ebd3f39def909d3e230b4fadce49b1f2a3f62daa41c2aa02e05e5c12e59a78b9d6c94b7436bca9b986c56b20053f49783
+  data.tar.gz: 9d7f93900e16f63f1ba593063a31b0d03eb3f218877c65ea6c5a9b13153bd9a3520c8e5555370f8d94e53d89741f0a709143483b8ffa196e0e2e23f9a42a4b31

data/CHANGELOG.md

@@ -1,5 +1,28 @@
 # Changelog
 
+## 0.1.29 (2026-10-16)
+
+- Reports stuck work, so DeployAngel can fail a release in which a job hangs
+  or the queue keeps growing while every job still succeeds. Each minute's
+  payload says how many jobs are still running and how long the oldest of
+  each class has been running (ActiveJob on any adapter, and native Sidekiq
+  jobs). Sidekiq and Solid Queue worker processes also read the queue backlog,
+  the jobs ready to run and waiting, once a minute from the reporter thread:
+  one pipelined Redis call for Sidekiq, one `COUNT` of
+  `solid_queue_ready_executions` for Solid Queue. Web processes don't read
+  it. Needs a DeployAngel cloud that understands the new fields; older ones
+  ignore them.
+
+## 0.1.28 (2026-10-12)
+
+- Clearer setup for coding agents, from a first-day rehearsal: `setup` says
+  the running app reads its token only from `DEPLOYANGEL_TOKEN` and points to
+  the per-host steps instead of printing Fly and Heroku examples; `install
+  agents` says the saved token is used here once `setup` has run; `doctor`'s
+  fix for an agent that hasn't reported says how to pass a saved app token;
+  and the help says `release` defaults to the current commit and when
+  `--format` defaults to json.
+
 ## 0.1.27 (2026-10-12)
 
 - The MCP server's instructions, and the ones `deployangel install agents`

data/README.md

@@ -88,6 +88,10 @@ the dashboard says so.
   queue latency, per job class. Works with every ActiveJob adapter (Solid
   Queue, Sidekiq, GoodJob, ...) and with native Sidekiq jobs through a server
   middleware. Failures that `retry_on` or `discard_on` handle still count.
+- Jobs still running at the end of each minute, and how long the oldest of
+  each class has been running, so a job that hangs is noticed. Sidekiq and
+  Solid Queue worker processes also send the queue backlog (how many jobs are
+  waiting), read once a minute off the request path.
 - Release identity, runtime versions, and a per-process instance ID.
 
 - Exceptions: a stable fingerprint, the exception class, the first line of the

data/lib/deployangel/agent.rb

@@ -86,6 +86,25 @@ module DeployAngel
       warn_once(:record_job, "DeployAngel failed to record a job: #{e.class}: #{e.message}")
     end
 
+    # Marks a job as running until #job_finished, so one that hangs is still
+    # reported. Returns the token to pass to it.
+    def job_started(job_class)
+      return unless @active
+
+      after_fork! if Process.pid != @pid
+      start_reporter
+      @running_jobs.start(job_class)
+    rescue StandardError => e
+      warn_once(:job_started, "DeployAngel failed to record a running job: #{e.class}: #{e.message}")
+      nil
+    end
+
+    def job_finished(token)
+      @running_jobs.finish(token) if token
+    rescue StandardError => e
+      warn_once(:job_finished, "DeployAngel failed to record a finished job: #{e.class}: #{e.message}")
+    end
+
     def record_discard(job_class:)
       return unless @active
 
@@ -135,7 +154,9 @@ module DeployAngel
     def flush(include_current: false)
       return 0 unless @active
 
-      @aggregator.drain(include_current: include_current, max_periods: config.max_queued_payloads).each do |period|
+      periods = @aggregator.drain(include_current: include_current, max_periods: config.max_queued_payloads)
+      add_stuck_work(periods.last, include_current) if periods.any?
+      periods.each do |period|
         @buffer.push(Transport.encode(Protocol.telemetry(period, instance: @instance, release: release,
           runtime: @runtime, capabilities: capabilities)))
       end
@@ -198,10 +219,24 @@ module DeployAngel
     end
 
     private
+      # What's still running, on the latest period only: as of its end, or
+      # now for the in-progress one at shutdown. The backlog is read here, in
+      # the reporter thread, but not at shutdown, which has seconds to send.
+      def add_stuck_work(period, include_current)
+        ended_at = [ period.started_at + Aggregator::PERIOD_SECONDS, @clock.call ].min
+        @running_jobs.snapshot(ended_at).each do |job_class, (count, oldest_ms)|
+          period.record_running(job_class, count, oldest_ms, config.max_routes)
+        end
+        period.queue_backlog = QueueBacklog.read unless include_current
+      rescue StandardError => e
+        warn_once(:stuck_work, "DeployAngel failed to read running jobs: #{e.class}: #{e.message}")
+      end
+
       def reset_process_state
         @pid = Process.pid
         @instance = Instance.new(env: @env, now: @clock.call)
         @aggregator = Aggregator.new(max_routes: config.max_routes, clock: @clock)
+        @running_jobs = RunningJobs.new(clock: @clock)
         @buffer = Buffer.new(config.max_queued_payloads)
         @paused_until = nil
         @metadata_sent = false

data/lib/deployangel/cli.rb

@@ -49,7 +49,8 @@ module DeployAngel
                    into a host's or CI's secrets
         mcp        Run the MCP server over stdio (for coding agents)
 
-      With no target, verify, plan, and check use the current git HEAD commit.
+      With no target, release, verify, plan, and check use the current git HEAD commit.
+      --format defaults to text in a terminal and json otherwise, such as in a pipe or an agent.
       In GitHub Actions, GitLab CI, CircleCI, and Buildkite, release fills in the
       commit, a build label (e.g. run-123), and a link to the run automatically.
       In a Kamal hook, it registers Kamal's release (KAMAL_VERSION).
@@ -123,10 +124,19 @@ module DeployAngel
       #{AGENTS_END}
     MARKDOWN
 
+    # After `deployangel setup`, the MCP server and CLI here use its saved token.
+    AGENTS_SAVED_TOKEN_HELP = <<~HELP.chomp
+      Here, the MCP server and CLI use the token `deployangel setup` saved in .deployangel/.
+      Teammates redeem their own setup code, or set DEPLOYANGEL_API_TOKEN to a
+      "CLI & coding agents" token from the app's Settings, for example in an .envrc
+      with direnv. These files hold no token and are meant to be committed.
+      Codex reads .codex/config.toml only in projects you've marked as trusted.
+    HELP
     AGENTS_TOKEN_HELP = <<~HELP.chomp
       The MCP server and CLI need DEPLOYANGEL_API_TOKEN, a "CLI & coding agents" token
       from the app's Settings, in the environment your agent runs in (for example in
-      an .envrc with direnv). Never put it in these files: they're meant to be committed.
+      an .envrc with direnv), or a token saved here by `deployangel setup`.
+      Never put it in these files: they're meant to be committed.
       Codex reads .codex/config.toml only in projects you've marked as trusted.
     HELP
 
@@ -414,7 +424,7 @@ module DeployAngel
           install_codex_mcp(command),
           *install_agent_instructions
         ]
-        @stdout.puts(*lines, "", AGENTS_TOKEN_HELP)
+        @stdout.puts(*lines, "", Credentials.find(@root) ? AGENTS_SAVED_TOKEN_HELP : AGENTS_TOKEN_HELP)
         0
       end
 
@@ -621,11 +631,11 @@ module DeployAngel
           Set up DeployAngel for #{app["name"]} (#{app["environment"]}). Saved its app token and a "CI deploys" token to
           #{relative}, kept out of git and not shown here. The deployangel command and MCP server here use them.
 
-          Give production the app token without printing it, for example:
-            fly secrets set DEPLOYANGEL_TOKEN=$(#{command_name} token app)
-            heroku config:set DEPLOYANGEL_TOKEN=$(#{command_name} token app)
-          and CI the "CI deploys" token:
-            #{command_name} token api | gh secret set DEPLOYANGEL_API_TOKEN
+          The running app reads its token only from DEPLOYANGEL_TOKEN in its environment, so give it the app
+          token without printing it: $(#{command_name} token app) in the command that starts it or sets your
+          host's secrets, or #{command_name} token app | <a command that reads a secret>. The same goes for CI and
+          the "CI deploys" token (#{command_name} token api). Steps for each host:
+          https://www.deployangel.com/setup-for-agents.md
         TEXT
         0
       end

data/lib/deployangel/core/aggregator.rb

@@ -19,9 +19,16 @@ module DeployAngel
       end
     end
 
-    JobStats = Struct.new(:processed, :failed, :discarded, :duration, :queue_latency) do
+    # running and oldest_running_ms: jobs still running as the period is
+    # sent, and how long the oldest had been running at its end.
+    JobStats = Struct.new(:processed, :failed, :discarded, :duration, :queue_latency, :running, :oldest_running_ms) do
       def self.empty
-        new(0, 0, 0, Histogram.new, Histogram.new)
+        new(0, 0, 0, Histogram.new, Histogram.new, 0, nil)
+      end
+
+      def add_running(count, oldest_ms)
+        self.running += count
+        self.oldest_running_ms = [ oldest_running_ms, oldest_ms ].compact.max
       end
 
       def record(failed, discarded, duration_ms, queue_latency_ms)
@@ -54,6 +61,8 @@ module DeployAngel
     class Period
       attr_reader :started_at, :requests, :status_counts, :unhandled_exceptions, :histogram, :routes,
         :jobs, :job_classes, :exceptions, :exceptions_truncated, :checkpoints
+      # Jobs waiting in the job backend's queues, read as the period is sent.
+      attr_accessor :queue_backlog
 
       def initialize(started_at)
         @started_at = started_at
@@ -97,6 +106,14 @@ module DeployAngel
         (@job_classes[key] ||= JobStats.empty).record(failed, discarded, duration_ms, queue_latency_ms)
       end
 
+      # A class with jobs running and none finished is listed too, so a job
+      # that hangs is reported.
+      def record_running(job_class, count, oldest_ms, max_classes)
+        @jobs.add_running(count, oldest_ms)
+        key = @job_classes.key?(job_class) || @job_classes.size < max_classes - 1 ? job_class : OTHER_ROUTE
+        (@job_classes[key] ||= JobStats.empty).add_running(count, oldest_ms)
+      end
+
       def record_discard(job_class, max_classes)
         @jobs.record_discard
         key = @job_classes.key?(job_class) || @job_classes.size < max_classes - 1 ? job_class : OTHER_ROUTE

data/lib/deployangel/core/protocol.rb

@@ -32,7 +32,7 @@ module DeployAngel
         end,
         "exceptions" => period.exceptions.values.map { |e| e.merge("sources" => e["sources"].to_h).compact },
         "exceptions_truncated" => period.exceptions_truncated,
-        "jobs" => job_stats(period.jobs),
+        "jobs" => job_stats(period.jobs).merge({ "queue_backlog" => period.queue_backlog }.compact),
         "job_classes" => period.job_classes.map { |key, stats| { "key" => key }.merge(job_stats(stats)) },
         # http and job are always sent, 0 when none, so the cloud can tell
         # this agent from one that doesn't say where checkpoints happened.
@@ -42,14 +42,18 @@ module DeployAngel
       }
     end
 
+    # running and oldest_running_ms only when jobs were running.
     def job_stats(stats)
-      {
+      fields = {
         "processed" => stats.processed,
         "failed" => stats.failed,
         "discarded" => stats.discarded,
         "duration_histogram" => stats.duration.to_protocol,
         "queue_latency_histogram" => stats.queue_latency.to_protocol
       }
+      return fields unless stats.running.positive?
+
+      fields.merge("running" => stats.running, "oldest_running_ms" => stats.oldest_running_ms)
     end
 
     def runtime(framework: nil, framework_version: nil)

data/lib/deployangel/core/running_jobs.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module DeployAngel
+  # The jobs this process is running now, so each payload can say how long
+  # the oldest of each class has been running: a job that hangs never
+  # finishes, so it never shows up in the counts. Starting and finishing is
+  # a hash update under a mutex.
+  class RunningJobs
+    # More than any process runs at once; past it, new jobs go untracked
+    # rather than growing memory.
+    MAX = 1_000
+
+    def initialize(clock:)
+      @clock = clock
+      @jobs = {}.compare_by_identity
+      @mutex = Mutex.new
+    end
+
+    # Returns the token to pass to #finish.
+    def start(job_class)
+      token = Object.new
+      @mutex.synchronize { @jobs[token] = [ job_class.to_s, @clock.call ] if @jobs.size < MAX }
+      token
+    end
+
+    def finish(token)
+      @mutex.synchronize { @jobs.delete(token) }
+    end
+
+    # { job_class => [running, oldest_running_ms] } for jobs that started by
+    # at and are still running.
+    def snapshot(at)
+      @mutex.synchronize do
+        @jobs.each_value.with_object({}) do |(job_class, started_at), running|
+          next if started_at > at
+
+          age_ms = ((at - started_at) * 1000).round
+          count, oldest = running[job_class]
+          running[job_class] = [ count.to_i + 1, [ oldest.to_i, age_ms ].max ]
+        end
+      end
+    end
+  end
+end

data/lib/deployangel/doctor.rb

@@ -2,6 +2,7 @@
 
 require "time"
 require_relative "configuration"
+require_relative "credentials"
 require_relative "metadata"
 require_relative "rails/metadata"
 require_relative "core/release"
@@ -99,8 +100,20 @@ module DeployAngel
           Check.new(status: :warn, message: "The agent last reported #{time(agent["last_received_at"])}, and nothing since",
             fix: "Check the app is running, and that DEPLOYANGEL_TOKEN is still set on its processes.")
         else
-          Check.new(status: :warn, message: "The agent hasn't reported yet",
-            fix: "Deploy with DEPLOYANGEL_TOKEN set to the app's agent token (Settings, Tokens: \"Deployed application\").")
+          Check.new(status: :warn, message: "The agent hasn't reported yet", fix: token_fix)
+        end
+      end
+
+      # The running app reads its token only from DEPLOYANGEL_TOKEN in its
+      # environment, never from the file `deployangel setup` saves.
+      def token_fix
+        if Credentials.find(@root)
+          command = File.exist?(File.join(@root, "Gemfile")) ? "bundle exec deployangel" : "deployangel"
+          "Start or deploy the app with DEPLOYANGEL_TOKEN in its processes' environment. The app token is saved here, " \
+            "so pass it without printing it: $(#{command} token app) in the start command or your host's secrets command."
+        else
+          "Start or deploy the app with DEPLOYANGEL_TOKEN in its processes' environment, set to its app token " \
+            "(\"Deployed application\" in the app's Settings, under API tokens)."
         end
       end
 

data/lib/deployangel/queue_backlog.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module DeployAngel
+  # Reads how many jobs are ready to run and waiting in the job backend's
+  # queues. Only job worker processes read it, once a minute from the
+  # reporter thread as they send telemetry, never in a request or job. Every
+  # worker sees the same queues, and the cloud keeps the largest reading.
+  module QueueBacklog
+    module_function
+
+    # Solid Queue runs each worker in its own process, forked by the
+    # supervisor, so it's marked as each one starts.
+    def install
+      return if @installed || !defined?(::SolidQueue) || !::SolidQueue.respond_to?(:on_worker_start)
+
+      @installed = true
+      ::SolidQueue.on_worker_start { worker!(:solid_queue) }
+    rescue StandardError
+      nil
+    end
+
+    # Called where the backend says this process works its queues: Sidekiq's
+    # server configuration, or a Solid Queue worker starting.
+    def worker!(backend)
+      @backend = backend
+    end
+
+    def backend
+      @backend
+    end
+
+    # Jobs waiting, or nil when this process doesn't work a queue it can read.
+    def read
+      case @backend
+      when :sidekiq then sidekiq
+      when :solid_queue then solid_queue
+      end
+    rescue StandardError, LoadError
+      nil
+    end
+
+    # One pipelined call for every queue's length.
+    def sidekiq
+      require "sidekiq/api"
+      ::Sidekiq::Stats.new.enqueued
+    end
+
+    # Ready executions only: scheduled and blocked jobs aren't waiting on a
+    # worker. The connection goes straight back to the pool.
+    def solid_queue
+      model = ::SolidQueue::ReadyExecution
+      model.connection_pool.with_connection { model.count }
+    end
+  end
+end

data/lib/deployangel/rails/active_job.rb

@@ -12,12 +12,14 @@ module DeployAngel
     module ActiveJob
       STATE = :@__deployangel_job_state
       PREVIOUS_CONTEXT = :@__deployangel_previous_context
+      RUNNING = :@__deployangel_running
 
       # Marks the fiber as running a job for as long as perform.active_job
-      # lasts, so checkpoints recorded by the job say so. ActiveSupport calls
-      # finish from an ensure, so the previous context (an HTTP request, for
-      # a job performed inline) comes back even when the job raises. It is
-      # kept on the job, so only a perform that started here restores.
+      # lasts, so checkpoints recorded by the job say so, and the job as
+      # running, so one that hangs is reported. ActiveSupport calls finish
+      # from an ensure, so the previous context (an HTTP request, for a job
+      # performed inline) comes back even when the job raises. Both are kept
+      # on the job, so only a perform that started here finishes them.
       module ContextListener
         module_function
 
@@ -26,6 +28,7 @@ module DeployAngel
           return unless job && DeployAngel.recording?
 
           job.instance_variable_set(PREVIOUS_CONTEXT, ExecutionContext.enter(ExecutionContext::JOB))
+          job.instance_variable_set(RUNNING, DeployAngel.job_started(job.class.name))
         rescue StandardError
           nil
         end
@@ -35,6 +38,7 @@ module DeployAngel
           return unless job&.instance_variable_defined?(PREVIOUS_CONTEXT)
 
           ExecutionContext.restore(job.remove_instance_variable(PREVIOUS_CONTEXT))
+          DeployAngel.job_finished(job.remove_instance_variable(RUNNING)) if job.instance_variable_defined?(RUNNING)
         rescue StandardError
           nil
         end

data/lib/deployangel/rails/railtie.rb

@@ -12,6 +12,7 @@ module DeployAngel
       initializer "deployangel.jobs" do
         ActiveSupport.on_load(:active_job) { DeployAngel::Rails::ActiveJob.install }
         DeployAngel::Sidekiq.install
+        DeployAngel::QueueBacklog.install
       end
 
       # Handled reports (Rails.error.handle / Rails.error.report) are shown for

data/lib/deployangel/sidekiq.rb

@@ -17,6 +17,7 @@ module DeployAngel
         started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
         latency = DeployAngel::Sidekiq.queue_latency_ms(job)
         previous_context = ExecutionContext.enter(ExecutionContext::JOB)
+        running = DeployAngel.job_started(job["class"])
         begin
           yield
         rescue Exception => e # rubocop:disable Lint/RescueException -- recorded, then re-raised untouched
@@ -25,6 +26,7 @@ module DeployAngel
           raise
         ensure
           ExecutionContext.restore(previous_context)
+          DeployAngel.job_finished(running)
         end
         record(job, started, latency, failed: false)
       end
@@ -46,6 +48,7 @@ module DeployAngel
       @installed = true
       DeployAngel.add_capability("jobs")
       ::Sidekiq.configure_server do |config|
+        QueueBacklog.worker!(:sidekiq)
         config.server_middleware { |chain| chain.add(ServerMiddleware) }
         # Jobs that exhausted their retries are discarded.
         config.death_handlers << ->(job, _error) { record_death(job) } if config.respond_to?(:death_handlers)

data/lib/deployangel/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module DeployAngel
-  VERSION = "0.1.27"
+  VERSION = "0.1.29"
 end

data/lib/deployangel.rb

@@ -8,6 +8,7 @@ require_relative "deployangel/core/fingerprint"
 require_relative "deployangel/core/release"
 require_relative "deployangel/core/instance"
 require_relative "deployangel/core/aggregator"
+require_relative "deployangel/core/running_jobs"
 require_relative "deployangel/core/protocol"
 require_relative "deployangel/core/execution_context"
 require_relative "deployangel/core/buffer"
@@ -57,6 +58,14 @@ module DeployAngel
       agent&.record_job(**attributes)
     end
 
+    def job_started(job_class)
+      agent&.job_started(job_class)
+    end
+
+    def job_finished(token)
+      agent&.job_finished(token)
+    end
+
     def record_exception(exception, **options)
       agent&.record_exception(exception, **options)
     end
@@ -128,6 +137,7 @@ require_relative "deployangel/rails/error_subscriber"
 require_relative "deployangel/tasks"
 require_relative "deployangel/metadata"
 require_relative "deployangel/rails/metadata"
+require_relative "deployangel/queue_backlog"
 require_relative "deployangel/sidekiq"
 require_relative "deployangel/apartment"
 require_relative "deployangel/rails/railtie" if defined?(::Rails::Railtie)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: deployangel
 version: !ruby/object:Gem::Version
-  version: 0.1.27
+  version: 0.1.29
 platform: ruby
 authors:
 - Jordan Owens
@@ -45,6 +45,7 @@ files:
 - lib/deployangel/core/protocol.rb
 - lib/deployangel/core/redaction.rb
 - lib/deployangel/core/release.rb
+- lib/deployangel/core/running_jobs.rb
 - lib/deployangel/core/transport.rb
 - lib/deployangel/credentials.rb
 - lib/deployangel/doctor.rb
@@ -52,6 +53,7 @@ files:
 - lib/deployangel/fork_hook.rb
 - lib/deployangel/mcp.rb
 - lib/deployangel/metadata.rb
+- lib/deployangel/queue_backlog.rb
 - lib/deployangel/rack/http.rb
 - lib/deployangel/rails/active_job.rb
 - lib/deployangel/rails/error_subscriber.rb
@@ -86,7 +88,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.0
+rubygems_version: 3.6.9
 specification_version: 4
 summary: 'DeployAngel agent for Rails: production verification for every deployment.'
 test_files: []