cronwatch 0.12.4 → 0.13.0

data/lib/cronwatch/client.rb

@@ -43,7 +43,7 @@ module Cronwatch
     # Run ids that start with this belong to the pg_cron source (Sources::PgCron).
     RESERVED_RUN_ID_PREFIX = "pgcron:"
     # The longest run id, in UTF-16 code units (JavaScript's string length):
-    # what start, resume and record_run take, and every store holds.
+    # what start, resume, and record_run take, and every store holds.
     MAX_RUN_ID = 200
     private_constant :CHANNEL_TIMEOUT_MS, :DEFAULT_OPTIONS, :FORK_LOCK, :HISTORY_MAX, :HISTORY_PAGE, :MAX_RUN_ID,
                      :PRUNE_INTERVAL_MS, :RESERVED_RUN_ID_PREFIX, :RETRY_BUDGET_MS, :SILENCE_OPTIONS, :STATE_ATTEMPTS,
@@ -77,19 +77,19 @@ module Cronwatch
 
     attr_reader :store, :alerts, :triage, :cron_secret, :retention_ms, :defaults, :sources
 
-    # store:       where jobs, runs and state live. Defaults to an in-memory store that forgets on restart.
+    # store:       where jobs, runs, and state live. Defaults to an in-memory store that forgets on restart.
     # alerts:      where alerts go: objects with #call(alert) and #name. Defaults to the console.
     # triage:      a callable taking a TriageContext and returning a short diagnosis, added to every alert but recoveries.
     # cron_secret: a second bearer Cronwatch::Web accepts for /api/check, for an outside cron. Defaults to
-    #              ENV["CRON_SECRET"]; an empty string or one of only whitespace, given here or in the
-    #              variable, counts as unset. Pass nil for none. Anything but a String or nil (false, a
+    #              ENV["CRON_SECRET"], trimmed of the whitespace around it; an empty string or one of
+    #              only whitespace, given here or in the variable, counts as unset. Pass nil for none. Anything but a String or nil (false, a
     #              number, a Symbol) raises TypeError.
     # retention:   how long finished runs are kept. Default "30d".
-    # defaults:    grace, timeout, timezone and failures_before_alert applied to every job unless it sets its own.
-    # redact:      applied to every run's output and error before it is stored, shown or sent to an alert
+    # defaults:    grace, timeout, timezone, and failures_before_alert applied to every job unless it sets its own.
+    # redact:      applied to every run's output and error before it is stored, shown, or sent to an alert
     #              channel or triage. The default (Output.redact_secrets) blanks values that look like secrets
     #              (password=..., Authorization headers, URL credentials, bearer tokens, JWTs, PEM private
-    #              keys, webhook URLs, AWS, GitHub, Slack, Stripe, Google and API key formats). Pass your own
+    #              keys, webhook URLs, AWS, GitHub, Slack, Stripe, Google, and API key formats). Pass your own
     #              callable, or false to keep output exactly as logged. A callable that raises or returns
     #              something other than a String is reported to on_error ("redact") and the default is used.
     # now:         the clock, a callable returning epoch milliseconds. Tests use this.
@@ -137,6 +137,9 @@ module Cronwatch
       @clock = now || -> { Process.clock_gettime(Process::CLOCK_REALTIME, :millisecond) }
       @on_error = on_error || method(:default_on_error)
       @definitions = {}
+      # Declarations forgotten here (forget), kept so that run and resume_run
+      # by name declare the job again as it was written, not bare.
+      @forgotten = {}
       @synced = Set.new
       @registry = Mutex.new
       @ready = false
@@ -159,18 +162,36 @@ module Cronwatch
     def job(name, **options)
       name = name.to_s if name.is_a?(Symbol)
       unless name.is_a?(String) && NAME_RE.match?(name)
-        raise ArgumentError, "job name \"#{name}\" must be 1 to 120 characters of letters, digits, \".\", \"_\", \":\" or \"-\""
+        raise ArgumentError, "job name \"#{name}\" must be 1 to 120 characters of letters, digits, \".\", \"_\", \":\", or \"-\""
       end
 
       definition = build_definition(name, options)
       validate_definition(definition)
       @registry.synchronize do
         @definitions[name] = definition
+        @forgotten.delete(name)
         @synced.delete(name)
       end
       JobHandle.new(self, definition)
     end
 
+    # The declaration of `name`: the standing one, else one forgotten here,
+    # declared again as it was.
+    def declared(name)
+      @registry.synchronize do
+        standing = @definitions[name]
+        next standing if standing
+
+        forgotten = @forgotten.delete(name)
+        if forgotten
+          @definitions[name] = forgotten
+          @synced.delete(name)
+        end
+        forgotten
+      end
+    end
+    private :declared
+
     # Run a job by name without keeping a handle, declaring it on first use
     # (or again, when options are given).
     #
@@ -184,8 +205,8 @@ module Cronwatch
         return get_run(name_or_id)
       end
 
-      declared = @registry.synchronize { @definitions[name_or_id.to_s] }
-      handle = options.any? || declared.nil? ? job(name_or_id, **options) : JobHandle.new(self, declared)
+      definition = options.any? ? nil : declared(name_or_id.to_s)
+      handle = definition.nil? ? job(name_or_id, **options) : JobHandle.new(self, definition)
       handle.run(&block)
     end
 
@@ -205,8 +226,14 @@ module Cronwatch
     # error message, or nil when the result is fine: an HTTP handler uses it to
     # count a 500 response as a failed run.
     #
+    # `requeued_on_shutdown` is for a queue that puts a job back when its
+    # worker shuts down (Sidekiq raises Sidekiq::Shutdown into the job and
+    # runs it again after the restart): a run stopped that way is recorded as
+    # failed but not judged, so a deploy opens no failed alert and does not
+    # count toward failures_before_alert.
+    #
     # @api private For integrations (JobHandle#run, ActiveJob, Sidekiq), not apps.
-    def execute(definition, trigger, failure: nil)
+    def execute(definition, trigger, failure: nil, requeued_on_shutdown: false)
       after_fork_check
       name = definition.name
       started_at = now
@@ -224,7 +251,7 @@ module Cronwatch
       # just before the block runs. The result is the same.
       if recorded
         begin
-          update_state(name) { |before| [Evaluate.on_run_start(before), nil] }
+          update_state(name) { |before| [Evaluate.hold_run(Evaluate.on_run_start(before), run.id), nil] }
         rescue StandardError => e
           report(e, "starting #{name}")
         end
@@ -252,7 +279,8 @@ module Cronwatch
 
       conclude(definition, run, result, error, threw, recorder.expect_text || returned, failure: failure)
       begin
-        ignored = record_finish(definition, run, recorded, finished_at)
+        judge = !(requeued_on_shutdown && threw && Client.shutdown?(error))
+        ignored = record_finish(definition, run, recorded, finished_at, judge: judge)
         report(RuntimeError.new("run #{run.id} of #{name} #{ignored}; ignored"), "finishing #{name}") if ignored
       rescue StandardError => e
         report(e, "recording #{name}")
@@ -267,7 +295,7 @@ module Cronwatch
     # job must be declared in this process, or it raises ArgumentError.
     def resume_run(name, run_id)
       name = name.to_s if name.is_a?(Symbol)
-      definition = @registry.synchronize { @definitions[name] }
+      definition = declared(name)
       raise ArgumentError, "resume_run: job \"#{name}\" is not declared; call job first" unless definition
 
       resume_handle(definition, run_id)
@@ -397,7 +425,7 @@ module Cronwatch
       end
       return [] unless evaluate
 
-      update_state(run.job) { |before| [Evaluate.on_run_start(before), nil] }
+      update_state(run.job) { |before| [Evaluate.hold_run(Evaluate.on_run_start(before), run.id), nil] }
       return [] if run.status == :running
 
       finish_run(definition, run, now)
@@ -496,13 +524,15 @@ module Cronwatch
     end
 
     # Remove a job and its runs from the store. A job still declared in code
-    # comes back: here on its next run, and in any other process that
-    # declares it on its next run there, or at that process's next check or
-    # dashboard read.
+    # comes back, with no history: here on its next run, declared as it was
+    # written (run and resume_run by name included), and in any other process
+    # that declares it on its next run there, or at that process's next check
+    # or dashboard read.
     def forget(name)
       ensure_ready
       @registry.synchronize do
-        @definitions.delete(name)
+        definition = @definitions.delete(name)
+        @forgotten[name] = definition if definition
         @synced.delete(name)
       end
       @store.delete_job(name)
@@ -618,7 +648,7 @@ module Cronwatch
       ticker
     end
 
-    # Locks, the check in flight, the interval thread and the channel and
+    # Locks, the check in flight, the interval thread, and the channel and
     # triage threads belong to one process. A forked child (Puma, Unicorn,
     # Sidekiq) starts with fresh ones, so start and check work there.
     def reset_process_state
@@ -718,7 +748,7 @@ module Cronwatch
       expect = definition.expect
       return if expect.nil? || expect.is_a?(String) || expect.is_a?(Regexp) || expect.respond_to?(:call)
 
-      raise ArgumentError, "job \"#{name}\": expect must be a string, a RegExp or a function"
+      raise ArgumentError, "job \"#{name}\": expect must be a string, a RegExp, or a function"
     end
 
     # String(value) as JavaScript writes it, for the messages above.
@@ -886,11 +916,24 @@ module Cronwatch
         report(RuntimeError.new("run #{run.id} of #{run.job} #{ignored}; ignored"), "recording #{run.job}")
         return []
       end
-      return [] if !evaluate || (late && run.status != :ok)
+      unless evaluate
+        # Stored unjudged on purpose: a check must not evaluate it as one whose evaluation was lost.
+        release_unjudged(run)
+        return []
+      end
+      return [] if late && run.status != :ok
 
       finish_run(definition, run, now)
     end
 
+    # Takes a run stored without judging out of the job's pending runs, so a
+    # check does not evaluate it as one whose evaluation was lost.
+    def release_unjudged(run)
+      update_state(run.job) { |current| [Evaluate.release_run(current, run.id), nil] }
+    rescue StandardError => e
+      report(e, "recording #{run.job}")
+    end
+
     # A conditional write (the store's update_run_if), or for a store
     # without one, a read then a plain write, which is safe only while one
     # process at a time finishes a given run.
@@ -926,7 +969,7 @@ module Cronwatch
       if threw
         run.status = :failed
         run.error = Output.describe_error(error)
-      elsif (problem = failure&.call(result))
+      elsif (problem = failure_of(failure, result))
         run.status = :failed
         run.error = Output.utf8(problem.to_s)
       else
@@ -945,18 +988,26 @@ module Cronwatch
       run.error = Output.redact_and_cap(run.error, @redact) unless run.error.nil?
     end
 
+    # What the failure: callable makes of the result, or nil. One that raises
+    # fails the run with its error described, so the run is still recorded.
+    def failure_of(failure, result)
+      failure&.call(result)
+    rescue StandardError => e
+      Output.describe_error(e)
+    end
+
     # Writes a finished run and evaluates it. `recorded` says whether its
     # start was written; if not, it is inserted now. Returns why nothing was
     # recorded (another process finished the run first, say), or nil. Raises
     # when the store does, so a handle can be finished again. Shared by
     # execute and RunHandle#finish.
-    def record_finish(definition, run, recorded, finished_at)
+    def record_finish(definition, run, recorded, finished_at, judge: true)
       unless recorded
         # The start was never written; the store may be back by now.
         sync(definition)
         begin
           @store.insert_run(run)
-          finish_run(Serialize.to_stored(definition), run, finished_at)
+          finish_run(Serialize.to_stored(definition), run, finished_at) if judge
           return nil
         rescue StandardError => e
           # Another process may have recorded a run with this id meanwhile.
@@ -972,10 +1023,22 @@ module Cronwatch
       late, ignored = claim_finish(run)
       return ignored if ignored
 
-      finish_run(Serialize.to_stored(definition), run, finished_at) if !late || run.status == :ok
+      if !judge
+        release_unjudged(run)
+      elsif !late || run.status == :ok
+        finish_run(Serialize.to_stored(definition), run, finished_at)
+      end
       nil
     end
 
+    # A job's worker shutting down, which its queue answers by running the job
+    # again: Sidekiq::Shutdown, when Sidekiq is loaded.
+    #
+    # @api private
+    def self.shutdown?(error)
+      defined?(::Sidekiq::Shutdown) && error.is_a?(::Sidekiq::Shutdown) ? true : false
+    end
+
     # The start of execute without the block: the run is inserted and missed
     # and stuck close (on_run_start). A store that fails is reported and the
     # handle inserts the finished run instead, as execute does.
@@ -1011,7 +1074,7 @@ module Cronwatch
       end
       if recorded
         begin
-          update_state(name) { |before| [Evaluate.on_run_start(before), nil] }
+          update_state(name) { |before| [Evaluate.hold_run(Evaluate.on_run_start(before), run.id), nil] }
         rescue StandardError => e
           report(e, "starting #{name}")
         end
@@ -1162,6 +1225,30 @@ module Cronwatch
       joined && Output.cap(joined)
     end
 
+    # Runs the job's state still holds as pending (Evaluate.hold_run) whose
+    # finish was written but never evaluated: the process that finished them
+    # stopped, or its state write failed, between the two writes. Each one
+    # finished more than SEND_LEASE_MS ago is evaluated now, as its finish
+    # would have been; one still running is left to the stuck check, and one
+    # the store no longer has (given back, or pruned) is let go. The wait
+    # leaves a finish under way in another process to write its own evaluation.
+    def evaluate_pending(stored, recent, state, at)
+      alerts = []
+      (state.pending_runs || []).each do |id|
+        run = recent.find { |r| r.id == id } || @store.get_run(id)
+        if run.nil? || run.job != stored.name
+          update_state(stored.name) { |current| [Evaluate.release_run(current, id), nil] }
+          next
+        end
+        next if run.status == :running || !run.finished_at.is_a?(Numeric) || run.finished_at > at - Evaluate::SEND_LEASE_MS
+
+        alerts.concat(finish_run(stored.definition, run, at))
+      rescue StandardError => e
+        report(e, "evaluating run #{id} of #{stored.name}")
+      end
+      alerts
+    end
+
     # Evaluate a finished run (ok, failed, or timed out by a check), already
     # written, against the job's state and send what that produces. The
     # alerts are written with that state (see outbox). Never raises.
@@ -1171,7 +1258,8 @@ module Cronwatch
         past = nil
         _, held = update_state(run.job) do |previous|
           past ||= history(run)
-          outbox(Evaluate.apply_silence(previous, Evaluate.on_run_finish(definition, run, previous, past, at), at), definition, at)
+          before = Evaluate.release_run(previous, run.id)
+          outbox(Evaluate.apply_silence(before, Evaluate.on_run_finish(definition, run, before, past, at), at), definition, at)
         end
       rescue StandardError => e
         report(e, "evaluating #{run.job}")
@@ -1280,6 +1368,7 @@ module Cronwatch
         report_dropped(stored.name, held.dropped)
         alerts.concat(retry_undelivered(stored.name, state, at, retries))
         alerts.concat(dispatch(stored.name, held.alerts, at))
+        alerts.concat(evaluate_pending(stored, recent, state, at))
         jobs << Evaluate.summarize(stored, recent, state, next_expected_at, at)
       rescue StandardError => e
         report(e, "checking #{stored.name}")
@@ -1338,7 +1427,7 @@ module Cronwatch
       Evaluate.unevaluable_summary(stored, recent, state, at)
     end
 
-    # Read, change and write one job's state, in turn with every other update to it.
+    # Read, change, and write one job's state, in turn with every other update to it.
     def patch_state(name)
       ensure_ready
       state, = update_state(name) do |current|
@@ -1376,22 +1465,36 @@ module Cronwatch
     RetryBudget = Struct.new(:spent_ms)
 
     # Send the alerts that no channel accepted last time, once each, oldest
-    # first. `state` is the job's state as this check left it: an alert that
-    # no longer describes it (Evaluate.stale_alert?) is dropped instead.
-    # Retries across a check share RETRY_BUDGET_MS of wall-clock time; once it
-    # is spent the rest stay queued for the next check.
+    # first. They are claimed first (Evaluate.claim_undelivered): moved into
+    # `sending` with a lease, in one conditional write, so when several
+    # processes check at once only the one whose write lands sends them. An
+    # alert that no longer describes the job (Evaluate.stale_alert?) is
+    # dropped instead. Retries across a check share RETRY_BUDGET_MS of
+    # wall-clock time; claimed alerts it has no time left for go back to the
+    # queue for the next check.
     def retry_undelivered(name, state, at, budget)
-      pending = state.undelivered || []
-      return [] if pending.empty? || Evaluate.silenced?(state, at) || @defer_delivery
+      return [] if (state.undelivered || []).empty? || @defer_delivery
+
+      begin
+        _, claimed = update_state(name) do |previous|
+          claim = Evaluate.claim_undelivered(previous, at, now + Evaluate::SEND_LEASE_MS)
+          [claim.state, claim.claimed]
+        end
+      rescue StandardError => e
+        report(e, "claiming undelivered alerts for #{name}")
+        return []
+      end
+      return [] if claimed.empty?
 
       delivered = []
       failed = []
-      dropped = pending.select { |alert| Evaluate.stale_alert?(alert, state) }
-      pending.each do |alert|
-        next if dropped.any? { |d| d.equal?(alert) }
-
+      claimed.each do |alert|
         left = @retry_budget_ms - budget.spent_ms
-        break if left <= 0
+        if left <= 0
+          # Not tried: back to the queue as it was.
+          failed << alert
+          next
+        end
 
         started = AbortSignal.monotonic
         # An alert queued by a process that delivers at check time was never
@@ -1400,7 +1503,7 @@ module Cronwatch
         (deliver(alert) ? delivered : failed) << alert
         budget.spent_ms += [0, ((AbortSignal.monotonic - started) * 1000).round].max
       end
-      record_delivery(name, delivered, failed, dropped, at)
+      record_delivery(name, delivered, failed, [], at)
       delivered
     end
 
@@ -1492,13 +1595,17 @@ module Cronwatch
     end
 
     # Sets the alert's triage to the diagnosis, or to nil (JSON null) when
-    # there is none (it raised, timed out or answered nil or ""), so it is
-    # tried once per alert. While a triage that timed out is still going,
-    # alerts go out without one rather than start another beside it.
+    # there is none (it raised, timed out, or answered nil or ""), so it is
+    # tried once per alert. While a triage that timed out on its full timeout
+    # is still going, alerts go out without one rather than start another
+    # beside it, and stay untried so a later retry can triage them. A timeout
+    # cut short by the retry budget says nothing about Claude, so it does not
+    # hold back the triages after it.
     def add_triage(alert, timeout_ms)
       signal = AbortSignal.new
       if @sending_lock.synchronize { @abandoned[:triage]&.alive? }
-        raise TimeoutError, "skipped: an earlier triage timed out and is still running"
+        report(TimeoutError.new("skipped: an earlier triage timed out and is still running"), "triage for #{alert.job}")
+        return
       end
 
       recent = @store.list_runs(alert.job, 5)
@@ -1512,7 +1619,7 @@ module Cronwatch
         outcome = [:error, e]
       end
       unless thread.join(timeout_ms / 1000.0)
-        @sending_lock.synchronize { @abandoned[:triage] = thread }
+        @sending_lock.synchronize { @abandoned[:triage] = thread } if timeout_ms >= @triage_timeout_ms
         raise TimeoutError, "timed out after #{timeout_ms}ms"
       end
       outcome ||= [:error, RuntimeError.new("the triage thread ended without an answer")]

data/lib/cronwatch/duration.rb

@@ -34,7 +34,7 @@ module Cronwatch
         return value
       end
       unless value.is_a?(String)
-        raise ArgumentError, "#{label} \"#{value}\" is not a duration like \"15m\", \"1h30m\" or \"90s\""
+        raise ArgumentError, "#{label} \"#{value}\" is not a duration like \"15m\", \"1h30m\", or \"90s\""
       end
 
       if value.length > MAX_LENGTH
@@ -52,7 +52,7 @@ module Cronwatch
         consumed << match[0]
       end
       if consumed.gsub(JS::SPACES, "") != text.gsub(JS::SPACES, "")
-        raise ArgumentError, "#{label} \"#{value}\" is not a duration like \"15m\", \"1h30m\" or \"90s\""
+        raise ArgumentError, "#{label} \"#{value}\" is not a duration like \"15m\", \"1h30m\", or \"90s\""
       end
 
       JS.round(total)

data/lib/cronwatch/environment.rb

@@ -4,19 +4,17 @@ module Cronwatch
   # The app's environment, read as every CronWatch library reads it: the
   # first of CRONWATCH_ENV, APP_ENV, then the app's own (Rails.env when Rails
   # is loaded, else RAILS_ENV, then RACK_ENV) that holds more than spaces,
-  # trimmed and lowercased, with "prod" read as "production" and "dev",
-  # "local", "test" and "testing" as "development". None set is neither,
-  # which is the safe reading. The SDK reads NODE_ENV where this reads the
-  # app's own.
+  # trimmed and lowercased, with "prod" read as "production" and "dev" and
+  # "local" as "development". "test" and "testing" are not development: a
+  # test run or CI needs CRON_SECRET or a token, as production does. None set
+  # is neither, which is the safe reading. The SDK reads NODE_ENV where this
+  # reads the app's own.
   #
   # @api private
   module Environment
     # The variables read before the app's own, in order.
     SHARED = %w[CRONWATCH_ENV APP_ENV].freeze
-    ALIASES = {
-      "prod" => "production", "dev" => "development", "local" => "development", "test" => "development",
-      "testing" => "development",
-    }.freeze
+    ALIASES = { "prod" => "production", "dev" => "development", "local" => "development" }.freeze
     private_constant :SHARED, :ALIASES
 
     module_function
@@ -59,20 +57,21 @@ module Cronwatch
       JS.trim(value.dup.force_encoding(Encoding::UTF_8).scrub).empty?
     end
 
-    # A secret from the environment (CRONWATCH_TOKEN, CRON_SECRET): nil when
-    # the variable is unset, empty or only whitespace, so a blank value counts
-    # as not set and the routes fail closed. Any other value is used as it
-    # is, untrimmed.
+    # A secret from the environment (CRONWATCH_TOKEN, CRON_SECRET), trimmed as
+    # JavaScript's String.prototype.trim trims (JS::WHITESPACE): a value read
+    # from a file often ends in a newline that no header can carry, since
+    # HTTP strips the whitespace around a header's value. nil when the
+    # variable is unset, empty, or only whitespace, so a blank value counts as
+    # not set and the routes fail closed.
     def secret(variable)
-      value = ENV.fetch(variable, nil)
-      present?(value) ? value : nil
+      trimmed(ENV.fetch(variable, nil))
     end
 
-    # A token or secret passed in code: a String, or nil. A String that is
-    # empty or only whitespace counts as not given (nil here; the caller
-    # tells that from nil the opt-out). Anything else (false, a number, a
-    # Symbol) raises TypeError naming the option, so it never becomes a
-    # password.
+    # A token or secret passed in code: a String, or nil. A String is
+    # trimmed, as secret trims, and one that is empty or only whitespace
+    # counts as not given (nil here; the caller tells that from nil the
+    # opt-out). Anything else (false, a number, a Symbol) raises TypeError
+    # naming the option, so it never becomes a password.
     def secret_option(value, what)
       return nil if value.nil?
       unless value.is_a?(String)
@@ -80,7 +79,19 @@ module Cronwatch
         raise TypeError, "#{what} must be a String, or nil to opt out, not #{kind}"
       end
 
-      blank?(value) ? nil : value
+      trimmed(value)
+    end
+
+    # A secret trimmed as JavaScript trims, or nil when nothing is left. Bytes
+    # that are not UTF-8 are kept as they are.
+    def trimmed(value)
+      return nil unless present?(value)
+
+      begin
+        JS.trim(value)
+      rescue ArgumentError
+        value
+      end
     end
 
     # The framework's own name for the environment, as it sets it: Rails.env
@@ -94,7 +105,7 @@ module Cronwatch
       [ENV.fetch("RAILS_ENV", nil), ENV.fetch("RACK_ENV", nil)].find { |value| value && !value.empty? }
     end
 
-    # Development (including test): Cronwatch::Web without a token serves only then.
+    # Development (not test): Cronwatch::Web without a token serves only then.
     def development?
       environment == "development"
     end
@@ -104,10 +115,10 @@ module Cronwatch
     end
 
     # Whether development was named by the app rather than by its server,
-    # for the dashboard's own token (Cronwatch::Web). Puma, Unicorn, Thin
+    # for the dashboard's own token (Cronwatch::Web). Puma, Unicorn, Thin,
     # and rackup set RACK_ENV to "development" when nothing names an
     # environment, so under one of them, in production too, RACK_ENV alone
-    # reads "development". There it takes CRONWATCH_ENV, APP_ENV, Rails.env
+    # reads "development". There it takes CRONWATCH_ENV, APP_ENV, Rails.env,
     # or RAILS_ENV to say development; any other value of RACK_ENV that reads
     # as development ("test", "dev") still counts, as no server sets it.
     def stated_development?