maintenance_on_steroids 0.2.0

data/app/models/maintenance_on_steroids/artifact.rb

@@ -0,0 +1,255 @@
+module MaintenanceOnSteroids
+  class Artifact < ApplicationRecord
+    belongs_to :run
+
+    attr_accessor :worker_token
+    around_save :guard_worker_write
+
+    def guard_worker_write(&block)
+      return yield unless worker_token
+
+      run.worker_token = worker_token
+      run.with_execution_lock(&block)
+    end
+
+    def check_buffer_size!(size)
+      limit = MaintenanceOnSteroids.max_artifact_size
+      return if limit.nil? || size <= limit
+
+      errors.add(:base, "artifact #{name.inspect} exceeds max_artifact_size (#{limit} bytes); use object storage for large outputs")
+      raise ActiveRecord::RecordInvalid, self
+    end
+
+    validates :name, presence: true
+    validates :kind, inclusion: { in: %w[input output] }
+    validates :artifact_type, inclusion: { in: %w[jsonb blob text csv] }
+
+    # Cap on how much of an artifact is materialised for an in-page preview.
+    # Parsing or HTML-escaping a payload costs several times its size in live
+    # objects, so an uncapped preview of a large export OOMs the web process
+    # to show the first hundred rows. The full payload is always downloadable.
+    # ponytail: the column itself is still SELECTed whole -- if multi-hundred-MB
+    # artifacts become normal, push the cap into SQL (substr) and out of Ruby.
+    PREVIEW_BYTES = 256 * 1024
+
+    # Top-level entries kept when previewing an oversized jsonb document.
+    # Slicing generated JSON would mean generating all of it first, which is
+    # exactly the allocation the byte cap exists to avoid -- so the structure
+    # is trimmed before it is rendered, and the byte cap is a second backstop.
+    PREVIEW_ENTRIES = 200
+
+    validate :payload_within_size_limit
+
+    def data
+      case artifact_type
+      when "jsonb"      then data_jsonb
+      when "blob", "csv" then data_blob
+      when "text"       then data_text
+      end
+    end
+
+    def downloadable?
+      %w[blob csv].include?(artifact_type) && data_blob.present?
+    end
+
+    # Parsed CSV rows for table preview, capped at PREVIEW_BYTES.
+    # nil for non-CSV artifacts.
+    def csv_rows
+      return nil unless artifact_type == "csv" && data_blob.present?
+      require "csv"
+      CSV.parse(preview_slice(data_blob))
+    rescue CSV::MalformedCSVError
+      nil
+    end
+
+    # Capped textual content for inline display; nil for types with no
+    # text preview (blob) or with nothing stored yet.
+    def preview_text
+      case artifact_type
+      when "jsonb" then preview_slice(JSON.pretty_generate(previewable_jsonb)) if data_jsonb.present?
+      when "text"  then preview_slice(data_text) if data_text.present?
+      end
+    end
+
+    # True when the inline preview shows less than the whole artifact -- either
+    # because the payload is over the byte cap or because a jsonb document had
+    # more than PREVIEW_ENTRIES top-level entries.
+    def preview_truncated?
+      return true if (byte_size || 0) > PREVIEW_BYTES
+
+      jsonb_over_entry_cap?
+    end
+
+    # Download attributes -- correct MIME and filename even for generated files
+    # whose content_type wasn't set explicitly.
+    def download_file_name
+      file_name.presence || "#{name}.#{artifact_type == 'csv' ? 'csv' : 'bin'}"
+    end
+
+    def download_content_type
+      content_type.presence ||
+        Rack::Mime.mime_type(::File.extname(download_file_name), "application/octet-stream")
+    end
+
+    def display_value
+      case artifact_type
+      when "jsonb"
+        data_jsonb.present? ? JSON.pretty_generate(data_jsonb) : "{}"
+      when "text"
+        data_text.to_s
+      when "csv"
+        data_blob.to_s
+      when "blob"
+        if file_name.present?
+          "#{file_name} (#{human_size})"
+        else
+          "Binary data (#{human_size})"
+        end
+      end
+    end
+
+    # Recomputes lightweight stats (entry/line count, byte size, timestamp)
+    # into the metadata column. Called on every artifact save so the UI can
+    # show size/counts without loading the full payload.
+    def refresh_metadata!
+      self.metadata = (metadata || {}).merge(computed_metadata)
+    end
+
+    def computed_metadata
+      base = { "generated_at" => Time.current.utc.iso8601 }
+      case artifact_type
+      when "jsonb"
+        data = data_jsonb || {}
+        base.merge("entries" => (data.respond_to?(:size) ? data.size : 1), "bytes" => data.to_json.bytesize)
+      when "text"
+        text = data_text || ""
+        base.merge("lines" => text.lines.size, "bytes" => text.bytesize)
+      when "csv"
+        blob = data_blob || ""
+        # Newline count == row count (CSV writes a trailing newline per row).
+        base.merge("rows" => (blob.empty? ? 0 : blob.count("\n")), "bytes" => blob.bytesize)
+      when "blob"
+        base.merge("bytes" => (data_blob ? data_blob.bytesize : 0))
+      else
+        base
+      end
+    end
+
+    # Prefers the precomputed metadata byte count; falls back to measuring the
+    # column that actually holds this type's payload. The old fallback only
+    # looked at data_blob, so jsonb and text rows written before metadata
+    # tracking reported nil -- which made preview_truncated? answer false and
+    # bypassed every preview cap that depends on it.
+    def byte_size
+      metadata&.dig("bytes") || measured_byte_size
+    end
+
+    def generated_at
+      ts = metadata&.dig("generated_at")
+      Time.iso8601(ts) if ts.present?
+    rescue ArgumentError
+      nil
+    end
+
+    # One-line human summary for the UI: "12 entries · 3.4 KB" etc.
+    def summary
+      case artifact_type
+      when "jsonb"
+        "#{metadata&.dig('entries') || 0} entries · #{human_size}"
+      when "text"
+        "#{metadata&.dig('lines') || 0} lines · #{human_size}"
+      when "csv"
+        "#{metadata&.dig('rows') || 0} rows · #{human_size}"
+      when "blob"
+        file_name.present? ? "#{file_name} (#{human_size})" : "Binary data (#{human_size})"
+      end
+    end
+
+    # data_jsonb trimmed to PREVIEW_ENTRIES top-level entries. Driven by the
+    # entry count itself rather than by preview_truncated?, so a row with no
+    # recorded byte size still gets trimmed instead of being generated whole.
+    # Scalars and small documents pass through untouched.
+    def previewable_jsonb
+      case data_jsonb
+      when Hash  then data_jsonb.size > PREVIEW_ENTRIES ? data_jsonb.first(PREVIEW_ENTRIES).to_h : data_jsonb
+      when Array then data_jsonb.size > PREVIEW_ENTRIES ? data_jsonb.first(PREVIEW_ENTRIES) : data_jsonb
+      else data_jsonb
+      end
+    end
+
+    def jsonb_over_entry_cap?
+      artifact_type == "jsonb" &&
+        data_jsonb.respond_to?(:size) &&
+        !data_jsonb.is_a?(String) &&
+        data_jsonb.size > PREVIEW_ENTRIES
+    end
+
+    # One choke point for every write path (explicit save, accumulator flush,
+    # file input). Failing the run with a legible message beats an OOM killer
+    # or a database-level error halfway through a long task.
+    def payload_within_size_limit
+      limit = MaintenanceOnSteroids.max_artifact_size
+      return if limit.nil?
+
+      size = measured_byte_size
+      return if size.nil? || size <= limit
+
+      errors.add(
+        :base,
+        "artifact #{name.inspect} is #{ActiveSupport::NumberHelper.number_to_human_size(size)}, " \
+        "over the #{ActiveSupport::NumberHelper.number_to_human_size(limit)} limit " \
+        "(MaintenanceOnSteroids.max_artifact_size). Write large outputs to object storage " \
+        "and store a reference instead."
+      )
+    end
+
+    def measured_byte_size
+      case artifact_type
+      when "jsonb"       then data_jsonb && data_jsonb.to_json.bytesize
+      when "text"        then data_text&.bytesize
+      when "blob", "csv" then data_blob&.bytesize
+      end
+    end
+
+    # At most PREVIEW_BYTES of a payload, always returned as valid UTF-8, cut
+    # back to the last complete line when it had to be truncated.
+    #
+    # Two encoding traps here, both of which used to reach the run page:
+    #   - data_blob comes back as ASCII-8BIT, and interpolating a BINARY string
+    #     holding non-ASCII bytes into the UTF-8 template raises
+    #     Encoding::CompatibilityError -- so any CSV export containing an
+    #     accented character took the page down, at any size.
+    #   - byteslice cuts on a byte boundary, landing inside a multibyte
+    #     character often enough to matter, and matching a Regexp against the
+    #     resulting invalid string raises ArgumentError.
+    # Slice first (bounded work), then transcode and scrub the small result.
+    def preview_slice(content)
+      truncated = content.bytesize > PREVIEW_BYTES
+      sliced = truncated ? content.byteslice(0, PREVIEW_BYTES) : content
+
+      sliced = sliced.dup.force_encoding(Encoding::UTF_8)
+      sliced = sliced.scrub("") unless sliced.valid_encoding?
+
+      return sliced unless truncated
+
+      # Drop the trailing partial line -- but only when there is an earlier
+      # newline to fall back to. A single-line payload (a minified blob, a log
+      # with no line breaks) has no newline in the slice at all, and the regex
+      # would match the whole thing and leave an empty preview.
+      trimmed = sliced.sub(/[^\n]*\z/, "")
+      trimmed.empty? ? sliced : trimmed
+    end
+
+    def human_size
+      size = byte_size
+      return "0 B" unless size && size.positive?
+      if size < 1024
+        "#{size} B"
+      elsif size < 1024 * 1024
+        "#{(size / 1024.0).round(1)} KB"
+      else
+        "#{(size / (1024.0 * 1024)).round(1)} MB"
+      end
+    end
+  end
+end

data/app/models/maintenance_on_steroids/run.rb

@@ -0,0 +1,310 @@
+module MaintenanceOnSteroids
+  class Run < ApplicationRecord
+    has_many :artifacts, dependent: :destroy
+
+    # Set only on the worker's instance, never accepted from request parameters.
+    attr_accessor :worker_token
+
+    STATUSES = %w[enqueued running pausing paused cancelling cancelled completed errored].freeze
+    ACTIVE_STATUSES = %w[enqueued running pausing paused].freeze
+    # Statuses an operator may restart from. "errored" is included because a
+    # run that failed can resume after its last committed checkpoint, retrying
+    # the failed record and any effects it made before checkpointing.
+    RESUMABLE_STATUSES = %w[paused errored].freeze
+    # Statuses that mean "a worker currently holds this run" -- candidates
+    # for staleness reaping when the worker died without updating the row.
+    STALE_CANDIDATE_STATUSES = %w[running pausing cancelling].freeze
+    # Runs that will never change again, and so are safe to prune.
+    TERMINAL_STATUSES = %w[completed cancelled errored].freeze
+
+    validates :task_class, presence: true
+    validates :status, inclusion: { in: STATUSES }
+
+    scope :recent, -> { order(created_at: :desc, id: :desc) }
+    scope :active, -> { where(status: ACTIVE_STATUSES) }
+
+    STATUSES.each do |s|
+      define_method(:"#{s}?") { status == s }
+    end
+
+    # Transitions runs stuck in an in-flight status to "errored" when the row
+    # hasn't been touched for `threshold`. RunJob updates the row at least
+    # once per processed record, so updated_at acts as a heartbeat. Call this
+    # periodically (cron, recurring job) to recover from worker crashes. Queued
+    # runs are included only with an explicit enqueued_threshold longer than
+    # normal queue latency. Revoking the token fences out a surviving worker.
+    # Returns the number of reaped runs.
+    def self.reap_stale!(threshold: 30.minutes, enqueued_threshold: nil)
+      stale = where(status: STALE_CANDIDATE_STATUSES).where(updated_at: ...threshold.ago)
+      if enqueued_threshold
+        stale = stale.or(where(status: "enqueued").where(updated_at: ...enqueued_threshold.ago))
+      end
+      stale.update_all(
+        status: "errored",
+        execution_token: nil,
+        error_message: "Run marked as stale: no progress before the configured timeout. The worker or dispatch may have failed.",
+        completed_at: Time.current,
+        updated_at: Time.current
+      )
+    end
+
+    # Deletes finished runs older than `older_than`, with their artifacts.
+    # Nothing expires these rows on its own, so a long-lived app accumulates
+    # every run, backtrace and stored blob forever. Call this periodically the
+    # same way as reap_stale!:
+    #
+    #   MaintenanceOnSteroids::Run.prune!(older_than: 90.days)
+    #
+    # Only terminal runs are eligible -- anything still active or paused is
+    # left alone regardless of age. Returns the number of runs deleted.
+    def self.prune!(older_than: 90.days, statuses: TERMINAL_STATUSES)
+      scope = where(status: Array(statuses) & TERMINAL_STATUSES)
+              .where(created_at: ...older_than.ago)
+
+      # destroy_all rather than delete_all so dependent artifacts go too;
+      # batched so pruning a large backlog doesn't build one huge transaction.
+      deleted = 0
+      scope.in_batches(of: 500) do |batch|
+        deleted += batch.destroy_all.size
+      end
+      deleted
+    end
+
+    def active?
+      ACTIVE_STATUSES.include?(status)
+    end
+
+    def stoppable?
+      %w[running pausing].include?(status)
+    end
+
+    def pausable?
+      status == "running"
+    end
+
+    def resumable?
+      RESUMABLE_STATUSES.include?(status)
+    end
+
+    def cancellable?
+      ACTIVE_STATUSES.include?(status)
+    end
+
+    def progress_percentage
+      return 0 if progress_total.zero?
+      [(progress_current.to_f / progress_total * 100).round(1), 100.0].min
+    end
+
+    def duration
+      return nil unless started_at
+      end_time = completed_at || Time.current
+      end_time - started_at
+    end
+
+    def formatted_duration
+      return "—" unless duration
+      seconds = duration.to_i
+      if seconds < 60
+        "#{seconds}s"
+      elsif seconds < 3600
+        "#{seconds / 60}m #{seconds % 60}s"
+      else
+        "#{seconds / 3600}h #{(seconds % 3600) / 60}m"
+      end
+    end
+
+    # Estimated time remaining for the pending records, extrapolated from the
+    # current processing rate. Only meaningful while progress is being tracked.
+    def estimated_duration
+      return nil unless running? && started_at && progress_total.positive? && progress_current.positive?
+      # No estimate once we've reached (or overshot) the total -- nothing
+      # pending, and progress_current > progress_total would go negative.
+      return nil unless progress_current < progress_total
+      duration * (progress_total - progress_current) / progress_current
+    end
+
+    def formatted_estimated_duration
+      total = estimated_duration
+      return nil unless total
+
+      total = total.round
+      days = total / 86_400
+      hours = (total % 86_400) / 3600
+      mins = (total % 3600) / 60
+      secs = total % 60
+
+      parts = []
+      parts << "#{days}d" if days.positive?
+      parts << "#{hours}h" if hours.positive? || days.positive?
+      parts << "#{mins}m" if mins.positive? || hours.positive? || days.positive?
+      parts << "#{secs}s"
+      parts.join(" ")
+    end
+
+    def task_instance
+      @task_instance ||= begin
+        klass = JobRegistry.find(task_class)
+        raise ArgumentError, "Unknown maintenance task: #{task_class}" unless klass
+        klass.new(self)
+      end
+    end
+
+    def enqueue!(job_id: SecureRandom.uuid)
+      previous_backtrace = error_backtrace
+      job_config = task_instance.class.job_config
+      job = RunJob.new(id)
+      job.job_id = job_id
+      job.enqueue_failure_backtrace = previous_backtrace
+      job.queue_name = job_config.queue_name if job_config.queue_name
+      job.priority = job_config.priority if job_config.priority
+
+      with_lock do
+        return false unless enqueued? && execution_token.nil? && (active_job_id.nil? || active_job_id == job_id)
+        # Publish the attempt identity and reset metadata before a worker can run.
+        update!(active_job_id: job_id, completed_at: nil, error_message: nil, error_backtrace: nil)
+      end
+      unless job.enqueue
+        raise EnqueueFailed, job.enqueue_error&.message || "Enqueue callback aborted the job"
+      end
+      safe_instrument(:enqueued)
+      true
+    rescue => e
+      # A fast worker may already have completed or failed. Never overwrite it,
+      # or a newer attempt, when dispatch reports an error.
+      self.class.where(id: id, status: "enqueued", execution_token: nil, active_job_id: [nil, job_id]).update_all(
+        status: "errored", error_message: "Failed to enqueue: #{e.message}",
+        error_backtrace: previous_backtrace, completed_at: Time.current, updated_at: Time.current
+      )
+      reload
+      raise EnqueueFailed, e.message
+    end
+
+    # All worker bookkeeping and buffered output commits are fenced by this
+    # token. The lock is short-lived; task side effects happen outside it.
+    def with_execution_lock
+      with_lock do
+        unless worker_token && execution_token == worker_token && STALE_CANDIDATE_STATUSES.include?(status)
+          raise ExecutionLost, "Run #{id} no longer belongs to this worker"
+        end
+        yield
+      end
+    end
+
+    # Compare-and-set, like #resume!: a check-then-update would let a pause
+    # clicked as the job finishes overwrite "completed" with "pausing", which
+    # nothing but reap_stale! would ever clear.
+    # Returns true when the transition was performed, false otherwise.
+    def pause!
+      claimed = self.class.where(id: id, status: "running").update_all(
+        status: "pausing",
+        updated_at: Time.current
+      ) == 1
+      reload if claimed
+      claimed
+    end
+
+    # Compare-and-set so two concurrent resumes can't both enqueue a job
+    # for the same run. Returns true when this call won the transition.
+    # Raises EnqueueFailed if the job could not be queued.
+    def resume!
+      job_id = SecureRandom.uuid
+      claimed = self.class.where(id: id, status: RESUMABLE_STATUSES).update_all(
+        status: "enqueued",
+        active_job_id: job_id,
+        execution_token: nil,
+        updated_at: Time.current
+      ) == 1
+      return false unless claimed
+
+      reload
+      return false unless enqueue!(job_id: job_id)
+      safe_instrument(:resumed)
+      true
+    end
+
+    # Returns true when the transition was performed, false otherwise.
+    # NOTE: an already-enqueued job cannot be portably removed from the queue
+    # via Active Job. RunJob#perform guards on terminal statuses, so a job
+    # that still fires for a cancelled run is a no-op.
+    def cancel!
+      # Compare-and-set on each group, for the same reason as #pause!.
+      if self.class.where(id: id, status: %w[enqueued paused]).update_all(
+           status: "cancelled", completed_at: Time.current, updated_at: Time.current
+         ) == 1
+        reload
+        safe_instrument(:cancelled)
+        true
+      elsif self.class.where(id: id, status: %w[running pausing]).update_all(
+              status: "cancelling", updated_at: Time.current
+            ) == 1
+        reload
+        true
+      else
+        false
+      end
+    end
+
+    # Store current user who triggered this run.
+    # Accepts either a user object directly, or resolves via the configured resolver.
+    def record_user!(user = nil)
+      user ||= resolve_current_user
+      return unless user
+
+      self.user_id = user.id.to_s
+      self.user_type = user.class.name
+      self.user_email = user.email if user.respond_to?(:email)
+    end
+
+    # Formatted user display for the UI.
+    # Uses the configured formatter or falls back to a sensible default.
+    def user_display
+      return nil if user_id.blank?
+
+      if MaintenanceOnSteroids.user_display_formatter
+        MaintenanceOnSteroids.user_display_formatter.call(self)
+      elsif user_email.present?
+        user_email
+      else
+        "#{user_type}##{user_id}"
+      end
+    rescue => e
+      Rails.logger.debug "[MaintenanceOnSteroids] user_display_formatter error: #{e.message}"
+      "#{user_type}##{user_id}"
+    end
+
+    def task_title
+      klass = JobRegistry.find(task_class)
+      klass&.task_title || task_class
+    rescue
+      task_class
+    end
+
+    def task_exists?
+      JobRegistry.find(task_class).present?
+    rescue
+      false
+    end
+
+    def output_artifacts
+      artifacts.where(kind: "output")
+    end
+
+    def input_artifacts
+      artifacts.where(kind: "input")
+    end
+
+    private
+
+    def safe_instrument(event, extra = {})
+      Instrumentation.safe_instrument(event, self, extra)
+    end
+
+    def resolve_current_user
+      resolver = MaintenanceOnSteroids.current_user_resolver
+      resolver&.call
+    rescue => e
+      Rails.logger.debug "[MaintenanceOnSteroids] Could not resolve current user: #{e.message}"
+      nil
+    end
+  end
+end

data/app/views/layouts/maintenance_on_steroids/application.html.erb

@@ -0,0 +1,48 @@
+<!DOCTYPE html>
+<html lang="en" data-theme="dark">
+<head>
+  <meta charset="utf-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1">
+  <title>Maintenance on Steroids</title>
+  <link rel="icon" href="/favicon.ico">
+  <%# Suppress the browser's automatic /favicon.ico request when the host app
+      has no favicon -- harmless if the file is absent. %>
+  <%= render "maintenance_on_steroids/shared/styles" %>
+  <script>
+    (function() {
+      var saved = localStorage.getItem('mos-theme') || 'dark';
+      document.documentElement.setAttribute('data-theme', saved);
+    })();
+  </script>
+</head>
+<body>
+  <div class="topbar">
+    <div class="topbar-inner">
+      <a href="<%= root_path %>" class="topbar-brand">
+        <span class="brand-main">Maintenance</span> <span class="brand-highlight">on Steroids</span>
+      </a>
+      <div class="topbar-nav">
+        <a href="<%= root_path %>" class="<%= "active" if request.path == root_path %>">Dashboard</a>
+        <a href="<%= jobs_path %>" class="<%= "active" if request.path.start_with?(jobs_path) %>">Tasks</a>
+        <div class="topbar-sep"></div>
+        <button class="theme-toggle" onclick="toggleTheme()" title="Toggle theme" id="theme-btn">
+          <span id="theme-icon"></span>
+        </button>
+      </div>
+    </div>
+  </div>
+
+  <div class="container">
+    <% if notice %>
+      <div class="flash flash-notice"><%= notice %></div>
+    <% end %>
+    <% if alert %>
+      <div class="flash flash-alert"><%= alert %></div>
+    <% end %>
+
+    <%= yield %>
+  </div>
+
+  <%= render "maintenance_on_steroids/shared/javascript" %>
+</body>
+</html>