maintenance_on_steroids 0.2.0

data/app/controllers/maintenance_on_steroids/application_controller.rb

@@ -0,0 +1,66 @@
+module MaintenanceOnSteroids
+  class ApplicationController < MaintenanceOnSteroids.parent_controller.constantize
+    protect_from_forgery with: :exception
+
+    layout "maintenance_on_steroids/application"
+
+    before_action :verify_http_basic_authentication
+    before_action :run_authentication_hook
+    before_action :verify_access
+
+    private
+
+    # Step 1: HTTP Basic auth (if enabled)
+    def verify_http_basic_authentication
+      return unless MaintenanceOnSteroids.http_basic_authentication_enabled
+
+      # A blank configured password would make secure_compare(supplied, "")
+      # succeed for anyone sending an empty password. Refuse outright rather
+      # than authenticate against nothing.
+      if MaintenanceOnSteroids.http_basic_authentication_password.to_s.empty?
+        Rails.logger.error(
+          "[MaintenanceOnSteroids] HTTP Basic is enabled but the password is blank; denying access. " \
+          "Set MaintenanceOnSteroids.http_basic_authentication_password."
+        )
+        return render plain: "Access denied", status: :forbidden
+      end
+
+      authenticate_or_request_with_http_basic("Maintenance on Steroids") do |username, password|
+        ActiveSupport::SecurityUtils.secure_compare(username, MaintenanceOnSteroids.http_basic_authentication_user_name.to_s) &
+          ActiveSupport::SecurityUtils.secure_compare(password, MaintenanceOnSteroids.http_basic_authentication_password.to_s)
+      end
+    end
+
+    # Step 2: General-purpose authentication hook (if configured)
+    def run_authentication_hook
+      return unless MaintenanceOnSteroids.authentication
+
+      instance_exec(&MaintenanceOnSteroids.authentication)
+    end
+
+    # Step 3: Proc-based access check (if configured)
+    def verify_access
+      return unless MaintenanceOnSteroids.verify_access_proc
+
+      unless MaintenanceOnSteroids.verify_access_proc.call(self)
+        render plain: "Access denied", status: :forbidden
+      end
+    end
+
+    # Resolve the current user in controller context where Devise/Warden methods are available.
+    # The resolver proc receives the controller so it can call current_user, etc.
+    def resolve_current_user
+      resolver = MaintenanceOnSteroids.current_user_resolver
+      return nil unless resolver
+
+      if resolver.arity == 0
+        resolver.call
+      else
+        resolver.call(self)
+      end
+    rescue => e
+      Rails.logger.debug "[MaintenanceOnSteroids] Could not resolve current user: #{e.message}"
+      nil
+    end
+  end
+end

data/app/controllers/maintenance_on_steroids/dashboard_controller.rb

@@ -0,0 +1,18 @@
+module MaintenanceOnSteroids
+  class DashboardController < ApplicationController
+    def index
+      @task_classes = MaintenanceOnSteroids.task_classes
+      @active_runs = Run.active.recent.limit(25)
+      @recent_runs = Run.recent.limit(25)
+      run_ids = @active_runs.map(&:id) | @recent_runs.map(&:id)
+      @artifact_counts = Artifact.where(run_id: run_ids, kind: "output").group(:run_id).count
+      status_counts = Run.group(:status).count
+      @stats = {
+        total_tasks: @task_classes.size,
+        active_runs: status_counts.values_at(*Run::ACTIVE_STATUSES).compact.sum,
+        completed: status_counts.fetch("completed", 0),
+        errored: status_counts.fetch("errored", 0)
+      }
+    end
+  end
+end

data/app/controllers/maintenance_on_steroids/jobs_controller.rb

@@ -0,0 +1,59 @@
+module MaintenanceOnSteroids
+  class JobsController < ApplicationController
+    before_action :set_task_class, only: %i[show source]
+
+    def index
+      @task_classes = MaintenanceOnSteroids.task_classes
+      @active_run_counts = MaintenanceOnSteroids::Run.active.group(:task_class).count
+      # Fetch only the latest run per task_class (MAX(id) per group), avoiding
+      # loading every historical run. Works on both SQLite and Postgres.
+      names = @task_classes.map(&:name)
+      @last_runs = MaintenanceOnSteroids::Run
+                   .where(id: MaintenanceOnSteroids::Run.where(task_class: names).group(:task_class).select("MAX(id)"))
+                   .index_by(&:task_class)
+      @sort = params[:sort] == "last_run" ? "last_run" : "name"
+      # Class name is the tiebreaker in both orders. sort_by is not stable, so
+      # without it two tasks sharing a title -- or any two never-executed tasks,
+      # which all share the same epoch timestamp -- swap places between page
+      # loads for no reason.
+      @task_classes =
+        if @sort == "last_run"
+          # Most recently executed first; never-executed tasks at the bottom.
+          @task_classes.sort_by { |tc| [-(@last_runs[tc.name]&.created_at.to_i || 0), tc.name.to_s] }
+        else
+          @task_classes.sort_by { |tc| [tc.task_title.to_s.downcase, tc.name.to_s] }
+        end
+    end
+
+    def show
+      @per_page = 50
+      scope = MaintenanceOnSteroids::Run.where(task_class: @task_class.name)
+      @total_runs = scope.count
+      @total_pages = [(@total_runs.to_f / @per_page).ceil, 1].max
+      # Clamp both ends so ?page=0/-1 and ?page=99999 don't render dead pages
+      # or trigger huge offset scans.
+      @page = params[:page].to_i.clamp(1, @total_pages)
+      @runs = scope.recent.limit(@per_page).offset((@page - 1) * @per_page)
+      @artifact_counts = MaintenanceOnSteroids::Artifact
+                         .where(run_id: @runs.map(&:id), kind: "output")
+                         .group(:run_id).count
+    end
+
+    def source
+      @source_file, @source_line = Object.const_source_location(@task_class.name)
+      @source_code =
+        if @source_file && @source_file.start_with?(Rails.root.to_s) && File.exist?(@source_file)
+          File.read(@source_file)
+        end
+    end
+
+    private
+
+    # JobRegistry.find only resolves registered Task subclasses; anything
+    # else (arbitrary user-supplied constants) returns nil -> 404.
+    def set_task_class
+      @task_class = MaintenanceOnSteroids::JobRegistry.find(params[:id])
+      raise ActiveRecord::RecordNotFound, "Unknown task: #{params[:id]}" unless @task_class
+    end
+  end
+end

data/app/controllers/maintenance_on_steroids/runs_controller.rb

@@ -0,0 +1,223 @@
+module MaintenanceOnSteroids
+  class RunsController < ApplicationController
+    before_action :set_run, only: %i[show pause resume cancel artifact_download]
+    before_action :set_task_class, only: %i[new create]
+
+    def new
+    end
+
+    def create
+      missing = missing_required_inputs
+      if missing.any?
+        flash.now[:alert] = "Missing required parameters: #{missing.map(&:label).join(', ')}."
+        return render :new, status: :unprocessable_entity
+      end
+
+      invalid = invalid_scalar_inputs
+      if invalid.any?
+        flash.now[:alert] = "Invalid value for: #{invalid.map(&:label).join(', ')}."
+        return render :new, status: :unprocessable_entity
+      end
+
+      if concurrency_exceeded?
+        limit = @task_class.job_config.concurrency
+        flash.now[:alert] =
+          "This task allows #{limit} concurrent #{'run'.pluralize(limit)} and that many are already active."
+        return render :new, status: :unprocessable_entity
+      end
+
+      oversized = oversized_file_inputs
+      if oversized.any?
+        max_mb = MaintenanceOnSteroids.max_upload_size / (1024 * 1024)
+        flash.now[:alert] = "File too large for: #{oversized.map(&:label).join(', ')} (max #{max_mb} MB)."
+        return render :new, status: :unprocessable_entity
+      end
+
+      run = MaintenanceOnSteroids::Run.new(
+        task_class: @task_class.name,
+        status: "enqueued",
+        params: extract_scalar_params,
+        progress_current: 0,
+        progress_total: 0
+      )
+
+      run.record_user!(resolve_current_user)
+
+      saved = MaintenanceOnSteroids::Run.transaction do
+        if run.save
+          store_file_params(run)
+          true
+        else
+          false
+        end
+      end
+
+      return render :new, status: :unprocessable_entity unless saved
+
+      begin
+        run.enqueue!
+      rescue MaintenanceOnSteroids::EnqueueFailed => e
+        return redirect_to run_path(run), alert: "Run could not be enqueued: #{e.message}"
+      end
+
+      redirect_to run_path(run), notice: "Task enqueued successfully."
+    end
+
+    def show
+      # May be nil when the task class no longer exists -- the view falls
+      # back to @run.task_title.
+      @task_class_obj = MaintenanceOnSteroids::JobRegistry.find(@run.task_class)
+    end
+
+    def pause
+      if @run.pause!
+        redirect_to run_path(@run), notice: "Task pause requested."
+      else
+        redirect_to run_path(@run), alert: "Task cannot be paused (status: #{@run.status})."
+      end
+    end
+
+    def resume
+      if @run.resume!
+        redirect_to run_path(@run), notice: "Task resumed."
+      else
+        redirect_to run_path(@run), alert: "Task cannot be resumed (status: #{@run.reload.status})."
+      end
+    rescue MaintenanceOnSteroids::EnqueueFailed => e
+      # Narrow on purpose: Run#resume! already rolled the run back to a terminal
+      # status, so show the operator why instead of a 500. Anything else still
+      # propagates to the host app's error reporting rather than being
+      # mislabelled as an enqueue failure.
+      redirect_to run_path(@run), alert: "Run could not be enqueued: #{e.message}"
+    end
+
+    def cancel
+      if @run.cancel!
+        redirect_to run_path(@run), notice: "Task cancelled."
+      else
+        redirect_to run_path(@run), alert: "Task cannot be cancelled (status: #{@run.status})."
+      end
+    end
+
+    # JSON endpoint for progress polling -- loads only the rendered columns.
+    def status
+      run = MaintenanceOnSteroids::Run
+              .select(:id, :status, :progress_current, :progress_total, :started_at, :completed_at, :error_message)
+              .find(params[:id])
+
+      render json: {
+        status: run.status,
+        progress_current: run.progress_current,
+        progress_total: run.progress_total,
+        progress_percentage: run.progress_percentage,
+        formatted_duration: run.formatted_duration,
+        formatted_estimated_duration: run.formatted_estimated_duration,
+        error_message: run.error_message
+      }
+    end
+
+    def artifact_download
+      artifact = @run.artifacts.find(params[:artifact_id])
+      # jsonb/text artifacts have no data_blob -- without this the response is
+      # a 200 carrying a zero-byte ".bin", which reads as "the task produced
+      # nothing" rather than "this artifact isn't a file".
+      raise ActiveRecord::RecordNotFound, "Artifact #{artifact.id} is not downloadable" unless artifact.downloadable?
+
+      send_data artifact.data_blob,
+                filename: artifact.download_file_name,
+                type: artifact.download_content_type,
+                disposition: "attachment"
+    end
+
+    private
+
+    def set_run
+      @run = MaintenanceOnSteroids::Run.find(params[:id])
+    end
+
+    # JobRegistry.find only resolves registered Task subclasses; anything
+    # else (arbitrary user-supplied constants) returns nil -> 404.
+    def set_task_class
+      @task_class = MaintenanceOnSteroids::JobRegistry.find(params[:job_id])
+      raise ActiveRecord::RecordNotFound, "Unknown task: #{params[:job_id]}" unless @task_class
+    end
+
+    def missing_required_inputs
+      @task_class.form_inputs.select(&:required).select do |input|
+        value = params.dig(:task_params, input.name)
+        input.blob? ? !value.respond_to?(:read) : value.blank?
+      end
+    end
+
+    # Advisory guard for `job { concurrency N }`. Racy by nature -- two
+    # simultaneous submissions can both pass -- but it catches the case this
+    # exists for: a double-clicked New Run starting a destructive task twice.
+    def concurrency_exceeded?
+      limit = @task_class.job_config.concurrency
+      return false if limit.nil? || limit <= 0
+
+      MaintenanceOnSteroids::Run.active.where(task_class: @task_class.name).count >= limit
+    end
+
+    # The form only constrains the browser -- the posted value is whatever the
+    # client sends. Two things are rejected rather than handed to the task:
+    # a <select> value outside its declared options, and a nested structure
+    # (`task_params[name][x]=1`) where a scalar was declared.
+    def invalid_scalar_inputs
+      @task_class.form_inputs.reject(&:blob?).select do |input|
+        value = params.dig(:task_params, input.name)
+        next false if value.nil?
+        next true unless scalar_param?(value)
+
+        input.type == :select && input.options.present? &&
+          value.present? && input.options.map(&:to_s).exclude?(value.to_s)
+      end
+    end
+
+    # Rails gives scalars as Strings; anything hash- or array-shaped came from
+    # a client building its own payload.
+    def scalar_param?(value)
+      !value.is_a?(Array) &&
+        !value.is_a?(Hash) &&
+        !value.is_a?(ActionController::Parameters)
+    end
+
+    def oversized_file_inputs
+      @task_class.form_inputs.select(&:blob?).select do |input|
+        file = params.dig(:task_params, input.name)
+        file.respond_to?(:size) && file.size.to_i > MaintenanceOnSteroids.max_upload_size
+      end
+    end
+
+    def extract_scalar_params
+      scalar_inputs = @task_class.form_inputs.reject(&:blob?)
+
+      result = {}
+      scalar_inputs.each do |input|
+        value = params.dig(:task_params, input.name)
+        result[input.name] = value if value.present?
+      end
+      result
+    end
+
+    def store_file_params(run)
+      blob_inputs = @task_class.form_inputs.select(&:blob?)
+
+      blob_inputs.each do |input|
+        file = params.dig(:task_params, input.name)
+        next unless file.respond_to?(:read)
+
+        artifact = run.artifacts.new(
+          name: input.name.to_s,
+          kind: "input",
+          artifact_type: "blob",
+          data_blob: file.read,
+          file_name: file.original_filename,
+          content_type: file.content_type
+        )
+        artifact.refresh_metadata!
+        artifact.save!
+      end
+    end
+  end
+end

data/app/jobs/maintenance_on_steroids/run_job.rb

@@ -0,0 +1,292 @@
+module MaintenanceOnSteroids
+  class RunJob < ActiveJob::Base
+    include ActiveJob::Continuable
+
+    # The run row is the record of truth. If it was deleted there is nothing
+    # left to do, and retrying only produces a RecordNotFound storm in the
+    # queue -- ~21 failing attempts over a day for work that can never succeed.
+    discard_on ActiveRecord::RecordNotFound
+
+    # Continuable would otherwise silently re-enqueue a job that raised after
+    # the continuation advanced. This gem surfaces errors to an operator
+    # instead: #perform marks the run "errored" and stops, and the operator
+    # resumes it from the dashboard (Run#resumable? includes "errored").
+    # Left on, that hidden retry fires against an already-terminal run and
+    # no-ops, so the run looks retried but never advances.
+    self.resume_errors_after_advancing = false
+    attr_writer :enqueue_failure_backtrace
+
+    # Continuable dispatches retries itself, bypassing Run#enqueue!. Preserve a
+    # visible, resumable failure if that dispatch is refused or the queue is down.
+    def enqueue(...)
+      result = super
+      record_dispatch_failure!(enqueue_error || EnqueueFailed.new("Enqueue callback aborted the job")) unless result
+      result
+    rescue => e
+      record_dispatch_failure!(e)
+      raise
+    end
+
+    def perform(run_id)
+      @run = Run.find(run_id)
+      return unless claim_run!
+
+      @task = @run.task_instance
+      @database_role = @task.class.job_config.database_role
+      @task.checkpoint_handler = method(:task_checkpoint!)
+
+      catch(:abort_run) do
+        if @first_start
+          safe_instrument(:started, @run)
+          safe_callback { @task.run_start_callbacks }
+        end
+        check_status!
+        if @task.collection_task?
+          process_collection
+        elsif @task.callable_task?
+          step :execute do |_step|
+            check_status!
+            @task.call
+          end
+        else
+          raise "Task #{@run.task_class} must define either collection+process or call"
+        end
+        complete_run!
+      end
+    rescue ExecutionLost
+      # A reaper or newer attempt owns the row. Never flush this worker's cache.
+      nil
+    rescue ActiveJob::Continuation::Interrupt
+      begin
+        @run.with_execution_lock do
+          flush_artifacts!
+          @run.update!(status: @run.running? ? "enqueued" : @run.status, execution_token: nil)
+        end
+      rescue ExecutionLost
+        return
+      rescue => e
+        record_error!(e)
+        raise e
+      end
+      raise
+    rescue => e
+      record_error!(e)
+      raise
+    end
+
+    private
+
+    def record_dispatch_failure!(error)
+      return unless arguments.first
+
+      Run.where(id: arguments.first, active_job_id: job_id, execution_token: nil,
+                status: %w[enqueued pausing cancelling]).update_all(
+        status: "errored", error_message: "Failed to enqueue: #{error.message}",
+        error_backtrace: @enqueue_failure_backtrace,
+        completed_at: Time.current, updated_at: Time.current
+      )
+    end
+
+    def claim_run!
+      @run.with_lock do
+        return false unless @run.execution_token.nil?
+        return false unless %w[enqueued pausing cancelling].include?(@run.status)
+        return false if @run.active_job_id && @run.active_job_id != job_id
+
+        @run.worker_token = SecureRandom.uuid
+        @first_start = @run.enqueued? && @run.started_at.nil?
+        attributes = { execution_token: @run.worker_token, active_job_id: job_id }
+        if @run.enqueued?
+          attributes.merge!(status: "running", started_at: @run.started_at || Time.current)
+        end
+        @run.update!(attributes)
+      end
+      true
+    end
+
+    def record_error!(error)
+      return unless @run&.worker_token
+
+      @run.with_execution_lock do
+        safe_callback { @task&.run_error_callbacks }
+        # Cleanup must not replace the original task error. A flush error on
+        # the normal completion path reaches here as the primary exception.
+        begin
+          flush_artifacts!
+        rescue => flush_error
+          Rails.logger.error "[MaintenanceOnSteroids] Artifact cleanup error: #{flush_error.message}"
+        end
+        @run.update!(status: "errored", execution_token: nil,
+                     error_message: error.message,
+                     error_backtrace: error.backtrace&.first(50)&.join("\n"),
+                     completed_at: Time.current)
+      end
+      safe_instrument(:errored, @run, error: error)
+    rescue ExecutionLost
+      nil
+    end
+
+    def process_collection
+      step :process_collection do |step|
+        @collection ||= with_collection_role { @task.collection }
+        warn_about_non_integer_primary_key(@collection)
+
+        if @run.progress_total.zero?
+          total = with_collection_role { @collection.count }
+          @run.with_execution_lock { @run.update!(progress_total: total) }
+        end
+
+        # The DB cursor (@run.cursor) is authoritative: it is written after
+        # every processed record and *before* step.set!, so it is always at
+        # least as advanced as the step cursor serialized with the job.
+        # step.cursor only matters when the DB cursor is absent.
+        effective_cursor = @run.cursor.presence || step.cursor
+
+        scope = if effective_cursor
+                  @collection.unscope(:order).where(@collection.model.arel_table[@collection.model.primary_key].gt(effective_cursor))
+                else
+                  @collection.unscope(:order)
+                end
+
+        # The role wraps the whole scan, so every batch query hits the replica.
+        # Each record's work steps back to :writing -- process may write, and
+        # the run's own cursor/progress bookkeeping always does.
+        with_collection_role do
+          scope.order(@collection.model.primary_key => :asc).find_each do |record|
+            with_writing_role do
+              check_status!
+
+              checkpointed = false
+              begin
+                @processing_record = true
+                @task.process(record)
+                cursor_value = record.public_send(record.class.primary_key)
+                # A crash commits both output and cursor, or neither. Host task
+                # side effects still need idempotency (they may use other DBs/APIs).
+                @run.with_execution_lock do
+                  flush_artifacts!
+                  advance_progress!(cursor_value)
+                end
+                checkpointed = true
+              ensure
+                @processing_record = false
+                # A failed record must not leak buffered rows into error cleanup.
+                @task.artifacts.discard! unless checkpointed
+              end
+              # Record the last processed pk as an exclusive cursor (resume uses gt(cursor)),
+              # matching @run.cursor semantics. advance! would store pk+1 and skip a record.
+              step.set!(cursor_value)
+            end
+          end
+        end
+      end
+    end
+
+    # Both are pass-throughs unless the task declared `job { database_role ... }`,
+    # so the default path pays nothing for connection switching.
+    def with_collection_role(&block)
+      return yield unless @database_role
+
+      ActiveRecord::Base.connected_to(role: @database_role, &block)
+    end
+
+    def with_writing_role(&block)
+      return yield unless @database_role
+
+      ActiveRecord::Base.connected_to(role: :writing, &block)
+    end
+
+    # Called while holding the execution lock and the output transaction.
+    def advance_progress!(cursor_value)
+      @run.update!(progress_current: @run.progress_current + 1, cursor: cursor_value.to_s)
+    end
+
+    def complete_run!
+      completed = false
+      @run.with_execution_lock do
+        if @run.running?
+          # Commit callback output and completion together. An operator waiting
+          # on this short finalization lock will then see the final state.
+          safe_callback { @task.run_complete_callbacks }
+          flush_artifacts!
+          @run.reload
+          if @run.running?
+            @run.update!(status: "completed", completed_at: Time.current, execution_token: nil)
+            completed = true
+          end
+        end
+      end
+      if completed
+        safe_instrument(:succeeded, @run)
+      else
+        check_status!
+      end
+    end
+
+    def warn_about_non_integer_primary_key(collection)
+      return if @pk_checked
+
+      @pk_checked = true
+      model = collection.model
+      pk_type = model.columns_hash[model.primary_key.to_s]&.type
+      return if pk_type == :integer
+
+      Rails.logger.warn(
+        "[MaintenanceOnSteroids] #{@run.task_class}: collection primary key " \
+        "#{model.primary_key.inspect} has type #{pk_type.inspect}. Cursor-based " \
+        "resumption relies on monotonically increasing primary keys; " \
+        "UUID/string keys can skip or repeat records on resume."
+      )
+    end
+
+    def task_checkpoint!
+      if @processing_record
+        # Do not persist partial record output without its cursor. Collection
+        # pause/cancel takes effect at the next record boundary.
+        @run.with_execution_lock { @run.touch }
+      else
+        check_status!
+      end
+    end
+
+    def check_status!
+      event = nil
+      @run.with_execution_lock do
+        case @run.status
+        when "pausing", "cancelling"
+          safe_callback { @task.run_interrupt_callbacks }
+          # Re-read in case a callback requested cancellation instead of pause.
+          @run.reload
+          event = @run.cancelling? ? :cancelled : :paused
+          safe_callback { event == :paused ? @task.run_pause_callbacks : @task.run_cancel_callbacks }
+          flush_artifacts!
+          @run.update!(status: event.to_s, execution_token: nil,
+                       completed_at: event == :cancelled ? Time.current : nil)
+        else
+          # Callable checkpoints are a real heartbeat and persist their output.
+          # Collection output is committed only alongside its record cursor.
+          flush_artifacts! unless @task.collection_task?
+          @run.touch
+        end
+      end
+      if event
+        safe_instrument(event, @run)
+        throw :abort_run
+      end
+    end
+
+    def safe_callback
+      yield
+    rescue => e
+      Rails.logger.error "[MaintenanceOnSteroids] Callback error: #{e.message}"
+    end
+
+    def safe_instrument(event, run, extra = {})
+      MaintenanceOnSteroids::Instrumentation.safe_instrument(event, run, extra)
+    end
+
+    def flush_artifacts!
+      @task&.artifacts&.flush!
+    end
+  end
+end

data/app/models/maintenance_on_steroids/application_record.rb

@@ -0,0 +1,6 @@
+module MaintenanceOnSteroids
+  class ApplicationRecord < ActiveRecord::Base
+    self.abstract_class = true
+    self.table_name_prefix = "maintenance_on_steroids_"
+  end
+end