solid_queue_lite 0.1.0

data/lib/solid_queue_lite/approximate_counter.rb

@@ -0,0 +1,87 @@
+require "json"
+
+module SolidQueueLite
+  module ApproximateCounter
+    module_function
+
+    def count(relation)
+      adapter_name = relation.connection.adapter_name.downcase
+
+      case adapter_name
+      when /postgres/
+        simple_relation?(relation) ? postgresql_table_estimate(relation) : postgresql_explain_estimate(relation)
+      when /mysql/, /trilogy/
+        simple_relation?(relation) ? mysql_table_estimate(relation) : mysql_explain_estimate(relation)
+      when /sqlite/
+        relation.except(:select, :order).count
+      else
+        raise NotImplementedError, "Unsupported adapter for approximate counts: #{relation.connection.adapter_name}"
+      end.to_i
+    end
+
+    def simple_relation?(relation)
+      relation.where_clause.empty? &&
+        relation.joins_values.empty? &&
+        relation.left_outer_joins_values.empty? &&
+        relation.group_values.empty? &&
+        relation.having_clause.empty? &&
+        relation.limit_value.nil? &&
+        relation.offset_value.nil? &&
+        !relation.distinct_value
+    end
+
+    def postgresql_table_estimate(relation)
+      relation.connection.select_value(<<~SQL.squish)&.to_i
+        SELECT COALESCE(reltuples, 0)
+        FROM pg_class
+        WHERE oid = #{relation.connection.quote(relation.table_name)}::regclass
+      SQL
+    end
+
+    def mysql_table_estimate(relation)
+      relation.connection.select_value(<<~SQL.squish)&.to_i
+        SELECT COALESCE(table_rows, 0)
+        FROM information_schema.tables
+        WHERE table_schema = DATABASE()
+          AND table_name = #{relation.connection.quote(relation.table_name)}
+      SQL
+    end
+
+    def postgresql_explain_estimate(relation)
+      plan_json = relation.connection.select_value(
+        "EXPLAIN (FORMAT JSON) #{relation.except(:select, :order).to_sql}"
+      )
+
+      JSON.parse(plan_json).dig(0, "Plan", "Plan Rows")
+    end
+
+    def mysql_explain_estimate(relation)
+      explain_json = relation.connection.select_value(
+        "EXPLAIN FORMAT=JSON #{relation.except(:select, :order).to_sql}"
+      )
+
+      extract_mysql_row_estimate(JSON.parse(explain_json))
+    end
+
+    def extract_mysql_row_estimate(node)
+      case node
+      when Hash
+        return node["rows_produced_per_join"] if node.key?("rows_produced_per_join")
+        return node["rows_examined_per_scan"] if node.key?("rows_examined_per_scan")
+        return node["rows"] if node.key?("rows")
+
+        node.each_value do |value|
+          estimate = extract_mysql_row_estimate(value)
+          return estimate if estimate
+        end
+      when Array
+        node.each do |value|
+          estimate = extract_mysql_row_estimate(value)
+          return estimate if estimate
+        end
+      end
+
+      nil
+    end
+  end
+end

data/lib/solid_queue_lite/engine.rb

@@ -0,0 +1,20 @@
+module SolidQueueLite
+  class Engine < ::Rails::Engine
+    isolate_namespace SolidQueueLite
+
+    initializer "solid_queue_lite.configuration" do
+      SolidQueueLite.configuration
+    end
+
+    config.after_initialize do
+      next unless SolidQueueLite.configuration.telemetry_backfill_on_boot
+      next unless ActiveRecord::Base.connected?
+      next unless ActiveRecord::Base.connection.data_source_exists?("solid_queue_lite_stats")
+
+      SolidQueueLite::Stat
+      SolidQueueLite::Telemetry.backfill!
+    rescue StandardError
+      nil
+    end
+  end
+end

data/lib/solid_queue_lite/install.rb

@@ -0,0 +1,107 @@
+require "fileutils"
+require "pathname"
+
+module SolidQueueLite
+  class Install
+    INITIALIZER_RELATIVE_PATH = "config/initializers/solid_queue_lite.rb"
+
+    def initialize(host_root: Rails.root, stdout: $stdout)
+      @host_root = Pathname(host_root)
+      @stdout = stdout
+    end
+
+    def run!(migrate: false)
+      install_migrations
+      install_initializer
+      run_migrations if migrate
+      print_next_steps(migrate: migrate)
+    end
+
+    private
+
+      attr_reader :host_root, :stdout
+
+      def install_migrations
+        with_env("FROM" => "solid_queue_lite") do
+          invoke_task("railties:install:migrations")
+        end
+      end
+
+      def install_initializer
+        if initializer_path.exist?
+          say "Skipped #{relative_initializer_path}; file already exists"
+          return
+        end
+
+        FileUtils.mkdir_p(initializer_path.dirname)
+        initializer_path.write(initializer_template)
+        say "Created #{relative_initializer_path}"
+      end
+
+      def run_migrations
+        invoke_task("db:migrate")
+      end
+
+      def print_next_steps(migrate:)
+        say ""
+        say "Solid Queue Lite install complete."
+
+        unless migrate
+          say "Run `bin/rails db:migrate` or rerun with `bin/rails solid_queue_lite:install MIGRATE=1`."
+        end
+
+        say "Mount the engine inside your host application's auth boundary, for example:"
+        say ""
+        say "authenticate :user, ->(user) { user.admin? } do"
+        say "  mount SolidQueueLite::Engine => \"/ops/jobs\""
+        say "end"
+        say ""
+        say "Schedule `SolidQueueLite::TelemetrySamplerJob` in `config/recurring.yml` if you want historical charts."
+      end
+
+      def invoke_task(task_name)
+        task = Rake::Task[task_name]
+        task.reenable
+        task.invoke
+      end
+
+      def with_env(updates)
+        previous_values = updates.transform_values { |_,| nil }
+
+        updates.each do |key, value|
+          previous_values[key] = ENV[key]
+          ENV[key] = value
+        end
+
+        yield
+      ensure
+        previous_values.each do |key, value|
+          ENV[key] = value
+        end
+      end
+
+      def initializer_path
+        host_root.join(relative_initializer_path)
+      end
+
+      def relative_initializer_path
+        INITIALIZER_RELATIVE_PATH
+      end
+
+      def initializer_template
+        <<~RUBY
+          SolidQueueLite.configure do |config|
+            config.tenant_scope = lambda do |relation|
+              relation
+            end
+
+            config.telemetry_backfill_on_boot = true
+          end
+        RUBY
+      end
+
+      def say(message)
+        stdout.puts(message)
+      end
+  end
+end

data/lib/solid_queue_lite/jobs.rb

@@ -0,0 +1,236 @@
+module SolidQueueLite
+  module Jobs
+    MAX_PAGES = 10
+    DEFAULT_PER_PAGE = 50
+    MAX_PER_PAGE = 100
+    SUPPORTED_STATES = {
+      "ready" => :ready,
+      "in_progress" => :claimed,
+      "claimed" => :claimed,
+      "scheduled" => :scheduled,
+      "failed" => :failed,
+      "recurring" => :recurring
+    }.freeze
+
+    module_function
+
+    def list(state_key:, page: 1, per_page: DEFAULT_PER_PAGE, queue_name: nil, include_details: false)
+      state = resolve_state(state_key)
+      relation = filtered_jobs_relation(state: state, queue_name: queue_name)
+      approximate_total = SolidQueueLite::ApproximateCounter.count(relation)
+      total_pages = estimated_total_pages(approximate_total, per_page)
+
+      listed_relation = relation
+        .reorder(created_at: :desc, id: :desc)
+        .limit(per_page)
+        .offset((page - 1) * per_page)
+
+      jobs = if include_details
+        listed_relation
+          .includes(
+            :ready_execution,
+            :claimed_execution,
+            :failed_execution,
+            :scheduled_execution,
+            :blocked_execution,
+            :recurring_execution
+          )
+          .map { |job| serialize_list_job(job) }
+      else
+        listed_relation
+          .pluck(:id, :class_name, :queue_name)
+          .map do |id, class_name, queue_name_value|
+            { id: id, class_name: class_name, queue_name: queue_name_value, state: state.to_s }
+          end
+      end
+
+      {
+        jobs: jobs,
+        selected_state: state_key,
+        selected_queue_name: queue_name.presence,
+        state_options: SUPPORTED_STATES.keys,
+        pagination: {
+          page: page,
+          per_page: per_page,
+          total_pages: total_pages,
+          approximate_total_count: approximate_total,
+          max_pages: MAX_PAGES
+        }
+      }
+    end
+
+    def find(id)
+      scoped_jobs(
+        ::SolidQueue::Job.includes(
+          :ready_execution,
+          :claimed_execution,
+          :failed_execution,
+          :scheduled_execution,
+          :blocked_execution,
+          :recurring_execution
+        )
+      ).find(id)
+    end
+
+    def retry!(id)
+      job = scoped_jobs(::SolidQueue::Job.includes(:failed_execution)).find(id)
+      raise StandardError, "Only failed jobs can be retried" unless job.failed_execution
+
+      job.failed_execution.retry
+      job
+    end
+
+    def discard!(id)
+      job = find(id)
+      previous_state = job.status
+      job.discard
+      [ job, previous_state ]
+    end
+
+    def bulk_retry!(job_ids:, state_key: "failed")
+      raise StandardError, "Bulk retry is only available for failed jobs" unless resolve_state(state_key) == :failed
+
+      jobs = selected_jobs(job_ids)
+      ::SolidQueue::FailedExecution.retry_all(jobs)
+      jobs.size
+    end
+
+    def bulk_discard!(job_ids:, state_key:)
+      jobs = selected_jobs(job_ids)
+      bulk_discard_execution_class(state_key).discard_all_from_jobs(jobs)
+      jobs.size
+    end
+
+    def serialize(job)
+      {
+        id: job.id,
+        active_job_id: job.active_job_id,
+        class_name: job.class_name,
+        queue_name: job.queue_name,
+        priority: job.priority,
+        scheduled_at: job.scheduled_at,
+        finished_at: job.finished_at,
+        created_at: job.created_at,
+        updated_at: job.updated_at,
+        concurrency_key: job.concurrency_key,
+        arguments: job.arguments,
+        state: job.status,
+        failed_execution: serialize_failed_execution(job.failed_execution),
+        recurring_execution: serialize_recurring_execution(job.recurring_execution)
+      }
+    end
+
+    def serialize_list_job(job)
+      serialize(job).merge(
+        state: job.status.to_s,
+        state_label: job.status.to_s.humanize,
+        error_label: job.failed_execution&.exception_class || job.status.to_s.humanize
+      )
+    end
+
+    def resolve_state(state_key)
+      SUPPORTED_STATES.fetch(state_key)
+    rescue KeyError
+      raise ::ActionController::BadRequest, "state must be one of: #{SUPPORTED_STATES.keys.join(', ')}"
+    end
+
+    def normalize_page(page)
+      value = page.to_i
+      raise ::ActionController::BadRequest, "page must be between 1 and #{MAX_PAGES}" if value < 1 || value > MAX_PAGES
+
+      value
+    end
+
+    def normalize_per_page(per_page)
+      value = per_page.to_i
+      return DEFAULT_PER_PAGE if value <= 0
+
+      [ value, MAX_PER_PAGE ].min
+    end
+
+    def selected_jobs(job_ids)
+      ids = Array(job_ids).map(&:to_i).uniq
+      raise ::ActionController::BadRequest, "At least one job must be selected" if ids.empty?
+
+      jobs = scoped_jobs(::SolidQueue::Job.where(id: ids)).to_a
+      raise ::ActionController::BadRequest, "Selected jobs could not be found" if jobs.empty?
+
+      jobs
+    end
+
+    def jobs_redirect_params(params, default_state: "failed")
+      {
+        state: params[:state].presence || default_state,
+        queue_name: params[:queue_name].presence,
+        per_page: params[:per_page].presence,
+        page: params[:page].presence
+      }.compact
+    end
+
+    def scoped_jobs(relation = ::SolidQueue::Job.all)
+      SolidQueueLite.apply_tenant_scope(relation)
+    end
+
+    def filtered_jobs_relation(state:, queue_name: nil)
+      relation = scoped_jobs(::SolidQueue::Job.all)
+      relation = relation.where(queue_name: queue_name) if queue_name.present?
+
+      case state
+      when :ready
+        relation.joins(:ready_execution)
+      when :claimed
+        relation.joins(:claimed_execution)
+      when :scheduled
+        relation.joins(:scheduled_execution)
+      when :failed
+        relation.joins(:failed_execution)
+      when :recurring
+        relation.joins(:recurring_execution)
+      else
+        raise ::ActionController::BadRequest, "Unsupported state filter: #{state}"
+      end
+    end
+
+    def serialize_failed_execution(failed_execution)
+      return unless failed_execution
+
+      {
+        id: failed_execution.id,
+        message: failed_execution.message,
+        exception_class: failed_execution.exception_class,
+        backtrace: failed_execution.backtrace,
+        created_at: failed_execution.created_at
+      }
+    end
+
+    def serialize_recurring_execution(recurring_execution)
+      return unless recurring_execution
+
+      {
+        id: recurring_execution.id,
+        task_key: recurring_execution.task_key,
+        run_at: recurring_execution.run_at,
+        created_at: recurring_execution.created_at
+      }
+    end
+
+    def estimated_total_pages(approximate_total, per_page)
+      pages = (approximate_total.to_f / per_page).ceil
+      pages = 1 if pages.zero?
+      [ pages, MAX_PAGES ].min
+    end
+
+    def bulk_discard_execution_class(state_key)
+      case resolve_state(state_key)
+      when :ready
+        ::SolidQueue::ReadyExecution
+      when :scheduled
+        ::SolidQueue::ScheduledExecution
+      when :failed
+        ::SolidQueue::FailedExecution
+      else
+        raise ::ActionController::BadRequest, "Bulk discard is supported only for ready, scheduled, and failed jobs"
+      end
+    end
+  end
+end

data/lib/solid_queue_lite/processes.rb

@@ -0,0 +1,156 @@
+module SolidQueueLite
+  module Processes
+    WILDCARD_QUEUE_NAME = "*"
+
+    module_function
+
+    def index_data
+      stale_cutoff = ::SolidQueue.process_alive_threshold.ago
+      dead_cutoff = (::SolidQueue.process_alive_threshold * 2).ago
+      available_queue_names = queue_names
+      processes = ::SolidQueue::Process
+        .order(:kind, last_heartbeat_at: :desc)
+        .pluck(:id, :kind, :name, :pid, :hostname, :last_heartbeat_at, :supervisor_id, :metadata)
+        .map do |id, kind, name, pid, hostname, last_heartbeat_at, supervisor_id, metadata|
+          status = process_status(last_heartbeat_at, stale_cutoff: stale_cutoff, dead_cutoff: dead_cutoff)
+
+          {
+            id: id,
+            kind: kind,
+            name: name,
+            pid: pid,
+            hostname: hostname,
+            last_heartbeat_at: last_heartbeat_at,
+            supervisor_id: supervisor_id,
+            stale: status != "active",
+            status: status,
+            metadata: metadata || {},
+            queue_names: queue_names_from_metadata(metadata, fallback_queue_names: available_queue_names)
+          }
+        end
+
+      {
+        processes: processes,
+        heartbeat: {
+          stale_after_seconds: ::SolidQueue.process_alive_threshold,
+          stale_cutoff: stale_cutoff,
+          dead_cutoff: dead_cutoff
+        },
+        queues: queue_rows(queue_names: available_queue_names)
+      }
+    end
+
+    def prune!
+      prunable_before = ::SolidQueue::Process.prunable.where(kind: [ "Worker", "Dispatcher", "Supervisor", "Supervisor(fork)", "Scheduler" ]).count
+      ::SolidQueue::Process.prune
+      prunable_before
+    end
+
+    def pause_queue!(queue_name)
+      queue(queue_name).pause
+    end
+
+    def resume_queue!(queue_name)
+      queue(queue_name).resume
+    end
+
+    def clear_queue!(queue_name)
+      queue(queue_name).clear
+    end
+
+    def queue_rows(queue_names: self.queue_names)
+      queue_names.map do |queue_name|
+        queue_record = queue(queue_name)
+
+        {
+          name: queue_name,
+          paused: queue_record.paused?,
+          ready_estimate: exact_queue_count(queue_name, :ready),
+          in_progress_count: exact_queue_count(queue_name, :claimed),
+          failed_count: exact_queue_count(queue_name, :failed),
+          scheduled_count: exact_queue_count(queue_name, :scheduled),
+          recurring_count: exact_queue_count(queue_name, :recurring),
+          total_jobs_count: exact_total_jobs_count(queue_name),
+          latency_seconds: queue_record.latency,
+          human_latency: queue_record.human_latency
+        }
+      end
+    end
+
+    def queue(queue_name)
+      ::SolidQueue::Queue.find_by_name(queue_name)
+    end
+
+    def exact_queue_count(queue_name, state)
+      relation = SolidQueueLite::Jobs.filtered_jobs_relation(state: state, queue_name: queue_name)
+      relation.except(:select, :order).count
+    end
+
+    def exact_total_jobs_count(queue_name)
+      SolidQueueLite::Jobs.scoped_jobs(::SolidQueue::Job.where(queue_name: queue_name)).count
+    end
+
+    def process_status(last_heartbeat_at, stale_cutoff:, dead_cutoff:)
+      return "dead" unless last_heartbeat_at
+      return "dead" if last_heartbeat_at <= dead_cutoff
+      return "stale" if last_heartbeat_at <= stale_cutoff
+
+      "active"
+    end
+
+    def queue_names_from_metadata(metadata, fallback_queue_names: [])
+      return [] unless metadata.is_a?(Hash)
+
+      value = metadata["queues"] || metadata[:queues] || metadata["queue_names"] || metadata[:queue_names]
+      queue_names = extract_queue_names(value)
+
+      if queue_names.include?(WILDCARD_QUEUE_NAME)
+        (queue_names - [ WILDCARD_QUEUE_NAME ] + fallback_queue_names).uniq
+      else
+        queue_names
+      end
+    end
+
+    def queue_names
+      (
+        ::SolidQueue::Queue.all.map(&:name) +
+        configured_queue_names +
+        configured_process_queue_names
+      ).reject { |queue_name| wildcard_queue_name?(queue_name) }.uniq.sort
+    end
+
+    def configured_queue_names
+      ::SolidQueue::Configuration.new.configured_processes.filter_map do |configured_process|
+        next unless configured_process.kind.to_sym == :worker
+
+        configured_process.attributes[:queues]
+      end.flat_map { |value| extract_queue_names(value) }
+    rescue StandardError
+      []
+    end
+
+    def configured_process_queue_names
+      ::SolidQueue::Process.pluck(:metadata).flat_map do |metadata|
+        next [] unless metadata.is_a?(Hash)
+
+        value = metadata["queues"] || metadata[:queues] || metadata["queue_names"] || metadata[:queue_names]
+        extract_queue_names(value)
+      end
+    end
+
+    def extract_queue_names(value)
+      case value
+      when String
+        value.split(/[\s,]+/).map(&:presence).compact
+      when Array
+        value.flat_map { |entry| extract_queue_names(entry) }
+      else
+        []
+      end
+    end
+
+    def wildcard_queue_name?(queue_name)
+      queue_name == WILDCARD_QUEUE_NAME
+    end
+  end
+end