flight_control 2.0.0.beta.1

data/lib/active_job/queue_adapters/solid_queue_ext.rb

@@ -0,0 +1,309 @@
+module ActiveJob::QueueAdapters::SolidQueueExt
+  include FlightControl::Adapter
+  include Workers
+  include RecurringTasks
+
+  def queues
+    queues = SolidQueue::Queue.all
+    pauses = SolidQueue::Pause.where(queue_name: queues.map(&:name)).index_by(&:queue_name)
+
+    queues.collect do |queue|
+      {
+        name: queue.name,
+        size: queue.size,
+        active: pauses[queue.name].nil?
+      }
+    end
+  end
+
+  def queue_size(queue_name)
+    find_queue_by_name(queue_name).size
+  end
+
+  def clear_queue(queue_name)
+    find_queue_by_name(queue_name).clear
+  end
+
+  def pause_queue(queue_name)
+    find_queue_by_name(queue_name).pause
+  end
+
+  def resume_queue(queue_name)
+    find_queue_by_name(queue_name).resume
+  end
+
+  def queue_paused?(queue_name)
+    find_queue_by_name(queue_name).paused?
+  end
+
+  def supported_job_statuses
+    SolidQueueJobs::STATUS_MAP.keys
+  end
+
+  def supported_job_filters(*)
+    [:queue_name, :job_class_name, :finished_at]
+  end
+
+  def jobs_count(jobs_relation)
+    SolidQueueJobs.new(jobs_relation).count
+  end
+
+  def fetch_jobs(jobs_relation)
+    SolidQueueJobs.new(jobs_relation).jobs.map do |job|
+      deserialize_and_proxy_solid_queue_job(job, jobs_relation.status)
+    end
+  end
+
+  def retry_all_jobs(jobs_relation)
+    SolidQueueJobs.new(jobs_relation).retry_all
+  end
+
+  def retry_job(job, jobs_relation)
+    find_solid_queue_job!(job.job_id, jobs_relation).retry
+  end
+
+  def discard_all_jobs(jobs_relation)
+    SolidQueueJobs.new(jobs_relation).discard_all
+  end
+
+  def discard_job(job, jobs_relation)
+    find_solid_queue_job!(job.job_id, jobs_relation).discard
+  end
+
+  def dispatch_job(job, jobs_relation)
+    dispatch_immediately find_solid_queue_job!(job.job_id, jobs_relation)
+  end
+
+  def find_job(job_id, *)
+    if (job = SolidQueue::Job.where(active_job_id: job_id).order(:id).last)
+      deserialize_and_proxy_solid_queue_job job
+    end
+  end
+
+  private
+
+  def find_queue_by_name(queue_name)
+    SolidQueue::Queue.find_by_name(queue_name)
+  end
+
+  def find_solid_queue_job!(job_id, jobs_relation)
+    find_solid_queue_job(job_id, jobs_relation) or raise ActiveJob::Errors::JobNotFoundError.new(job_id, jobs_relation)
+  end
+
+  def find_solid_queue_job(job_id, jobs_relation)
+    SolidQueueJobs.new(jobs_relation).find_job(job_id)
+  end
+
+  def deserialize_and_proxy_solid_queue_job(solid_queue_job, job_status = nil)
+    job_status ||= status_from_solid_queue_job(solid_queue_job)
+
+    ActiveJob::JobProxy.new(solid_queue_job.arguments).tap do |job|
+      job.status = job_status
+      job.last_execution_error = execution_error_from_solid_queue_job(solid_queue_job) if job_status == :failed
+      job.raw_data = solid_queue_job.as_json
+      job.filtered_raw_data = filter_raw_data_arguments(job.raw_data)
+      job.failed_at = solid_queue_job&.failed_execution&.created_at if job_status == :failed
+      job.finished_at = solid_queue_job.finished_at
+      job.blocked_by = solid_queue_job.concurrency_key
+      job.blocked_until = solid_queue_job&.blocked_execution&.expires_at if job_status == :blocked
+      job.worker_id = solid_queue_job&.claimed_execution&.process_id if job_status == :in_progress
+      job.started_at = solid_queue_job&.claimed_execution&.created_at if job_status == :in_progress
+      job.scheduled_at = solid_queue_job.scheduled_at
+    end
+  end
+
+  def filter_raw_data_arguments(raw_data)
+    raw_data.deep_dup.tap do |filtered_raw_data|
+      filtered_raw_data["arguments"]["arguments"] = FlightControl.job_arguments_filter.apply_to(filtered_raw_data.dig("arguments", "arguments"))
+    end
+  end
+
+  def status_from_solid_queue_job(solid_queue_job)
+    SolidQueueJobs::STATUS_MAP.invert[solid_queue_job.status]
+  end
+
+  def execution_error_from_solid_queue_job(solid_queue_job)
+    if solid_queue_job.failed?
+      ActiveJob::ExecutionError.new \
+        error_class: solid_queue_job.failed_execution.exception_class,
+        message: solid_queue_job.failed_execution.message,
+        backtrace: solid_queue_job.failed_execution.backtrace || []
+    end
+  end
+
+  def dispatch_immediately(job)
+    if job.blocked?
+      SolidQueue::Job.transaction do
+        job.dispatch_bypassing_concurrency_limits
+        job.blocked_execution.destroy!
+      end
+    else
+      job.scheduled_execution.update!(scheduled_at: Time.now)
+    end
+  end
+
+  class SolidQueueJobs
+    STATUS_MAP = {
+      pending: :ready,
+      failed: :failed,
+      in_progress: :claimed,
+      blocked: :blocked,
+      scheduled: :scheduled,
+      finished: :finished
+    }
+
+    def initialize(jobs_relation)
+      @jobs_relation = jobs_relation
+    end
+
+    def jobs
+      solid_queue_status.finished? ? order_finished_jobs(finished_jobs) : order_executions(executions).map(&:job).compact
+    end
+
+    def count
+      limit_value_provided? ? direct_count : internally_limited_count
+    end
+
+    def find_job(active_job_id)
+      if (job = SolidQueue::Job.where(active_job_id: active_job_id).order(:id).last)
+        job if matches_relation_filters?(job)
+      end
+    end
+
+    def discard_all
+      execution_class_by_status.discard_all_from_jobs(jobs)
+    end
+
+    def retry_all
+      SolidQueue::FailedExecution.retry_all(jobs)
+    end
+
+    private
+
+    attr_reader :jobs_relation
+
+    delegate :queue_name, :limit_value, :limit_value_provided?, :offset_value, :job_class_name,
+      :default_page_size, :worker_id, :recurring_task_id, :finished_at, to: :jobs_relation
+
+    def executions
+      execution_class_by_status
+        .then { |executions| include_execution_association(executions) }
+        .then { |executions| filter_executions_by_queue(executions) }
+        .then { |executions| filter_executions_by_class(executions) }
+        .then { |executions| filter_executions_by_process_id(executions) }
+        .then { |executions| filter_executions_by_task_key(executions) }
+        .then { |executions| limit(executions) }
+        .then { |executions| offset(executions) }
+    end
+
+    def finished_jobs
+      SolidQueue::Job.finished
+        .then { |jobs| filter_jobs_by_queue(jobs) }
+        .then { |jobs| filter_jobs_by_class(jobs) }
+        .then { |jobs| filter_jobs_by_finished_at(jobs) }
+        .then { |jobs| limit(jobs) }
+        .then { |jobs| offset(jobs) }
+    end
+
+    def order_finished_jobs(jobs)
+      jobs.order(finished_at: :desc)
+    end
+
+    def order_executions(executions)
+      # Follow polling order for scheduled executions, the rest newest first
+      if solid_queue_status.scheduled?
+        executions.ordered
+      else
+        executions.order(job_id: :desc)
+      end
+    end
+
+    def matches_relation_filters?(job)
+      matches_status?(job) && matches_queue_name?(job)
+    end
+
+    def direct_count
+      solid_queue_status.finished? ? finished_jobs.count : executions.count
+    end
+
+    def internally_limited_count
+      count_limit = FlightControl.internal_query_count_limit + 1
+      limited_count = solid_queue_status.finished? ? finished_jobs.limit(count_limit).count : executions.limit(count_limit).count
+      (limited_count == count_limit) ? Float::INFINITY : limited_count
+    end
+
+    def execution_class_by_status
+      if recurring_task_id.present?
+        SolidQueue::RecurringExecution
+      elsif solid_queue_status.present? && !solid_queue_status.finished?
+        "SolidQueue::#{solid_queue_status.capitalize}Execution".safe_constantize
+      else
+        raise ActiveJob::Errors::QueryError, "Status not supported: #{jobs_relation.status}"
+      end
+    end
+
+    def include_execution_association(executions)
+      solid_queue_status.present? ? executions.includes(job: "#{solid_queue_status}_execution") : executions.includes(:job)
+    end
+
+    def filter_executions_by_queue(executions)
+      return executions unless queue_name.present?
+
+      if solid_queue_status.ready?
+        executions.where(queue_name: queue_name)
+      else
+        executions.where(job: {queue_name: queue_name})
+      end
+    end
+
+    def filter_jobs_by_queue(jobs)
+      queue_name.present? ? jobs.where(queue_name: queue_name) : jobs
+    end
+
+    def filter_executions_by_class(executions)
+      job_class_name.present? ? executions.where(job: {class_name: job_class_name}) : executions
+    end
+
+    def filter_executions_by_process_id(executions)
+      return executions unless worker_id.present?
+
+      if solid_queue_status.claimed?
+        executions.where(process_id: worker_id)
+      else
+        raise ActiveJob::Errors::QueryError, "Filtering by worker id is not supported for status #{jobs_relation.status}"
+      end
+    end
+
+    def filter_executions_by_task_key(executions)
+      recurring_task_id.present? ? executions.where(task_key: recurring_task_id) : executions
+    end
+
+    def filter_jobs_by_class(jobs)
+      job_class_name.present? ? jobs.where(class_name: job_class_name) : jobs
+    end
+
+    def filter_jobs_by_finished_at(jobs)
+      finished_at.present? ? jobs.where(finished_at: finished_at) : jobs
+    end
+
+    def limit(executions_or_jobs)
+      limit_value.present? ? executions_or_jobs.limit(limit_value) : executions_or_jobs
+    end
+
+    def offset(executions_or_jobs)
+      offset_value.present? ? executions_or_jobs.offset(offset_value) : executions_or_jobs
+    end
+
+    def matches_status?(job)
+      solid_queue_status.blank? || job.public_send("#{solid_queue_status}?")
+    end
+
+    def matches_queue_name?(job)
+      queue_name.blank? || job.queue_name == queue_name
+    end
+
+    def solid_queue_status
+      STATUS_MAP[jobs_relation.status].to_s.inquiry
+    end
+  end
+end

data/lib/active_job/queues.rb

@@ -0,0 +1,34 @@
+# An enumerable collection of queues that supports direct access to queues by name.
+#
+#   queue_1 = ApplicationJob::Queue.new("queue_1")
+#   queue_2 = ApplicationJob::Queue.new("queue_2")
+#   queues = ApplicationJob::Queues.new([queue_1, queue_2])
+#
+#   queues[:queue_1] #=> queue_1
+#   queues[:queue_2] #=> queue_2
+#   queues.to_a #=> [ queue_1, queue_2 ] # Enumerable
+#
+# See +ActiveJob::Queue+.
+class ActiveJob::Queues
+  include Enumerable
+
+  delegate :each, to: :values
+  delegate :values, to: :queues_by_id, private: true
+  delegate :size, :length, :to_s, :inspect, to: :queues_by_id
+
+  def initialize(queues)
+    @queues_by_id = queues.index_by(&:id).with_indifferent_access
+  end
+
+  def to_h
+    queues_by_id.dup
+  end
+
+  def [](name)
+    queues_by_id[name.to_s.parameterize]
+  end
+
+  private
+
+  attr_reader :queues_by_id
+end

data/lib/flight_control/adapter.rb

@@ -0,0 +1,155 @@
+module FlightControl::Adapter
+  def activating(&block)
+    block.call
+  end
+
+  def supports_job_status?(status)
+    supported_job_statuses.include?(status)
+  end
+
+  def supported_job_statuses
+    # All adapters need to support these at a minimum
+    [:pending, :failed]
+  end
+
+  def supports_job_filter?(jobs_relation, filter)
+    supported_job_filters(jobs_relation).include?(filter)
+  end
+
+  # List of filters supported natively. Non-supported filters are done in memory.
+  def supported_job_filters(jobs_relation)
+    []
+  end
+
+  def supports_queue_pausing?
+    true
+  end
+
+  def exposes_workers?
+    false
+  end
+
+  def supports_recurring_tasks?
+    false
+  end
+
+  # Returns an array with the list of recurring tasks. Each task is represented as a hash
+  # with these attributes:
+  #   {
+  #     id: "periodic-job",
+  #     job_class_name: "MyJob",
+  #     arguments: [ 123, { arg: :value }]
+  #     schedule: "every monday at 9 am",
+  #     last_enqueued_at: Fri, 26 Jan 2024 20:31:09.652174000 UTC +00:00,
+  #   }
+  def recurring_tasks
+    if supports_recurring_tasks?
+      raise_incompatible_adapter_error_from :recurring_tasks
+    end
+  end
+
+  # Returns a recurring task represented by a hash as indicated above
+  def find_recurring_task(recurring_task_id)
+    if supports_recurring_tasks?
+      raise_incompatible_adapter_error_from :find_recurring_task
+    end
+  end
+
+  # Returns an array with the list of workers. Each worker is represented as a hash
+  # with these attributes:
+  #   {
+  #     id: 123,
+  #     name: "worker-name",
+  #     hostname: "hey-default-101",
+  #     last_heartbeat_at: Fri, 26 Jan 2024 20:31:09.652174000 UTC +00:00,
+  #     configuration: { ... }
+  #     raw_data: { ... }
+  #   }
+  def workers
+    if exposes_workers?
+      raise_incompatible_adapter_error_from :workers
+    end
+  end
+
+  # Returns a worker represented by a hash as indicated above
+  def find_worker(worker_id)
+    if exposes_workers?
+      raise_incompatible_adapter_error_from :find_worker
+    end
+  end
+
+  # Returns an array with the list of queues. Each queue is represented as a hash
+  # with these attributes:
+  #   {
+  #     name: "queue_name",
+  #     size: 1,
+  #     active: true
+  #   }
+  def queues
+    raise_incompatible_adapter_error_from :queue_names
+  end
+
+  def queue_size(queue_name)
+    raise_incompatible_adapter_error_from :queue_size
+  end
+
+  def clear_queue(queue_name)
+    raise_incompatible_adapter_error_from :clear_queue
+  end
+
+  def pause_queue(queue_name)
+    if supports_queue_pausing?
+      raise_incompatible_adapter_error_from :pause_queue
+    end
+  end
+
+  def resume_queue(queue_name)
+    if supports_queue_pausing?
+      raise_incompatible_adapter_error_from :resume_queue
+    end
+  end
+
+  def queue_paused?(queue_name)
+    if supports_queue_pausing?
+      raise_incompatible_adapter_error_from :queue_paused?
+    end
+  end
+
+  def jobs_count(jobs_relation)
+    raise_incompatible_adapter_error_from :jobs_count
+  end
+
+  def fetch_jobs(jobs_relation)
+    raise_incompatible_adapter_error_from :fetch_jobs
+  end
+
+  def retry_all_jobs(jobs_relation)
+    raise_incompatible_adapter_error_from :retry_all_jobs
+  end
+
+  def retry_job(job, jobs_relation)
+    raise_incompatible_adapter_error_from :retry_job
+  end
+
+  def discard_all_jobs(jobs_relation)
+    raise_incompatible_adapter_error_from :discard_all_jobs
+  end
+
+  def discard_job(job, jobs_relation)
+    raise_incompatible_adapter_error_from :discard_job
+  end
+
+  def dispatch_job(job, jobs_relation)
+    raise_incompatible_adapter_error_from :dispatch_job
+  end
+
+  def find_job(job_id, *)
+    raise_incompatible_adapter_error_from :find_job
+  end
+
+  private
+
+  def raise_incompatible_adapter_error_from(method_name)
+    raise FlightControl::Errors::IncompatibleAdapter, "Adapter #{ActiveJob.adapter_name(self)} must implement `#{method_name}`"
+  end
+end

data/lib/flight_control/application.rb

@@ -0,0 +1,20 @@
+# An application containing backend jobs servers
+class FlightControl::Application
+  include FlightControl::IdentifiedByName
+
+  attr_reader :servers
+
+  def initialize(name:)
+    super
+    @servers = FlightControl::IdentifiedElements.new
+  end
+
+  def add_servers(queue_adapters_by_name)
+    queue_adapters_by_name.each do |name, queue_adapter|
+      adapter, cleaner = queue_adapter
+
+      servers << FlightControl::Server.new(name: name.to_s, queue_adapter: adapter,
+        backtrace_cleaner: cleaner, application: self)
+    end
+  end
+end

data/lib/flight_control/applications.rb

@@ -0,0 +1,8 @@
+# A container to register applications
+class FlightControl::Applications < FlightControl::IdentifiedElements
+  def add(name, queue_adapters_by_name = {})
+    self << FlightControl::Application.new(name: name).tap do |application|
+      application.add_servers(queue_adapters_by_name)
+    end
+  end
+end

data/lib/flight_control/arguments_filter.rb

@@ -0,0 +1,25 @@
+# Replaces argument values with [FILTERED] for any keys that match a filter.
+class FlightControl::ArgumentsFilter
+  FILTERED = "[FILTERED]"
+
+  def initialize(filter)
+    @filter = filter
+  end
+
+  def apply_to(arguments)
+    case arguments
+    when Array
+      arguments.map { |a| apply_to(a) }
+    when Hash
+      arguments.map do |k, v|
+        [k, filter.include?(k.to_s) ? FILTERED : v]
+      end.to_h
+    else
+      arguments
+    end
+  end
+
+  private
+
+  attr_reader :filter
+end

data/lib/flight_control/authentication.rb

@@ -0,0 +1,68 @@
+require "rails/command"
+
+class FlightControl::Authentication < Rails::Command::Base
+  def self.configure
+    new.configure
+  end
+
+  def configure
+    if credentials_accessible?
+      if authentication_configured?
+        say "HTTP Basic Authentication is already configured for `#{Rails.env}`. You can edit it using `credentials:edit`"
+      else
+        say "Setting up credentials for HTTP Basic Authentication for `#{Rails.env}` environment."
+        say ""
+
+        username = ask "Enter username: "
+        password = SecureRandom.base58(64)
+
+        store_credentials(username, password)
+        say "Username and password stored in Rails encrypted credentials."
+        say ""
+        say "You can now access Flight Control with: "
+        say ""
+        say " - Username: #{username}"
+        say " - password: #{password}"
+        say ""
+        say "You can also edit these in the future via `credentials:edit`"
+      end
+    else
+      say "Rails credentials haven't been configured or aren't accessible. Configure them following the instructions in `credentials:help`"
+    end
+  end
+
+  private
+
+  attr_reader :environment
+
+  def credentials_accessible?
+    credentials.read.present?
+  end
+
+  def authentication_configured?
+    %i[http_basic_auth_user http_basic_auth_password].any? do |key|
+      credentials.dig(:flight_control, key).present?
+    end
+  end
+
+  def store_credentials(username, password)
+    content = credentials.read + "\n" + http_authentication_entry(username, password) + "\n"
+    credentials.write(content)
+  end
+
+  def credentials
+    @credentials ||= Rails.application.encrypted(config.content_path, key_path: config.key_path)
+  end
+
+  def config
+    Rails.application.config.credentials
+  end
+
+  def http_authentication_entry(username, password)
+    <<~ENTRY
+      flight_control:
+        http_basic_auth_user: #{username}
+        http_basic_auth_password: #{password}
+    ENTRY
+  end
+end

data/lib/flight_control/console/connect_to.rb

@@ -0,0 +1,15 @@
+require "irb/command"
+
+module FlightControl::Console
+  class ConnectTo < IRB::Command::Base
+    category "Flight Control jobs"
+    description "Connect to a job server"
+
+    def execute(server_locator)
+      server = FlightControl::Server.from_global_id(server_locator)
+      FlightControl::Current.server = server
+
+      puts "Connected to #{server_locator}"
+    end
+  end
+end

data/lib/flight_control/console/context.rb

@@ -0,0 +1,11 @@
+module FlightControl::Console::Context
+  mattr_accessor :jobs_server
+
+  def evaluate(*)
+    if FlightControl::Current.server
+      FlightControl::Current.server.activating { super }
+    else
+      super
+    end
+  end
+end

data/lib/flight_control/console/jobs_help.rb

@@ -0,0 +1,23 @@
+require "irb/command"
+
+module FlightControl::Console
+  class JobsHelp < IRB::Command::Base
+    category "Flight Control jobs"
+    description "Show help for managing jobs"
+
+    def execute(*)
+      puts "You are currently connected to #{FlightControl::Current.server}" if FlightControl::Current.server
+
+      puts "You can connect to a job server with"
+      puts %(  connect_to <app_id>:<server_id>\n\n)
+
+      puts "Available job servers:\n"
+
+      FlightControl.applications.each do |application|
+        application.servers.each do |server|
+          puts "  * #{server.to_global_id}"
+        end
+      end
+    end
+  end
+end