source_monitor 0.1.1

data/lib/source_monitor/items/retention_pruner.rb

@@ -0,0 +1,146 @@
+# frozen_string_literal: true
+
+require "source_monitor/instrumentation"
+require "source_monitor/items/retention_strategies"
+require "source_monitor/items/retention_strategies/destroy"
+require "source_monitor/items/retention_strategies/soft_delete"
+
+module SourceMonitor
+  module Items
+    # Removes items that fall outside the configured retention rules for a source.
+    # Supports age-based (items_retention_days) and count-based (max_items) limits.
+    class RetentionPruner
+      Result = Struct.new(:removed_by_age, :removed_by_limit, :removed_total, keyword_init: true) do
+        def applied?
+          removed_total.positive?
+        end
+      end
+
+      STRATEGY_CLASSES = {
+        destroy: SourceMonitor::Items::RetentionStrategies::Destroy,
+        soft_delete: SourceMonitor::Items::RetentionStrategies::SoftDelete
+      }.freeze
+
+      def self.call(source:, now: Time.current, strategy: nil)
+        new(source:, now:, strategy:).call
+      end
+
+      def initialize(source:, now: Time.current, strategy: nil)
+        @source = source
+        @now = now
+        @strategy_name = normalize_strategy(strategy)
+        @strategy_handler = STRATEGY_CLASSES.fetch(@strategy_name).new(source: source)
+      end
+
+      def call
+        removed_by_age = prune_by_age
+        removed_by_limit = prune_by_limit
+        removed_total = removed_by_age + removed_by_limit
+
+        if removed_total.positive?
+          SourceMonitor::Instrumentation.item_retention(
+            source_id: source.id,
+            removed_by_age:,
+            removed_by_limit:,
+            removed_total:,
+            items_retention_days: items_retention_days,
+            max_items: max_items_limit
+          )
+        end
+
+        Result.new(
+          removed_by_age:,
+          removed_by_limit:,
+          removed_total:
+        )
+      end
+
+      private
+
+      attr_reader :source, :now, :strategy_name, :strategy_handler
+
+      def prune_by_age
+        days = items_retention_days
+        return 0 unless days.present?
+
+        return 0 if days <= 0
+
+        cutoff = now - days.days
+
+        timestamp_expression = Arel::Nodes::NamedFunction.new(
+          "COALESCE",
+          [
+            SourceMonitor::Item.arel_table[:published_at],
+            SourceMonitor::Item.arel_table[:created_at]
+          ]
+        )
+
+        scope = source.items.where(timestamp_expression.lteq(cutoff))
+        remove_scope(scope)
+      end
+
+      def prune_by_limit
+        limit = max_items_limit
+        return 0 unless limit.present?
+
+        return 0 if limit <= 0
+
+        ids_to_keep = source.items
+          .order(Arel.sql("published_at DESC NULLS LAST, created_at DESC"))
+          .limit(limit)
+          .pluck(:id)
+
+        scope =
+          if ids_to_keep.empty?
+            source.items.none
+          else
+            source.items.where.not(id: ids_to_keep)
+          end
+
+        remove_scope(scope)
+      end
+
+      def remove_scope(scope)
+        return 0 if scope.none?
+
+        removed = 0
+        scope.in_batches(of: 100) do |batch|
+          removed += strategy_handler.apply(batch:, now:)
+        end
+        removed
+      end
+
+      def normalize_strategy(value)
+        value = SourceMonitor.config.retention.strategy if value.nil?
+
+        value = value.to_sym if value.respond_to?(:to_sym)
+        return value if STRATEGY_CLASSES.key?(value)
+
+        valid = STRATEGY_CLASSES.keys.join(", ")
+        raise ArgumentError, "Invalid retention strategy #{value.inspect}. Valid strategies: #{valid}"
+      end
+
+      def items_retention_days
+        @items_retention_days ||= begin
+          value = source.items_retention_days
+          value = SourceMonitor.config.retention.items_retention_days if value.nil?
+          convert_to_integer(value)
+        end
+      end
+
+      def max_items_limit
+        @max_items_limit ||= begin
+          value = source.max_items
+          value = SourceMonitor.config.retention.max_items if value.nil?
+          convert_to_integer(value)
+        end
+      end
+
+      def convert_to_integer(value)
+        return nil if value.nil?
+
+        value.respond_to?(:to_i) ? value.to_i : value
+      end
+    end
+  end
+end

data/lib/source_monitor/items/retention_strategies/destroy.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  module Items
+    module RetentionStrategies
+      class Destroy
+        def initialize(source:)
+          @source = source
+        end
+
+        def apply(batch:, now: Time.current) # rubocop:disable Lint/UnusedMethodArgument
+          count = 0
+          batch.each do |item|
+            item.destroy!
+            count += 1
+          end
+          count
+        end
+
+        private
+
+        attr_reader :source
+      end
+    end
+  end
+end

data/lib/source_monitor/items/retention_strategies/soft_delete.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  module Items
+    module RetentionStrategies
+      class SoftDelete
+        def initialize(source:)
+          @source = source
+        end
+
+        def apply(batch:, now: Time.current)
+          ids = Array(batch.pluck(:id))
+          return 0 if ids.empty?
+
+          timestamp = normalized_timestamp(now)
+
+          # Use with_deleted to update items that may already be marked as deleted
+          SourceMonitor::Item.with_deleted.where(id: ids).update_all(
+            deleted_at: timestamp,
+            updated_at: timestamp
+          )
+
+          adjust_source_counter(ids.length)
+          ids.length
+        end
+
+        private
+
+        attr_reader :source
+
+        def normalized_timestamp(now)
+          return Time.current if now.nil?
+
+          now.respond_to?(:in_time_zone) ? now.in_time_zone : now
+        end
+
+        def adjust_source_counter(amount)
+          return unless source&.id
+
+          SourceMonitor::Source.update_counters(source.id, items_count: -amount)
+
+          return unless source.respond_to?(:items_count) && !source.items_count.nil?
+
+          source.items_count -= amount
+          source.items_count = 0 if source.items_count.negative?
+        end
+      end
+    end
+  end
+end

data/lib/source_monitor/items/retention_strategies.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  module Items
+    module RetentionStrategies
+      # Namespace for item retention strategy implementations.
+    end
+  end
+end

data/lib/source_monitor/jobs/cleanup_options.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+require "time"
+
+module SourceMonitor
+  module Jobs
+    module CleanupOptions
+      module_function
+
+      def normalize(options)
+        case options
+        when nil
+          {}
+        when Hash
+          options.respond_to?(:symbolize_keys) ? options.symbolize_keys : symbolize_keys(options)
+        else
+          {}
+        end
+      end
+
+      def resolve_time(value, default: Time.current)
+        case value
+        when nil
+          default
+        when Time
+          value
+        when String
+          parse_time(value, default)
+        else
+          value.respond_to?(:to_time) ? value.to_time : default
+        end
+      end
+
+      def extract_ids(value)
+        Array(value)
+          .flat_map do |entry|
+            case entry
+            when Integer
+              [ entry ]
+            when Array
+              entry
+            else
+              entry.to_s.split(",")
+            end
+          end
+          .map { |entry| entry.is_a?(String) ? entry.strip : entry }
+          .reject { |entry| entry.respond_to?(:blank?) ? entry.blank? : entry.nil? }
+          .map { |entry| integer(entry) }
+          .compact
+          .reject(&:zero?)
+      end
+
+      def integer(value)
+        return value if value.is_a?(Integer)
+
+        Integer(value, exception: false)
+      end
+
+      def batch_size(options, default:)
+        value = integer(options[:batch_size])
+        return default unless value&.positive?
+
+        value
+      end
+
+      def symbolize_keys(hash)
+        hash.each_with_object({}) do |(key, value), memo|
+          memo[key.respond_to?(:to_sym) ? key.to_sym : key] = value
+        end
+      end
+      private_class_method :symbolize_keys
+
+      def parse_time(value, default)
+        if Time.zone
+          Time.zone.parse(value) || default
+        else
+          Time.parse(value)
+        end
+      rescue ArgumentError, TypeError
+        default
+      end
+      private_class_method :parse_time
+    end
+  end
+end

data/lib/source_monitor/jobs/fetch_failure_subscriber.rb

@@ -0,0 +1,129 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  module Jobs
+    module FetchFailureCallbacks
+      extend ActiveSupport::Concern
+
+      included do
+        after_create :notify_source_monitor_of_failure
+      end
+
+      private
+
+      def notify_source_monitor_of_failure
+        SourceMonitor::Jobs::FetchFailureSubscriber.handle_failed_execution(self)
+      end
+    end
+
+    class FetchFailureSubscriber
+      PROCESS_FAILURE_CLASSES = %w[
+        SolidQueue::Processes::ProcessExitError
+        SolidQueue::Processes::ProcessPrunedError
+      ].freeze
+
+      class << self
+        def setup!
+          register_on_load_hook!
+          attach_callbacks! if solid_queue_loaded?
+        end
+
+        def handle_failed_execution(failed_execution)
+          job = failed_execution.job
+          return unless job
+          return unless job.queue_name == SourceMonitor.queue_name(:fetch)
+
+          error = failed_execution.error || {}
+          return unless PROCESS_FAILURE_CLASSES.include?(error["exception_class"])
+
+          source_id = extract_source_id(job)
+          return unless source_id
+
+          source = SourceMonitor::Source.find_by(id: source_id)
+          return unless source
+
+          now = Time.current
+
+          source.with_lock do
+            source.reload
+            update_attributes = {
+              fetch_status: "failed",
+              last_error: formatted_error_message(error),
+              last_error_at: now,
+              failure_count: source.failure_count.to_i + 1,
+              next_fetch_at: now,
+              backoff_until: nil
+            }
+            source.update!(update_attributes)
+          end
+
+          SourceMonitor::Realtime.broadcast_source(source)
+        rescue StandardError => exception
+          log_failure(source_id, exception)
+        end
+
+        def attach_callbacks!
+          failed_execution_class = load_failed_execution_class
+          return unless failed_execution_class
+          return if failed_execution_class < FetchFailureCallbacks
+
+          failed_execution_class.include(FetchFailureCallbacks)
+        end
+
+        private
+
+        def solid_queue_loaded?
+          !!load_failed_execution_class
+        end
+
+        def register_on_load_hook!
+          return if @hook_registered
+
+          ActiveSupport.on_load(:solid_queue) do
+            SourceMonitor::Jobs::FetchFailureSubscriber.attach_callbacks!
+          end
+
+          @hook_registered = true
+        end
+
+        def load_failed_execution_class
+          ::SolidQueue::FailedExecution
+        rescue NameError
+          begin
+            require "solid_queue/failed_execution"
+          rescue LoadError
+            return nil
+          end
+
+          ::SolidQueue::FailedExecution
+        end
+
+        def extract_source_id(job)
+          arguments = job.arguments
+          return unless arguments.is_a?(Hash)
+
+          args_array = arguments["arguments"]
+          return unless args_array.is_a?(Array)
+
+          source_arg = args_array.first
+          Integer(source_arg)
+        rescue ArgumentError, TypeError
+          nil
+        end
+
+        def formatted_error_message(error)
+          message = error["message"] || "Fetch job terminated unexpectedly"
+          "#{error['exception_class']}: #{message}"
+        end
+
+        def log_failure(source_id, exception)
+          return unless defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger
+
+          Rails.logger.error(
+            "[SourceMonitor::Jobs::FetchFailureSubscriber] Failed to handle process failure for source #{source_id.inspect}: #{exception.class}: #{exception.message}"
+          )
+        end
+      end
+    end
+  end
+end

data/lib/source_monitor/jobs/solid_queue_metrics.rb

@@ -0,0 +1,199 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  module Jobs
+    class SolidQueueMetrics
+      QueueSummary = Struct.new(
+        :queue_name,
+        :ready_count,
+        :scheduled_count,
+        :failed_count,
+        :recurring_count,
+        :paused,
+        :last_enqueued_at,
+        :last_started_at,
+        :last_finished_at,
+        :available,
+        keyword_init: true
+      ) do
+        def total_count
+          ready_count + scheduled_count + failed_count
+        end
+      end
+
+      DEFAULT_QUEUE_NAME = "default"
+
+      def self.call(queue_names:)
+        new(queue_names).call
+      end
+
+      def initialize(queue_names)
+        @queue_names = Array(queue_names).map(&:to_s)
+      end
+
+      def call
+        metrics = initialize_metrics
+
+        return metrics unless solid_queue_supported?
+
+        populate_counts(metrics)
+        populate_timestamps(metrics)
+        populate_pause_state(metrics)
+
+        metrics
+      rescue ActiveRecord::StatementInvalid, ActiveRecord::ConnectionNotEstablished
+        @solid_queue_supported = false
+        initialize_metrics
+      end
+
+      private
+
+      attr_reader :queue_names
+
+      def initialize_metrics
+        queue_names.index_with do |queue_name|
+          QueueSummary.new(
+            queue_name: queue_name,
+            ready_count: 0,
+            scheduled_count: 0,
+            failed_count: 0,
+            recurring_count: 0,
+            paused: false,
+            last_enqueued_at: nil,
+            last_started_at: nil,
+            last_finished_at: nil,
+            available: solid_queue_supported?
+          )
+        end
+      end
+
+      def solid_queue_supported?
+        return @solid_queue_supported if defined?(@solid_queue_supported)
+
+        @solid_queue_supported =
+          defined?(::SolidQueue::Job) &&
+          defined?(::SolidQueue::ReadyExecution) &&
+          defined?(::SolidQueue::ScheduledExecution) &&
+          table_exists?(::SolidQueue::Job) &&
+          table_exists?(::SolidQueue::ReadyExecution) &&
+          table_exists?(::SolidQueue::ScheduledExecution)
+      end
+
+      def populate_counts(metrics)
+        merge_counts(metrics, ready_counts, :ready_count)
+        merge_counts(metrics, scheduled_counts, :scheduled_count)
+        merge_counts(metrics, failed_counts, :failed_count)
+        merge_counts(metrics, recurring_counts, :recurring_count)
+      end
+
+      def populate_timestamps(metrics)
+        merge_counts(metrics, last_enqueued_times, :last_enqueued_at)
+        merge_counts(metrics, last_started_times, :last_started_at)
+        merge_counts(metrics, last_finished_times, :last_finished_at)
+      end
+
+      def populate_pause_state(metrics)
+        return unless table_exists?(::SolidQueue::Pause)
+
+        paused_queue_names.each do |queue_name|
+          summary = metrics[queue_name]
+          next unless summary
+
+          summary.paused = true
+        end
+      end
+
+      def merge_counts(metrics, collection, attribute)
+        collection.each do |queue_name, value|
+          summary = metrics[queue_name.to_s]
+          next unless summary
+
+          summary.public_send("#{attribute}=", value)
+        end
+      end
+
+      def ready_counts
+        return {} unless table_exists?(::SolidQueue::ReadyExecution)
+
+        ::SolidQueue::ReadyExecution.
+          where(queue_name: queue_names).
+          group(:queue_name).
+          count
+      end
+
+      def scheduled_counts
+        return {} unless table_exists?(::SolidQueue::ScheduledExecution)
+
+        ::SolidQueue::ScheduledExecution.
+          where(queue_name: queue_names).
+          group(:queue_name).
+          count
+      end
+
+      def failed_counts
+        return {} unless defined?(::SolidQueue::FailedExecution) && table_exists?(::SolidQueue::FailedExecution)
+
+        ::SolidQueue::FailedExecution.
+          joins(:job).
+          where(::SolidQueue::Job.arel_table[:queue_name].in(queue_names)).
+          group(::SolidQueue::Job.arel_table[:queue_name]).
+          count
+      end
+
+      def recurring_counts
+        return {} unless defined?(::SolidQueue::RecurringTask) && table_exists?(::SolidQueue::RecurringTask)
+
+        ::SolidQueue::RecurringTask.
+          group(:queue_name).
+          count.
+          transform_keys { |queue_name| normalize_queue_name(queue_name) }.
+          select { |queue_name, _| queue_names.include?(queue_name) }
+      end
+
+      def last_enqueued_times
+        ::SolidQueue::Job.
+          where(queue_name: queue_names).
+          group(:queue_name).
+          maximum(:created_at)
+      end
+
+      def last_started_times
+        return {} unless defined?(::SolidQueue::ClaimedExecution) && table_exists?(::SolidQueue::ClaimedExecution)
+
+        ::SolidQueue::ClaimedExecution.
+          joins(:job).
+          where(::SolidQueue::Job.arel_table[:queue_name].in(queue_names)).
+          group(::SolidQueue::Job.arel_table[:queue_name]).
+          maximum(arel_table_for(::SolidQueue::ClaimedExecution)[:created_at])
+      end
+
+      def last_finished_times
+        ::SolidQueue::Job.
+          where(queue_name: queue_names).
+          where.not(finished_at: nil).
+          group(:queue_name).
+          maximum(:finished_at)
+      end
+
+      def paused_queue_names
+        ::SolidQueue::Pause.
+          where(queue_name: queue_names).
+          pluck(:queue_name)
+      end
+
+      def table_exists?(model)
+        model.respond_to?(:table_exists?) && model.table_exists?
+      rescue ActiveRecord::StatementInvalid, ActiveRecord::NoDatabaseError
+        false
+      end
+
+      def normalize_queue_name(name)
+        (name.presence || DEFAULT_QUEUE_NAME).to_s
+      end
+
+      def arel_table_for(model)
+        model.arel_table
+      end
+    end
+  end
+end