source_monitor 0.1.1

data/lib/source_monitor/jobs/visibility.rb

@@ -0,0 +1,133 @@
+# frozen_string_literal: true
+
+require "active_support/notifications"
+require "monitor"
+
+module SourceMonitor
+  module Jobs
+    module Visibility
+      module_function
+
+      def setup!
+        return if @subscribed || !SourceMonitor.config.job_metrics_enabled
+
+        subscribe_enqueue
+        subscribe_perform_start
+        subscribe_perform
+
+        @subscribed = true
+      end
+
+      def adapter_name
+        ActiveJob::Base.queue_adapter_name
+      end
+
+      def queue_depth(queue_name)
+        state_for(queue_name)[:depth]
+      end
+
+      def last_enqueued_at(queue_name)
+        state_for(queue_name)[:last_enqueued_at]
+      end
+
+      def last_started_at(queue_name)
+        state_for(queue_name)[:last_started_at]
+      end
+
+      def last_finished_at(queue_name)
+        state_for(queue_name)[:last_finished_at]
+      end
+
+      def reset!
+        synchronize do
+          @queue_state = nil
+        end
+      end
+
+      def state_snapshot
+        synchronize do
+          queue_state.transform_values(&:dup)
+        end
+      end
+
+      def trackable_job?(job)
+        job.class.name.start_with?("SourceMonitor::")
+      rescue StandardError
+        false
+      end
+
+      def queue_identifier(job)
+        (job.queue_name || SourceMonitor.config.fetch_queue_name).to_s
+      end
+
+      def queue_state
+        @queue_state ||= Hash.new do |hash, key|
+          hash[key] = { depth: 0, last_enqueued_at: nil, last_started_at: nil, last_finished_at: nil }
+        end
+      end
+
+      def state_for(name)
+        synchronize do
+          queue_state[name.to_s]
+        end
+      end
+
+      def synchronize(&block)
+        monitor.synchronize(&block)
+      end
+
+      def monitor
+        @monitor ||= Monitor.new
+      end
+
+      def subscribe_enqueue
+        ActiveSupport::Notifications.subscribe("enqueue.active_job") do |_event, _start, _finish, _id, payload|
+          job = payload[:job]
+          next unless trackable_job?(job)
+
+          queue_name = queue_identifier(job)
+
+          synchronize do
+            state = queue_state[queue_name]
+            state[:depth] += 1
+            state[:last_enqueued_at] = Time.current
+            SourceMonitor::Metrics.gauge("jobs_queue_depth_#{queue_name}", state[:depth])
+            SourceMonitor::Metrics.gauge("jobs_last_enqueued_at_#{queue_name}", state[:last_enqueued_at])
+          end
+        end
+      end
+
+      def subscribe_perform_start
+        ActiveSupport::Notifications.subscribe("perform_start.active_job") do |_event, _start, _finish, _id, payload|
+          job = payload[:job]
+          next unless trackable_job?(job)
+
+          queue_name = queue_identifier(job)
+
+          synchronize do
+            state = queue_state[queue_name]
+            state[:depth] = [ state[:depth] - 1, 0 ].max
+            state[:last_started_at] = Time.current
+            SourceMonitor::Metrics.gauge("jobs_queue_depth_#{queue_name}", state[:depth])
+            SourceMonitor::Metrics.gauge("jobs_last_started_at_#{queue_name}", state[:last_started_at])
+          end
+        end
+      end
+
+      def subscribe_perform
+        ActiveSupport::Notifications.subscribe("perform.active_job") do |_event, _start, _finish, _id, payload|
+          job = payload[:job]
+          next unless trackable_job?(job)
+
+          queue_name = queue_identifier(job)
+
+          synchronize do
+            state = queue_state[queue_name]
+            state[:last_finished_at] = Time.current
+            SourceMonitor::Metrics.gauge("jobs_last_finished_at_#{queue_name}", state[:last_finished_at])
+          end
+        end
+      end
+    end
+  end
+end

data/lib/source_monitor/logs/entry_sync.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  module Logs
+    class EntrySync
+      def self.call(loggable)
+        new(loggable).call
+      end
+
+      def initialize(loggable)
+        @loggable = loggable
+      end
+
+      def call
+        return unless loggable&.persisted?
+        return unless loggable.respond_to?(:log_entry)
+
+        entry = loggable.log_entry || loggable.build_log_entry
+        entry.assign_attributes(entry_attributes)
+        entry.save!
+      rescue ActiveRecord::RecordInvalid, ActiveRecord::RecordNotSaved
+        Rails.logger&.error("[SourceMonitor::Logs::EntrySync] Failed to sync log entry for #{loggable.class.name}##{loggable.id}")
+        nil
+      end
+
+      private
+
+      attr_reader :loggable
+
+      def entry_attributes
+        {
+          source: loggable.source,
+          item: extract_item,
+          success: boolean_success,
+          started_at: loggable.started_at,
+          completed_at: loggable.respond_to?(:completed_at) ? loggable.completed_at : nil,
+          http_status: safe_attribute(:http_status),
+          duration_ms: safe_attribute(:duration_ms),
+          items_created: safe_attribute(:items_created),
+          items_updated: safe_attribute(:items_updated),
+          items_failed: safe_attribute(:items_failed),
+          scraper_adapter: safe_attribute(:scraper_adapter),
+          content_length: safe_attribute(:content_length),
+          error_class: safe_attribute(:error_class),
+          error_message: safe_attribute(:error_message)
+        }
+      end
+
+      def extract_item
+        return nil unless loggable.respond_to?(:item)
+
+        loggable.item
+      end
+
+      def safe_attribute(attribute)
+        loggable.respond_to?(attribute) ? loggable.public_send(attribute) : nil
+      end
+
+      def boolean_success
+        return false unless loggable.respond_to?(:success)
+
+        value = loggable.public_send(:success)
+        return false if value.nil?
+
+        ActiveModel::Type::Boolean.new.cast(value)
+      end
+    end
+  end
+end

data/lib/source_monitor/logs/filter_set.rb

@@ -0,0 +1,163 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  module Logs
+    class FilterSet
+      STATUS_MAP = {
+        "success" => true,
+        "failed" => false
+      }.freeze
+
+      LOG_TYPE_MAP = {
+        "fetch" => "SourceMonitor::FetchLog",
+        "scrape" => "SourceMonitor::ScrapeLog",
+        "health_check" => "SourceMonitor::HealthCheckLog"
+      }.freeze
+
+      TIMEFRAME_MAP = {
+        "24h" => 24.hours,
+        "7d" => 7.days,
+        "30d" => 30.days
+      }.freeze
+
+      MAX_PER_PAGE = 100
+      DEFAULT_PER_PAGE = 25
+
+      attr_reader :raw_params
+
+      def initialize(params:)
+        @raw_params = params || {}
+      end
+
+      def status
+        @status ||= begin
+          value = sanitize_string(raw_params[:status])
+          STATUS_MAP.key?(value) ? value : nil
+        end
+      end
+
+      def success_flag
+        STATUS_MAP[status]
+      end
+
+      def log_type
+        @log_type ||= begin
+          value = sanitize_string(raw_params[:log_type])
+          LOG_TYPE_MAP.key?(value) ? value : nil
+        end
+      end
+
+      def loggable_type
+        LOG_TYPE_MAP[log_type]
+      end
+
+      def timeframe
+        @timeframe ||= begin
+          value = sanitize_string(raw_params[:timeframe])
+          TIMEFRAME_MAP.key?(value) ? value : nil
+        end
+      end
+
+      def timeframe_start
+        return nil unless timeframe
+
+        current_time - TIMEFRAME_MAP.fetch(timeframe)
+      end
+
+      def started_after
+        @started_after ||= parse_time_param(raw_params[:started_after])
+      end
+
+      def started_before
+        @started_before ||= parse_time_param(raw_params[:started_before])
+      end
+
+      def effective_started_after
+        [ timeframe_start, started_after ].compact.max
+      end
+
+      def source_id
+        @source_id ||= integer_param(raw_params[:source_id])
+      end
+
+      def item_id
+        @item_id ||= integer_param(raw_params[:item_id])
+      end
+
+      def search
+        @search ||= begin
+          value = sanitize_string(raw_params[:search])
+          value.presence
+        end
+      end
+
+      def page
+        @page ||= begin
+          integer = integer_param(raw_params[:page])
+          integer.present? && integer.positive? ? integer : 1
+        end
+      end
+
+      def per_page
+        @per_page ||= begin
+          integer = integer_param(raw_params[:per_page])
+          return DEFAULT_PER_PAGE unless integer.present?
+
+          integer = DEFAULT_PER_PAGE if integer <= 0
+          [ integer, MAX_PER_PAGE ].min
+        end
+      end
+
+      def to_params
+        {
+          status: status,
+          log_type: log_type,
+          timeframe: timeframe,
+          started_after: started_after&.iso8601,
+          started_before: started_before&.iso8601,
+          source_id: source_id,
+          item_id: item_id,
+          search: search,
+          page: page,
+          per_page: per_page
+        }.compact
+      end
+
+      private
+
+      def sanitize_string(value)
+        return "" if value.nil?
+
+        SourceMonitor::Security::ParameterSanitizer.sanitize(value.to_s)
+      end
+
+      def integer_param(value)
+        return nil if value.nil?
+
+        sanitized = sanitize_string(value)
+        return nil unless sanitized.match?(/\A\d+\z/)
+
+        sanitized.to_i
+      end
+
+      def parse_time_param(value)
+        return nil if value.nil?
+
+        sanitized = sanitize_string(value)
+        return nil if sanitized.blank?
+
+        Time.iso8601(sanitized)
+      rescue ArgumentError
+        begin
+          Time.zone.parse(sanitized)
+        rescue ArgumentError, TypeError
+          nil
+        end
+      end
+
+      def current_time
+        @current_time ||= Time.zone.now
+      end
+    end
+  end
+end

data/lib/source_monitor/logs/query.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  module Logs
+    class Query
+      Result = Struct.new(
+        :entries,
+        :page,
+        :per_page,
+        :has_next_page,
+        :has_previous_page,
+        :filter_set,
+        keyword_init: true
+      ) do
+        def has_next_page?
+          !!self[:has_next_page]
+        end
+
+        def has_previous_page?
+          !!self[:has_previous_page]
+        end
+      end
+
+      def initialize(params:)
+        @filter_set = SourceMonitor::Logs::FilterSet.new(params:)
+      end
+
+      def call
+        pagination_result = SourceMonitor::Pagination::Paginator.new(
+          scope: filtered_scope,
+          page: filter_set.page,
+          per_page: filter_set.per_page
+        ).paginate
+
+        Result.new(
+          entries: pagination_result.records,
+          page: pagination_result.page,
+          per_page: pagination_result.per_page,
+          has_next_page: pagination_result.has_next_page?,
+          has_previous_page: pagination_result.has_previous_page?,
+          filter_set:
+        )
+      end
+
+      private
+
+      attr_reader :filter_set
+
+      def filtered_scope
+        scope = SourceMonitor::LogEntry.includes(:source, :item, :loggable).recent
+        scope = scope.where(success: filter_set.success_flag) unless filter_set.success_flag.nil?
+        scope = scope.where(loggable_type: filter_set.loggable_type) if filter_set.loggable_type
+        scope = scope.where(source_id: filter_set.source_id) if filter_set.source_id
+        scope = scope.where(item_id: filter_set.item_id) if filter_set.item_id
+        scope = scope.where("sourcemon_log_entries.started_at >= ?", filter_set.effective_started_after) if filter_set.effective_started_after
+        scope = scope.where("sourcemon_log_entries.started_at <= ?", filter_set.started_before) if filter_set.started_before
+        scope = apply_search(scope, filter_set.search) if filter_set.search
+        scope.order(started_at: :desc, id: :desc)
+      end
+
+      def apply_search(scope, term)
+        normalized = "%#{term.to_s.downcase}%"
+
+        scope.
+          left_outer_joins(:source).
+          left_outer_joins(:item).
+          where(
+            <<~SQL.squish,
+              (LOWER(sourcemon_log_entries.error_message) LIKE :query) OR
+              (LOWER(sourcemon_log_entries.error_class) LIKE :query) OR
+              (CAST(sourcemon_log_entries.http_status AS TEXT) LIKE :query) OR
+              (LOWER(sourcemon_log_entries.scraper_adapter) LIKE :query) OR
+              (LOWER(sourcemon_sources.name) LIKE :query) OR
+              (LOWER(sourcemon_items.title) LIKE :query)
+            SQL
+            query: normalized
+          )
+      end
+    end
+  end
+end

data/lib/source_monitor/logs/table_presenter.rb

@@ -0,0 +1,161 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  module Logs
+    class TablePresenter
+      class Row
+        def initialize(entry, url_helpers)
+          @entry = entry
+          @url_helpers = url_helpers
+        end
+
+        def dom_id
+          "#{type_slug}-#{entry.loggable_id}"
+        end
+
+        def type_label
+          if fetch?
+            "Fetch"
+          elsif scrape?
+            "Scrape"
+          else
+            "Health Check"
+          end
+        end
+
+        def type_variant
+          if fetch?
+            :fetch
+          elsif scrape?
+            :scrape
+          else
+            :health_check
+          end
+        end
+
+        def status_label
+          entry.success? ? "Success" : "Failure"
+        end
+
+        def status_variant
+          entry.success? ? :success : :failure
+        end
+
+        def started_at
+          entry.started_at
+        end
+
+        def primary_label
+          if scrape?
+            entry.item&.title.presence || "(untitled)"
+          else
+            entry.source&.name
+          end
+        end
+
+        def primary_path
+          if scrape? && entry.item
+            url_helpers.item_path(entry.item)
+          elsif entry.source
+            url_helpers.source_path(entry.source)
+          end
+        end
+
+        def source_label
+          entry.source&.name
+        end
+
+        def source_path
+          url_helpers.source_path(entry.source) if entry.source
+        end
+
+        def http_summary
+          if fetch?
+            entry.http_status.present? ? entry.http_status.to_s : "—"
+          elsif scrape?
+            parts = []
+            parts << entry.http_status.to_s if entry.http_status
+            parts << entry.scraper_adapter if entry.scraper_adapter.present?
+            parts.compact.join(" · ").presence || "—"
+          else
+            entry.http_status.present? ? entry.http_status.to_s : "—"
+          end
+        end
+
+        def metrics_summary
+          if fetch?
+            "+#{entry.items_created.to_i} / ~#{entry.items_updated.to_i} / ✕#{entry.items_failed.to_i}"
+          else
+            entry.duration_ms.present? ? "#{entry.duration_ms} ms" : "—"
+          end
+        end
+
+        def detail_path
+          case entry.loggable
+          when SourceMonitor::FetchLog
+            url_helpers.fetch_log_path(entry.loggable)
+          when SourceMonitor::ScrapeLog
+            url_helpers.scrape_log_path(entry.loggable)
+          else
+            nil
+          end
+        end
+
+        def adapter
+          entry.scraper_adapter
+        end
+
+        def success?
+          entry.success?
+        end
+
+        def failure?
+          !success?
+        end
+
+        def error_message
+          entry.error_message
+        end
+
+        def type_slug
+          if fetch?
+            "fetch"
+          elsif scrape?
+            "scrape"
+          else
+            "health-check"
+          end
+        end
+
+        def fetch?
+          entry.fetch?
+        end
+
+        def scrape?
+          entry.scrape?
+        end
+
+        def health_check?
+          entry.health_check?
+        end
+
+        private
+
+        attr_reader :entry, :url_helpers
+      end
+
+      def initialize(entries:, url_helpers:)
+        @entries = entries
+        @url_helpers = url_helpers
+      end
+
+      def rows
+        entries.map { |entry| Row.new(entry, url_helpers) }
+      end
+
+      private
+
+      attr_reader :entries, :url_helpers
+    end
+  end
+end

data/lib/source_monitor/metrics.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+require "active_support/notifications"
+
+module SourceMonitor
+  module Metrics
+    module_function
+
+    def increment(name, value = 1)
+      counters[name.to_s] += value
+    end
+
+    def gauge(name, value)
+      gauges[name.to_s] = value
+    end
+
+    def counter(name)
+      counters[name.to_s]
+    end
+
+    def gauge_value(name)
+      gauges[name.to_s]
+    end
+
+    def snapshot
+      { counters: counters.dup, gauges: gauges.dup }
+    end
+
+    def reset!
+      @counters = Hash.new(0)
+      @gauges = {}
+    end
+
+    def setup_subscribers!
+      return if defined?(@subscribed) && @subscribed
+
+      ActiveSupport::Notifications.subscribe("source_monitor.fetch.start") do |_name, _start, _finish, _id, payload|
+        increment(:fetch_started_total)
+        gauge(:last_fetch_source_id, payload[:source_id]) if payload.key?(:source_id)
+      end
+
+      ActiveSupport::Notifications.subscribe("source_monitor.fetch.finish") do |_name, start, finish, _id, payload|
+        increment(:fetch_finished_total)
+        success = payload.fetch(:success, true)
+        if success
+          increment(:fetch_success_total)
+        else
+          increment(:fetch_failure_total)
+        end
+
+        duration_ms = payload[:duration_ms] || ((finish - start) * 1000.0).round(2)
+        gauge(:last_fetch_duration_ms, duration_ms)
+      end
+
+      ActiveSupport::Notifications.subscribe("source_monitor.scheduler.run") do |_name, start, finish, _id, payload|
+        enqueued = payload[:enqueued_count].to_i
+        duration_ms = payload[:duration_ms] || ((finish - start) * 1000.0).round(2)
+
+        increment(:scheduler_runs_total)
+        increment(:scheduler_sources_enqueued_total, enqueued)
+        gauge(:scheduler_last_enqueued_count, enqueued)
+        gauge(:scheduler_last_duration_ms, duration_ms)
+        gauge(:scheduler_last_run_at_epoch, finish.to_f)
+      end
+
+      @subscribed = true
+    end
+
+    def counters
+      @counters ||= Hash.new(0)
+    end
+
+    def gauges
+      @gauges ||= {}
+    end
+  end
+end