source_monitor 0.1.1

data/lib/source_monitor/dashboard/queries.rb

@@ -0,0 +1,356 @@
+# frozen_string_literal: true
+
+require "active_support/notifications"
+require "source_monitor/dashboard/upcoming_fetch_schedule"
+
+module SourceMonitor
+  module Dashboard
+    class Queries
+      def initialize(reference_time: Time.current)
+        @reference_time = reference_time
+        @cache = Cache.new
+      end
+
+      def stats
+        cache.fetch(:stats) do
+          measure(:stats) do
+            StatsQuery.new(reference_time:).call
+          end
+        end
+      end
+
+      def recent_activity(limit: DEFAULT_RECENT_ACTIVITY_LIMIT)
+        cache.fetch([ :recent_activity, limit ]) do
+          measure(:recent_activity, limit:) do
+            RecentActivityQuery.new(limit:).call
+          end
+        end
+      end
+
+      def quick_actions
+        QUICK_ACTIONS
+      end
+
+      def job_metrics(queue_names: queue_name_map.values)
+        measure(:job_metrics, queue_names:) do
+          summaries = SourceMonitor::Jobs::SolidQueueMetrics.call(queue_names:)
+
+          queue_name_map.map do |role, queue_name|
+            summary = summaries[queue_name.to_s] || SourceMonitor::Jobs::SolidQueueMetrics::QueueSummary.new(
+              queue_name: queue_name.to_s,
+              ready_count: 0,
+              scheduled_count: 0,
+              failed_count: 0,
+              recurring_count: 0,
+              paused: false,
+              last_enqueued_at: nil,
+              last_started_at: nil,
+              last_finished_at: nil,
+              available: false
+            )
+
+            {
+              role: role,
+              queue_name: queue_name,
+              summary: summary
+            }
+          end
+        end
+      end
+
+      def upcoming_fetch_schedule
+        cache.fetch(:upcoming_fetch_schedule) do
+          measure(:upcoming_fetch_schedule) do
+            SourceMonitor::Dashboard::UpcomingFetchSchedule.new(scope: SourceMonitor::Source.active)
+          end
+        end
+      end
+
+      private
+
+      DEFAULT_RECENT_ACTIVITY_LIMIT = 8
+
+      attr_reader :reference_time, :cache
+
+      def measure(name, metadata = {})
+        started_at = monotonic_time
+        result = yield
+        duration_ms = ((monotonic_time - started_at) * 1000.0).round(2)
+        recorded_at = Time.current
+
+        payload = metadata.merge(duration_ms:, recorded_at:)
+        ActiveSupport::Notifications.instrument("source_monitor.dashboard.#{name}", payload)
+        record_metrics(name, result, duration_ms:, recorded_at:, metadata:)
+
+        result
+      end
+
+      def record_metrics(name, result, duration_ms:, recorded_at:, metadata:)
+        SourceMonitor::Metrics.gauge(:"dashboard_#{name}_duration_ms", duration_ms)
+        SourceMonitor::Metrics.gauge(:"dashboard_#{name}_last_run_at_epoch", recorded_at.to_f)
+
+        case name
+        when :stats
+          record_stats_metrics(result)
+        when :recent_activity
+          SourceMonitor::Metrics.gauge(:dashboard_recent_activity_events_count, result.size)
+          SourceMonitor::Metrics.gauge(:dashboard_recent_activity_limit, metadata[:limit]) if metadata[:limit]
+        when :job_metrics
+          SourceMonitor::Metrics.gauge(:dashboard_job_metrics_queue_count, result.size)
+        when :upcoming_fetch_schedule
+          SourceMonitor::Metrics.gauge(:dashboard_fetch_schedule_group_count, result.groups.size)
+        end
+      end
+
+      def record_stats_metrics(stats)
+        SourceMonitor::Metrics.gauge(:dashboard_stats_total_sources, stats[:total_sources])
+        SourceMonitor::Metrics.gauge(:dashboard_stats_active_sources, stats[:active_sources])
+        SourceMonitor::Metrics.gauge(:dashboard_stats_failed_sources, stats[:failed_sources])
+        SourceMonitor::Metrics.gauge(:dashboard_stats_total_items, stats[:total_items])
+        SourceMonitor::Metrics.gauge(:dashboard_stats_fetches_today, stats[:fetches_today])
+      end
+
+      def queue_name_map
+        @queue_name_map ||= {
+          fetch: SourceMonitor.queue_name(:fetch),
+          scrape: SourceMonitor.queue_name(:scrape)
+        }
+      end
+
+      def monotonic_time
+        Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      end
+
+      class Cache
+        def initialize
+          @store = {}
+        end
+
+        def fetch(key)
+          if store.key?(key)
+            store.fetch(key)
+          else
+            store[key] = yield
+          end
+        end
+
+        private
+
+        attr_reader :store
+      end
+
+      class StatsQuery
+        def initialize(reference_time:)
+          @reference_time = reference_time
+        end
+
+        def call
+          {
+            total_sources: integer_value(source_counts["total_sources"]),
+            active_sources: integer_value(source_counts["active_sources"]),
+            failed_sources: integer_value(source_counts["failed_sources"]),
+            total_items: total_items_count,
+            fetches_today: fetches_today_count
+          }
+        end
+
+        private
+
+        attr_reader :reference_time
+
+        def source_counts
+          @source_counts ||= begin
+            SourceMonitor::Source.connection.exec_query(source_counts_sql).first || {}
+          end
+        end
+
+        def total_items_count
+          SourceMonitor::Item.connection.select_value(total_items_sql).to_i
+        end
+
+        def fetches_today_count
+          SourceMonitor::FetchLog.where("started_at >= ?", start_of_day).count
+        end
+
+        def source_counts_sql
+          <<~SQL.squish
+            SELECT
+              COUNT(*) AS total_sources,
+              SUM(CASE WHEN active THEN 1 ELSE 0 END) AS active_sources,
+              SUM(CASE WHEN (#{failure_condition}) THEN 1 ELSE 0 END) AS failed_sources
+            FROM #{SourceMonitor::Source.quoted_table_name}
+          SQL
+        end
+
+        def failure_condition
+          [
+            "#{SourceMonitor::Source.quoted_table_name}.failure_count > 0",
+            "#{SourceMonitor::Source.quoted_table_name}.last_error IS NOT NULL",
+            "#{SourceMonitor::Source.quoted_table_name}.last_error_at IS NOT NULL"
+          ].join(" OR ")
+        end
+
+        def total_items_sql
+          "SELECT COUNT(*) FROM #{SourceMonitor::Item.quoted_table_name}"
+        end
+
+        def start_of_day
+          reference_time.in_time_zone.beginning_of_day
+        end
+
+        def integer_value(value)
+          value.to_i
+        end
+      end
+
+      class RecentActivityQuery
+        EVENT_TYPE_FETCH = "fetch_log"
+        EVENT_TYPE_SCRAPE = "scrape_log"
+        EVENT_TYPE_ITEM = "item"
+
+        def initialize(limit:)
+          @limit = limit
+        end
+
+        def call
+          rows = connection.exec_query(sanitized_sql)
+          rows.map { |row| build_event(row) }
+        end
+
+        private
+
+        attr_reader :limit
+
+        def connection
+          ActiveRecord::Base.connection
+        end
+
+        def build_event(row)
+          SourceMonitor::Dashboard::RecentActivity::Event.new(
+            type: row["resource_type"].to_sym,
+            id: row["resource_id"],
+            occurred_at: row["occurred_at"],
+            success: row["success_flag"].to_i == 1,
+            items_created: row["items_created"],
+            items_updated: row["items_updated"],
+            scraper_adapter: row["scraper_adapter"],
+            item_title: row["item_title"],
+            item_url: row["item_url"],
+            source_name: row["source_name"],
+            source_id: row["source_id"]
+          )
+        end
+
+        def sanitized_sql
+          ActiveRecord::Base.send(:sanitize_sql_array, [ unified_sql_template, limit ])
+        end
+
+        def unified_sql_template
+          <<~SQL
+            SELECT resource_type,
+                   resource_id,
+                   occurred_at,
+                   success_flag,
+                   items_created,
+                   items_updated,
+                   scraper_adapter,
+                   item_title,
+                   item_url,
+                   source_name,
+                   source_id
+            FROM (
+              #{fetch_log_sql}
+              UNION ALL
+              #{scrape_log_sql}
+              UNION ALL
+              #{item_sql}
+            ) AS dashboard_events
+            WHERE occurred_at IS NOT NULL
+            ORDER BY occurred_at DESC
+            LIMIT ?
+          SQL
+        end
+
+        def fetch_log_sql
+          <<~SQL
+            SELECT
+              '#{EVENT_TYPE_FETCH}' AS resource_type,
+              #{SourceMonitor::FetchLog.quoted_table_name}.id AS resource_id,
+              #{SourceMonitor::FetchLog.quoted_table_name}.started_at AS occurred_at,
+              CASE WHEN #{SourceMonitor::FetchLog.quoted_table_name}.success THEN 1 ELSE 0 END AS success_flag,
+              #{SourceMonitor::FetchLog.quoted_table_name}.items_created AS items_created,
+              #{SourceMonitor::FetchLog.quoted_table_name}.items_updated AS items_updated,
+              NULL AS scraper_adapter,
+              NULL AS item_title,
+              NULL AS item_url,
+              NULL AS source_name,
+              #{SourceMonitor::FetchLog.quoted_table_name}.source_id AS source_id
+            FROM #{SourceMonitor::FetchLog.quoted_table_name}
+          SQL
+        end
+
+        def scrape_log_sql
+          <<~SQL
+            SELECT
+              '#{EVENT_TYPE_SCRAPE}' AS resource_type,
+              #{SourceMonitor::ScrapeLog.quoted_table_name}.id AS resource_id,
+              #{SourceMonitor::ScrapeLog.quoted_table_name}.started_at AS occurred_at,
+              CASE WHEN #{SourceMonitor::ScrapeLog.quoted_table_name}.success THEN 1 ELSE 0 END AS success_flag,
+              NULL AS items_created,
+              NULL AS items_updated,
+              #{SourceMonitor::ScrapeLog.quoted_table_name}.scraper_adapter AS scraper_adapter,
+              NULL AS item_title,
+              NULL AS item_url,
+              #{SourceMonitor::Source.quoted_table_name}.#{quoted_source_name} AS source_name,
+              #{SourceMonitor::ScrapeLog.quoted_table_name}.source_id AS source_id
+            FROM #{SourceMonitor::ScrapeLog.quoted_table_name}
+            LEFT JOIN #{SourceMonitor::Source.quoted_table_name}
+              ON #{SourceMonitor::Source.quoted_table_name}.id = #{SourceMonitor::ScrapeLog.quoted_table_name}.source_id
+          SQL
+        end
+
+        def item_sql
+          <<~SQL
+            SELECT
+              '#{EVENT_TYPE_ITEM}' AS resource_type,
+              #{SourceMonitor::Item.quoted_table_name}.id AS resource_id,
+              #{SourceMonitor::Item.quoted_table_name}.created_at AS occurred_at,
+              1 AS success_flag,
+              NULL AS items_created,
+              NULL AS items_updated,
+              NULL AS scraper_adapter,
+              #{SourceMonitor::Item.quoted_table_name}.title AS item_title,
+              #{SourceMonitor::Item.quoted_table_name}.url AS item_url,
+              #{SourceMonitor::Source.quoted_table_name}.#{quoted_source_name} AS source_name,
+              #{SourceMonitor::Item.quoted_table_name}.source_id AS source_id
+            FROM #{SourceMonitor::Item.quoted_table_name}
+            LEFT JOIN #{SourceMonitor::Source.quoted_table_name}
+              ON #{SourceMonitor::Source.quoted_table_name}.id = #{SourceMonitor::Item.quoted_table_name}.source_id
+          SQL
+        end
+
+        def quoted_source_name
+          ActiveRecord::Base.connection.quote_column_name("name")
+        end
+      end
+
+      QUICK_ACTIONS = [
+        SourceMonitor::Dashboard::QuickAction.new(
+          label: "Add Source",
+          description: "Create a new feed source",
+          route_name: :new_source_path
+        ).freeze,
+        SourceMonitor::Dashboard::QuickAction.new(
+          label: "View Sources",
+          description: "Manage existing sources",
+          route_name: :sources_path
+        ).freeze,
+        SourceMonitor::Dashboard::QuickAction.new(
+          label: "Check Health",
+          description: "Verify engine status",
+          route_name: :health_path
+        ).freeze
+      ].freeze
+    end
+  end
+end

data/lib/source_monitor/dashboard/quick_action.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  module Dashboard
+    QuickAction = Struct.new(:label, :description, :route_name, keyword_init: true)
+  end
+end

data/lib/source_monitor/dashboard/quick_actions_presenter.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  module Dashboard
+    class QuickActionsPresenter
+      def initialize(actions, url_helpers:)
+        @actions = actions
+        @url_helpers = url_helpers
+      end
+
+      def to_a
+        actions.map do |action|
+          {
+            label: action.label,
+            description: action.description,
+            path: url_helpers.public_send(action.route_name)
+          }
+        end
+      end
+
+      private
+
+      attr_reader :actions, :url_helpers
+    end
+  end
+end

data/lib/source_monitor/dashboard/recent_activity.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  module Dashboard
+    module RecentActivity
+      Event = Struct.new(
+        :type,
+        :id,
+        :occurred_at,
+        :success,
+        :items_created,
+        :items_updated,
+        :scraper_adapter,
+        :item_title,
+        :item_url,
+        :source_name,
+        :source_id,
+        keyword_init: true
+      ) do
+        def type
+          self[:type]&.to_sym
+        end
+
+        def success?
+          !!self[:success]
+        end
+      end
+    end
+  end
+end

data/lib/source_monitor/dashboard/recent_activity_presenter.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  module Dashboard
+    class RecentActivityPresenter
+      def initialize(events, url_helpers:)
+        @events = events
+        @url_helpers = url_helpers
+      end
+
+      def to_a
+        events.map { |event| build_view_model(event) }
+      end
+
+      private
+
+      attr_reader :events, :url_helpers
+
+      def build_view_model(event)
+        case event.type
+        when :fetch_log
+          fetch_event(event)
+        when :scrape_log
+          scrape_event(event)
+        when :item
+          item_event(event)
+        else
+          fallback_event(event)
+        end
+      end
+
+      def fetch_event(event)
+        {
+          label: "Fetch ##{event.id}",
+          description: "#{event.items_created.to_i} created / #{event.items_updated.to_i} updated",
+          status: event.success? ? :success : :failure,
+          type: :fetch,
+          time: event.occurred_at,
+          path: url_helpers.fetch_log_path(event.id)
+        }
+      end
+
+      def scrape_event(event)
+        {
+          label: "Scrape ##{event.id}",
+          description: (event.scraper_adapter.presence || "Scraper"),
+          status: event.success? ? :success : :failure,
+          type: :scrape,
+          time: event.occurred_at,
+          path: url_helpers.scrape_log_path(event.id)
+        }
+      end
+
+      def item_event(event)
+        {
+          label: event.item_title.presence || "New Item",
+          description: event.source_name.presence || event.item_url.presence || "New feed item",
+          status: :success,
+          type: :item,
+          time: event.occurred_at,
+          path: url_helpers.item_path(event.id)
+        }
+      end
+
+      def fallback_event(event)
+        {
+          label: "Event ##{event.id}",
+          description: event.source_name.presence || "No additional details recorded.",
+          status: event.success? ? :success : :failure,
+          type: event.type || :unknown,
+          time: event.occurred_at,
+          path: nil
+        }
+      end
+    end
+  end
+end

data/lib/source_monitor/dashboard/turbo_broadcaster.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  module Dashboard
+    module TurboBroadcaster
+      STREAM_NAME = "source_monitor_dashboard"
+
+      module_function
+
+      def setup!
+        return unless turbo_streams_available?
+
+        register_callback(:after_fetch_completed, fetch_callback)
+        register_callback(:after_item_created, item_callback)
+      end
+
+      def fetch_callback
+        @fetch_callback ||= lambda { |_event| broadcast_dashboard_updates }
+      end
+
+      def item_callback
+        @item_callback ||= lambda { |_event| broadcast_dashboard_updates }
+      end
+
+      def broadcast_dashboard_updates
+        return unless turbo_streams_available?
+
+        queries = SourceMonitor::Dashboard::Queries.new
+        url_helpers = SourceMonitor::Engine.routes.url_helpers
+
+        Turbo::StreamsChannel.broadcast_replace_to(
+          STREAM_NAME,
+          target: "source_monitor_dashboard_stats",
+          html: render_partial("source_monitor/dashboard/stats", stats: queries.stats)
+        )
+
+        Turbo::StreamsChannel.broadcast_replace_to(
+          STREAM_NAME,
+          target: "source_monitor_dashboard_recent_activity",
+          html: render_partial(
+            "source_monitor/dashboard/recent_activity",
+            recent_activity: SourceMonitor::Dashboard::RecentActivityPresenter.new(
+              queries.recent_activity,
+              url_helpers:
+            ).to_a
+          )
+        )
+
+        fetch_schedule = queries.upcoming_fetch_schedule
+        Turbo::StreamsChannel.broadcast_replace_to(
+          STREAM_NAME,
+          target: "source_monitor_dashboard_fetch_schedule",
+          html: render_partial(
+            "source_monitor/dashboard/fetch_schedule",
+            groups: fetch_schedule.groups,
+            reference_time: fetch_schedule.reference_time
+          )
+        )
+      rescue StandardError => error
+        Rails.logger.error(
+          "[SourceMonitor] Turbo stream broadcast failed: #{error.class}: #{error.message}"
+        )
+      end
+
+      def turbo_streams_available?
+        defined?(Turbo::StreamsChannel)
+      end
+      private_class_method :turbo_streams_available?
+
+      def render_partial(partial, locals)
+        SourceMonitor::DashboardController.render(
+          partial:,
+          locals:
+        )
+      end
+      private_class_method :render_partial
+
+      def register_callback(name, callback)
+        callbacks = SourceMonitor.config.events.callbacks_for(name)
+        return if callbacks.include?(callback)
+
+        SourceMonitor.config.events.public_send(name, callback)
+      end
+      private_class_method :register_callback
+    end
+  end
+end