source_monitor 0.1.1

data/lib/source_monitor/dashboard/upcoming_fetch_schedule.rb

@@ -0,0 +1,126 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  module Dashboard
+    class UpcomingFetchSchedule
+      Group = Struct.new(
+        :key,
+        :label,
+        :min_minutes,
+        :max_minutes,
+        :window_start,
+        :window_end,
+        :include_unscheduled,
+        :sources,
+        keyword_init: true
+      ) do
+        def empty?
+          sources.blank?
+        end
+      end
+
+      INTERVAL_DEFINITIONS = [
+        { key: "0-30", label: "Within 30 minutes", min_minutes: 0, max_minutes: 30 },
+        { key: "30-60", label: "30-60 minutes", min_minutes: 30, max_minutes: 60 },
+        { key: "60-120", label: "60-120 minutes", min_minutes: 60, max_minutes: 120 },
+        { key: "120-240", label: "120-240 minutes", min_minutes: 120, max_minutes: 240 },
+        { key: "240+", label: "240 minutes +", min_minutes: 240, max_minutes: nil, include_unscheduled: true }
+      ].freeze
+
+      attr_reader :scope, :reference_time
+
+      def initialize(scope: SourceMonitor::Source.active, reference_time: Time.current)
+        @scope = scope
+        @reference_time = reference_time
+      end
+
+      def groups
+        @groups ||= build_groups
+      end
+
+      private
+
+      def build_groups
+        definitions = build_definitions
+        scheduled_sources.each do |source|
+          definition = definition_for(source.next_fetch_at)
+          definitions[definition[:key]][:sources] << source if definition
+        end
+
+        unscheduled_sources.each do |source|
+          definition = definitions.values.find { |value| value[:include_unscheduled] }
+          next unless definition
+
+          definition[:sources] << source
+        end
+
+        definitions.values.map do |definition|
+          Group.new(
+            key: definition[:key],
+            label: definition[:label],
+            min_minutes: definition[:min_minutes],
+            max_minutes: definition[:max_minutes],
+            window_start: window_start_for(definition[:min_minutes]),
+            window_end: window_end_for(definition[:max_minutes]),
+            include_unscheduled: definition[:include_unscheduled],
+            sources: sort_sources(definition[:sources])
+          )
+        end
+      end
+
+      def build_definitions
+        INTERVAL_DEFINITIONS.each_with_object({}) do |definition, memo|
+          memo[definition[:key]] = definition.merge(sources: [])
+        end
+      end
+
+      def scheduled_sources
+        scope.where.not(next_fetch_at: nil).order(:next_fetch_at)
+      end
+
+      def unscheduled_sources
+        scope.where(next_fetch_at: nil).order(:name)
+      end
+
+      def definition_for(next_fetch_at)
+        minutes = minutes_until(next_fetch_at)
+
+        INTERVAL_DEFINITIONS.find do |definition|
+          min = definition[:min_minutes]
+          max = definition[:max_minutes]
+
+          minutes >= min && (max.nil? || minutes < max)
+        end
+      end
+
+      def minutes_until(timestamp)
+        return Float::INFINITY if timestamp.blank?
+
+        minutes = (timestamp - reference_time) / 60.0
+        return 0 if minutes.negative?
+
+        minutes
+      end
+
+      def window_start_for(min_minutes)
+        return nil if min_minutes.nil? || min_minutes.infinite?
+
+        reference_time + min_minutes.minutes
+      end
+
+      def window_end_for(max_minutes)
+        return nil if max_minutes.nil? || max_minutes.infinite?
+
+        reference_time + max_minutes.minutes
+      end
+
+      def sort_sources(sources)
+        future_cap = reference_time + 100.years
+
+        sources.sort_by do |source|
+          [ source.next_fetch_at || future_cap, source.name.to_s ]
+        end
+      end
+    end
+  end
+end

data/lib/source_monitor/engine.rb

@@ -0,0 +1,107 @@
+module SourceMonitor
+  class Engine < ::Rails::Engine
+    isolate_namespace SourceMonitor
+    require "source_monitor/assets/bundler"
+    require "source_monitor/jobs/fetch_failure_subscriber"
+
+    def self.table_name_prefix
+      SourceMonitor.config.models.table_name_prefix
+    end
+
+    initializer "source_monitor.assets" do |app|
+      next unless app.config.respond_to?(:assets)
+
+      engine_root = SourceMonitor::Engine.root
+
+      app.config.assets.paths << engine_root.join("app/assets/builds").to_s
+      app.config.assets.paths << engine_root.join("app/assets/images").to_s
+      app.config.assets.paths << engine_root.join("app/assets/svgs").to_s
+    end
+
+    initializer "source_monitor.assets.sprockets" do |app|
+      next unless app.config.respond_to?(:assets)
+
+      manifest_entry = "source_monitor_manifest.js"
+      app.config.assets.precompile << manifest_entry unless app.config.assets.precompile.include?(manifest_entry)
+      app.config.assets.precompile.concat(SourceMonitor::Engine.asset_precompile_entries)
+      app.config.assets.precompile.uniq!
+    end
+
+    initializer "source_monitor.metrics" do
+      SourceMonitor::Metrics.setup_subscribers!
+    end
+
+    initializer "source_monitor.dashboard_streams" do
+      config.to_prepare do
+        SourceMonitor::Health.setup!
+        SourceMonitor::Realtime.setup!
+        SourceMonitor::Dashboard::TurboBroadcaster.setup!
+      end
+    end
+
+    initializer "source_monitor.jobs" do |app|
+      SourceMonitor::Jobs::Visibility.setup!
+      SourceMonitor::Jobs::FetchFailureSubscriber.setup!
+
+      if defined?(::SolidQueue)
+        adapter_name = ActiveJob::Base.queue_adapter_name.to_s
+        if adapter_name.empty? || adapter_name == "async"
+          ActiveJob::Base.queue_adapter = :solid_queue
+        end
+
+        if defined?(::SolidQueue::RecurringTask)
+          job_class_config = SourceMonitor.config.recurring_command_job_class
+          if job_class_config.present?
+            resolved_class = job_class_config.is_a?(String) ? job_class_config.constantize : job_class_config
+            SolidQueue::RecurringTask.default_job_class = resolved_class
+          end
+        end
+
+        if defined?(MissionControl::Jobs)
+          adapters = MissionControl::Jobs.adapters
+          if adapters.respond_to?(:add)
+            adapters.add(:solid_queue)
+            adapters.delete(:async)
+          elsif adapters.respond_to?(:<<)
+            adapters << :solid_queue unless adapters.include?(:solid_queue)
+            adapters.delete(:async) if adapters.respond_to?(:delete)
+          end
+
+          if defined?(ActiveJob::QueueAdapters::SolidQueueExt) &&
+              !(ActiveJob::QueueAdapters::SolidQueueAdapter < ActiveJob::QueueAdapters::SolidQueueExt)
+            ActiveJob::QueueAdapters::SolidQueueAdapter.prepend ActiveJob::QueueAdapters::SolidQueueExt
+          end
+
+          MissionControl::Jobs.applications.each do |application|
+            next if application.servers.any? { |server| server.queue_adapter_name == :solid_queue }
+
+            solid_queue_adapter = ActiveJob::QueueAdapters.lookup(:solid_queue).new
+            application.add_servers(solid_queue: solid_queue_adapter)
+          end
+        end
+
+        app.config.after_initialize do
+          SourceMonitor::Jobs::Visibility.setup!
+        end
+      end
+    end
+    class << self
+      def asset_precompile_entries
+        engine_root = SourceMonitor::Engine.root
+        asset_roots = {
+          images: engine_root.join("app/assets/images"),
+          svgs: engine_root.join("app/assets/svgs")
+        }
+
+        asset_roots.flat_map do |_, base_path|
+          Dir[base_path.join("source_monitor/**/*").to_s].filter_map do |absolute_path|
+            next unless File.file?(absolute_path)
+            next if File.basename(absolute_path).start_with?(".")
+
+            Pathname.new(absolute_path).relative_path_from(base_path).to_s
+          end
+        end
+      end
+    end
+  end
+end

data/lib/source_monitor/events.rb

@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+
+require "active_support/core_ext/time"
+
+module SourceMonitor
+  module Events
+    ItemCreatedEvent = Struct.new(:item, :source, :entry, :result, :status, :occurred_at, keyword_init: true) do
+      def created?
+        status.to_s == "created"
+      end
+    end
+
+    ItemScrapedEvent = Struct.new(:item, :source, :result, :log, :status, :occurred_at, keyword_init: true) do
+      def success?
+        status.to_s != "failed"
+      end
+    end
+
+    FetchCompletedEvent = Struct.new(:source, :result, :status, :occurred_at, keyword_init: true)
+
+    ItemProcessorContext = Struct.new(:item, :source, :entry, :result, :status, :occurred_at, keyword_init: true)
+
+    module_function
+
+    def after_item_created(item:, source:, entry:, result:)
+      event = ItemCreatedEvent.new(
+        item: item,
+        source: source,
+        entry: entry,
+        result: result,
+        status: result&.status,
+        occurred_at: Time.current
+      )
+
+      dispatch(:after_item_created, event)
+    end
+
+    def after_item_scraped(result)
+      item = result&.item
+      source = item&.source
+      event = ItemScrapedEvent.new(
+        item: item,
+        source: source,
+        result: result,
+        log: result&.log,
+        status: result&.status,
+        occurred_at: Time.current
+      )
+
+      dispatch(:after_item_scraped, event)
+    end
+
+    def after_fetch_completed(source:, result:)
+      event = FetchCompletedEvent.new(
+        source: source,
+        result: result,
+        status: result&.status,
+        occurred_at: Time.current
+      )
+
+      dispatch(:after_fetch_completed, event)
+    end
+
+    def run_item_processors(source:, entry:, result:)
+      item = result&.item
+      context = ItemProcessorContext.new(
+        item: item,
+        source: source,
+        entry: entry,
+        result: result,
+        status: result&.status,
+        occurred_at: Time.current
+      )
+
+      SourceMonitor.config.events.item_processors.each do |processor|
+        invoke(processor, context)
+      rescue StandardError => error
+        log_handler_error(:item_processor, processor, error)
+      end
+    end
+
+    def dispatch(event_name, event)
+      SourceMonitor.config.events.callbacks_for(event_name).each do |callback|
+        invoke(callback, event)
+      rescue StandardError => error
+        log_handler_error(event_name, callback, error)
+      end
+    end
+
+    def invoke(callable, event)
+      if callable.respond_to?(:arity) && callable.arity.zero?
+        callable.call
+      else
+        callable.call(event)
+      end
+    end
+
+    def log_handler_error(kind, handler, error)
+      message = "[SourceMonitor] #{kind} handler #{handler.inspect} failed: #{error.class}: #{error.message}"
+
+      if defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger
+        Rails.logger.error(message)
+      else
+        warn(message)
+      end
+    rescue StandardError
+      warn(message)
+    end
+  end
+end

data/lib/source_monitor/feedjira_extensions.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+require "feedjira"
+require "sax-machine"
+
+module SourceMonitor
+  module FeedjiraExtensions
+    class MediaThumbnail
+      include SAXMachine
+
+      attribute :url
+      attribute :width
+      attribute :height
+    end
+
+    class MediaContent
+      include SAXMachine
+
+      attribute :url
+      attribute :type
+      attribute :medium
+      attribute :height
+      attribute :width
+      attribute :"fileSize", as: :file_size
+      attribute :duration
+      attribute :expression
+    end
+
+    class Enclosure
+      include SAXMachine
+
+      attribute :url
+      attribute :type
+      attribute :length
+    end
+
+    class AtomAuthor
+      include SAXMachine
+
+      element :name
+      element :email
+      element :uri
+    end
+
+    class AtomLink
+      include SAXMachine
+
+      attribute :href
+      attribute :rel
+      attribute :type
+      attribute :length
+    end
+
+    module_function
+
+    def apply!
+      return if @applied
+
+      extend_rss_entry
+      extend_atom_entry
+
+      @applied = true
+    end
+
+    def extend_rss_entry
+      Feedjira::Parser::RSSEntry.element :"media:keywords", as: :media_keywords_raw
+      Feedjira::Parser::RSSEntry.element :"itunes:keywords", as: :itunes_keywords_raw
+      Feedjira::Parser::RSSEntry.element :"slash:comments", as: :slash_comments_raw
+      Feedjira::Parser::RSSEntry.elements :"media:thumbnail",
+        as: :media_thumbnail_nodes,
+        class: MediaThumbnail
+      Feedjira::Parser::RSSEntry.elements :"media:content",
+        as: :media_content_nodes,
+        class: MediaContent
+      Feedjira::Parser::RSSEntry.elements :enclosure,
+        as: :enclosure_nodes,
+        class: Enclosure
+
+      Feedjira::Parser::RSSEntry.prepend(RSSAuthorCapture)
+    end
+
+    def extend_atom_entry
+      Feedjira::Parser::AtomEntry.elements :author,
+        as: :author_nodes,
+        class: AtomAuthor
+      Feedjira::Parser::AtomEntry.elements :link,
+        as: :link_nodes,
+        class: AtomLink
+    end
+    module RSSAuthorCapture
+      def author=(value)
+        (@source_monitor_rss_authors ||= []) << value if value
+        super
+      end
+
+      def rss_authors
+        Array(@source_monitor_rss_authors)
+      end
+    end
+  end
+end
+
+SourceMonitor::FeedjiraExtensions.apply!

data/lib/source_monitor/fetching/advisory_lock.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  module Fetching
+    # Wraps Postgres advisory lock usage to provide a small, testable collaborator
+    # for coordinating fetch execution across processes.
+    class AdvisoryLock
+      NotAcquiredError = Class.new(StandardError)
+
+      def initialize(namespace:, key:, connection_pool: ActiveRecord::Base.connection_pool)
+        @namespace = namespace
+        @key = key
+        @connection_pool = connection_pool
+      end
+
+      def with_lock
+        connection_pool.with_connection do |connection|
+          locked = try_lock(connection)
+          raise NotAcquiredError, "advisory lock #{namespace}/#{key} busy" unless locked
+
+          begin
+            yield
+          ensure
+            release(connection)
+          end
+        end
+      end
+
+      private
+
+      attr_reader :namespace, :key, :connection_pool
+
+      def try_lock(connection)
+        result = connection.exec_query(
+          "SELECT pg_try_advisory_lock(#{namespace.to_i}, #{key.to_i})"
+        )
+
+        truthy?(result.rows.dig(0, 0))
+      end
+
+      def release(connection)
+        connection.exec_query(
+          "SELECT pg_advisory_unlock(#{namespace.to_i}, #{key.to_i})"
+        )
+      rescue StandardError
+        nil
+      end
+
+      def truthy?(value)
+        value == true || value.to_s == "t"
+      end
+    end
+  end
+end

data/lib/source_monitor/fetching/completion/event_publisher.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  module Fetching
+    module Completion
+      # Publishes fetch completion events to the configured event dispatcher.
+      class EventPublisher
+        def initialize(dispatcher: SourceMonitor::Events)
+          @dispatcher = dispatcher
+        end
+
+        def call(source:, result:)
+          dispatcher.after_fetch_completed(source: source, result: result)
+        end
+
+        private
+
+        attr_reader :dispatcher
+      end
+    end
+  end
+end

data/lib/source_monitor/fetching/completion/follow_up_handler.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  module Fetching
+    module Completion
+      # Enqueues follow-up scraping work for items created during a fetch.
+      class FollowUpHandler
+        def initialize(enqueuer_class: SourceMonitor::Scraping::Enqueuer, job_class: SourceMonitor::ScrapeItemJob)
+          @enqueuer_class = enqueuer_class
+          @job_class = job_class
+        end
+
+        def call(source:, result:)
+          return unless should_enqueue?(source:, result:)
+
+          Array(result.item_processing&.created_items).each do |item|
+            next unless item.present? && item.scraped_at.nil?
+
+            enqueuer_class.enqueue(item:, source:, job_class:, reason: :auto)
+          end
+        end
+
+        private
+
+        attr_reader :enqueuer_class, :job_class
+
+        def should_enqueue?(source:, result:)
+          return false unless result
+          return false unless result.status == :fetched
+          return false unless source.scraping_enabled? && source.auto_scrape?
+
+          result.item_processing&.created.to_i.positive?
+        end
+      end
+    end
+  end
+end

data/lib/source_monitor/fetching/completion/retention_handler.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  module Fetching
+    module Completion
+      # Applies item retention after a fetch completes.
+      class RetentionHandler
+        def initialize(pruner: SourceMonitor::Items::RetentionPruner)
+          @pruner = pruner
+        end
+
+        def call(source:, result:) # rubocop:disable Lint/UnusedMethodArgument
+          pruner.call(
+            source: source,
+            strategy: SourceMonitor.config.retention.strategy
+          )
+        rescue StandardError => error
+          Rails.logger.error(
+            "[SourceMonitor] Retention pruning failed for source #{source.id}: #{error.class} - #{error.message}"
+          )
+          nil
+        end
+
+        private
+
+        attr_reader :pruner
+      end
+    end
+  end
+end