source_monitor 0.3.3 → 0.5.0

data/lib/generators/source_monitor/install/install_generator.rb

@@ -49,7 +49,48 @@ module SourceMonitor
         end
       end
 
+      def patch_procfile_dev
+        procfile_path = File.join(destination_root, "Procfile.dev")
+
+        if File.exist?(procfile_path)
+          content = File.read(procfile_path)
+          if content.match?(/^jobs:/)
+            say_status :skip, "Procfile.dev (jobs entry already present)", :yellow
+            return
+          end
+
+          File.open(procfile_path, "a") { |f| f.puts("", PROCFILE_JOBS_ENTRY) }
+          say_status :append, "Procfile.dev", :green
+        else
+          File.write(procfile_path, "web: bin/rails server -p 3000\n#{PROCFILE_JOBS_ENTRY}\n")
+          say_status :create, "Procfile.dev", :green
+        end
+      end
+
+      def configure_queue_dispatcher
+        queue_path = File.join(destination_root, "config/queue.yml")
+
+        unless File.exist?(queue_path)
+          say_status :skip, "config/queue.yml (file not found — create it or run rails app:update to generate)", :yellow
+          return
+        end
+
+        parsed = YAML.safe_load(File.read(queue_path), aliases: true) || {}
+
+        if queue_config_has_recurring_schedule?(parsed)
+          say_status :skip, "config/queue.yml (recurring_schedule already configured)", :yellow
+          return
+        end
+
+        add_recurring_schedule_to_dispatchers!(parsed)
+        File.write(queue_path, YAML.dump(parsed))
+        say_status :append, "config/queue.yml (added recurring_schedule to dispatchers)", :green
+      end
+
       def print_next_steps
+        say_status :info,
+          "Procfile.dev configured — run bin/dev to start both web server and Solid Queue workers.",
+          :green
         say_status :info,
           "Recurring jobs configured in config/recurring.yml — they'll run automatically with bin/dev or bin/jobs.",
           :green
@@ -60,6 +101,8 @@ module SourceMonitor
 
       private
 
+      PROCFILE_JOBS_ENTRY = "jobs: bundle exec rake solid_queue:start"
+
       RECURRING_ENTRIES = {
         "source_monitor_schedule_fetches" => {
           "class" => "SourceMonitor::ScheduleFetchesJob",
@@ -154,6 +197,64 @@ module SourceMonitor
         path = (raw_path && !raw_path.strip.empty?) ? raw_path.strip : "/source_monitor"
         path.start_with?("/") ? path : "/#{path}"
       end
+
+      RECURRING_SCHEDULE_VALUE = "config/recurring.yml"
+
+      DEFAULT_DISPATCHER = {
+        "polling_interval" => 1,
+        "batch_size" => 500,
+        "recurring_schedule" => RECURRING_SCHEDULE_VALUE
+      }.freeze
+
+      def queue_config_has_recurring_schedule?(parsed)
+        parsed.each_value do |value|
+          next unless value.is_a?(Hash)
+
+          dispatchers = value["dispatchers"] || value[:dispatchers]
+          if dispatchers.is_a?(Array)
+            return true if dispatchers.any? { |d| d.is_a?(Hash) && d.key?("recurring_schedule") }
+          end
+
+          return true if queue_config_has_recurring_schedule?(value)
+        end
+
+        # Check top-level dispatchers (flat config)
+        if parsed.key?("dispatchers") && parsed["dispatchers"].is_a?(Array)
+          return true if parsed["dispatchers"].any? { |d| d.is_a?(Hash) && d.key?("recurring_schedule") }
+        end
+
+        false
+      end
+
+      def add_recurring_schedule_to_dispatchers!(parsed)
+        found_dispatchers = false
+
+        parsed.each_value do |value|
+          next unless value.is_a?(Hash)
+
+          if value.key?("dispatchers") && value["dispatchers"].is_a?(Array)
+            value["dispatchers"].each do |dispatcher|
+              next unless dispatcher.is_a?(Hash)
+              dispatcher["recurring_schedule"] ||= RECURRING_SCHEDULE_VALUE
+            end
+            found_dispatchers = true
+          end
+        end
+
+        # Check top-level dispatchers (flat config)
+        if parsed.key?("dispatchers") && parsed["dispatchers"].is_a?(Array)
+          parsed["dispatchers"].each do |dispatcher|
+            next unless dispatcher.is_a?(Hash)
+            dispatcher["recurring_schedule"] ||= RECURRING_SCHEDULE_VALUE
+          end
+          found_dispatchers = true
+        end
+
+        # No dispatchers found at all — add a default section
+        unless found_dispatchers
+          parsed["dispatchers"] = [ DEFAULT_DISPATCHER.dup ]
+        end
+      end
     end
   end
 end

data/lib/source_monitor/configuration/deprecation_registry.rb

@@ -0,0 +1,237 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  class DeprecatedOptionError < StandardError; end
+
+  class Configuration
+    # Registry for deprecated configuration options.
+    #
+    # Engine developers register deprecations at boot time via the DSL:
+    #
+    #   SourceMonitor::Configuration::DeprecationRegistry.register(
+    #     "http.old_proxy_url",
+    #     removed_in: "0.5.0",
+    #     replacement: "http.proxy",
+    #     severity: :warning,
+    #     message: "Use config.http.proxy instead"
+    #   )
+    #
+    # When a host app's initializer accesses a deprecated option, the
+    # trapping method fires automatically:
+    #   - :warning  -- logs via Rails.logger.warn and forwards to replacement
+    #   - :error    -- raises SourceMonitor::DeprecatedOptionError
+    #
+    class DeprecationRegistry
+      # Maps settings accessor names (as used on Configuration) to their classes.
+      SETTINGS_CLASSES = {
+        "http" => "HTTPSettings",
+        "fetching" => "FetchingSettings",
+        "health" => "HealthSettings",
+        "scraping" => "ScrapingSettings",
+        "retention" => "RetentionSettings",
+        "realtime" => "RealtimeSettings",
+        "authentication" => "AuthenticationSettings",
+        "images" => "ImagesSettings",
+        "scrapers" => "ScraperRegistry",
+        "events" => "Events",
+        "models" => "Models"
+      }.freeze
+
+      class << self
+        # Register a deprecated configuration option.
+        #
+        # @param path [String] dot-notation path, e.g. "http.old_proxy_url" or "old_queue_prefix"
+        # @param removed_in [String] version in which the option was deprecated
+        # @param replacement [String, nil] dot-notation path to the replacement option
+        # @param severity [:warning, :error] :warning logs + forwards, :error raises
+        # @param message [String, nil] additional migration guidance
+        def register(path, removed_in:, replacement: nil, severity: :warning, message: nil)
+          segments = path.split(".")
+          source_prefix = nil
+          if segments.length == 1
+            target_class = Configuration
+            option_name = segments.first
+          else
+            source_prefix = segments.first
+            option_name = segments.last
+            class_name = SETTINGS_CLASSES[source_prefix]
+            raise ArgumentError, "Unknown settings accessor: #{source_prefix}" unless class_name
+
+            target_class = Configuration.const_get(class_name)
+          end
+
+          deprecation_message = build_message(path, removed_in, replacement, message)
+
+          if target_class.method_defined?(:"#{option_name}=") || target_class.method_defined?(option_name.to_sym)
+            warn "[SourceMonitor] DeprecationRegistry: '#{path}' already exists on #{target_class.name}. " \
+                 "Skipping trap definition -- the option is not yet removed/renamed."
+            entries[path] = { path: path, removed_in: removed_in, replacement: replacement,
+                              severity: severity, message: deprecation_message, skipped: true }
+            return
+          end
+
+          define_trap_methods(target_class, option_name, deprecation_message, severity, replacement,
+                              source_prefix: source_prefix)
+
+          entries[path] = { path: path, removed_in: removed_in, replacement: replacement,
+                            severity: severity, message: deprecation_message, skipped: false }
+        end
+
+        # Remove all registered deprecation traps and clear state.
+        # Essential for test isolation.
+        def clear!
+          defined_methods.each do |target_class, method_name|
+            target_class.remove_method(method_name) if target_class.method_defined?(method_name)
+          rescue NameError
+            # Method was already removed or never defined; ignore.
+          end
+
+          @entries = {}
+          @defined_methods = []
+        end
+
+        # Returns a duplicate of the entries hash for inspection.
+        def entries
+          @entries ||= {}
+        end
+
+        # Check if a path is registered.
+        def registered?(path)
+          entries.key?(path)
+        end
+
+        # No-op hook for future "default changed" checks.
+        # Called by Configuration#check_deprecations! after the configure block.
+        def check_defaults!(_config)
+          # Reserved for future use. Phases may add checks like:
+          # "option X changed its default from A to B in version Y"
+        end
+
+        private
+
+        def defined_methods
+          @defined_methods ||= []
+        end
+
+        def build_message(path, removed_in, replacement, extra_message)
+          parts = +"[SourceMonitor] DEPRECATION: '#{path}' was deprecated in v#{removed_in}"
+          parts << " and replaced by '#{replacement}'" if replacement
+          parts << ". #{extra_message}" if extra_message
+          parts << "." unless parts.end_with?(".")
+          parts.freeze
+        end
+
+        def define_trap_methods(target_class, option_name, deprecation_message, severity, replacement, source_prefix: nil)
+          writer_name = :"#{option_name}="
+          reader_name = option_name.to_sym
+
+          case severity
+          when :warning
+            define_warning_writer(target_class, writer_name, deprecation_message, replacement, source_prefix)
+            define_warning_reader(target_class, reader_name, deprecation_message, replacement, source_prefix)
+          when :error
+            define_error_method(target_class, writer_name, deprecation_message)
+            define_error_method(target_class, reader_name, deprecation_message)
+          else
+            raise ArgumentError, "Unknown severity: #{severity}. Must be :warning or :error."
+          end
+
+          defined_methods.push([ target_class, writer_name ], [ target_class, reader_name ])
+        end
+
+        def define_warning_writer(target_class, writer_name, deprecation_message, replacement, source_prefix)
+          replacement_writer = replacement_setter_for(replacement, source_prefix)
+
+          target_class.define_method(writer_name) do |value|
+            Rails.logger.warn(deprecation_message)
+            if replacement_writer
+              resolve_replacement_target(replacement_writer[:target]).public_send(
+                replacement_writer[:setter], value
+              )
+            end
+          end
+        end
+
+        def define_warning_reader(target_class, reader_name, deprecation_message, replacement, source_prefix)
+          replacement_reader = replacement_getter_for(replacement, source_prefix)
+
+          target_class.define_method(reader_name) do
+            Rails.logger.warn(deprecation_message)
+            if replacement_reader
+              resolve_replacement_target(replacement_reader[:target]).public_send(
+                replacement_reader[:getter]
+              )
+            end
+          end
+        end
+
+        def define_error_method(target_class, method_name, deprecation_message)
+          target_class.define_method(method_name) do |*|
+            raise SourceMonitor::DeprecatedOptionError, deprecation_message
+          end
+        end
+
+        # Parse replacement path into target accessor chain and setter name.
+        # When source_prefix matches the replacement prefix, the target is nil
+        # (replacement is on the same settings class).
+        #
+        # "http.proxy" with source_prefix "http" => { target: nil, setter: "proxy=" }
+        # "queue_namespace" => { target: nil, setter: "queue_namespace=" }
+        # "http.proxy" with source_prefix nil => { target: :http, setter: "proxy=" }
+        def replacement_setter_for(replacement, source_prefix = nil)
+          return nil unless replacement
+
+          segments = replacement.split(".")
+          if segments.length == 1
+            { target: nil, setter: :"#{segments.first}=" }
+          elsif source_prefix && segments.first == source_prefix
+            { target: nil, setter: :"#{segments.last}=" }
+          else
+            { target: segments.first.to_sym, setter: :"#{segments.last}=" }
+          end
+        end
+
+        # Parse replacement path into target accessor chain and getter name.
+        def replacement_getter_for(replacement, source_prefix = nil)
+          return nil unless replacement
+
+          segments = replacement.split(".")
+          if segments.length == 1
+            { target: nil, getter: segments.first.to_sym }
+          elsif source_prefix && segments.first == source_prefix
+            { target: nil, getter: segments.last.to_sym }
+          else
+            { target: segments.first.to_sym, getter: segments.last.to_sym }
+          end
+        end
+      end
+    end
+  end
+end
+
+# Add a helper method to settings classes and Configuration for resolving
+# replacement targets. This allows "http.proxy" to resolve as self.http.proxy
+# from within a Configuration instance, or as self.proxy from within an
+# HTTPSettings instance.
+module SourceMonitor
+  class Configuration
+    private
+
+    def resolve_replacement_target(accessor)
+      accessor ? public_send(accessor) : self
+    end
+  end
+end
+
+# Add the same helper to all settings classes so forwarding works
+# when the deprecated method is defined on a nested settings class
+# and the replacement is on the same class (e.g. "http.old_proxy" -> "http.proxy").
+SourceMonitor::Configuration::DeprecationRegistry::SETTINGS_CLASSES.each_value do |class_name|
+  klass = SourceMonitor::Configuration.const_get(class_name)
+  unless klass.method_defined?(:resolve_replacement_target, false)
+    klass.define_method(:resolve_replacement_target) do |accessor|
+      accessor ? public_send(accessor) : self
+    end
+    klass.send(:private, :resolve_replacement_target)
+  end
+end

data/lib/source_monitor/configuration/http_settings.rb

@@ -13,7 +13,10 @@ module SourceMonitor
         :retry_interval,
         :retry_interval_randomness,
         :retry_backoff_factor,
-        :retry_statuses
+        :retry_statuses,
+        :ssl_ca_file,
+        :ssl_ca_path,
+        :ssl_verify
 
       def initialize
         reset!
@@ -31,6 +34,9 @@ module SourceMonitor
         @retry_interval_randomness = 0.5
         @retry_backoff_factor = 2
         @retry_statuses = nil
+        @ssl_ca_file = nil
+        @ssl_ca_path = nil
+        @ssl_verify = true
       end
 
       private

data/lib/source_monitor/configuration/images_settings.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  class Configuration
+    class ImagesSettings
+      attr_accessor :download_to_active_storage,
+        :max_download_size,
+        :download_timeout,
+        :allowed_content_types
+
+      DEFAULT_MAX_DOWNLOAD_SIZE = 10 * 1024 * 1024 # 10 MB
+      DEFAULT_DOWNLOAD_TIMEOUT = 30 # seconds
+      DEFAULT_ALLOWED_CONTENT_TYPES = %w[
+        image/jpeg
+        image/png
+        image/gif
+        image/webp
+        image/svg+xml
+      ].freeze
+
+      def initialize
+        reset!
+      end
+
+      def reset!
+        @download_to_active_storage = false
+        @max_download_size = DEFAULT_MAX_DOWNLOAD_SIZE
+        @download_timeout = DEFAULT_DOWNLOAD_TIMEOUT
+        @allowed_content_types = DEFAULT_ALLOWED_CONTENT_TYPES.dup
+      end
+
+      def download_enabled?
+        !!download_to_active_storage
+      end
+    end
+  end
+end

data/lib/source_monitor/configuration.rb

@@ -8,11 +8,13 @@ require "source_monitor/configuration/scraping_settings"
 require "source_monitor/configuration/realtime_settings"
 require "source_monitor/configuration/retention_settings"
 require "source_monitor/configuration/authentication_settings"
+require "source_monitor/configuration/images_settings"
 require "source_monitor/configuration/scraper_registry"
 require "source_monitor/configuration/events"
 require "source_monitor/configuration/validation_definition"
 require "source_monitor/configuration/model_definition"
 require "source_monitor/configuration/models"
+require "source_monitor/configuration/deprecation_registry"
 
 module SourceMonitor
   class Configuration
@@ -26,7 +28,7 @@ module SourceMonitor
       :mission_control_enabled,
       :mission_control_dashboard_path
 
-    attr_reader :http, :scrapers, :retention, :events, :models, :realtime, :fetching, :health, :authentication, :scraping
+    attr_reader :http, :scrapers, :retention, :events, :models, :realtime, :fetching, :health, :authentication, :scraping, :images
 
     DEFAULT_QUEUE_NAMESPACE = "source_monitor"
 
@@ -50,6 +52,7 @@ module SourceMonitor
       @health = HealthSettings.new
       @authentication = AuthenticationSettings.new
       @scraping = ScrapingSettings.new
+      @images = ImagesSettings.new
     end
 
     def queue_name_for(role)
@@ -83,5 +86,12 @@ module SourceMonitor
         raise ArgumentError, "unknown queue role #{role.inspect}"
       end
     end
+
+    # Post-configure hook for deprecation validation.
+    # Delegates to DeprecationRegistry.check_defaults! for future
+    # "default changed" checks. Currently a no-op.
+    def check_deprecations!
+      DeprecationRegistry.check_defaults!(self)
+    end
   end
 end

data/lib/source_monitor/dashboard/queries/recent_activity_query.rb

@@ -37,7 +37,8 @@ module SourceMonitor
             item_title: row["item_title"],
             item_url: row["item_url"],
             source_name: row["source_name"],
-            source_id: row["source_id"]
+            source_id: row["source_id"],
+            source_feed_url: row["source_feed_url"]
           )
         end
 
@@ -57,7 +58,8 @@ module SourceMonitor
                    item_title,
                    item_url,
                    source_name,
-                   source_id
+                   source_id,
+                   source_feed_url
             FROM (
               #{fetch_log_sql}
               UNION ALL
@@ -83,9 +85,12 @@ module SourceMonitor
               NULL AS scraper_adapter,
               NULL AS item_title,
               NULL AS item_url,
-              NULL AS source_name,
-              #{SourceMonitor::FetchLog.quoted_table_name}.source_id AS source_id
+              #{SourceMonitor::Source.quoted_table_name}.#{quoted_source_name} AS source_name,
+              #{SourceMonitor::FetchLog.quoted_table_name}.source_id AS source_id,
+              #{SourceMonitor::Source.quoted_table_name}.feed_url AS source_feed_url
             FROM #{SourceMonitor::FetchLog.quoted_table_name}
+            LEFT JOIN #{SourceMonitor::Source.quoted_table_name}
+              ON #{SourceMonitor::Source.quoted_table_name}.id = #{SourceMonitor::FetchLog.quoted_table_name}.source_id
           SQL
         end
 
@@ -100,12 +105,15 @@ module SourceMonitor
               NULL AS items_updated,
               #{SourceMonitor::ScrapeLog.quoted_table_name}.scraper_adapter AS scraper_adapter,
               NULL AS item_title,
-              NULL AS item_url,
+              #{SourceMonitor::Item.quoted_table_name}.url AS item_url,
               #{SourceMonitor::Source.quoted_table_name}.#{quoted_source_name} AS source_name,
-              #{SourceMonitor::ScrapeLog.quoted_table_name}.source_id AS source_id
+              #{SourceMonitor::ScrapeLog.quoted_table_name}.source_id AS source_id,
+              NULL AS source_feed_url
             FROM #{SourceMonitor::ScrapeLog.quoted_table_name}
             LEFT JOIN #{SourceMonitor::Source.quoted_table_name}
               ON #{SourceMonitor::Source.quoted_table_name}.id = #{SourceMonitor::ScrapeLog.quoted_table_name}.source_id
+            LEFT JOIN #{SourceMonitor::Item.quoted_table_name}
+              ON #{SourceMonitor::Item.quoted_table_name}.id = #{SourceMonitor::ScrapeLog.quoted_table_name}.item_id
           SQL
         end
 
@@ -122,7 +130,8 @@ module SourceMonitor
               #{SourceMonitor::Item.quoted_table_name}.title AS item_title,
               #{SourceMonitor::Item.quoted_table_name}.url AS item_url,
               #{SourceMonitor::Source.quoted_table_name}.#{quoted_source_name} AS source_name,
-              #{SourceMonitor::Item.quoted_table_name}.source_id AS source_id
+              #{SourceMonitor::Item.quoted_table_name}.source_id AS source_id,
+              NULL AS source_feed_url
             FROM #{SourceMonitor::Item.quoted_table_name}
             LEFT JOIN #{SourceMonitor::Source.quoted_table_name}
               ON #{SourceMonitor::Source.quoted_table_name}.id = #{SourceMonitor::Item.quoted_table_name}.source_id

data/lib/source_monitor/dashboard/recent_activity.rb

@@ -15,6 +15,7 @@ module SourceMonitor
         :item_url,
         :source_name,
         :source_id,
+        :source_feed_url,
         keyword_init: true
       ) do
         def type

data/lib/source_monitor/dashboard/recent_activity_presenter.rb

@@ -30,13 +30,16 @@ module SourceMonitor
       end
 
       def fetch_event(event)
+        domain = source_domain(event.source_feed_url)
         {
           label: "Fetch ##{event.id}",
           description: "#{event.items_created.to_i} created / #{event.items_updated.to_i} updated",
           status: event.success? ? :success : :failure,
           type: :fetch,
           time: event.occurred_at,
-          path: url_helpers.fetch_log_path(event.id)
+          path: url_helpers.fetch_log_path(event.id),
+          url_display: domain,
+          url_href: event.source_feed_url
         }
       end
 
@@ -47,10 +50,20 @@ module SourceMonitor
           status: event.success? ? :success : :failure,
           type: :scrape,
           time: event.occurred_at,
-          path: url_helpers.scrape_log_path(event.id)
+          path: url_helpers.scrape_log_path(event.id),
+          url_display: event.item_url,
+          url_href: event.item_url
         }
       end
 
+      def source_domain(feed_url)
+        return nil if feed_url.blank?
+
+        URI.parse(feed_url.to_s).host
+      rescue URI::InvalidURIError
+        nil
+      end
+
       def item_event(event)
         {
           label: event.item_title.presence || "New Item",

data/lib/source_monitor/fetching/feed_fetcher/entry_processor.rb

@@ -38,6 +38,7 @@ module SourceMonitor
                 created += 1
                 created_items << result.item
                 SourceMonitor::Events.after_item_created(item: result.item, source:, entry:, result: result)
+                enqueue_image_download(result.item)
               else
                 updated += 1
                 updated_items << result.item
@@ -61,6 +62,18 @@ module SourceMonitor
 
         private
 
+        def enqueue_image_download(item)
+          return unless SourceMonitor.config.images.download_enabled?
+          return if item.content.blank?
+
+          SourceMonitor::DownloadContentImagesJob.perform_later(item.id)
+        rescue StandardError => error
+          # Image download enqueue failure must never break feed processing
+          if defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger
+            Rails.logger.error("[SourceMonitor] Failed to enqueue image download for item #{item.id}: #{error.message}")
+          end
+        end
+
         def normalize_item_error(entry, error)
           {
             guid: safe_entry_guid(entry),

data/lib/source_monitor/http.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require "openssl"
 require "faraday"
 require "faraday/retry"
 require "faraday/follow_redirects"
@@ -57,9 +58,31 @@ module SourceMonitor
           connection.headers[key] = value
         end
 
+        configure_ssl(connection, settings)
+
         connection.adapter Faraday.default_adapter
       end
 
+      # Configure SSL to use a proper cert store. Without this, some systems
+      # fail to verify certificate chains that depend on intermediate CAs
+      # (e.g., Medium/Netflix on AWS). OpenSSL::X509::Store#set_default_paths
+      # loads all system-trusted CAs including intermediates.
+      def configure_ssl(connection, settings)
+        connection.ssl.verify = settings.ssl_verify != false
+
+        if settings.ssl_ca_file
+          connection.ssl.ca_file = settings.ssl_ca_file
+        elsif settings.ssl_ca_path
+          connection.ssl.ca_path = settings.ssl_ca_path
+        else
+          connection.ssl.cert_store = default_cert_store
+        end
+      end
+
+      def default_cert_store
+        OpenSSL::X509::Store.new.tap(&:set_default_paths)
+      end
+
       def default_headers(settings)
         base_headers = {
           "User-Agent" => resolve_callable(settings.user_agent).presence || DEFAULT_USER_AGENT,