source_monitor 0.7.1 → 0.8.0

data/lib/source_monitor/configuration/favicons_settings.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  class Configuration
+    class FaviconsSettings
+      attr_accessor :enabled,
+        :fetch_timeout,
+        :max_download_size,
+        :retry_cooldown_days,
+        :allowed_content_types
+
+      DEFAULT_FETCH_TIMEOUT = 5 # seconds
+      DEFAULT_MAX_DOWNLOAD_SIZE = 1 * 1024 * 1024 # 1 MB
+      DEFAULT_RETRY_COOLDOWN_DAYS = 7
+      DEFAULT_ALLOWED_CONTENT_TYPES = %w[
+        image/x-icon
+        image/vnd.microsoft.icon
+        image/png
+        image/jpeg
+        image/gif
+        image/svg+xml
+        image/webp
+      ].freeze
+
+      def initialize
+        reset!
+      end
+
+      def reset!
+        @enabled = true
+        @fetch_timeout = DEFAULT_FETCH_TIMEOUT
+        @max_download_size = DEFAULT_MAX_DOWNLOAD_SIZE
+        @retry_cooldown_days = DEFAULT_RETRY_COOLDOWN_DAYS
+        @allowed_content_types = DEFAULT_ALLOWED_CONTENT_TYPES.dup
+      end
+
+      def enabled?
+        !!enabled && !!defined?(ActiveStorage)
+      end
+    end
+  end
+end

data/lib/source_monitor/configuration/http_settings.rb

@@ -42,7 +42,7 @@ module SourceMonitor
       private
 
       def default_user_agent
-        "SourceMonitor/#{SourceMonitor::VERSION}"
+        "Mozilla/5.0 (compatible; SourceMonitor/#{SourceMonitor::VERSION})"
       end
     end
   end

data/lib/source_monitor/configuration/scraping_settings.rb

@@ -5,7 +5,7 @@ module SourceMonitor
     class ScrapingSettings
       attr_accessor :max_in_flight_per_source, :max_bulk_batch_size
 
-      DEFAULT_MAX_IN_FLIGHT = 25
+      DEFAULT_MAX_IN_FLIGHT = nil
       DEFAULT_MAX_BULK_BATCH_SIZE = 100
 
       def initialize

data/lib/source_monitor/configuration.rb

@@ -9,6 +9,7 @@ require "source_monitor/configuration/realtime_settings"
 require "source_monitor/configuration/retention_settings"
 require "source_monitor/configuration/authentication_settings"
 require "source_monitor/configuration/images_settings"
+require "source_monitor/configuration/favicons_settings"
 require "source_monitor/configuration/scraper_registry"
 require "source_monitor/configuration/events"
 require "source_monitor/configuration/validation_definition"
@@ -28,7 +29,7 @@ module SourceMonitor
       :mission_control_enabled,
       :mission_control_dashboard_path
 
-    attr_reader :http, :scrapers, :retention, :events, :models, :realtime, :fetching, :health, :authentication, :scraping, :images
+    attr_reader :http, :scrapers, :retention, :events, :models, :realtime, :fetching, :health, :authentication, :scraping, :images, :favicons
 
     DEFAULT_QUEUE_NAMESPACE = "source_monitor"
 
@@ -53,6 +54,7 @@ module SourceMonitor
       @authentication = AuthenticationSettings.new
       @scraping = ScrapingSettings.new
       @images = ImagesSettings.new
+      @favicons = FaviconsSettings.new
     end
 
     def queue_name_for(role)

data/lib/source_monitor/favicons/discoverer.rb

@@ -0,0 +1,196 @@
+# frozen_string_literal: true
+
+require "faraday"
+require "faraday/follow_redirects"
+require "securerandom"
+require "nokogiri"
+
+module SourceMonitor
+  module Favicons
+    class Discoverer
+      Result = Struct.new(:io, :filename, :content_type, :url, keyword_init: true)
+
+      attr_reader :website_url, :settings
+
+      def initialize(website_url, settings: nil)
+        @website_url = website_url
+        @settings = settings || SourceMonitor.config.favicons
+      end
+
+      def call
+        return if website_url.blank?
+
+        try_html_link_tags || try_google_favicon_api || try_favicon_ico
+      rescue Faraday::Error, URI::InvalidURIError, Timeout::Error
+        nil
+      end
+
+      private
+
+      def try_favicon_ico
+        uri = URI.parse(website_url)
+        favicon_url = "#{uri.scheme}://#{uri.host}/favicon.ico"
+        download_favicon(favicon_url)
+      rescue URI::InvalidURIError
+        nil
+      end
+
+      def try_html_link_tags
+        response = html_client.get(website_url)
+        return unless response.status == 200
+
+        doc = Nokogiri::HTML(response.body)
+        candidates = extract_icon_candidates(doc)
+        return if candidates.empty?
+
+        candidates.each do |candidate_url|
+          result = download_favicon(candidate_url)
+          return result if result
+        end
+        nil
+      rescue Faraday::Error, Nokogiri::SyntaxError
+        nil
+      end
+
+      def try_google_favicon_api
+        uri = URI.parse(website_url)
+        api_url = "https://www.google.com/s2/favicons?domain=#{uri.host}&sz=64"
+        download_favicon(api_url)
+      rescue URI::InvalidURIError
+        nil
+      end
+
+      def extract_icon_candidates(doc)
+        candidates = []
+
+        # Search link[rel] tags for icon types
+        icon_selectors = [
+          'link[rel*="icon"]',
+          'link[rel="apple-touch-icon"]',
+          'link[rel="apple-touch-icon-precomposed"]',
+          'link[rel="mask-icon"]'
+        ]
+
+        icon_selectors.each do |selector|
+          doc.css(selector).each do |link|
+            href = link["href"]
+            next if href.blank?
+
+            absolute_url = resolve_url(href)
+            next unless absolute_url
+
+            sizes = parse_sizes(link["sizes"])
+            candidates << { url: absolute_url, size: sizes }
+          end
+        end
+
+        # Search meta tags for msapplication-TileImage
+        doc.css('meta[name="msapplication-TileImage"]').each do |meta|
+          content = meta["content"]
+          next if content.blank?
+
+          absolute_url = resolve_url(content)
+          candidates << { url: absolute_url, size: 0 } if absolute_url
+        end
+
+        # og:image as last resort
+        doc.css('meta[property="og:image"]').each do |meta|
+          content = meta["content"]
+          next if content.blank?
+
+          absolute_url = resolve_url(content)
+          candidates << { url: absolute_url, size: -1 } if absolute_url
+        end
+
+        # Sort by size descending (prefer larger), deduplicate by URL
+        candidates
+          .sort_by { |c| -(c[:size] || 0) }
+          .uniq { |c| c[:url] }
+          .map { |c| c[:url] }
+      end
+
+      def parse_sizes(sizes_attr)
+        return 0 if sizes_attr.blank?
+        return 0 if sizes_attr.casecmp("any").zero?
+
+        # Parse "32x32", "256x256", etc. -- take the max dimension
+        match = sizes_attr.match(/(\d+)x(\d+)/i)
+        return 0 unless match
+
+        [ match[1].to_i, match[2].to_i ].max
+      end
+
+      def resolve_url(href)
+        return nil if href.blank?
+
+        uri = URI.parse(href)
+        if uri.absolute?
+          href
+        else
+          URI.join(website_url, href).to_s
+        end
+      rescue URI::InvalidURIError, URI::BadURIError
+        nil
+      end
+
+      def download_favicon(url)
+        response = image_client.get(url)
+        return unless response.status == 200
+
+        content_type = response.headers["content-type"]&.split(";")&.first&.strip&.downcase
+        return unless content_type && settings.allowed_content_types.include?(content_type)
+
+        body = response.body
+        return unless body && body.bytesize > 0
+        return if body.bytesize > settings.max_download_size
+
+        filename = derive_filename(url, content_type)
+
+        Result.new(
+          io: StringIO.new(body),
+          filename: filename,
+          content_type: content_type,
+          url: url
+        )
+      rescue Faraday::Error
+        nil
+      end
+
+      def derive_filename(favicon_url, content_type)
+        uri = URI.parse(favicon_url)
+        basename = File.basename(uri.path) if uri.path.present?
+
+        if basename.present? && basename.include?(".")
+          basename
+        else
+          ext = Rack::Mime::MIME_TYPES.invert[content_type] || ".ico"
+          "favicon-#{SecureRandom.hex(8)}#{ext}"
+        end
+      rescue URI::InvalidURIError
+        ext = Rack::Mime::MIME_TYPES.invert[content_type] || ".ico"
+        "favicon-#{SecureRandom.hex(8)}#{ext}"
+      end
+
+      def html_client
+        build_client("text/html, application/xhtml+xml")
+      end
+
+      def image_client
+        build_client("image/*")
+      end
+
+      def build_client(accept_header)
+        timeout = settings.fetch_timeout
+
+        Faraday.new do |f|
+          f.response :follow_redirects, limit: 3
+          f.options.timeout = timeout
+          f.options.open_timeout = [ timeout / 2, 3 ].min
+          f.headers["User-Agent"] = SourceMonitor.config.http.user_agent || "SourceMonitor/#{SourceMonitor::VERSION}"
+          f.headers["Accept"] = accept_header
+          f.adapter Faraday.default_adapter
+        end
+      end
+    end
+  end
+end

data/lib/source_monitor/fetching/feed_fetcher/source_updater.rb

@@ -37,6 +37,7 @@ module SourceMonitor
           attributes[:metadata] = updated_metadata(feed_signature: feed_signature, entries_digest: entries_digest)
           reset_retry_state!(attributes)
           source.update!(attributes)
+          enqueue_favicon_fetch_if_needed
         end
 
         def update_source_for_not_modified(response, duration_ms)
@@ -62,6 +63,7 @@ module SourceMonitor
           attributes[:metadata] = updated_metadata
           reset_retry_state!(attributes)
           source.update!(attributes)
+          enqueue_favicon_fetch_if_needed
         end
 
         def update_source_for_failure(error, duration_ms)
@@ -137,6 +139,25 @@ module SourceMonitor
           attributes[:fetch_circuit_until] = nil
         end
 
+        def enqueue_favicon_fetch_if_needed
+          return unless defined?(ActiveStorage)
+          return unless SourceMonitor.config.favicons.enabled?
+          return if source.website_url.blank?
+          return if source.respond_to?(:favicon) && source.favicon.attached?
+
+          last_attempt = source.metadata&.dig("favicon_last_attempted_at")
+          if last_attempt.present?
+            cooldown_days = SourceMonitor.config.favicons.retry_cooldown_days
+            return if Time.parse(last_attempt) > cooldown_days.days.ago
+          end
+
+          SourceMonitor::FaviconFetchJob.perform_later(source.id)
+        rescue StandardError => error
+          Rails.logger.warn(
+            "[SourceMonitor::SourceUpdater] Failed to enqueue favicon fetch for source #{source.id}: #{error.message}"
+          ) if defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger
+        end
+
         def apply_retry_strategy!(attributes, error, now)
           decision = SourceMonitor::Fetching::RetryPolicy.new(source:, error:, now:).decision
 

data/lib/source_monitor/fetching/feed_fetcher.rb

@@ -103,6 +103,7 @@ module SourceMonitor
 
       def request_headers
         headers = (source.custom_headers || {}).transform_keys { |key| key.to_s }
+        headers["Referer"] = source.website_url if source.website_url.present?
         headers["If-None-Match"] = source.etag if source.etag.present?
         if source.last_modified.present?
           headers["If-Modified-Since"] = source.last_modified.httpdate

data/lib/source_monitor/http.rb

@@ -14,7 +14,7 @@ module SourceMonitor
     DEFAULT_TIMEOUT = 15
     DEFAULT_OPEN_TIMEOUT = 5
     DEFAULT_MAX_REDIRECTS = 5
-    DEFAULT_USER_AGENT = "SourceMonitor/#{SourceMonitor::VERSION}"
+    DEFAULT_USER_AGENT = "Mozilla/5.0 (compatible; SourceMonitor/#{SourceMonitor::VERSION})"
     RETRY_STATUSES = [ 429, 500, 502, 503, 504 ].freeze
 
     class << self
@@ -89,8 +89,10 @@ module SourceMonitor
       def default_headers(settings)
         base_headers = {
           "User-Agent" => resolve_callable(settings.user_agent).presence || DEFAULT_USER_AGENT,
-          "Accept" => "application/rss+xml, application/atom+xml, application/json;q=0.9, text/xml;q=0.8",
-          "Accept-Encoding" => "gzip,deflate"
+          "Accept" => "text/html, application/rss+xml, application/atom+xml, application/json;q=0.9, text/xml;q=0.8",
+          "Accept-Encoding" => "gzip,deflate",
+          "Accept-Language" => "en-US,en;q=0.9",
+          "DNT" => "1"
         }
 
         base_headers.merge(settings.headers || {})

data/lib/source_monitor/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module SourceMonitor
-  VERSION = "0.7.1"
+  VERSION = "0.8.0"
 end

data/lib/source_monitor.rb

@@ -87,6 +87,10 @@ module SourceMonitor
     autoload :HealthCheckBroadcaster, "source_monitor/import_sessions/health_check_broadcaster"
   end
 
+  module Favicons
+    autoload :Discoverer, "source_monitor/favicons/discoverer"
+  end
+
   module Images
     autoload :ContentRewriter, "source_monitor/images/content_rewriter"
     autoload :Downloader, "source_monitor/images/downloader"

data/source_monitor.gemspec

@@ -23,7 +23,7 @@ Gem::Specification.new do |spec|
   spec.files = Dir.chdir(File.expand_path(__dir__)) do
     tracked_files = `git ls-files -z`.split("\x0")
     tracked_files.reject do |file|
-      file.start_with?(".ai/", ".github/", "coverage/", "node_modules/", "pkg/", "spec/", "test/", "tmp/", "vendor/", "examples/", "bin/")
+      file.start_with?(".ai/", ".github/", ".vbw-planning/", "coverage/", "node_modules/", "pkg/", "spec/", "test/", "tmp/", "vendor/", "examples/", "bin/")
     end
   end
   spec.files += [ "CHANGELOG.md" ].select { |path| File.exist?(File.join(__dir__, path)) }

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: source_monitor
 version: !ruby/object:Gem::Version
-  version: 0.7.1
+  version: 0.8.0
 platform: ruby
 authors:
 - dchuk
@@ -342,111 +342,6 @@ files:
 - ".gitignore"
 - ".rubocop.yml"
 - ".ruby-version"
-- ".vbw-planning/PROJECT.md"
-- ".vbw-planning/ROADMAP.md"
-- ".vbw-planning/SHIPPED.md"
-- ".vbw-planning/STATE.md"
-- ".vbw-planning/codebase/ARCHITECTURE.md"
-- ".vbw-planning/codebase/CONCERNS.md"
-- ".vbw-planning/codebase/CONVENTIONS.md"
-- ".vbw-planning/codebase/DEPENDENCIES.md"
-- ".vbw-planning/codebase/INDEX.md"
-- ".vbw-planning/codebase/META.md"
-- ".vbw-planning/codebase/PATTERNS.md"
-- ".vbw-planning/codebase/STACK.md"
-- ".vbw-planning/codebase/STRUCTURE.md"
-- ".vbw-planning/codebase/TESTING.md"
-- ".vbw-planning/config.json"
-- ".vbw-planning/discovery.json"
-- ".vbw-planning/milestones/default/ROADMAP.md"
-- ".vbw-planning/milestones/default/STATE.md"
-- ".vbw-planning/milestones/default/phases/01-coverage-analysis-quick-wins/PLAN-01-SUMMARY.md"
-- ".vbw-planning/milestones/default/phases/01-coverage-analysis-quick-wins/PLAN-01.md"
-- ".vbw-planning/milestones/default/phases/01-coverage-analysis-quick-wins/PLAN-02-SUMMARY.md"
-- ".vbw-planning/milestones/default/phases/01-coverage-analysis-quick-wins/PLAN-02.md"
-- ".vbw-planning/milestones/default/phases/02-critical-path-test-coverage/PLAN-01-SUMMARY.md"
-- ".vbw-planning/milestones/default/phases/02-critical-path-test-coverage/PLAN-01.md"
-- ".vbw-planning/milestones/default/phases/02-critical-path-test-coverage/PLAN-02-SUMMARY.md"
-- ".vbw-planning/milestones/default/phases/02-critical-path-test-coverage/PLAN-02.md"
-- ".vbw-planning/milestones/default/phases/02-critical-path-test-coverage/PLAN-03-SUMMARY.md"
-- ".vbw-planning/milestones/default/phases/02-critical-path-test-coverage/PLAN-03.md"
-- ".vbw-planning/milestones/default/phases/02-critical-path-test-coverage/PLAN-04-SUMMARY.md"
-- ".vbw-planning/milestones/default/phases/02-critical-path-test-coverage/PLAN-04.md"
-- ".vbw-planning/milestones/default/phases/02-critical-path-test-coverage/PLAN-05-SUMMARY.md"
-- ".vbw-planning/milestones/default/phases/02-critical-path-test-coverage/PLAN-05.md"
-- ".vbw-planning/milestones/default/phases/03-large-file-refactoring/03-VERIFICATION-wave1.md"
-- ".vbw-planning/milestones/default/phases/03-large-file-refactoring/03-VERIFICATION.md"
-- ".vbw-planning/milestones/default/phases/03-large-file-refactoring/PLAN-01-SUMMARY.md"
-- ".vbw-planning/milestones/default/phases/03-large-file-refactoring/PLAN-01.md"
-- ".vbw-planning/milestones/default/phases/03-large-file-refactoring/PLAN-02-SUMMARY.md"
-- ".vbw-planning/milestones/default/phases/03-large-file-refactoring/PLAN-02.md"
-- ".vbw-planning/milestones/default/phases/03-large-file-refactoring/PLAN-03-SUMMARY.md"
-- ".vbw-planning/milestones/default/phases/03-large-file-refactoring/PLAN-03.md"
-- ".vbw-planning/milestones/default/phases/03-large-file-refactoring/PLAN-04-SUMMARY.md"
-- ".vbw-planning/milestones/default/phases/03-large-file-refactoring/PLAN-04.md"
-- ".vbw-planning/milestones/default/phases/04-code-quality-conventions-cleanup/04-CONTEXT.md"
-- ".vbw-planning/milestones/default/phases/04-code-quality-conventions-cleanup/PLAN-01-SUMMARY.md"
-- ".vbw-planning/milestones/default/phases/04-code-quality-conventions-cleanup/PLAN-01.md"
-- ".vbw-planning/milestones/default/phases/04-code-quality-conventions-cleanup/PLAN-02-SUMMARY.md"
-- ".vbw-planning/milestones/default/phases/04-code-quality-conventions-cleanup/PLAN-02.md"
-- ".vbw-planning/milestones/default/phases/04-code-quality-conventions-cleanup/PLAN-03-SUMMARY.md"
-- ".vbw-planning/milestones/default/phases/04-code-quality-conventions-cleanup/PLAN-03.md"
-- ".vbw-planning/milestones/generator-enhancements/REQUIREMENTS.md"
-- ".vbw-planning/milestones/generator-enhancements/ROADMAP.md"
-- ".vbw-planning/milestones/generator-enhancements/SHIPPED.md"
-- ".vbw-planning/milestones/generator-enhancements/STATE.md"
-- ".vbw-planning/milestones/generator-enhancements/phases/01-generator-steps/01-CONTEXT.md"
-- ".vbw-planning/milestones/generator-enhancements/phases/01-generator-steps/01-VERIFICATION.md"
-- ".vbw-planning/milestones/generator-enhancements/phases/01-generator-steps/PLAN-01-SUMMARY.md"
-- ".vbw-planning/milestones/generator-enhancements/phases/01-generator-steps/PLAN-01.md"
-- ".vbw-planning/milestones/generator-enhancements/phases/02-verification/02-VERIFICATION.md"
-- ".vbw-planning/milestones/generator-enhancements/phases/02-verification/PLAN-01-SUMMARY.md"
-- ".vbw-planning/milestones/generator-enhancements/phases/02-verification/PLAN-01.md"
-- ".vbw-planning/milestones/generator-enhancements/phases/03-docs-alignment/03-VERIFICATION.md"
-- ".vbw-planning/milestones/generator-enhancements/phases/03-docs-alignment/PLAN-01-SUMMARY.md"
-- ".vbw-planning/milestones/generator-enhancements/phases/03-docs-alignment/PLAN-01.md"
-- ".vbw-planning/milestones/generator-enhancements/phases/04-dashboard-ux/04-VERIFICATION.md"
-- ".vbw-planning/milestones/generator-enhancements/phases/04-dashboard-ux/PLAN-01-SUMMARY.md"
-- ".vbw-planning/milestones/generator-enhancements/phases/04-dashboard-ux/PLAN-01.md"
-- ".vbw-planning/milestones/generator-enhancements/phases/05-active-storage-images/05-VERIFICATION.md"
-- ".vbw-planning/milestones/generator-enhancements/phases/05-active-storage-images/PLAN-01-SUMMARY.md"
-- ".vbw-planning/milestones/generator-enhancements/phases/05-active-storage-images/PLAN-01.md"
-- ".vbw-planning/milestones/generator-enhancements/phases/05-active-storage-images/PLAN-02-SUMMARY.md"
-- ".vbw-planning/milestones/generator-enhancements/phases/05-active-storage-images/PLAN-02.md"
-- ".vbw-planning/milestones/generator-enhancements/phases/06-netflix-feed-fix/06-VERIFICATION.md"
-- ".vbw-planning/milestones/generator-enhancements/phases/06-netflix-feed-fix/PLAN-01-SUMMARY.md"
-- ".vbw-planning/milestones/generator-enhancements/phases/06-netflix-feed-fix/PLAN-01.md"
-- ".vbw-planning/milestones/upgrade-assurance/REQUIREMENTS.md"
-- ".vbw-planning/milestones/upgrade-assurance/ROADMAP.md"
-- ".vbw-planning/milestones/upgrade-assurance/STATE.md"
-- ".vbw-planning/milestones/upgrade-assurance/phases/01-upgrade-command/01-VERIFICATION.md"
-- ".vbw-planning/milestones/upgrade-assurance/phases/01-upgrade-command/PLAN-01-SUMMARY.md"
-- ".vbw-planning/milestones/upgrade-assurance/phases/01-upgrade-command/PLAN-01.md"
-- ".vbw-planning/milestones/upgrade-assurance/phases/02-config-deprecation/PLAN-01-SUMMARY.md"
-- ".vbw-planning/milestones/upgrade-assurance/phases/02-config-deprecation/PLAN-01.md"
-- ".vbw-planning/milestones/upgrade-assurance/phases/03-upgrade-skill-docs/03-VERIFICATION.md"
-- ".vbw-planning/milestones/upgrade-assurance/phases/03-upgrade-skill-docs/PLAN-01-SUMMARY.md"
-- ".vbw-planning/milestones/upgrade-assurance/phases/03-upgrade-skill-docs/PLAN-01.md"
-- ".vbw-planning/phases/01-aia-certificate-resolution/.context-dev.md"
-- ".vbw-planning/phases/01-aia-certificate-resolution/PLAN-01-SUMMARY.md"
-- ".vbw-planning/phases/01-aia-certificate-resolution/PLAN-01.md"
-- ".vbw-planning/phases/01-aia-certificate-resolution/PLAN-02-SUMMARY.md"
-- ".vbw-planning/phases/01-aia-certificate-resolution/PLAN-02.md"
-- ".vbw-planning/phases/01-aia-certificate-resolution/PLAN-03-SUMMARY.md"
-- ".vbw-planning/phases/01-aia-certificate-resolution/PLAN-03.md"
-- ".vbw-planning/phases/02-test-performance/.context-dev.md"
-- ".vbw-planning/phases/02-test-performance/.context-lead.md"
-- ".vbw-planning/phases/02-test-performance/.context-qa.md"
-- ".vbw-planning/phases/02-test-performance/02-RESEARCH.md"
-- ".vbw-planning/phases/02-test-performance/02-VERIFICATION.md"
-- ".vbw-planning/phases/02-test-performance/PLAN-01-SUMMARY.md"
-- ".vbw-planning/phases/02-test-performance/PLAN-01.md"
-- ".vbw-planning/phases/02-test-performance/PLAN-02-SUMMARY.md"
-- ".vbw-planning/phases/02-test-performance/PLAN-02.md"
-- ".vbw-planning/phases/02-test-performance/PLAN-03-SUMMARY.md"
-- ".vbw-planning/phases/02-test-performance/PLAN-03.md"
-- ".vbw-planning/phases/02-test-performance/PLAN-04-SUMMARY.md"
-- ".vbw-planning/phases/02-test-performance/PLAN-04.md"
 - AGENTS.md
 - CHANGELOG.md
 - CLAUDE.md
@@ -468,6 +363,7 @@ files:
 - app/assets/javascripts/source_monitor/controllers/confirm_navigation_controller.js
 - app/assets/javascripts/source_monitor/controllers/dropdown_controller.js
 - app/assets/javascripts/source_monitor/controllers/modal_controller.js
+- app/assets/javascripts/source_monitor/controllers/notification_container_controller.js
 - app/assets/javascripts/source_monitor/controllers/notification_controller.js
 - app/assets/javascripts/source_monitor/controllers/select_all_controller.js
 - app/assets/javascripts/source_monitor/turbo_actions.js
@@ -488,6 +384,7 @@ files:
 - app/controllers/source_monitor/logs_controller.rb
 - app/controllers/source_monitor/scrape_logs_controller.rb
 - app/controllers/source_monitor/source_bulk_scrapes_controller.rb
+- app/controllers/source_monitor/source_favicon_fetches_controller.rb
 - app/controllers/source_monitor/source_fetches_controller.rb
 - app/controllers/source_monitor/source_health_checks_controller.rb
 - app/controllers/source_monitor/source_health_resets_controller.rb
@@ -499,6 +396,7 @@ files:
 - app/helpers/source_monitor/table_sort_helper.rb
 - app/jobs/source_monitor/application_job.rb
 - app/jobs/source_monitor/download_content_images_job.rb
+- app/jobs/source_monitor/favicon_fetch_job.rb
 - app/jobs/source_monitor/fetch_feed_job.rb
 - app/jobs/source_monitor/import_opml_job.rb
 - app/jobs/source_monitor/import_session_health_check_job.rb
@@ -612,6 +510,7 @@ files:
 - lib/source_monitor/configuration/authentication_settings.rb
 - lib/source_monitor/configuration/deprecation_registry.rb
 - lib/source_monitor/configuration/events.rb
+- lib/source_monitor/configuration/favicons_settings.rb
 - lib/source_monitor/configuration/fetching_settings.rb
 - lib/source_monitor/configuration/health_settings.rb
 - lib/source_monitor/configuration/http_settings.rb
@@ -634,6 +533,7 @@ files:
 - lib/source_monitor/dashboard/upcoming_fetch_schedule.rb
 - lib/source_monitor/engine.rb
 - lib/source_monitor/events.rb
+- lib/source_monitor/favicons/discoverer.rb
 - lib/source_monitor/feedjira_extensions.rb
 - lib/source_monitor/fetching/advisory_lock.rb
 - lib/source_monitor/fetching/completion/event_publisher.rb

data/.vbw-planning/PROJECT.md

@@ -1,51 +0,0 @@
-<!-- VBW PROJECT TEMPLATE (ARTF-04) -- Human-facing project definition -->
-<!-- Created by /vbw init, maintained by Architect agent -->
-
-# SourceMonitor
-
-## What This Is
-
-SourceMonitor is a mountable Rails 8 engine for ingesting RSS/Atom/JSON feeds, scraping article content via pluggable adapters, and providing Solid Queue-powered dashboards for monitoring and remediation. It is distributed as a RubyGem and integrates with host Rails applications.
-
-## Core Value
-
-A drop-in Rails engine that gives any Rails application feed monitoring, content scraping, and operational dashboards without building the plumbing from scratch.
-
-## Requirements
-
-### Validated
-
-None yet.
-
-### Active
-
-- [ ] Close test coverage gaps identified in the coverage baseline
-- [ ] Refactor large files for maintainability and single-responsibility
-- [ ] Ensure codebase follows Rails best practices and conventions throughout
-
-### Out of Scope
-
-- Multi-database support (MySQL/SQLite) -- Keep PostgreSQL-only for now
-- Built-in authentication -- Continue relying on host app for auth
-
-## Context
-
-This is a brownfield Rails engine at v0.2.1 with 530 source files (325 Ruby, 48 ERB). The codebase has 130 test files, CI/CD via GitHub Actions, and a coverage baseline tracking 2329 lines of uncovered code. Key technical debt includes large files (FeedFetcher 627 lines, Configuration 655 lines, ImportSessionsController 792 lines) and coverage gaps in critical paths.
-
-## Constraints
-
-- **Ruby**: >= 3.4.0
-- **Rails**: >= 8.0.3, < 9.0
-- **Database**: PostgreSQL only
-- **Testing**: Minitest (not RSpec), branch coverage via SimpleCov
-
-## Key Decisions
-
-| Decision | Rationale | Outcome |
-|----------|-----------|---------|
-| Focus on coverage + refactoring before new features | Stabilize existing code before adding complexity | Pending |
-| Keep PostgreSQL-only | Not worth the complexity of multi-DB support at this stage | Confirmed |
-| Keep host-app auth | Engine should be composable, not opinionated about auth | Confirmed |
-
----
-*Last updated: 2026-02-09 after VBW bootstrap*

data/.vbw-planning/ROADMAP.md

@@ -1,53 +0,0 @@
-# Roadmap
-
-## Milestone: aia-ssl-fix
-
-### Phases
-
-1. [x] **AIA Certificate Resolution** -- Fix SSL failures for feeds with missing intermediate certificates by implementing AIA (Authority Information Access) resolution
-2. [x] **Test Performance** -- Reduce test suite runtime from ~133s to ~50s by splitting monolithic test classes, enabling parallelism, reducing log IO, and adopting before_all
-
-### Phase Details
-
-#### Phase 1: AIA Certificate Resolution
-
-**Goal:** Implement automatic AIA intermediate certificate fetching so feeds like netflixtechblog.com (served via Medium/AWS with wrong intermediates) succeed without manual cert configuration.
-
-**Requirements:**
-- REQ-AIA-01: Create AIAResolver module with thread-safe cache and 1-hour TTL
-- REQ-AIA-02: Add cert_store: parameter to HTTP.client for custom cert stores
-- REQ-AIA-03: On Faraday::SSLError, attempt AIA resolution before failing
-- REQ-AIA-04: Best-effort only -- never make things worse (rescue StandardError -> nil)
-
-**Success Criteria:**
-- [ ] AIAResolver.resolve(hostname) fetches leaf cert, extracts AIA URL, downloads intermediate
-- [ ] HTTP.client(cert_store:) accepts and uses custom cert stores
-- [ ] FeedFetcher retries once with AIA-resolved cert store on SSL failure
-- [ ] All existing tests pass (1003+), new tests cover AIA paths
-- [ ] RuboCop zero offenses, Brakeman zero warnings
-
-#### Phase 2: Test Performance
-
-**Goal:** Reduce test suite wall-clock time from ~133s to ~50s through structural optimizations. The 3-agent investigation identified that FeedFetcherTest (71 tests, 84.8s, 64% of total) is a monolithic class that cannot be parallelized, integration tests add 31s, and 95MB of debug logging adds 5-15s.
-
-**Requirements:**
-- REQ-PERF-01: Split FeedFetcherTest into 5+ smaller classes by concern (success paths, error handling, adaptive interval, dirty-check, content fingerprint, utilities)
-- REQ-PERF-02: Set test log level to :warn in test/dummy/config/environments/test.rb (eliminates 95MB log IO)
-- REQ-PERF-03: Tag integration tests (host_install_flow, release_packaging) so they can be excluded during dev with --exclude-pattern
-- REQ-PERF-04: Switch default parallelism from forks to threads (avoids PG segfault, enables splitting benefit)
-- REQ-PERF-05: Adopt before_all/setup_once in top DB-heavy test files (dashboard/queries_test.rb, etc.)
-
-**Success Criteria:**
-- [ ] FeedFetcherTest split into 5+ files, each independently runnable
-- [ ] All 1031+ tests pass with PARALLEL_WORKERS=1 and default workers
-- [ ] Test suite completes in <70s locally (down from 133s)
-- [ ] `bin/rails test --exclude-pattern="**/integration/**"` runs <50s
-- [ ] RuboCop zero offenses, Brakeman zero warnings
-- [ ] No test isolation regressions (parallel runs still green)
-
-### Progress
-
-| Phase | Status | Plans | Completed |
-|-------|--------|-------|-----------|
-| 1. AIA Certificate Resolution | Complete | 3 | 3 |
-| 2. Test Performance | Complete | 4 | 4 |