source_monitor 0.5.3 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 21aedfc8713e8179626900475c725eeb1fca98e169b5477d1ee45e557cdebf7e
-  data.tar.gz: 30e532cc201cf97c31209010125de15643dc9e53529e6aedbbbf07e67fee7d9d
+  metadata.gz: 4710a57d0da9bd7c73f4f7b3a48a4bbfd31fa1927cf79423c9e8955f4b433378
+  data.tar.gz: e0847be423ba3022b3c853902de2061ed7b5b476801dc9f83aefdb957df239f4
 SHA512:
-  metadata.gz: cc9da6548d8c8873e0aa989b21cd9fb85b76c14ab9a1335952e873a0be3694674c8aca74cbdb03cdb05e3c00db3638d519147f9410ee13439a6b71b31fecf29d
-  data.tar.gz: 34e16d32cfef7444234f812c4c27359a7f73d964859630c79bccccdeda43eb65a50669639e17ffecf9de492f82dbf654706986ac489d65635fcc9ad964615133
+  metadata.gz: ccccf11d1adb294ca141cb4efb8459f1948ad339d09feb2b55140d7983fd04ffdb772d5a75cfc3b73c7ec40c085e80026c8791357837f8184fee59df06efe5d8
+  data.tar.gz: 56224653d622b47a309f36bda5b3860118e4743fbafee7d657e09b7f6876377631d17d9a118d803e9c37bed69724852f5706e2cbabff5ef438a58febf0231831

data/.vbw-planning/milestones/default/STATE.md

@@ -50,7 +50,6 @@ Progress: [##########] 100%
 - [phase-4]: Fix-everything approach for public API convention violations
 - [phase-4]: 3 files slightly exceed 300 lines (entry_parser 390, queries 356, application_helper 346) -- all single-responsibility, cannot be split further
 
-### Pending Todos
 
 None
 

data/CHANGELOG.md

@@ -15,6 +15,21 @@ All notable changes to this project are documented below. The format follows [Ke
 
 - No unreleased changes yet.
 
+## [0.6.0] - 2026-02-17
+
+### Added
+
+- AIA (Authority Information Access) certificate resolution for SSL failures. When feed fetching or scraping encounters `certificate verify failed` errors due to missing intermediate certificates, the engine now automatically fetches the missing intermediate via AIA URLs and retries the request. This fixes feeds hosted on servers with incomplete certificate chains (e.g., Medium/Netflix Tech Blog on AWS).
+- `SourceMonitor::HTTP::AIAResolver` module with thread-safe hostname-keyed cache (1-hour TTL), SNI support, and DER/PEM certificate parsing.
+- `cert_store:` parameter on `SourceMonitor::HTTP.client` for passing custom certificate stores.
+- Brakeman ignore configuration (`config/brakeman.ignore`) for the intentional `VERIFY_NONE` in the AIA resolver's leaf certificate fetch.
+
+### Testing
+
+- 1,028 tests, 0 failures (up from 1,003 in 0.5.x).
+- RuboCop: 0 offenses.
+- Brakeman: 0 warnings (1 intentional ignore).
+
 ## [0.5.3] - 2026-02-16
 
 ### Fixed

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    source_monitor (0.5.3)
+    source_monitor (0.6.0)
       cssbundling-rails (~> 1.4)
       faraday (~> 2.9)
       faraday-follow_redirects (~> 0.4)

data/VERSION

@@ -1 +1 @@
-0.5.3
+0.6.0

data/config/brakeman.ignore

@@ -0,0 +1,17 @@
+{
+  "ignored_warnings": [
+    {
+      "warning_type": "SSL Verification Bypass",
+      "warning_code": 71,
+      "fingerprint": "17da2beb8f8ecf05b0ca1e3da89ee27d593395c711197f2f7dd38df759ea3720",
+      "check_name": "SSLVerify",
+      "message": "SSL certificate verification was bypassed",
+      "file": "lib/source_monitor/http/aia_resolver.rb",
+      "line": 77,
+      "code": "OpenSSL::SSL::SSLContext.new.verify_mode = OpenSSL::SSL::VERIFY_NONE",
+      "note": "Intentional: AIA resolver must connect without verification to fetch the leaf certificate from servers with broken certificate chains. This is the core purpose of the module -- it only uses VERIFY_NONE to read the cert, never to transmit data."
+    }
+  ],
+  "updated": "2026-02-17",
+  "brakeman_version": "8.0.2"
+}

data/lib/source_monitor/fetching/feed_fetcher.rb

@@ -81,8 +81,11 @@ module SourceMonitor
         raise error
       rescue Faraday::TimeoutError => error
         raise TimeoutError.new(error.message, original_error: error)
-      rescue Faraday::ConnectionFailed, Faraday::SSLError => error
+      rescue Faraday::ConnectionFailed => error
         raise ConnectionError.new(error.message, original_error: error)
+      rescue Faraday::SSLError => error
+        attempt_aia_recovery(error, started_at, instrumentation_payload) ||
+          raise(ConnectionError.new(error.message, original_error: error))
       rescue Faraday::ClientError => error
         raise build_http_error_from_faraday(error)
       rescue Faraday::Error => error
@@ -236,6 +239,24 @@ module SourceMonitor
         )
       end
 
+      def attempt_aia_recovery(_error, started_at, instrumentation_payload)
+        return if @aia_attempted
+
+        @aia_attempted = true
+        hostname = URI.parse(source.feed_url).host
+        intermediate = SourceMonitor::HTTP::AIAResolver.resolve(hostname)
+        return unless intermediate
+
+        store = SourceMonitor::HTTP::AIAResolver.enhanced_cert_store([ intermediate ])
+        @connection = SourceMonitor::HTTP.client(cert_store: store, headers: request_headers)
+        instrumentation_payload[:aia_resolved] = true
+
+        response = perform_request
+        handle_response(response, started_at, instrumentation_payload)
+      rescue StandardError
+        nil
+      end
+
       def build_http_error_from_faraday(error)
         response_hash = error.response || {}
         headers = response_hash[:headers] || response_hash[:response_headers] || {}

data/lib/source_monitor/http/aia_resolver.rb

@@ -0,0 +1,128 @@
+# frozen_string_literal: true
+
+require "openssl"
+require "net/http"
+require "socket"
+
+module SourceMonitor
+  module HTTP
+    module AIAResolver
+      CONNECT_TIMEOUT = 5
+      DOWNLOAD_TIMEOUT = 5
+      CACHE_TTL = 3600 # 1 hour
+
+      class << self
+        def resolve(hostname, port: 443)
+          cached = cache_lookup(hostname)
+          return cached if cached
+
+          cert = fetch_leaf_certificate(hostname, port)
+          return unless cert
+
+          url = extract_aia_url(cert)
+          return unless url
+
+          intermediate = download_certificate(url)
+          return unless intermediate
+
+          cache_store(hostname, intermediate)
+          intermediate
+        rescue StandardError
+          nil
+        end
+
+        def enhanced_cert_store(additional_certs)
+          store = OpenSSL::X509::Store.new
+          store.set_default_paths
+
+          Array(additional_certs).each do |cert|
+            store.add_cert(cert)
+          rescue OpenSSL::X509::StoreError
+            # Already in store or invalid -- skip
+          end
+
+          store
+        end
+
+        def clear_cache!
+          @mutex.synchronize { @cache.clear }
+        end
+
+        def cache_size
+          @mutex.synchronize { @cache.size }
+        end
+
+        private
+
+        def cache_lookup(hostname)
+          @mutex.synchronize do
+            entry = @cache[hostname]
+            return unless entry
+            return entry[:cert] if entry[:expires_at] > Time.now
+
+            @cache.delete(hostname)
+            nil
+          end
+        end
+
+        def cache_store(hostname, cert)
+          @mutex.synchronize do
+            @cache[hostname] = { cert: cert, expires_at: Time.now + CACHE_TTL }
+          end
+        end
+
+        def fetch_leaf_certificate(hostname, port)
+          tcp = Socket.tcp(hostname, port, connect_timeout: CONNECT_TIMEOUT)
+          ssl_context = OpenSSL::SSL::SSLContext.new
+          ssl_context.verify_mode = OpenSSL::SSL::VERIFY_NONE
+
+          ssl = OpenSSL::SSL::SSLSocket.new(tcp, ssl_context)
+          ssl.hostname = hostname
+          ssl.connect
+
+          ssl.peer_cert
+        rescue StandardError
+          nil
+        ensure
+          ssl&.close rescue nil # rubocop:disable Style/RescueModifier
+          tcp&.close rescue nil # rubocop:disable Style/RescueModifier
+        end
+
+        def extract_aia_url(cert)
+          return unless cert.respond_to?(:ca_issuer_uris)
+
+          uris = cert.ca_issuer_uris
+          return if uris.nil? || uris.empty?
+
+          uris.first.to_s
+        rescue StandardError
+          nil
+        end
+
+        def download_certificate(url)
+          uri = URI.parse(url)
+          http = Net::HTTP.new(uri.host, uri.port)
+          http.open_timeout = DOWNLOAD_TIMEOUT
+          http.read_timeout = DOWNLOAD_TIMEOUT
+
+          response = http.get(uri.request_uri)
+          return unless response.is_a?(Net::HTTPSuccess)
+
+          body = response.body
+          parse_certificate(body)
+        rescue StandardError
+          nil
+        end
+
+        def parse_certificate(body)
+          OpenSSL::X509::Certificate.new(body) # tries DER first, then PEM
+        rescue OpenSSL::X509::CertificateError
+          nil
+        end
+      end
+
+      @mutex = Mutex.new
+      @cache = {}
+    end
+  end
+end

data/lib/source_monitor/http.rb

@@ -9,6 +9,8 @@ require "active_support/core_ext/object/blank"
 
 module SourceMonitor
   module HTTP
+    autoload :AIAResolver, "source_monitor/http/aia_resolver"
+
     DEFAULT_TIMEOUT = 15
     DEFAULT_OPEN_TIMEOUT = 5
     DEFAULT_MAX_REDIRECTS = 5
@@ -16,7 +18,7 @@ module SourceMonitor
     RETRY_STATUSES = [ 429, 500, 502, 503, 504 ].freeze
 
     class << self
-      def client(proxy: nil, headers: {}, timeout: nil, open_timeout: nil, retry_requests: true)
+      def client(proxy: nil, headers: {}, timeout: nil, open_timeout: nil, retry_requests: true, cert_store: nil)
         settings = SourceMonitor.config.http
 
         effective_proxy = resolve_proxy(proxy, settings)
@@ -30,14 +32,15 @@ module SourceMonitor
             timeout: effective_timeout,
             open_timeout: effective_open_timeout,
             settings: settings,
-            enable_retry: retry_requests
+            enable_retry: retry_requests,
+            cert_store: cert_store
           )
         end
       end
 
       private
 
-      def configure_request(connection, headers, timeout:, open_timeout:, settings:, enable_retry:) # rubocop:disable Metrics/MethodLength
+      def configure_request(connection, headers, timeout:, open_timeout:, settings:, enable_retry:, cert_store: nil) # rubocop:disable Metrics/MethodLength
         if enable_retry
           connection.request :retry,
                              max: settings.retry_max || 4,
@@ -58,7 +61,7 @@ module SourceMonitor
           connection.headers[key] = value
         end
 
-        configure_ssl(connection, settings)
+        configure_ssl(connection, settings, cert_store: cert_store)
 
         connection.adapter Faraday.default_adapter
       end
@@ -67,7 +70,7 @@ module SourceMonitor
       # fail to verify certificate chains that depend on intermediate CAs
       # (e.g., Medium/Netflix on AWS). OpenSSL::X509::Store#set_default_paths
       # loads all system-trusted CAs including intermediates.
-      def configure_ssl(connection, settings)
+      def configure_ssl(connection, settings, cert_store: nil)
         connection.ssl.verify = settings.ssl_verify != false
 
         if settings.ssl_ca_file
@@ -75,7 +78,7 @@ module SourceMonitor
         elsif settings.ssl_ca_path
           connection.ssl.ca_path = settings.ssl_ca_path
         else
-          connection.ssl.cert_store = default_cert_store
+          connection.ssl.cert_store = cert_store || default_cert_store
         end
       end
 

data/lib/source_monitor/scrapers/fetchers/http_fetcher.rb

@@ -10,6 +10,7 @@ module SourceMonitor
 
         def initialize(http: SourceMonitor::HTTP)
           @http = http
+          @aia_attempted = false
         end
 
         def fetch(url:, settings: nil)
@@ -25,6 +26,11 @@ module SourceMonitor
               message: "Non-success HTTP status"
             )
           end
+        rescue Faraday::SSLError => error
+          result = attempt_aia_recovery(url, settings)
+          return result if result
+
+          Result.new(status: :failed, error: error.class.name, message: error.message)
         rescue Faraday::ClientError => error
           Result.new(
             status: :failed,
@@ -40,13 +46,34 @@ module SourceMonitor
 
         attr_reader :http
 
-        def connection(settings)
+        def attempt_aia_recovery(url, settings)
+          return if @aia_attempted
+
+          @aia_attempted = true
+          hostname = URI.parse(url).host
+          intermediate = SourceMonitor::HTTP::AIAResolver.resolve(hostname)
+          return unless intermediate
+
+          store = SourceMonitor::HTTP::AIAResolver.enhanced_cert_store([ intermediate ])
+          response = connection(settings, cert_store: store).get(url)
+
+          if success_status?(response.status)
+            Result.new(status: :success, body: response.body, headers: response.headers, http_status: response.status)
+          else
+            Result.new(status: :failed, http_status: response.status, error: "http_error", message: "Non-success HTTP status")
+          end
+        rescue StandardError
+          nil
+        end
+
+        def connection(settings, cert_store: nil)
           normalized = normalize_settings(settings)
           http.client(
             proxy: normalized[:proxy],
             headers: normalized[:headers],
             timeout: normalized[:timeout] || SourceMonitor::HTTP::DEFAULT_TIMEOUT,
-            open_timeout: normalized[:open_timeout] || SourceMonitor::HTTP::DEFAULT_OPEN_TIMEOUT
+            open_timeout: normalized[:open_timeout] || SourceMonitor::HTTP::DEFAULT_OPEN_TIMEOUT,
+            cert_store: cert_store
           )
         end
 

data/lib/source_monitor/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module SourceMonitor
-  VERSION = "0.5.3"
+  VERSION = "0.6.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: source_monitor
 version: !ruby/object:Gem::Version
-  version: 0.5.3
+  version: 0.6.0
 platform: ruby
 authors:
 - dchuk
@@ -540,6 +540,7 @@ files:
 - app/views/source_monitor/sources/index.html.erb
 - app/views/source_monitor/sources/new.html.erb
 - app/views/source_monitor/sources/show.html.erb
+- config/brakeman.ignore
 - config/coverage_baseline.json
 - config/initializers/feedjira.rb
 - config/routes.rb
@@ -630,6 +631,7 @@ files:
 - lib/source_monitor/health/source_health_monitor.rb
 - lib/source_monitor/health/source_health_reset.rb
 - lib/source_monitor/http.rb
+- lib/source_monitor/http/aia_resolver.rb
 - lib/source_monitor/images/content_rewriter.rb
 - lib/source_monitor/images/downloader.rb
 - lib/source_monitor/import_sessions/entry_normalizer.rb