source_monitor 0.3.2 → 0.4.0

data/.vbw-planning/phases/06-netflix-feed-fix/PLAN-01.md

@@ -0,0 +1,345 @@
+---
+phase: 6
+plan: "01"
+title: ssl-cert-store-configuration
+type: execute
+wave: 1
+depends_on: []
+cross_phase_deps: []
+autonomous: true
+effort_override: thorough
+skills_used: []
+files_modified:
+  - lib/source_monitor/configuration/http_settings.rb
+  - lib/source_monitor/http.rb
+  - test/lib/source_monitor/http_test.rb
+  - test/lib/source_monitor/fetching/feed_fetcher_test.rb
+  - test/vcr_cassettes/source_monitor/fetching/netflix_medium_rss.yml
+must_haves:
+  truths:
+    - "Running `PARALLEL_WORKERS=1 bin/rails test test/lib/source_monitor/http_test.rb` exits 0 with 0 failures"
+    - "Running `PARALLEL_WORKERS=1 bin/rails test test/lib/source_monitor/fetching/feed_fetcher_test.rb` exits 0 with 0 failures"
+    - "Running `bin/rubocop lib/source_monitor/http.rb lib/source_monitor/configuration/http_settings.rb test/lib/source_monitor/http_test.rb test/lib/source_monitor/fetching/feed_fetcher_test.rb` exits 0 with no offenses"
+    - "Running `bin/rails test` (full suite) exits 0 with 0 failures"
+    - "`grep -r 'cert_store' lib/source_monitor/http.rb` returns at least one match"
+    - "`grep -r 'ssl_ca_file' lib/source_monitor/configuration/http_settings.rb` returns at least one match"
+    - "The VCR cassette at test/vcr_cassettes/source_monitor/fetching/netflix_medium_rss.yml exists and contains 'netflixtechblog'"
+  artifacts:
+    - path: "lib/source_monitor/http.rb"
+      provides: "SSL cert store configuration on Faraday connections"
+      contains: "cert_store"
+    - path: "lib/source_monitor/configuration/http_settings.rb"
+      provides: "Configurable SSL options (ca_file, ca_path, verify)"
+      contains: "ssl_ca_file"
+    - path: "test/lib/source_monitor/http_test.rb"
+      provides: "Tests for SSL configuration on HTTP client"
+      contains: "ssl"
+    - path: "test/lib/source_monitor/fetching/feed_fetcher_test.rb"
+      provides: "Netflix Tech Blog VCR regression test"
+      contains: "netflix"
+    - path: "test/vcr_cassettes/source_monitor/fetching/netflix_medium_rss.yml"
+      provides: "Recorded VCR cassette from real Netflix Tech Blog feed"
+      contains: "netflixtechblog"
+  key_links:
+    - from: "http.rb#configure_ssl"
+      to: "REQ-25"
+      via: "Configures Faraday SSL with system cert store to fix certificate verification failures"
+    - from: "http_settings.rb#ssl_ca_file"
+      to: "REQ-25"
+      via: "Exposes configurable SSL CA file/path for environments with non-standard cert locations"
+    - from: "feed_fetcher_test.rb#netflix_regression"
+      to: "REQ-25"
+      via: "VCR cassette proves Netflix Tech Blog feed parses successfully"
+---
+<objective>
+Fix SSL certificate verification failures (like Netflix Tech Blog's "unable to get local issuer certificate") by configuring the Faraday HTTP client to use a properly initialized OpenSSL cert store with system default paths. Add configurable SSL options to HTTPSettings so users can override CA file/path in non-standard environments. Record a VCR cassette from the real Netflix Tech Blog feed as a regression test. REQ-25.
+</objective>
+<context>
+@lib/source_monitor/http.rb -- The HTTP client module. Creates Faraday connections via `HTTP.client`. Currently configures timeouts, retry, gzip, follow-redirects, and headers -- but does NOT configure any SSL options. Faraday's `connection.ssl` is left at defaults, which means it relies on the underlying adapter (net_http) to find CA certs. On some systems (macOS with Homebrew Ruby, Docker Alpine, custom OpenSSL builds), the compiled-in `OpenSSL::X509::DEFAULT_CERT_FILE` path may not include all intermediate certificates -- causing "certificate verify failed" for sites like Netflix Tech Blog whose chain depends on intermediates served by the TLS handshake being validated against a complete CA bundle. The fix is to explicitly set `connection.ssl.cert_store` to an `OpenSSL::X509::Store` initialized with `set_default_paths`, which loads both `DEFAULT_CERT_FILE` and `DEFAULT_CERT_DIR`. Optionally, if the user configures `ssl_ca_file` or `ssl_ca_path`, those override the default store.
+
+@lib/source_monitor/configuration/http_settings.rb -- The settings class for HTTP configuration. Has 11 `attr_accessor` fields for timeout, retry, proxy, headers, etc. New SSL settings (`ssl_ca_file`, `ssl_ca_path`, `ssl_verify`) should be added here following the same pattern. Default: `ssl_verify = true` (never disable verification), `ssl_ca_file = nil`, `ssl_ca_path = nil` (nil means use system defaults via cert_store).
+
+@test/lib/source_monitor/http_test.rb -- 8 existing tests for the HTTP client. Tests inspect `@connection.builder.handlers`, `@connection.options`, and `@connection.headers`. New SSL tests should inspect `@connection.ssl.cert_store`, `@connection.ssl.verify`, and optionally `@connection.ssl.ca_file` when configured.
+
+@test/lib/source_monitor/fetching/feed_fetcher_test.rb -- Existing tests use VCR cassettes for RSS, Atom, and JSON feeds (ruby-lang.org, W3C, json_sample). The Netflix regression test should follow the same pattern: `VCR.use_cassette("source_monitor/fetching/netflix_medium_rss")` with a source pointing at `https://netflixtechblog.com/feed`.
+
+@test/vcr_cassettes/ -- Contains 3 existing cassettes (rss_success, atom_success, json_success). The Netflix cassette should be recorded with `VCR.use_cassette(..., record: :new_episodes)` during the first test run against the real feed (with WebMock allowing the Netflix host temporarily), then committed as a fixture for CI. This requires temporarily allowing net connect to netflixtechblog.com during recording.
+
+@lib/source_monitor/fetching/feed_fetcher.rb -- Lines 84-85 already catch `Faraday::SSLError` and wrap it as `ConnectionError`. This error path will stop triggering once SSL is properly configured, but the error handling remains as a safety net for genuinely invalid certificates.
+
+**Root cause analysis:**
+The Netflix Tech Blog (Medium-hosted at netflixtechblog.com, IP 52.1.173.203) serves a TLS certificate chain that requires the client to have Amazon's intermediate CA in its trust store. Ruby's compiled-in `OpenSSL::X509::DEFAULT_CERT_FILE` may point to a cert bundle that is missing this intermediate, or the system's cert directory may not be indexed. By explicitly creating an `OpenSSL::X509::Store` with `set_default_paths` and assigning it to the Faraday connection's `ssl.cert_store`, we ensure Ruby loads all available system certificates -- which on a properly maintained system includes Amazon/AWS intermediates. This is the standard, general fix for SSL verification issues in Ruby HTTP clients.
+
+**Key design decisions:**
+1. Use `OpenSSL::X509::Store.new.tap(&:set_default_paths)` as the default cert store -- this is the most cross-platform approach
+2. Add `ssl_ca_file` and `ssl_ca_path` as optional overrides in HTTPSettings -- when set, they configure `connection.ssl.ca_file` / `connection.ssl.ca_path` instead of using the cert store
+3. Keep `ssl_verify = true` as default and do NOT add a way to disable verification globally -- security-first design
+4. The cert store is created fresh per `HTTP.client` call (Faraday connections are short-lived and not shared across threads)
+5. For recording the VCR cassette: use a dedicated recording script or a test with `record: :new_episodes` and temporarily permit net connect
+</context>
+<tasks>
+<task type="auto">
+  <name>add-ssl-settings-to-http-settings</name>
+  <files>
+    lib/source_monitor/configuration/http_settings.rb
+  </files>
+  <action>
+Add three new `attr_accessor` fields to `HTTPSettings` for SSL configuration:
+
+1. `ssl_ca_file` -- Path to a CA certificate file (PEM format). When set, Faraday uses this instead of the default cert store. Default: `nil`.
+2. `ssl_ca_path` -- Path to a directory of CA certificates. When set, Faraday uses this. Default: `nil`.
+3. `ssl_verify` -- Whether to verify SSL certificates. Default: `true`. This exists for completeness but should almost never be set to `false`.
+
+Add the three new fields to the `attr_accessor` list (after `retry_statuses`):
+
+```ruby
+attr_accessor :timeout,
+  :open_timeout,
+  :max_redirects,
+  :user_agent,
+  :proxy,
+  :headers,
+  :retry_max,
+  :retry_interval,
+  :retry_interval_randomness,
+  :retry_backoff_factor,
+  :retry_statuses,
+  :ssl_ca_file,
+  :ssl_ca_path,
+  :ssl_verify
+```
+
+In `reset!`, add after `@retry_statuses = nil`:
+
+```ruby
+@ssl_ca_file = nil
+@ssl_ca_path = nil
+@ssl_verify = true
+```
+  </action>
+  <verify>
+Read `lib/source_monitor/configuration/http_settings.rb` and confirm: (a) all three new attr_accessors are present, (b) `reset!` initializes them with correct defaults, (c) `ssl_verify` defaults to `true`.
+  </verify>
+  <done>
+HTTPSettings now has ssl_ca_file, ssl_ca_path, and ssl_verify configuration options with safe defaults.
+  </done>
+</task>
+<task type="auto">
+  <name>configure-faraday-ssl-cert-store</name>
+  <files>
+    lib/source_monitor/http.rb
+  </files>
+  <action>
+Modify the `HTTP` module to configure SSL on every Faraday connection. Add a `require "openssl"` at the top of the file (after the existing requires).
+
+In the `configure_request` method, add SSL configuration BEFORE the adapter line (`connection.adapter Faraday.default_adapter`):
+
+```ruby
+configure_ssl(connection, settings)
+```
+
+Add a new private method `configure_ssl`:
+
+```ruby
+def configure_ssl(connection, settings)
+  connection.ssl.verify = settings.ssl_verify != false
+
+  if settings.ssl_ca_file
+    connection.ssl.ca_file = settings.ssl_ca_file
+  elsif settings.ssl_ca_path
+    connection.ssl.ca_path = settings.ssl_ca_path
+  else
+    connection.ssl.cert_store = default_cert_store
+  end
+end
+
+def default_cert_store
+  OpenSSL::X509::Store.new.tap(&:set_default_paths)
+end
+```
+
+The logic:
+1. Always set `verify = true` unless explicitly configured to `false` (defense in depth).
+2. If user specifies `ssl_ca_file`, use that (overrides cert store).
+3. Else if user specifies `ssl_ca_path`, use that (overrides cert store).
+4. Otherwise, create a fresh `OpenSSL::X509::Store` with `set_default_paths` -- this is the key fix that resolves the Netflix SSL error by loading all system CA certificates including intermediates.
+
+Note: `ca_file` and `ca_path` take precedence over `cert_store` in Faraday/net_http, so we only set one path.
+  </action>
+  <verify>
+Read `lib/source_monitor/http.rb` and confirm: (a) `require "openssl"` is present, (b) `configure_ssl` is called in `configure_request`, (c) the method creates an `OpenSSL::X509::Store` with `set_default_paths` as the default, (d) `ssl_ca_file` and `ssl_ca_path` override the store when set, (e) `ssl.verify` is always explicitly set. Run `bin/rubocop lib/source_monitor/http.rb` to confirm no offenses.
+  </verify>
+  <done>
+The HTTP client now explicitly configures SSL with a proper cert store. By default, every Faraday connection gets an OpenSSL::X509::Store initialized with system default paths, which resolves certificate chain verification failures like the Netflix Tech Blog issue.
+  </done>
+</task>
+<task type="auto">
+  <name>add-ssl-unit-tests</name>
+  <files>
+    test/lib/source_monitor/http_test.rb
+  </files>
+  <action>
+Add the following tests to `HTTPTest`:
+
+1. **"configures SSL with default cert store"** -- Create a default client, assert `@connection.ssl.verify` is truthy, assert `@connection.ssl.cert_store` is an instance of `OpenSSL::X509::Store`, assert `@connection.ssl.ca_file` is nil (not overridden).
+
+2. **"uses configured ssl_ca_file when set"** -- Configure `config.http.ssl_ca_file = "/path/to/custom/ca.pem"`, create a client, assert `connection.ssl.ca_file` equals the configured path, assert `connection.ssl.cert_store` is nil (ca_file takes precedence).
+
+3. **"uses configured ssl_ca_path when set"** -- Configure `config.http.ssl_ca_path = "/path/to/certs"`, create a client, assert `connection.ssl.ca_path` equals the configured path.
+
+4. **"ssl verify defaults to true"** -- Create a default client, assert `connection.ssl.verify` is `true`.
+
+5. **"respects ssl_verify configuration"** -- Configure `config.http.ssl_verify = false`, create a client, assert `connection.ssl.verify` is `false`. (This tests the escape hatch exists, even though it should rarely be used.)
+
+Add `require "openssl"` at the top of the test file if not already present.
+
+Each test should follow the existing pattern: create a connection via `SourceMonitor::HTTP.client`, then inspect the `connection.ssl` object.
+  </action>
+  <verify>
+Run `PARALLEL_WORKERS=1 bin/rails test test/lib/source_monitor/http_test.rb` and confirm all tests pass (8 existing + 5 new = 13 tests). Run `bin/rubocop test/lib/source_monitor/http_test.rb` and confirm no offenses.
+  </verify>
+  <done>
+5 new SSL configuration tests added. All 13 HTTP client tests pass. The cert store, ca_file, ca_path, and verify options are all verified.
+  </done>
+</task>
+<task type="auto">
+  <name>record-netflix-vcr-cassette-and-regression-test</name>
+  <files>
+    test/lib/source_monitor/fetching/feed_fetcher_test.rb
+    test/vcr_cassettes/source_monitor/fetching/netflix_medium_rss.yml
+  </files>
+  <action>
+This task records a VCR cassette from the real Netflix Tech Blog feed and adds a regression test.
+
+**Step 1: Record the VCR cassette.**
+
+Create a temporary recording script or use a one-off test run. The simplest approach: add the test first (below), then run it once with `VCR_RECORD=new_episodes` or equivalent to record the cassette. The cassette will be committed as a test fixture.
+
+To record, temporarily allow net connect for the Netflix host. You can do this by running:
+
+```bash
+PARALLEL_WORKERS=1 bin/rails test test/lib/source_monitor/fetching/feed_fetcher_test.rb -n test_fetches_netflix_tech_blog_feed_via_medium_rss
+```
+
+with VCR configured to `record: :new_episodes` for this specific cassette. If WebMock blocks the request, temporarily use `WebMock.allow_net_connect!` inside the test during recording, then remove it after the cassette is committed.
+
+**Alternative recording approach:** Use a standalone Ruby script to fetch the feed and manually create the VCR cassette YAML:
+
+```ruby
+require "faraday"
+require "openssl"
+require "yaml"
+
+conn = Faraday.new do |f|
+  f.ssl.cert_store = OpenSSL::X509::Store.new.tap(&:set_default_paths)
+  f.request :gzip
+  f.response :follow_redirects, limit: 5
+  f.adapter :net_http
+end
+
+response = conn.get("https://netflixtechblog.com/feed")
+# Save as VCR cassette format...
+```
+
+After recording, verify the cassette file exists at `test/vcr_cassettes/source_monitor/fetching/netflix_medium_rss.yml` and contains a 200 response with RSS/XML body containing Netflix blog entries.
+
+**Step 2: Add the regression test.**
+
+Add a new test to `FeedFetcherTest`:
+
+```ruby
+test "fetches Netflix Tech Blog feed via Medium RSS" do
+  source = build_source(
+    name: "Netflix Tech Blog",
+    feed_url: "https://netflixtechblog.com/feed"
+  )
+
+  result = nil
+  VCR.use_cassette("source_monitor/fetching/netflix_medium_rss") do
+    result = FeedFetcher.new(source: source, jitter: ->(_) { 0 }).call
+  end
+
+  assert_equal :fetched, result.status
+  assert_not_nil result.feed
+  assert_kind_of Feedjira::Parser::RSS, result.feed
+  assert result.feed.entries.any?, "Expected at least one feed entry"
+  assert_match(/netflix/i, result.feed.title.to_s)
+end
+```
+
+This test uses the recorded VCR cassette so it works in CI without network access. It validates that the feed parses as RSS and contains Netflix entries.
+
+**Important:** The `build_source` helper is already available in this test file. Check existing test patterns to confirm the helper signature.
+  </action>
+  <verify>
+Confirm: (a) the VCR cassette file exists at `test/vcr_cassettes/source_monitor/fetching/netflix_medium_rss.yml`, (b) it contains `netflixtechblog` in the request URI, (c) the response status is 200, (d) the response body contains RSS/XML content. Run `PARALLEL_WORKERS=1 bin/rails test test/lib/source_monitor/fetching/feed_fetcher_test.rb -n test_fetches_Netflix_Tech_Blog_feed_via_Medium_RSS` and confirm it passes.
+  </verify>
+  <done>
+VCR cassette recorded from real Netflix Tech Blog feed. Regression test passes using the cassette. The feed parses as RSS with Netflix blog entries, proving the SSL fix resolves the original "certificate verify failed" error.
+  </done>
+</task>
+<task type="auto">
+  <name>full-suite-verification-and-documentation</name>
+  <files>
+    lib/source_monitor/http.rb
+    lib/source_monitor/configuration/http_settings.rb
+    test/lib/source_monitor/http_test.rb
+    test/lib/source_monitor/fetching/feed_fetcher_test.rb
+  </files>
+  <action>
+Run the full verification suite:
+
+1. `PARALLEL_WORKERS=1 bin/rails test test/lib/source_monitor/http_test.rb test/lib/source_monitor/fetching/feed_fetcher_test.rb` -- all targeted tests pass
+2. `bin/rails test` -- full suite passes with 874+ runs and 0 failures
+3. `bin/rubocop` -- zero offenses
+4. `bin/brakeman --no-pager` -- zero warnings
+
+Review all modified files for:
+- `http.rb`: `require "openssl"` present, `configure_ssl` called in `configure_request`, `default_cert_store` creates `OpenSSL::X509::Store` with `set_default_paths`
+- `http_settings.rb`: three new attr_accessors (`ssl_ca_file`, `ssl_ca_path`, `ssl_verify`), initialized in `reset!`
+- `http_test.rb`: 5 new SSL tests covering cert_store default, ca_file override, ca_path override, verify default, verify override
+- `feed_fetcher_test.rb`: Netflix regression test using VCR cassette
+- VCR cassette: valid YAML with Netflix feed content
+
+If any failures or offenses are found, fix them before completing.
+
+Add a brief inline comment in `http.rb` above `configure_ssl` documenting the root cause:
+
+```ruby
+# Configure SSL to use a proper cert store. Without this, some systems
+# fail to verify certificate chains that depend on intermediate CAs
+# (e.g., Medium/Netflix on AWS). OpenSSL::X509::Store#set_default_paths
+# loads all system-trusted CAs including intermediates.
+```
+  </action>
+  <verify>
+`bin/rails test` exits 0 with 874+ runs, 0 failures. `bin/rubocop` exits 0 with 0 offenses. `bin/brakeman --no-pager` exits 0 with 0 warnings. All modified files are clean and well-documented.
+  </verify>
+  <done>
+Full suite passes. All quality gates green. SSL cert store fix is general (not Netflix-specific), configurable via HTTPSettings, documented inline, and regression-tested with a VCR cassette.
+  </done>
+</task>
+</tasks>
+<verification>
+1. `PARALLEL_WORKERS=1 bin/rails test test/lib/source_monitor/http_test.rb` -- 13+ tests pass (8 existing + 5 new SSL tests)
+2. `PARALLEL_WORKERS=1 bin/rails test test/lib/source_monitor/fetching/feed_fetcher_test.rb` -- all tests pass including Netflix regression
+3. `bin/rails test` -- 874+ runs, 0 failures
+4. `bin/rubocop` -- 0 offenses
+5. `bin/brakeman --no-pager` -- 0 warnings
+6. `grep -n 'cert_store' lib/source_monitor/http.rb` returns matches for configure_ssl and default_cert_store
+7. `grep -n 'ssl_ca_file' lib/source_monitor/configuration/http_settings.rb` returns match in attr_accessor and reset!
+8. `test -f test/vcr_cassettes/source_monitor/fetching/netflix_medium_rss.yml` exits 0
+9. `grep 'netflixtechblog' test/vcr_cassettes/source_monitor/fetching/netflix_medium_rss.yml` returns matches
+</verification>
+<success_criteria>
+- Root cause identified: missing intermediate CA certs when OpenSSL cert store not explicitly initialized (REQ-25)
+- General fix applied: Faraday SSL configured with OpenSSL::X509::Store#set_default_paths on every connection (REQ-25)
+- Configurable: ssl_ca_file, ssl_ca_path, ssl_verify exposed via HTTPSettings for non-standard environments (REQ-25)
+- Netflix Tech Blog feed fetches successfully via VCR cassette regression test (REQ-25)
+- No regressions: existing SSL error wrapping (Faraday::SSLError -> ConnectionError) still works (REQ-25)
+- VCR cassette recorded from real Netflix feed and committed as test fixture (REQ-25)
+- All tests pass, RuboCop clean, Brakeman clean (REQ-25)
+</success_criteria>
+<output>
+.vbw-planning/phases/06-netflix-feed-fix/PLAN-01-SUMMARY.md
+</output>

data/CHANGELOG.md

@@ -15,6 +15,49 @@ All notable changes to this project are documented below. The format follows [Ke
 
 - No unreleased changes yet.
 
+## [0.4.0] - 2026-02-12
+
+### Added
+
+- Install generator now auto-patches `Procfile.dev` with Solid Queue `jobs:` entry and `queue.yml` with `recurring_schedule` dispatcher wiring (idempotent, skip if already present).
+- `RecurringScheduleVerifier` checks that recurring tasks are registered with Solid Queue dispatchers; `SolidQueueVerifier` remediation now mentions `Procfile.dev` for `bin/dev` users.
+- Dashboard fetch log entries display source URL (domain for RSS, item URL for scrapes) alongside existing summary.
+- External links across dashboard, logs, sources, and items open in new tab with visual indicator icon.
+- Configurable Active Storage image downloads: `config.images.download_to_active_storage` (default `false`) detects inline images in feed content, downloads them via background job, and rewrites `<img>` src attributes with Active Storage URLs.
+- `Images::ContentRewriter` extracts and rewrites image URLs from HTML content using Nokolexbor.
+- `Images::Downloader` service validates content type and size before downloading images.
+- `DownloadContentImagesJob` orchestrates the download/attach/rewrite pipeline per item.
+- SSL certificate store configuration: every Faraday connection gets an `OpenSSL::X509::Store` initialized with `set_default_paths`, resolving "unable to get local issuer certificate" errors on systems with incomplete CA bundles.
+- Configurable SSL options in `HTTPSettings`: `ssl_ca_file`, `ssl_ca_path`, `ssl_verify` for non-standard certificate environments.
+- Netflix Tech Blog VCR cassette regression test proving Medium-hosted RSS feeds parse correctly with the SSL fix.
+
+### Fixed
+
+- SSL certificate verification failures for feeds hosted on services requiring intermediate CAs (e.g., Netflix Tech Blog via Medium/AWS).
+- Setup documentation now includes `Procfile.dev` and `recurring_schedule` guidance.
+
+### Changed
+
+- Updated `sm-host-setup`, `sm-configure`, and setup documentation to reflect that the generator handles Procfile.dev and recurring_schedule automatically.
+
+### Testing
+
+- 973 tests, 3,114 assertions, 0 failures (up from 841 tests in 0.3.3).
+- RuboCop: 389 files, 0 offenses.
+- Brakeman: 0 warnings.
+
+## [0.3.3] - 2026-02-11
+
+### Fixed
+
+- Added missing `recurring.yml` configuration to the install generator so host apps get Solid Queue recurring job config on install.
+- Fixed YAML alias parsing in install generator so merging into existing `recurring.yml` files with `<<: *default` anchors works correctly.
+
+### Changed
+
+- Updated `sm-host-setup` and `sm-job` skills to reflect latest conventions.
+- Updated setup documentation with current installation steps.
+
 ## [0.3.2] - 2026-02-10
 
 ### Fixed

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    source_monitor (0.3.2)
+    source_monitor (0.4.0)
       cssbundling-rails (~> 1.4)
       faraday (~> 2.9)
       faraday-follow_redirects (~> 0.4)

data/VERSION

@@ -1 +1 @@
-0.3.2
+0.4.0

data/app/assets/builds/source_monitor/application.css

@@ -705,6 +705,10 @@ video {
   margin-right: 0.25rem;
 }
 
+.fm-admin .mt-0\.5 {
+  margin-top: 0.125rem;
+}
+
 .fm-admin .mt-1 {
   margin-top: 0.25rem;
 }
@@ -1789,6 +1793,11 @@ video {
   --tw-ring-color: transparent;
 }
 
+.fm-admin .invert {
+  --tw-invert: invert(100%);
+  filter: var(--tw-blur) var(--tw-brightness) var(--tw-contrast) var(--tw-grayscale) var(--tw-hue-rotate) var(--tw-invert) var(--tw-saturate) var(--tw-sepia) var(--tw-drop-shadow);
+}
+
 .fm-admin .filter {
   filter: var(--tw-blur) var(--tw-brightness) var(--tw-contrast) var(--tw-grayscale) var(--tw-hue-rotate) var(--tw-invert) var(--tw-saturate) var(--tw-sepia) var(--tw-drop-shadow);
 }

data/app/helpers/source_monitor/application_helper.rb

@@ -212,8 +212,46 @@ module SourceMonitor
       end
     end
 
+    # Renders a clickable link that opens in a new tab with an external-link icon.
+    # Returns the label as plain text if the URL is blank.
+    def external_link_to(label, url, **options)
+      return label if url.blank?
+
+      css = options.delete(:class) || "text-blue-600 hover:text-blue-500"
+      link_to(url, target: "_blank", rel: "noopener noreferrer", class: css, title: url, **options) do
+        safe_join([ label, " ", external_link_icon ])
+      end
+    end
+
+    # Extracts the domain from a URL, returning nil if parsing fails.
+    def domain_from_url(url)
+      return nil if url.blank?
+
+      URI.parse(url.to_s).host
+    rescue URI::InvalidURIError
+      nil
+    end
+
     private
 
+    def external_link_icon
+      tag.svg(
+        class: "inline-block h-3 w-3 text-slate-400",
+        xmlns: "http://www.w3.org/2000/svg",
+        fill: "none",
+        viewBox: "0 0 24 24",
+        stroke_width: "2",
+        stroke: "currentColor",
+        aria: { hidden: "true" }
+      ) do
+        tag.path(
+          stroke_linecap: "round",
+          stroke_linejoin: "round",
+          d: "M13.5 6H5.25A2.25 2.25 0 003 8.25v10.5A2.25 2.25 0 005.25 21h10.5A2.25 2.25 0 0018 18.75V10.5m-10.5 6L21 3m0 0h-5.25M21 3v5.25"
+        )
+      end
+    end
+
     def derive_item_scrape_status(item:, source: nil)
       return "idle" unless item
 

data/app/jobs/source_monitor/download_content_images_job.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  class DownloadContentImagesJob < ApplicationJob
+    source_monitor_queue :fetch
+
+    discard_on ActiveJob::DeserializationError
+
+    def perform(item_id)
+      item = SourceMonitor::Item.find_by(id: item_id)
+      return unless item
+      return unless SourceMonitor.config.images.download_enabled?
+
+      html = item.content
+      return if html.blank?
+
+      # Build or find item_content for attachment storage
+      item_content = item.item_content || item.build_item_content
+
+      # Skip if images already attached (idempotency)
+      return if item_content.persisted? && item_content.images.attached?
+
+      base_url = item.url
+      rewriter = SourceMonitor::Images::ContentRewriter.new(html, base_url: base_url)
+      image_urls = rewriter.image_urls
+      return if image_urls.empty?
+
+      # Save item_content first so we can attach blobs to it
+      item_content.save! unless item_content.persisted?
+
+      # Download images and build URL mapping
+      url_mapping = download_images(item_content, image_urls)
+      return if url_mapping.empty?
+
+      # Rewrite HTML with Active Storage URLs
+      rewritten_html = rewriter.rewrite do |original_url|
+        url_mapping[original_url]
+      end
+
+      # Update the item content with rewritten HTML
+      item.update!(content: rewritten_html)
+    end
+
+    private
+
+    def download_images(item_content, image_urls)
+      url_mapping = {}
+      settings = SourceMonitor.config.images
+
+      image_urls.each do |image_url|
+        result = SourceMonitor::Images::Downloader.new(image_url, settings: settings).call
+        next unless result
+
+        blob = ActiveStorage::Blob.create_and_upload!(
+          io: result.io,
+          filename: result.filename,
+          content_type: result.content_type
+        )
+        item_content.images.attach(blob)
+
+        # Generate a serving URL for the blob
+        url_mapping[image_url] = Rails.application.routes.url_helpers.rails_blob_path(blob, only_path: true)
+      rescue StandardError
+        # Individual image failure should not block others.
+        # Original URL will be preserved (graceful fallback).
+        next
+      end
+
+      url_mapping
+    end
+  end
+end

data/app/models/source_monitor/item_content.rb

@@ -6,6 +6,8 @@ module SourceMonitor
 
     validates :item, presence: true
 
+    has_many_attached :images if defined?(ActiveStorage)
+
     SourceMonitor::ModelExtensions.register(self, :item_content)
   end
 end

data/app/views/source_monitor/dashboard/_recent_activity.html.erb

@@ -21,6 +21,15 @@
             <div class="mt-1 text-xs text-slate-500">
               <%= event[:description].presence || "No additional details recorded." %>
             </div>
+            <% if event[:url_display].present? %>
+              <div class="mt-0.5 text-xs text-slate-400 truncate max-w-sm" data-testid="event-url-display">
+                <% if event[:url_href].present? %>
+                  <%= external_link_to event[:url_display], event[:url_href], class: "text-slate-400 hover:text-blue-500" %>
+                <% else %>
+                  <%= event[:url_display] %>
+                <% end %>
+              </div>
+            <% end %>
           </div>
           <div class="text-right">
             <span class="inline-flex items-center rounded-full px-3 py-1 text-xs font-semibold <%= event[:status] == :success ? "bg-green-100 text-green-700" : "bg-rose-100 text-rose-700" %>">

data/app/views/source_monitor/items/_details.html.erb

@@ -53,8 +53,8 @@
           <% details = {
                "GUID" => item.guid,
                "Content Fingerprint" => item.content_fingerprint || "—",
-               "URL" => item.url,
-               "Canonical URL" => item.canonical_url || "—",
+               "URL" => (item.url.present? ? external_link_to(item.url, item.url, class: "text-slate-900 hover:text-blue-500") : "\u2014"),
+               "Canonical URL" => (item.canonical_url.present? ? external_link_to(item.canonical_url, item.canonical_url, class: "text-slate-900 hover:text-blue-500") : "\u2014"),
                "Author" => item.author || "—",
                "Published At" => (item.published_at&.strftime("%b %d, %Y %H:%M %Z") || "—"),
                "Updated At (Source)" => (item.updated_at_source&.strftime("%b %d, %Y %H:%M %Z") || "—"),

data/app/views/source_monitor/logs/index.html.erb

@@ -134,6 +134,15 @@
               <% else %>
                 <%= row.primary_label %>
               <% end %>
+              <% if row.url_label.present? %>
+                <div class="mt-0.5 text-xs text-slate-400 truncate max-w-xs">
+                  <% if row.url_href.present? %>
+                    <%= external_link_to row.url_label, row.url_href, class: "text-slate-400 hover:text-blue-500" %>
+                  <% else %>
+                    <%= row.url_label %>
+                  <% end %>
+                </div>
+              <% end %>
             </td>
             <td class="px-6 py-4 text-sm">
               <% if row.source_path %>

data/app/views/source_monitor/sources/_details.html.erb

@@ -25,7 +25,7 @@
           </span>
         <% end %>
       </div>
-      <p class="mt-2 text-sm text-slate-500">Feed URL: <%= source.feed_url %></p>
+      <p class="mt-2 text-sm text-slate-500">Feed URL: <%= external_link_to source.feed_url, source.feed_url, class: "text-slate-500 hover:text-blue-500" %></p>
     </div>
     <div class="flex flex-wrap items-center justify-end gap-3">
       <% fetch_disabled = %w[queued fetching].include?(source.fetch_status) %>
@@ -137,7 +137,7 @@
                end
 
              details = {
-               "Website" => (source.website_url.presence || "—"),
+               "Website" => (source.website_url.present? ? external_link_to(source.website_url, source.website_url, class: "text-slate-900 hover:text-blue-500") : "\u2014"),
                "Fetch interval" => "#{source.fetch_interval_minutes} minutes (~#{interval_hours} hours)",
                "Adaptive interval" => source.adaptive_fetching_enabled? ? "Auto" : "Fixed",
                "Scraper" => source.scraper_adapter,

data/app/views/source_monitor/sources/_row.html.erb

@@ -29,7 +29,7 @@
             class: "text-slate-900 hover:text-blue-600 focus:outline-none focus:ring-2 focus:ring-blue-500 focus:ring-offset-2",
             data: { turbo_frame: "_top" } %>
     </div>
-    <div class="text-xs text-slate-500 truncate max-w-xs"><%= source.feed_url %></div>
+    <div class="text-xs text-slate-500 truncate max-w-xs"><%= external_link_to source.feed_url, source.feed_url, class: "text-slate-500 hover:text-blue-500" %></div>
   </td>
   <td class="px-6 py-4">
     <div class="flex flex-col gap-2 text-xs">