source_monitor 0.3.3 → 0.5.0

data/lib/source_monitor/images/content_rewriter.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+require "nokolexbor"
+require "uri"
+
+module SourceMonitor
+  module Images
+    class ContentRewriter
+      attr_reader :html, :base_url
+
+      def initialize(html, base_url: nil)
+        @html = html.to_s
+        @base_url = base_url
+      end
+
+      # Returns an array of absolute image URLs found in <img> tags.
+      # Skips data: URIs, blank src, and invalid URLs.
+      def image_urls
+        return [] if html.blank?
+
+        doc = parse_fragment
+        urls = []
+
+        doc.css("img[src]").each do |img|
+          url = resolve_url(img["src"])
+          urls << url if url && downloadable_url?(url)
+        end
+
+        urls.uniq
+      end
+
+      # Rewrites <img src="..."> attributes by yielding each original URL
+      # to the block and replacing with the block's return value.
+      # Returns the rewritten HTML string.
+      # If the block returns nil, the original URL is preserved (graceful fallback).
+      def rewrite
+        return html if html.blank?
+
+        doc = parse_fragment
+
+        doc.css("img[src]").each do |img|
+          original_url = resolve_url(img["src"])
+          next unless original_url && downloadable_url?(original_url)
+
+          new_url = yield(original_url)
+          img["src"] = new_url if new_url.present?
+        end
+
+        doc.to_html
+      end
+
+      private
+
+      def parse_fragment
+        Nokolexbor::DocumentFragment.parse(html)
+      end
+
+      def resolve_url(src)
+        src = src.to_s.strip
+        return nil if src.blank?
+        return nil if src.start_with?("data:")
+
+        uri = URI.parse(src)
+        if uri.relative? && base_url.present?
+          URI.join(base_url, src).to_s
+        elsif uri.absolute?
+          src
+        end
+      rescue URI::InvalidURIError
+        nil
+      end
+
+      def downloadable_url?(url)
+        uri = URI.parse(url)
+        uri.is_a?(URI::HTTP) || uri.is_a?(URI::HTTPS)
+      rescue URI::InvalidURIError
+        false
+      end
+    end
+  end
+end

data/lib/source_monitor/images/downloader.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+require "faraday"
+require "securerandom"
+
+module SourceMonitor
+  module Images
+    class Downloader
+      Result = Struct.new(:io, :filename, :content_type, :byte_size, keyword_init: true)
+
+      attr_reader :url, :settings
+
+      def initialize(url, settings: nil)
+        @url = url
+        @settings = settings || SourceMonitor.config.images
+      end
+
+      # Downloads the image and returns a Result, or nil if download fails
+      # or the image does not meet validation criteria.
+      def call
+        response = fetch_image
+        return unless response
+
+        content_type = response.headers["content-type"]&.split(";")&.first&.strip&.downcase
+        return unless allowed_content_type?(content_type)
+
+        body = response.body
+        return unless body && body.bytesize > 0
+        return if body.bytesize > settings.max_download_size
+
+        filename = derive_filename(url, content_type)
+
+        Result.new(
+          io: StringIO.new(body),
+          filename: filename,
+          content_type: content_type,
+          byte_size: body.bytesize
+        )
+      rescue Faraday::Error, URI::InvalidURIError, Timeout::Error
+        nil
+      end
+
+      private
+
+      def fetch_image
+        connection = Faraday.new do |f|
+          f.options.timeout = settings.download_timeout
+          f.options.open_timeout = [ settings.download_timeout / 2, 5 ].min
+          f.headers["User-Agent"] = SourceMonitor.config.http.user_agent || "SourceMonitor/#{SourceMonitor::VERSION}"
+          f.headers["Accept"] = "image/*"
+          f.adapter Faraday.default_adapter
+        end
+
+        response = connection.get(url)
+        return response if response.status == 200
+
+        nil
+      end
+
+      def allowed_content_type?(content_type)
+        return false if content_type.blank?
+
+        settings.allowed_content_types.include?(content_type)
+      end
+
+      def derive_filename(image_url, content_type)
+        uri = URI.parse(image_url)
+        basename = File.basename(uri.path) if uri.path.present?
+
+        if basename.present? && basename.include?(".")
+          basename
+        else
+          ext = Rack::Mime::MIME_TYPES.invert[content_type] || ".bin"
+          "image-#{SecureRandom.hex(8)}#{ext}"
+        end
+      rescue URI::InvalidURIError
+        ext = Rack::Mime::MIME_TYPES.invert[content_type] || ".bin"
+        "image-#{SecureRandom.hex(8)}#{ext}"
+      end
+    end
+  end
+end

data/lib/source_monitor/logs/table_presenter.rb

@@ -61,6 +61,22 @@ module SourceMonitor
           end
         end
 
+        def url_label
+          if fetch?
+            domain_from_feed_url
+          elsif scrape?
+            entry.item&.url
+          end
+        end
+
+        def url_href
+          if fetch?
+            entry.source&.feed_url
+          elsif scrape?
+            entry.item&.url
+          end
+        end
+
         def source_label
           entry.source&.name
         end
@@ -142,6 +158,15 @@ module SourceMonitor
         private
 
         attr_reader :entry, :url_helpers
+
+        def domain_from_feed_url
+          feed_url = entry.source&.feed_url
+          return nil if feed_url.blank?
+
+          URI.parse(feed_url.to_s).host
+        rescue URI::InvalidURIError
+          nil
+        end
       end
 
       def initialize(entries:, url_helpers:)

data/lib/source_monitor/setup/cli.rb

@@ -23,6 +23,13 @@ module SourceMonitor
         handle_summary(summary)
       end
 
+      desc "upgrade", "Upgrade SourceMonitor after a gem version change"
+      def upgrade
+        command = UpgradeCommand.new
+        summary = command.call
+        handle_summary(summary)
+      end
+
       private
 
       def handle_summary(summary)

data/lib/source_monitor/setup/procfile_patcher.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require "pathname"
+
+module SourceMonitor
+  module Setup
+    class ProcfilePatcher
+      JOBS_ENTRY = "jobs: bundle exec rake solid_queue:start"
+
+      def initialize(path: "Procfile.dev")
+        @path = Pathname.new(path)
+      end
+
+      def patch
+        if path.exist?
+          content = path.read
+          return false if content.match?(/^jobs:/)
+
+          path.open("a") { |f| f.puts("", JOBS_ENTRY) }
+        else
+          path.write("web: bin/rails server -p 3000\n#{JOBS_ENTRY}\n")
+        end
+        true
+      end
+
+      private
+
+      attr_reader :path
+    end
+  end
+end

data/lib/source_monitor/setup/queue_config_patcher.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+require "pathname"
+require "yaml"
+
+module SourceMonitor
+  module Setup
+    class QueueConfigPatcher
+      RECURRING_SCHEDULE_VALUE = "config/recurring.yml"
+
+      DEFAULT_DISPATCHER = {
+        "polling_interval" => 1,
+        "batch_size" => 500,
+        "recurring_schedule" => RECURRING_SCHEDULE_VALUE
+      }.freeze
+
+      def initialize(path: "config/queue.yml")
+        @path = Pathname.new(path)
+      end
+
+      def patch
+        return false unless path.exist?
+
+        parsed = YAML.safe_load(path.read, aliases: true) || {}
+        return false if has_recurring_schedule?(parsed)
+
+        add_recurring_schedule!(parsed)
+        path.write(YAML.dump(parsed))
+        true
+      end
+
+      private
+
+      attr_reader :path
+
+      def has_recurring_schedule?(parsed)
+        parsed.each_value do |value|
+          next unless value.is_a?(Hash)
+
+          dispatchers = value["dispatchers"]
+          if dispatchers.is_a?(Array)
+            return true if dispatchers.any? { |d| d.is_a?(Hash) && d.key?("recurring_schedule") }
+          end
+
+          return true if has_recurring_schedule?(value)
+        end
+
+        if parsed.key?("dispatchers") && parsed["dispatchers"].is_a?(Array)
+          return true if parsed["dispatchers"].any? { |d| d.is_a?(Hash) && d.key?("recurring_schedule") }
+        end
+
+        false
+      end
+
+      def add_recurring_schedule!(parsed)
+        found_dispatchers = false
+
+        parsed.each_value do |value|
+          next unless value.is_a?(Hash)
+
+          if value.key?("dispatchers") && value["dispatchers"].is_a?(Array)
+            value["dispatchers"].each do |dispatcher|
+              next unless dispatcher.is_a?(Hash)
+              dispatcher["recurring_schedule"] ||= RECURRING_SCHEDULE_VALUE
+            end
+            found_dispatchers = true
+          end
+        end
+
+        if parsed.key?("dispatchers") && parsed["dispatchers"].is_a?(Array)
+          parsed["dispatchers"].each do |dispatcher|
+            next unless dispatcher.is_a?(Hash)
+            dispatcher["recurring_schedule"] ||= RECURRING_SCHEDULE_VALUE
+          end
+          found_dispatchers = true
+        end
+
+        unless found_dispatchers
+          parsed["dispatchers"] = [ DEFAULT_DISPATCHER.dup ]
+        end
+      end
+    end
+  end
+end

data/lib/source_monitor/setup/skills_installer.rb

@@ -11,6 +11,7 @@ module SourceMonitor
       CONSUMER_SKILLS = %w[
         sm-host-setup sm-configure sm-scraper-adapter
         sm-event-handler sm-model-extension sm-dashboard-widget
+        sm-upgrade
       ].freeze
 
       CONTRIBUTOR_SKILLS = %w[

data/lib/source_monitor/setup/upgrade_command.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  module Setup
+    class UpgradeCommand
+      def initialize(
+        migration_installer: MigrationInstaller.new,
+        install_generator: InstallGenerator.new,
+        verifier: Verification::Runner.new,
+        version_file: File.join(Dir.pwd, ".source_monitor_version"),
+        current_version: SourceMonitor::VERSION
+      )
+        @migration_installer = migration_installer
+        @install_generator = install_generator
+        @verifier = verifier
+        @version_file = version_file
+        @current_version = current_version
+      end
+
+      def call
+        stored = read_stored_version
+
+        if stored == current_version
+          return up_to_date_summary
+        end
+
+        migration_installer.install
+        install_generator.run
+        summary = verifier.call
+        write_version_marker
+        summary
+      end
+
+      private
+
+      attr_reader :migration_installer, :install_generator, :verifier, :version_file, :current_version
+
+      def read_stored_version
+        return nil unless File.exist?(version_file)
+
+        File.read(version_file).strip
+      end
+
+      def write_version_marker
+        File.write(version_file, current_version)
+      end
+
+      def up_to_date_summary
+        result = Verification::Result.new(
+          key: :upgrade,
+          name: "Upgrade",
+          status: :ok,
+          details: "Already up to date (v#{current_version})"
+        )
+        Verification::Summary.new([ result ])
+      end
+    end
+  end
+end

data/lib/source_monitor/setup/verification/pending_migrations_verifier.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  module Setup
+    module Verification
+      class PendingMigrationsVerifier
+        MIGRATION_TIMESTAMP_PATTERN = /\A\d+_/
+
+        def initialize(
+          engine_migrations_path: default_engine_migrations_path,
+          host_migrations_path: default_host_migrations_path,
+          connection: default_connection
+        )
+          @engine_migrations_path = engine_migrations_path
+          @host_migrations_path = host_migrations_path
+          @connection = connection
+        end
+
+        def call
+          engine_names = source_monitor_migration_names(engine_migrations_path)
+          return ok_result("No SourceMonitor engine migrations found") if engine_names.empty?
+
+          host_names = migration_names(host_migrations_path)
+          missing = engine_names - host_names
+
+          if missing.any?
+            warning_result(
+              "#{missing.size} SourceMonitor migration(s) not copied to host: #{missing.join(', ')}",
+              "Run `bin/source_monitor upgrade` or `bin/rails railties:install:migrations FROM=source_monitor`"
+            )
+          elsif connection.migration_context.needs_migration?
+            warning_result(
+              "All SourceMonitor migrations are copied but some migrations are pending",
+              "Run `bin/rails db:migrate` to apply pending migrations"
+            )
+          else
+            ok_result("All SourceMonitor migrations are present and up to date")
+          end
+        rescue StandardError => e
+          error_result(
+            "Migration verification failed: #{e.message}",
+            "Check database connectivity and migration file permissions"
+          )
+        end
+
+        private
+
+        attr_reader :engine_migrations_path, :host_migrations_path, :connection
+
+        def default_engine_migrations_path
+          SourceMonitor::Engine.root.join("db/migrate")
+        end
+
+        def default_host_migrations_path
+          Rails.root.join("db/migrate")
+        end
+
+        def default_connection
+          ActiveRecord::Base.connection
+        end
+
+        def source_monitor_migration_names(path)
+          migration_names(path).select { |name| name.include?("source_monitor") }
+        end
+
+        def migration_names(path)
+          return [] unless File.directory?(path.to_s)
+
+          Dir.children(path.to_s)
+            .select { |f| f.end_with?(".rb") }
+            .map { |f| strip_timestamp(f) }
+        end
+
+        def strip_timestamp(filename)
+          filename.sub(MIGRATION_TIMESTAMP_PATTERN, "").delete_suffix(".rb")
+        end
+
+        def ok_result(details)
+          Result.new(key: :pending_migrations, name: "Pending Migrations", status: :ok, details: details)
+        end
+
+        def warning_result(details, remediation)
+          Result.new(key: :pending_migrations, name: "Pending Migrations", status: :warning, details: details, remediation: remediation)
+        end
+
+        def error_result(details, remediation)
+          Result.new(key: :pending_migrations, name: "Pending Migrations", status: :error, details: details, remediation: remediation)
+        end
+      end
+    end
+  end
+end

data/lib/source_monitor/setup/verification/recurring_schedule_verifier.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  module Setup
+    module Verification
+      class RecurringScheduleVerifier
+        SOURCE_MONITOR_KEY_PREFIX = "source_monitor_"
+        SOURCE_MONITOR_NAMESPACE = "SourceMonitor::"
+
+        def initialize(task_relation: default_task_relation, connection: default_connection)
+          @task_relation = task_relation
+          @connection = connection
+        end
+
+        def call
+          return missing_gem_result unless task_relation
+          return missing_tables_result unless tables_present?
+
+          tasks = all_tasks
+          sm_tasks = source_monitor_tasks(tasks)
+
+          if sm_tasks.any?
+            ok_result("#{sm_tasks.size} SourceMonitor recurring task(s) registered")
+          elsif tasks.any?
+            warning_result(
+              "Recurring tasks exist but none belong to SourceMonitor",
+              "Add SourceMonitor entries to config/recurring.yml and ensure the dispatcher has `recurring_schedule: config/recurring.yml`"
+            )
+          else
+            warning_result(
+              "No recurring tasks are registered with Solid Queue",
+              "Configure a dispatcher with `recurring_schedule: config/recurring.yml` in config/queue.yml and ensure recurring.yml contains SourceMonitor task entries"
+            )
+          end
+        rescue StandardError => e
+          error_result(
+            "Recurring schedule verification failed: #{e.message}",
+            "Verify Solid Queue migrations are up to date and the dispatcher is configured with recurring_schedule"
+          )
+        end
+
+        private
+
+        attr_reader :task_relation, :connection
+
+        def default_task_relation
+          SolidQueue::RecurringTask if defined?(SolidQueue::RecurringTask)
+        end
+
+        def default_connection
+          SolidQueue::RecurringTask.connection if defined?(SolidQueue::RecurringTask)
+        rescue StandardError
+          nil
+        end
+
+        def tables_present?
+          return false unless connection
+
+          connection.table_exists?(task_relation.table_name)
+        end
+
+        def all_tasks
+          task_relation.all.to_a
+        end
+
+        def source_monitor_tasks(tasks)
+          tasks.select do |task|
+            task.key.start_with?(SOURCE_MONITOR_KEY_PREFIX) ||
+              task.class_name.to_s.start_with?(SOURCE_MONITOR_NAMESPACE) ||
+              task.command.to_s.include?(SOURCE_MONITOR_NAMESPACE)
+          end
+        end
+
+        def missing_gem_result
+          error_result(
+            "Solid Queue gem is not available",
+            "Add `solid_queue` to your Gemfile and bundle install"
+          )
+        end
+
+        def missing_tables_result
+          error_result(
+            "Solid Queue recurring tasks table is missing",
+            "Run `rails solid_queue:install` or copy the engine's Solid Queue migration"
+          )
+        end
+
+        def ok_result(details)
+          Result.new(key: :recurring_schedule, name: "Recurring Schedule", status: :ok, details: details)
+        end
+
+        def warning_result(details, remediation)
+          Result.new(key: :recurring_schedule, name: "Recurring Schedule", status: :warning, details: details, remediation: remediation)
+        end
+
+        def error_result(details, remediation)
+          Result.new(key: :recurring_schedule, name: "Recurring Schedule", status: :error, details: details, remediation: remediation)
+        end
+      end
+    end
+  end
+end

data/lib/source_monitor/setup/verification/runner.rb

@@ -18,7 +18,7 @@ module SourceMonitor
         attr_reader :verifiers
 
         def default_verifiers
-          [ SolidQueueVerifier.new, ActionCableVerifier.new ]
+          [ PendingMigrationsVerifier.new, SolidQueueVerifier.new, RecurringScheduleVerifier.new, ActionCableVerifier.new ]
         end
       end
     end

data/lib/source_monitor/setup/verification/solid_queue_verifier.rb

@@ -21,7 +21,7 @@ module SourceMonitor
           if recent
             ok_result("Solid Queue workers are reporting heartbeats")
           else
-            warning_result("No Solid Queue workers have reported in the last #{DEFAULT_HEARTBEAT_THRESHOLD.inspect}", "Start a Solid Queue worker with `bin/rails solid_queue:start` and ensure it stays running")
+            warning_result("No Solid Queue workers have reported in the last #{DEFAULT_HEARTBEAT_THRESHOLD.inspect}", "Start a Solid Queue worker with `bin/rails solid_queue:start` or add `jobs: bundle exec rake solid_queue:start` to Procfile.dev and run `bin/dev`")
           end
         rescue StandardError => e
           error_result("Solid Queue verification failed: #{e.message}", "Verify Solid Queue migrations are up to date and workers can access the database")

data/lib/source_monitor/setup/workflow.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 require "pathname"
+require_relative "procfile_patcher"
+require_relative "queue_config_patcher"
 require_relative "skills_installer"
 
 module SourceMonitor
@@ -35,6 +37,8 @@ module SourceMonitor
         install_generator: InstallGenerator.new,
         migration_installer: MigrationInstaller.new,
         initializer_patcher: InitializerPatcher.new,
+        procfile_patcher: ProcfilePatcher.new,
+        queue_config_patcher: QueueConfigPatcher.new,
         devise_detector: method(:default_devise_detector),
         verifier: Verification::Runner.new,
         skills_installer: SkillsInstaller.new
@@ -47,6 +51,8 @@ module SourceMonitor
         @install_generator = install_generator
         @migration_installer = migration_installer
         @initializer_patcher = initializer_patcher
+        @procfile_patcher = procfile_patcher
+        @queue_config_patcher = queue_config_patcher
         @devise_detector = devise_detector
         @verifier = verifier
         @skills_installer = skills_installer
@@ -64,6 +70,8 @@ module SourceMonitor
         install_generator.run(mount_path: mount_path)
         migration_installer.install
         initializer_patcher.ensure_navigation_hint(mount_path: mount_path)
+        procfile_patcher.patch
+        queue_config_patcher.patch
 
         if devise_available? && prompter.yes?("Wire Devise authentication hooks into SourceMonitor?", default: true)
           initializer_patcher.ensure_devise_hooks
@@ -89,6 +97,8 @@ module SourceMonitor
         :install_generator,
         :migration_installer,
         :initializer_patcher,
+        :procfile_patcher,
+        :queue_config_patcher,
         :devise_detector,
         :verifier,
         :skills_installer

data/lib/source_monitor/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module SourceMonitor
-  VERSION = "0.3.3"
+  VERSION = "0.5.0"
 end