source_monitor 0.3.2 → 0.4.0

data/lib/source_monitor/images/content_rewriter.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+require "nokolexbor"
+require "uri"
+
+module SourceMonitor
+  module Images
+    class ContentRewriter
+      attr_reader :html, :base_url
+
+      def initialize(html, base_url: nil)
+        @html = html.to_s
+        @base_url = base_url
+      end
+
+      # Returns an array of absolute image URLs found in <img> tags.
+      # Skips data: URIs, blank src, and invalid URLs.
+      def image_urls
+        return [] if html.blank?
+
+        doc = parse_fragment
+        urls = []
+
+        doc.css("img[src]").each do |img|
+          url = resolve_url(img["src"])
+          urls << url if url && downloadable_url?(url)
+        end
+
+        urls.uniq
+      end
+
+      # Rewrites <img src="..."> attributes by yielding each original URL
+      # to the block and replacing with the block's return value.
+      # Returns the rewritten HTML string.
+      # If the block returns nil, the original URL is preserved (graceful fallback).
+      def rewrite
+        return html if html.blank?
+
+        doc = parse_fragment
+
+        doc.css("img[src]").each do |img|
+          original_url = resolve_url(img["src"])
+          next unless original_url && downloadable_url?(original_url)
+
+          new_url = yield(original_url)
+          img["src"] = new_url if new_url.present?
+        end
+
+        doc.to_html
+      end
+
+      private
+
+      def parse_fragment
+        Nokolexbor::DocumentFragment.parse(html)
+      end
+
+      def resolve_url(src)
+        src = src.to_s.strip
+        return nil if src.blank?
+        return nil if src.start_with?("data:")
+
+        uri = URI.parse(src)
+        if uri.relative? && base_url.present?
+          URI.join(base_url, src).to_s
+        elsif uri.absolute?
+          src
+        end
+      rescue URI::InvalidURIError
+        nil
+      end
+
+      def downloadable_url?(url)
+        uri = URI.parse(url)
+        uri.is_a?(URI::HTTP) || uri.is_a?(URI::HTTPS)
+      rescue URI::InvalidURIError
+        false
+      end
+    end
+  end
+end

data/lib/source_monitor/images/downloader.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+require "faraday"
+require "securerandom"
+
+module SourceMonitor
+  module Images
+    class Downloader
+      Result = Struct.new(:io, :filename, :content_type, :byte_size, keyword_init: true)
+
+      attr_reader :url, :settings
+
+      def initialize(url, settings: nil)
+        @url = url
+        @settings = settings || SourceMonitor.config.images
+      end
+
+      # Downloads the image and returns a Result, or nil if download fails
+      # or the image does not meet validation criteria.
+      def call
+        response = fetch_image
+        return unless response
+
+        content_type = response.headers["content-type"]&.split(";")&.first&.strip&.downcase
+        return unless allowed_content_type?(content_type)
+
+        body = response.body
+        return unless body && body.bytesize > 0
+        return if body.bytesize > settings.max_download_size
+
+        filename = derive_filename(url, content_type)
+
+        Result.new(
+          io: StringIO.new(body),
+          filename: filename,
+          content_type: content_type,
+          byte_size: body.bytesize
+        )
+      rescue Faraday::Error, URI::InvalidURIError, Timeout::Error
+        nil
+      end
+
+      private
+
+      def fetch_image
+        connection = Faraday.new do |f|
+          f.options.timeout = settings.download_timeout
+          f.options.open_timeout = [ settings.download_timeout / 2, 5 ].min
+          f.headers["User-Agent"] = SourceMonitor.config.http.user_agent || "SourceMonitor/#{SourceMonitor::VERSION}"
+          f.headers["Accept"] = "image/*"
+          f.adapter Faraday.default_adapter
+        end
+
+        response = connection.get(url)
+        return response if response.status == 200
+
+        nil
+      end
+
+      def allowed_content_type?(content_type)
+        return false if content_type.blank?
+
+        settings.allowed_content_types.include?(content_type)
+      end
+
+      def derive_filename(image_url, content_type)
+        uri = URI.parse(image_url)
+        basename = File.basename(uri.path) if uri.path.present?
+
+        if basename.present? && basename.include?(".")
+          basename
+        else
+          ext = Rack::Mime::MIME_TYPES.invert[content_type] || ".bin"
+          "image-#{SecureRandom.hex(8)}#{ext}"
+        end
+      rescue URI::InvalidURIError
+        ext = Rack::Mime::MIME_TYPES.invert[content_type] || ".bin"
+        "image-#{SecureRandom.hex(8)}#{ext}"
+      end
+    end
+  end
+end

data/lib/source_monitor/logs/table_presenter.rb

@@ -61,6 +61,22 @@ module SourceMonitor
           end
         end
 
+        def url_label
+          if fetch?
+            domain_from_feed_url
+          elsif scrape?
+            entry.item&.url
+          end
+        end
+
+        def url_href
+          if fetch?
+            entry.source&.feed_url
+          elsif scrape?
+            entry.item&.url
+          end
+        end
+
         def source_label
           entry.source&.name
         end
@@ -142,6 +158,15 @@ module SourceMonitor
         private
 
         attr_reader :entry, :url_helpers
+
+        def domain_from_feed_url
+          feed_url = entry.source&.feed_url
+          return nil if feed_url.blank?
+
+          URI.parse(feed_url.to_s).host
+        rescue URI::InvalidURIError
+          nil
+        end
       end
 
       def initialize(entries:, url_helpers:)

data/lib/source_monitor/setup/procfile_patcher.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require "pathname"
+
+module SourceMonitor
+  module Setup
+    class ProcfilePatcher
+      JOBS_ENTRY = "jobs: bundle exec rake solid_queue:start"
+
+      def initialize(path: "Procfile.dev")
+        @path = Pathname.new(path)
+      end
+
+      def patch
+        if path.exist?
+          content = path.read
+          return false if content.match?(/^jobs:/)
+
+          path.open("a") { |f| f.puts("", JOBS_ENTRY) }
+        else
+          path.write("web: bin/rails server -p 3000\n#{JOBS_ENTRY}\n")
+        end
+        true
+      end
+
+      private
+
+      attr_reader :path
+    end
+  end
+end

data/lib/source_monitor/setup/queue_config_patcher.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+require "pathname"
+require "yaml"
+
+module SourceMonitor
+  module Setup
+    class QueueConfigPatcher
+      RECURRING_SCHEDULE_VALUE = "config/recurring.yml"
+
+      DEFAULT_DISPATCHER = {
+        "polling_interval" => 1,
+        "batch_size" => 500,
+        "recurring_schedule" => RECURRING_SCHEDULE_VALUE
+      }.freeze
+
+      def initialize(path: "config/queue.yml")
+        @path = Pathname.new(path)
+      end
+
+      def patch
+        return false unless path.exist?
+
+        parsed = YAML.safe_load(path.read, aliases: true) || {}
+        return false if has_recurring_schedule?(parsed)
+
+        add_recurring_schedule!(parsed)
+        path.write(YAML.dump(parsed))
+        true
+      end
+
+      private
+
+      attr_reader :path
+
+      def has_recurring_schedule?(parsed)
+        parsed.each_value do |value|
+          next unless value.is_a?(Hash)
+
+          dispatchers = value["dispatchers"]
+          if dispatchers.is_a?(Array)
+            return true if dispatchers.any? { |d| d.is_a?(Hash) && d.key?("recurring_schedule") }
+          end
+
+          return true if has_recurring_schedule?(value)
+        end
+
+        if parsed.key?("dispatchers") && parsed["dispatchers"].is_a?(Array)
+          return true if parsed["dispatchers"].any? { |d| d.is_a?(Hash) && d.key?("recurring_schedule") }
+        end
+
+        false
+      end
+
+      def add_recurring_schedule!(parsed)
+        found_dispatchers = false
+
+        parsed.each_value do |value|
+          next unless value.is_a?(Hash)
+
+          if value.key?("dispatchers") && value["dispatchers"].is_a?(Array)
+            value["dispatchers"].each do |dispatcher|
+              next unless dispatcher.is_a?(Hash)
+              dispatcher["recurring_schedule"] ||= RECURRING_SCHEDULE_VALUE
+            end
+            found_dispatchers = true
+          end
+        end
+
+        if parsed.key?("dispatchers") && parsed["dispatchers"].is_a?(Array)
+          parsed["dispatchers"].each do |dispatcher|
+            next unless dispatcher.is_a?(Hash)
+            dispatcher["recurring_schedule"] ||= RECURRING_SCHEDULE_VALUE
+          end
+          found_dispatchers = true
+        end
+
+        unless found_dispatchers
+          parsed["dispatchers"] = [ DEFAULT_DISPATCHER.dup ]
+        end
+      end
+    end
+  end
+end

data/lib/source_monitor/setup/verification/recurring_schedule_verifier.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  module Setup
+    module Verification
+      class RecurringScheduleVerifier
+        SOURCE_MONITOR_KEY_PREFIX = "source_monitor_"
+        SOURCE_MONITOR_NAMESPACE = "SourceMonitor::"
+
+        def initialize(task_relation: default_task_relation, connection: default_connection)
+          @task_relation = task_relation
+          @connection = connection
+        end
+
+        def call
+          return missing_gem_result unless task_relation
+          return missing_tables_result unless tables_present?
+
+          tasks = all_tasks
+          sm_tasks = source_monitor_tasks(tasks)
+
+          if sm_tasks.any?
+            ok_result("#{sm_tasks.size} SourceMonitor recurring task(s) registered")
+          elsif tasks.any?
+            warning_result(
+              "Recurring tasks exist but none belong to SourceMonitor",
+              "Add SourceMonitor entries to config/recurring.yml and ensure the dispatcher has `recurring_schedule: config/recurring.yml`"
+            )
+          else
+            warning_result(
+              "No recurring tasks are registered with Solid Queue",
+              "Configure a dispatcher with `recurring_schedule: config/recurring.yml` in config/queue.yml and ensure recurring.yml contains SourceMonitor task entries"
+            )
+          end
+        rescue StandardError => e
+          error_result(
+            "Recurring schedule verification failed: #{e.message}",
+            "Verify Solid Queue migrations are up to date and the dispatcher is configured with recurring_schedule"
+          )
+        end
+
+        private
+
+        attr_reader :task_relation, :connection
+
+        def default_task_relation
+          SolidQueue::RecurringTask if defined?(SolidQueue::RecurringTask)
+        end
+
+        def default_connection
+          SolidQueue::RecurringTask.connection if defined?(SolidQueue::RecurringTask)
+        rescue StandardError
+          nil
+        end
+
+        def tables_present?
+          return false unless connection
+
+          connection.table_exists?(task_relation.table_name)
+        end
+
+        def all_tasks
+          task_relation.all.to_a
+        end
+
+        def source_monitor_tasks(tasks)
+          tasks.select do |task|
+            task.key.start_with?(SOURCE_MONITOR_KEY_PREFIX) ||
+              task.class_name.to_s.start_with?(SOURCE_MONITOR_NAMESPACE) ||
+              task.command.to_s.include?(SOURCE_MONITOR_NAMESPACE)
+          end
+        end
+
+        def missing_gem_result
+          error_result(
+            "Solid Queue gem is not available",
+            "Add `solid_queue` to your Gemfile and bundle install"
+          )
+        end
+
+        def missing_tables_result
+          error_result(
+            "Solid Queue recurring tasks table is missing",
+            "Run `rails solid_queue:install` or copy the engine's Solid Queue migration"
+          )
+        end
+
+        def ok_result(details)
+          Result.new(key: :recurring_schedule, name: "Recurring Schedule", status: :ok, details: details)
+        end
+
+        def warning_result(details, remediation)
+          Result.new(key: :recurring_schedule, name: "Recurring Schedule", status: :warning, details: details, remediation: remediation)
+        end
+
+        def error_result(details, remediation)
+          Result.new(key: :recurring_schedule, name: "Recurring Schedule", status: :error, details: details, remediation: remediation)
+        end
+      end
+    end
+  end
+end

data/lib/source_monitor/setup/verification/runner.rb

@@ -18,7 +18,7 @@ module SourceMonitor
         attr_reader :verifiers
 
         def default_verifiers
-          [ SolidQueueVerifier.new, ActionCableVerifier.new ]
+          [ SolidQueueVerifier.new, RecurringScheduleVerifier.new, ActionCableVerifier.new ]
         end
       end
     end

data/lib/source_monitor/setup/verification/solid_queue_verifier.rb

@@ -21,7 +21,7 @@ module SourceMonitor
           if recent
             ok_result("Solid Queue workers are reporting heartbeats")
           else
-            warning_result("No Solid Queue workers have reported in the last #{DEFAULT_HEARTBEAT_THRESHOLD.inspect}", "Start a Solid Queue worker with `bin/rails solid_queue:start` and ensure it stays running")
+            warning_result("No Solid Queue workers have reported in the last #{DEFAULT_HEARTBEAT_THRESHOLD.inspect}", "Start a Solid Queue worker with `bin/rails solid_queue:start` or add `jobs: bundle exec rake solid_queue:start` to Procfile.dev and run `bin/dev`")
           end
         rescue StandardError => e
           error_result("Solid Queue verification failed: #{e.message}", "Verify Solid Queue migrations are up to date and workers can access the database")

data/lib/source_monitor/setup/workflow.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 require "pathname"
+require_relative "procfile_patcher"
+require_relative "queue_config_patcher"
 require_relative "skills_installer"
 
 module SourceMonitor
@@ -35,6 +37,8 @@ module SourceMonitor
         install_generator: InstallGenerator.new,
         migration_installer: MigrationInstaller.new,
         initializer_patcher: InitializerPatcher.new,
+        procfile_patcher: ProcfilePatcher.new,
+        queue_config_patcher: QueueConfigPatcher.new,
         devise_detector: method(:default_devise_detector),
         verifier: Verification::Runner.new,
         skills_installer: SkillsInstaller.new
@@ -47,6 +51,8 @@ module SourceMonitor
         @install_generator = install_generator
         @migration_installer = migration_installer
         @initializer_patcher = initializer_patcher
+        @procfile_patcher = procfile_patcher
+        @queue_config_patcher = queue_config_patcher
         @devise_detector = devise_detector
         @verifier = verifier
         @skills_installer = skills_installer
@@ -64,6 +70,8 @@ module SourceMonitor
         install_generator.run(mount_path: mount_path)
         migration_installer.install
         initializer_patcher.ensure_navigation_hint(mount_path: mount_path)
+        procfile_patcher.patch
+        queue_config_patcher.patch
 
         if devise_available? && prompter.yes?("Wire Devise authentication hooks into SourceMonitor?", default: true)
           initializer_patcher.ensure_devise_hooks
@@ -89,6 +97,8 @@ module SourceMonitor
         :install_generator,
         :migration_installer,
         :initializer_patcher,
+        :procfile_patcher,
+        :queue_config_patcher,
         :devise_detector,
         :verifier,
         :skills_installer

data/lib/source_monitor/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module SourceMonitor
-  VERSION = "0.3.2"
+  VERSION = "0.4.0"
 end

data/lib/source_monitor.rb

@@ -87,6 +87,11 @@ module SourceMonitor
     autoload :HealthCheckBroadcaster, "source_monitor/import_sessions/health_check_broadcaster"
   end
 
+  module Images
+    autoload :ContentRewriter, "source_monitor/images/content_rewriter"
+    autoload :Downloader, "source_monitor/images/downloader"
+  end
+
   module Items
     autoload :ItemCreator, "source_monitor/items/item_creator"
     autoload :RetentionPruner, "source_monitor/items/retention_pruner"
@@ -161,6 +166,8 @@ module SourceMonitor
     autoload :InstallGenerator, "source_monitor/setup/install_generator"
     autoload :MigrationInstaller, "source_monitor/setup/migration_installer"
     autoload :InitializerPatcher, "source_monitor/setup/initializer_patcher"
+    autoload :ProcfilePatcher, "source_monitor/setup/procfile_patcher"
+    autoload :QueueConfigPatcher, "source_monitor/setup/queue_config_patcher"
     autoload :Workflow, "source_monitor/setup/workflow"
     autoload :CLI, "source_monitor/setup/cli"
 
@@ -169,6 +176,7 @@ module SourceMonitor
       autoload :Summary, "source_monitor/setup/verification/result"
       autoload :SolidQueueVerifier, "source_monitor/setup/verification/solid_queue_verifier"
       autoload :ActionCableVerifier, "source_monitor/setup/verification/action_cable_verifier"
+      autoload :RecurringScheduleVerifier, "source_monitor/setup/verification/recurring_schedule_verifier"
       autoload :Runner, "source_monitor/setup/verification/runner"
       autoload :Printer, "source_monitor/setup/verification/printer"
       autoload :TelemetryLogger, "source_monitor/setup/verification/telemetry_logger"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: source_monitor
 version: !ruby/object:Gem::Version
-  version: 0.3.2
+  version: 0.4.0
 platform: ruby
 authors:
 - dchuk
@@ -238,6 +238,8 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".claude/agent-memory/vbw-vbw-dev/MEMORY.md"
+- ".claude/agent-memory/vbw-vbw-lead/MEMORY.md"
 - ".claude/agents/rails-concern.md"
 - ".claude/agents/rails-controller.md"
 - ".claude/agents/rails-hotwire.md"
@@ -256,6 +258,7 @@ files:
 - ".claude/agents/rails-tdd.md"
 - ".claude/agents/rails-test.md"
 - ".claude/agents/rails-view-component.md"
+- ".claude/commands/release.md"
 - ".claude/hooks/block-secrets.sh"
 - ".claude/settings.json"
 - ".claude/skills/action-cable-patterns/SKILL.md"
@@ -336,11 +339,11 @@ files:
 - ".gitignore"
 - ".rubocop.yml"
 - ".ruby-version"
-- ".vbw-planning/.notification-log.jsonl"
-- ".vbw-planning/.session-log.jsonl"
 - ".vbw-planning/PROJECT.md"
 - ".vbw-planning/REQUIREMENTS.md"
+- ".vbw-planning/ROADMAP.md"
 - ".vbw-planning/SHIPPED.md"
+- ".vbw-planning/STATE.md"
 - ".vbw-planning/codebase/ARCHITECTURE.md"
 - ".vbw-planning/codebase/CONCERNS.md"
 - ".vbw-planning/codebase/CONVENTIONS.md"
@@ -386,6 +389,27 @@ files:
 - ".vbw-planning/milestones/default/phases/04-code-quality-conventions-cleanup/PLAN-02.md"
 - ".vbw-planning/milestones/default/phases/04-code-quality-conventions-cleanup/PLAN-03-SUMMARY.md"
 - ".vbw-planning/milestones/default/phases/04-code-quality-conventions-cleanup/PLAN-03.md"
+- ".vbw-planning/phases/01-generator-steps/01-CONTEXT.md"
+- ".vbw-planning/phases/01-generator-steps/01-VERIFICATION.md"
+- ".vbw-planning/phases/01-generator-steps/PLAN-01-SUMMARY.md"
+- ".vbw-planning/phases/01-generator-steps/PLAN-01.md"
+- ".vbw-planning/phases/02-verification/02-VERIFICATION.md"
+- ".vbw-planning/phases/02-verification/PLAN-01-SUMMARY.md"
+- ".vbw-planning/phases/02-verification/PLAN-01.md"
+- ".vbw-planning/phases/03-docs-alignment/03-VERIFICATION.md"
+- ".vbw-planning/phases/03-docs-alignment/PLAN-01-SUMMARY.md"
+- ".vbw-planning/phases/03-docs-alignment/PLAN-01.md"
+- ".vbw-planning/phases/04-dashboard-ux/04-VERIFICATION.md"
+- ".vbw-planning/phases/04-dashboard-ux/PLAN-01-SUMMARY.md"
+- ".vbw-planning/phases/04-dashboard-ux/PLAN-01.md"
+- ".vbw-planning/phases/05-active-storage-images/05-VERIFICATION.md"
+- ".vbw-planning/phases/05-active-storage-images/PLAN-01-SUMMARY.md"
+- ".vbw-planning/phases/05-active-storage-images/PLAN-01.md"
+- ".vbw-planning/phases/05-active-storage-images/PLAN-02-SUMMARY.md"
+- ".vbw-planning/phases/05-active-storage-images/PLAN-02.md"
+- ".vbw-planning/phases/06-netflix-feed-fix/06-VERIFICATION.md"
+- ".vbw-planning/phases/06-netflix-feed-fix/PLAN-01-SUMMARY.md"
+- ".vbw-planning/phases/06-netflix-feed-fix/PLAN-01.md"
 - AGENTS.md
 - CHANGELOG.md
 - CLAUDE.md
@@ -437,6 +461,7 @@ files:
 - app/helpers/source_monitor/health_badge_helper.rb
 - app/helpers/source_monitor/table_sort_helper.rb
 - app/jobs/source_monitor/application_job.rb
+- app/jobs/source_monitor/download_content_images_job.rb
 - app/jobs/source_monitor/fetch_feed_job.rb
 - app/jobs/source_monitor/import_opml_job.rb
 - app/jobs/source_monitor/import_session_health_check_job.rb
@@ -550,6 +575,7 @@ files:
 - lib/source_monitor/configuration/fetching_settings.rb
 - lib/source_monitor/configuration/health_settings.rb
 - lib/source_monitor/configuration/http_settings.rb
+- lib/source_monitor/configuration/images_settings.rb
 - lib/source_monitor/configuration/model_definition.rb
 - lib/source_monitor/configuration/models.rb
 - lib/source_monitor/configuration/realtime_settings.rb
@@ -587,6 +613,8 @@ files:
 - lib/source_monitor/health/source_health_monitor.rb
 - lib/source_monitor/health/source_health_reset.rb
 - lib/source_monitor/http.rb
+- lib/source_monitor/images/content_rewriter.rb
+- lib/source_monitor/images/downloader.rb
 - lib/source_monitor/import_sessions/entry_normalizer.rb
 - lib/source_monitor/import_sessions/health_check_broadcaster.rb
 - lib/source_monitor/instrumentation.rb
@@ -640,12 +668,15 @@ files:
 - lib/source_monitor/setup/install_generator.rb
 - lib/source_monitor/setup/migration_installer.rb
 - lib/source_monitor/setup/node_installer.rb
+- lib/source_monitor/setup/procfile_patcher.rb
 - lib/source_monitor/setup/prompter.rb
+- lib/source_monitor/setup/queue_config_patcher.rb
 - lib/source_monitor/setup/requirements.rb
 - lib/source_monitor/setup/shell_runner.rb
 - lib/source_monitor/setup/skills_installer.rb
 - lib/source_monitor/setup/verification/action_cable_verifier.rb
 - lib/source_monitor/setup/verification/printer.rb
+- lib/source_monitor/setup/verification/recurring_schedule_verifier.rb
 - lib/source_monitor/setup/verification/result.rb
 - lib/source_monitor/setup/verification/runner.rb
 - lib/source_monitor/setup/verification/solid_queue_verifier.rb