source_monitor 0.3.3 → 0.4.0

data/lib/source_monitor/logs/table_presenter.rb

@@ -61,6 +61,22 @@ module SourceMonitor
           end
         end
 
+        def url_label
+          if fetch?
+            domain_from_feed_url
+          elsif scrape?
+            entry.item&.url
+          end
+        end
+
+        def url_href
+          if fetch?
+            entry.source&.feed_url
+          elsif scrape?
+            entry.item&.url
+          end
+        end
+
         def source_label
           entry.source&.name
         end
@@ -142,6 +158,15 @@ module SourceMonitor
         private
 
         attr_reader :entry, :url_helpers
+
+        def domain_from_feed_url
+          feed_url = entry.source&.feed_url
+          return nil if feed_url.blank?
+
+          URI.parse(feed_url.to_s).host
+        rescue URI::InvalidURIError
+          nil
+        end
       end
 
       def initialize(entries:, url_helpers:)

data/lib/source_monitor/setup/procfile_patcher.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require "pathname"
+
+module SourceMonitor
+  module Setup
+    class ProcfilePatcher
+      JOBS_ENTRY = "jobs: bundle exec rake solid_queue:start"
+
+      def initialize(path: "Procfile.dev")
+        @path = Pathname.new(path)
+      end
+
+      def patch
+        if path.exist?
+          content = path.read
+          return false if content.match?(/^jobs:/)
+
+          path.open("a") { |f| f.puts("", JOBS_ENTRY) }
+        else
+          path.write("web: bin/rails server -p 3000\n#{JOBS_ENTRY}\n")
+        end
+        true
+      end
+
+      private
+
+      attr_reader :path
+    end
+  end
+end

data/lib/source_monitor/setup/queue_config_patcher.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+require "pathname"
+require "yaml"
+
+module SourceMonitor
+  module Setup
+    class QueueConfigPatcher
+      RECURRING_SCHEDULE_VALUE = "config/recurring.yml"
+
+      DEFAULT_DISPATCHER = {
+        "polling_interval" => 1,
+        "batch_size" => 500,
+        "recurring_schedule" => RECURRING_SCHEDULE_VALUE
+      }.freeze
+
+      def initialize(path: "config/queue.yml")
+        @path = Pathname.new(path)
+      end
+
+      def patch
+        return false unless path.exist?
+
+        parsed = YAML.safe_load(path.read, aliases: true) || {}
+        return false if has_recurring_schedule?(parsed)
+
+        add_recurring_schedule!(parsed)
+        path.write(YAML.dump(parsed))
+        true
+      end
+
+      private
+
+      attr_reader :path
+
+      def has_recurring_schedule?(parsed)
+        parsed.each_value do |value|
+          next unless value.is_a?(Hash)
+
+          dispatchers = value["dispatchers"]
+          if dispatchers.is_a?(Array)
+            return true if dispatchers.any? { |d| d.is_a?(Hash) && d.key?("recurring_schedule") }
+          end
+
+          return true if has_recurring_schedule?(value)
+        end
+
+        if parsed.key?("dispatchers") && parsed["dispatchers"].is_a?(Array)
+          return true if parsed["dispatchers"].any? { |d| d.is_a?(Hash) && d.key?("recurring_schedule") }
+        end
+
+        false
+      end
+
+      def add_recurring_schedule!(parsed)
+        found_dispatchers = false
+
+        parsed.each_value do |value|
+          next unless value.is_a?(Hash)
+
+          if value.key?("dispatchers") && value["dispatchers"].is_a?(Array)
+            value["dispatchers"].each do |dispatcher|
+              next unless dispatcher.is_a?(Hash)
+              dispatcher["recurring_schedule"] ||= RECURRING_SCHEDULE_VALUE
+            end
+            found_dispatchers = true
+          end
+        end
+
+        if parsed.key?("dispatchers") && parsed["dispatchers"].is_a?(Array)
+          parsed["dispatchers"].each do |dispatcher|
+            next unless dispatcher.is_a?(Hash)
+            dispatcher["recurring_schedule"] ||= RECURRING_SCHEDULE_VALUE
+          end
+          found_dispatchers = true
+        end
+
+        unless found_dispatchers
+          parsed["dispatchers"] = [ DEFAULT_DISPATCHER.dup ]
+        end
+      end
+    end
+  end
+end

data/lib/source_monitor/setup/verification/recurring_schedule_verifier.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  module Setup
+    module Verification
+      class RecurringScheduleVerifier
+        SOURCE_MONITOR_KEY_PREFIX = "source_monitor_"
+        SOURCE_MONITOR_NAMESPACE = "SourceMonitor::"
+
+        def initialize(task_relation: default_task_relation, connection: default_connection)
+          @task_relation = task_relation
+          @connection = connection
+        end
+
+        def call
+          return missing_gem_result unless task_relation
+          return missing_tables_result unless tables_present?
+
+          tasks = all_tasks
+          sm_tasks = source_monitor_tasks(tasks)
+
+          if sm_tasks.any?
+            ok_result("#{sm_tasks.size} SourceMonitor recurring task(s) registered")
+          elsif tasks.any?
+            warning_result(
+              "Recurring tasks exist but none belong to SourceMonitor",
+              "Add SourceMonitor entries to config/recurring.yml and ensure the dispatcher has `recurring_schedule: config/recurring.yml`"
+            )
+          else
+            warning_result(
+              "No recurring tasks are registered with Solid Queue",
+              "Configure a dispatcher with `recurring_schedule: config/recurring.yml` in config/queue.yml and ensure recurring.yml contains SourceMonitor task entries"
+            )
+          end
+        rescue StandardError => e
+          error_result(
+            "Recurring schedule verification failed: #{e.message}",
+            "Verify Solid Queue migrations are up to date and the dispatcher is configured with recurring_schedule"
+          )
+        end
+
+        private
+
+        attr_reader :task_relation, :connection
+
+        def default_task_relation
+          SolidQueue::RecurringTask if defined?(SolidQueue::RecurringTask)
+        end
+
+        def default_connection
+          SolidQueue::RecurringTask.connection if defined?(SolidQueue::RecurringTask)
+        rescue StandardError
+          nil
+        end
+
+        def tables_present?
+          return false unless connection
+
+          connection.table_exists?(task_relation.table_name)
+        end
+
+        def all_tasks
+          task_relation.all.to_a
+        end
+
+        def source_monitor_tasks(tasks)
+          tasks.select do |task|
+            task.key.start_with?(SOURCE_MONITOR_KEY_PREFIX) ||
+              task.class_name.to_s.start_with?(SOURCE_MONITOR_NAMESPACE) ||
+              task.command.to_s.include?(SOURCE_MONITOR_NAMESPACE)
+          end
+        end
+
+        def missing_gem_result
+          error_result(
+            "Solid Queue gem is not available",
+            "Add `solid_queue` to your Gemfile and bundle install"
+          )
+        end
+
+        def missing_tables_result
+          error_result(
+            "Solid Queue recurring tasks table is missing",
+            "Run `rails solid_queue:install` or copy the engine's Solid Queue migration"
+          )
+        end
+
+        def ok_result(details)
+          Result.new(key: :recurring_schedule, name: "Recurring Schedule", status: :ok, details: details)
+        end
+
+        def warning_result(details, remediation)
+          Result.new(key: :recurring_schedule, name: "Recurring Schedule", status: :warning, details: details, remediation: remediation)
+        end
+
+        def error_result(details, remediation)
+          Result.new(key: :recurring_schedule, name: "Recurring Schedule", status: :error, details: details, remediation: remediation)
+        end
+      end
+    end
+  end
+end

data/lib/source_monitor/setup/verification/runner.rb

@@ -18,7 +18,7 @@ module SourceMonitor
         attr_reader :verifiers
 
         def default_verifiers
-          [ SolidQueueVerifier.new, ActionCableVerifier.new ]
+          [ SolidQueueVerifier.new, RecurringScheduleVerifier.new, ActionCableVerifier.new ]
         end
       end
     end

data/lib/source_monitor/setup/verification/solid_queue_verifier.rb

@@ -21,7 +21,7 @@ module SourceMonitor
           if recent
             ok_result("Solid Queue workers are reporting heartbeats")
           else
-            warning_result("No Solid Queue workers have reported in the last #{DEFAULT_HEARTBEAT_THRESHOLD.inspect}", "Start a Solid Queue worker with `bin/rails solid_queue:start` and ensure it stays running")
+            warning_result("No Solid Queue workers have reported in the last #{DEFAULT_HEARTBEAT_THRESHOLD.inspect}", "Start a Solid Queue worker with `bin/rails solid_queue:start` or add `jobs: bundle exec rake solid_queue:start` to Procfile.dev and run `bin/dev`")
           end
         rescue StandardError => e
           error_result("Solid Queue verification failed: #{e.message}", "Verify Solid Queue migrations are up to date and workers can access the database")

data/lib/source_monitor/setup/workflow.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 require "pathname"
+require_relative "procfile_patcher"
+require_relative "queue_config_patcher"
 require_relative "skills_installer"
 
 module SourceMonitor
@@ -35,6 +37,8 @@ module SourceMonitor
         install_generator: InstallGenerator.new,
         migration_installer: MigrationInstaller.new,
         initializer_patcher: InitializerPatcher.new,
+        procfile_patcher: ProcfilePatcher.new,
+        queue_config_patcher: QueueConfigPatcher.new,
         devise_detector: method(:default_devise_detector),
         verifier: Verification::Runner.new,
         skills_installer: SkillsInstaller.new
@@ -47,6 +51,8 @@ module SourceMonitor
         @install_generator = install_generator
         @migration_installer = migration_installer
         @initializer_patcher = initializer_patcher
+        @procfile_patcher = procfile_patcher
+        @queue_config_patcher = queue_config_patcher
         @devise_detector = devise_detector
         @verifier = verifier
         @skills_installer = skills_installer
@@ -64,6 +70,8 @@ module SourceMonitor
         install_generator.run(mount_path: mount_path)
         migration_installer.install
         initializer_patcher.ensure_navigation_hint(mount_path: mount_path)
+        procfile_patcher.patch
+        queue_config_patcher.patch
 
         if devise_available? && prompter.yes?("Wire Devise authentication hooks into SourceMonitor?", default: true)
           initializer_patcher.ensure_devise_hooks
@@ -89,6 +97,8 @@ module SourceMonitor
         :install_generator,
         :migration_installer,
         :initializer_patcher,
+        :procfile_patcher,
+        :queue_config_patcher,
         :devise_detector,
         :verifier,
         :skills_installer

data/lib/source_monitor/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module SourceMonitor
-  VERSION = "0.3.3"
+  VERSION = "0.4.0"
 end

data/lib/source_monitor.rb

@@ -87,6 +87,11 @@ module SourceMonitor
     autoload :HealthCheckBroadcaster, "source_monitor/import_sessions/health_check_broadcaster"
   end
 
+  module Images
+    autoload :ContentRewriter, "source_monitor/images/content_rewriter"
+    autoload :Downloader, "source_monitor/images/downloader"
+  end
+
   module Items
     autoload :ItemCreator, "source_monitor/items/item_creator"
     autoload :RetentionPruner, "source_monitor/items/retention_pruner"
@@ -161,6 +166,8 @@ module SourceMonitor
     autoload :InstallGenerator, "source_monitor/setup/install_generator"
     autoload :MigrationInstaller, "source_monitor/setup/migration_installer"
     autoload :InitializerPatcher, "source_monitor/setup/initializer_patcher"
+    autoload :ProcfilePatcher, "source_monitor/setup/procfile_patcher"
+    autoload :QueueConfigPatcher, "source_monitor/setup/queue_config_patcher"
     autoload :Workflow, "source_monitor/setup/workflow"
     autoload :CLI, "source_monitor/setup/cli"
 
@@ -169,6 +176,7 @@ module SourceMonitor
       autoload :Summary, "source_monitor/setup/verification/result"
       autoload :SolidQueueVerifier, "source_monitor/setup/verification/solid_queue_verifier"
       autoload :ActionCableVerifier, "source_monitor/setup/verification/action_cable_verifier"
+      autoload :RecurringScheduleVerifier, "source_monitor/setup/verification/recurring_schedule_verifier"
       autoload :Runner, "source_monitor/setup/verification/runner"
       autoload :Printer, "source_monitor/setup/verification/printer"
       autoload :TelemetryLogger, "source_monitor/setup/verification/telemetry_logger"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: source_monitor
 version: !ruby/object:Gem::Version
-  version: 0.3.3
+  version: 0.4.0
 platform: ruby
 authors:
 - dchuk
@@ -341,7 +341,9 @@ files:
 - ".ruby-version"
 - ".vbw-planning/PROJECT.md"
 - ".vbw-planning/REQUIREMENTS.md"
+- ".vbw-planning/ROADMAP.md"
 - ".vbw-planning/SHIPPED.md"
+- ".vbw-planning/STATE.md"
 - ".vbw-planning/codebase/ARCHITECTURE.md"
 - ".vbw-planning/codebase/CONCERNS.md"
 - ".vbw-planning/codebase/CONVENTIONS.md"
@@ -387,6 +389,27 @@ files:
 - ".vbw-planning/milestones/default/phases/04-code-quality-conventions-cleanup/PLAN-02.md"
 - ".vbw-planning/milestones/default/phases/04-code-quality-conventions-cleanup/PLAN-03-SUMMARY.md"
 - ".vbw-planning/milestones/default/phases/04-code-quality-conventions-cleanup/PLAN-03.md"
+- ".vbw-planning/phases/01-generator-steps/01-CONTEXT.md"
+- ".vbw-planning/phases/01-generator-steps/01-VERIFICATION.md"
+- ".vbw-planning/phases/01-generator-steps/PLAN-01-SUMMARY.md"
+- ".vbw-planning/phases/01-generator-steps/PLAN-01.md"
+- ".vbw-planning/phases/02-verification/02-VERIFICATION.md"
+- ".vbw-planning/phases/02-verification/PLAN-01-SUMMARY.md"
+- ".vbw-planning/phases/02-verification/PLAN-01.md"
+- ".vbw-planning/phases/03-docs-alignment/03-VERIFICATION.md"
+- ".vbw-planning/phases/03-docs-alignment/PLAN-01-SUMMARY.md"
+- ".vbw-planning/phases/03-docs-alignment/PLAN-01.md"
+- ".vbw-planning/phases/04-dashboard-ux/04-VERIFICATION.md"
+- ".vbw-planning/phases/04-dashboard-ux/PLAN-01-SUMMARY.md"
+- ".vbw-planning/phases/04-dashboard-ux/PLAN-01.md"
+- ".vbw-planning/phases/05-active-storage-images/05-VERIFICATION.md"
+- ".vbw-planning/phases/05-active-storage-images/PLAN-01-SUMMARY.md"
+- ".vbw-planning/phases/05-active-storage-images/PLAN-01.md"
+- ".vbw-planning/phases/05-active-storage-images/PLAN-02-SUMMARY.md"
+- ".vbw-planning/phases/05-active-storage-images/PLAN-02.md"
+- ".vbw-planning/phases/06-netflix-feed-fix/06-VERIFICATION.md"
+- ".vbw-planning/phases/06-netflix-feed-fix/PLAN-01-SUMMARY.md"
+- ".vbw-planning/phases/06-netflix-feed-fix/PLAN-01.md"
 - AGENTS.md
 - CHANGELOG.md
 - CLAUDE.md
@@ -438,6 +461,7 @@ files:
 - app/helpers/source_monitor/health_badge_helper.rb
 - app/helpers/source_monitor/table_sort_helper.rb
 - app/jobs/source_monitor/application_job.rb
+- app/jobs/source_monitor/download_content_images_job.rb
 - app/jobs/source_monitor/fetch_feed_job.rb
 - app/jobs/source_monitor/import_opml_job.rb
 - app/jobs/source_monitor/import_session_health_check_job.rb
@@ -551,6 +575,7 @@ files:
 - lib/source_monitor/configuration/fetching_settings.rb
 - lib/source_monitor/configuration/health_settings.rb
 - lib/source_monitor/configuration/http_settings.rb
+- lib/source_monitor/configuration/images_settings.rb
 - lib/source_monitor/configuration/model_definition.rb
 - lib/source_monitor/configuration/models.rb
 - lib/source_monitor/configuration/realtime_settings.rb
@@ -588,6 +613,8 @@ files:
 - lib/source_monitor/health/source_health_monitor.rb
 - lib/source_monitor/health/source_health_reset.rb
 - lib/source_monitor/http.rb
+- lib/source_monitor/images/content_rewriter.rb
+- lib/source_monitor/images/downloader.rb
 - lib/source_monitor/import_sessions/entry_normalizer.rb
 - lib/source_monitor/import_sessions/health_check_broadcaster.rb
 - lib/source_monitor/instrumentation.rb
@@ -641,12 +668,15 @@ files:
 - lib/source_monitor/setup/install_generator.rb
 - lib/source_monitor/setup/migration_installer.rb
 - lib/source_monitor/setup/node_installer.rb
+- lib/source_monitor/setup/procfile_patcher.rb
 - lib/source_monitor/setup/prompter.rb
+- lib/source_monitor/setup/queue_config_patcher.rb
 - lib/source_monitor/setup/requirements.rb
 - lib/source_monitor/setup/shell_runner.rb
 - lib/source_monitor/setup/skills_installer.rb
 - lib/source_monitor/setup/verification/action_cable_verifier.rb
 - lib/source_monitor/setup/verification/printer.rb
+- lib/source_monitor/setup/verification/recurring_schedule_verifier.rb
 - lib/source_monitor/setup/verification/result.rb
 - lib/source_monitor/setup/verification/runner.rb
 - lib/source_monitor/setup/verification/solid_queue_verifier.rb