source_monitor 0.11.1 → 0.12.0

data/lib/source_monitor/scraping/runner.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  module Scraping
+    # Orchestrates a single item scrape: pre-flight checks (scraping enabled?),
+    # state management (mark_processing!, mark_failed!, clear_inflight!),
+    # and delegation to ItemScraper. Extracted from ScrapeItemJob so scraping
+    # logic can be invoked synchronously.
+    class Runner
+      def initialize(item)
+        @item = item
+        @source = item.source
+      end
+
+      def call
+        unless source&.scraping_enabled?
+          log("runner:skipped_scraping_disabled")
+          State.clear_inflight!(item)
+          return
+        end
+
+        State.mark_processing!(item)
+        SourceMonitor::Scraping::ItemScraper.new(item: item, source: source).call
+        log("runner:completed", status: item.scrape_status)
+      rescue StandardError => error
+        log("runner:error", error: error.message)
+        State.mark_failed!(item)
+        raise
+      ensure
+        State.clear_inflight!(item) if item
+      end
+
+      private
+
+      attr_reader :item, :source
+
+      def log(stage, **extra)
+        return unless defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger
+
+        payload = {
+          stage: "SourceMonitor::Scraping::Runner##{stage}",
+          item_id: item&.id,
+          source_id: source&.id
+        }.merge(extra.compact)
+        level = stage.to_s.include?("error") ? :error : :info
+        Rails.logger.public_send(level, "[SourceMonitor::Scraping::Runner] #{payload.to_json}")
+      rescue StandardError
+        nil
+      end
+    end
+  end
+end

data/lib/source_monitor/scraping/scheduler.rb

@@ -24,6 +24,11 @@ module SourceMonitor
           result = SourceMonitor::Scraping::Enqueuer.enqueue(item: item, source: item.source, reason: :auto)
           result.enqueued? ? 1 : 0
         end
+      rescue StandardError => error
+        Rails.logger.warn(
+          "[SourceMonitor::Scraping::Scheduler] Scheduler run failed: #{error.class} - #{error.message}"
+        ) if defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger
+        0
       end
 
       private

data/lib/source_monitor/scraping/state.rb

@@ -71,8 +71,10 @@ module SourceMonitor
 
       def broadcast_item(item)
         SourceMonitor::Realtime.broadcast_item(item)
-      rescue StandardError
-        nil
+      rescue StandardError => e
+        if defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger
+          Rails.logger.warn("[SourceMonitor::Scraping::State] Broadcast failed: #{e.class}: #{e.message}")
+        end
       end
     end
   end

data/lib/source_monitor/security/parameter_sanitizer.rb

@@ -7,6 +7,13 @@ module SourceMonitor
     module ParameterSanitizer
       module_function
 
+      def safe_redirect_path(raw_value)
+        return if raw_value.blank?
+
+        sanitized = sanitize(raw_value.to_s)
+        sanitized.start_with?("/") ? sanitized : nil
+      end
+
       def sanitize(value)
         case value
         when ActionController::Parameters

data/lib/source_monitor/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module SourceMonitor
-  VERSION = "0.11.1"
+  VERSION = "0.12.0"
 end

data/lib/source_monitor.rb

@@ -50,6 +50,7 @@ require "source_monitor/feedjira_extensions"
 
 module SourceMonitor
   autoload :HTTP, "source_monitor/http"
+  autoload :Queries, "source_monitor/queries"
   autoload :Scheduler, "source_monitor/scheduler"
   autoload :Assets, "source_monitor/assets"
 
@@ -79,6 +80,7 @@ module SourceMonitor
     autoload :UnexpectedResponseError, "source_monitor/fetching/fetch_error"
     autoload :FeedFetcher, "source_monitor/fetching/feed_fetcher"
     autoload :FetchRunner, "source_monitor/fetching/fetch_runner"
+    autoload :RetryOrchestrator, "source_monitor/fetching/retry_orchestrator"
     autoload :RetryPolicy, "source_monitor/fetching/retry_policy"
     autoload :StalledFetchReconciler, "source_monitor/fetching/stalled_fetch_reconciler"
     autoload :AdvisoryLock, "source_monitor/fetching/advisory_lock"
@@ -88,16 +90,20 @@ module SourceMonitor
   module ImportSessions
     autoload :EntryNormalizer, "source_monitor/import_sessions/entry_normalizer"
     autoload :HealthCheckBroadcaster, "source_monitor/import_sessions/health_check_broadcaster"
+    autoload :HealthCheckUpdater, "source_monitor/import_sessions/health_check_updater"
+    autoload :OPMLImporter, "source_monitor/import_sessions/opml_importer"
   end
 
   module Favicons
     autoload :Discoverer, "source_monitor/favicons/discoverer"
+    autoload :Fetcher, "source_monitor/favicons/fetcher"
     autoload :SvgConverter, "source_monitor/favicons/svg_converter"
   end
 
   module Images
     autoload :ContentRewriter, "source_monitor/images/content_rewriter"
     autoload :Downloader, "source_monitor/images/downloader"
+    autoload :Processor, "source_monitor/images/processor"
   end
 
   module Items
@@ -152,6 +158,7 @@ module SourceMonitor
     autoload :Enqueuer, "source_monitor/scraping/enqueuer"
     autoload :BulkSourceScraper, "source_monitor/scraping/bulk_source_scraper"
     autoload :BulkResultPresenter, "source_monitor/scraping/bulk_result_presenter"
+    autoload :Runner, "source_monitor/scraping/runner"
     autoload :State, "source_monitor/scraping/state"
     autoload :Scheduler, "source_monitor/scraping/scheduler"
     autoload :ItemScraper, "source_monitor/scraping/item_scraper"

data/source_monitor.gemspec

@@ -46,4 +46,5 @@ Gem::Specification.new do |spec|
   spec.add_dependency "solid_queue", ">= 0.3", "< 3.0"
   spec.add_dependency "solid_cable", ">= 3.0", "< 4.0"
   spec.add_dependency "ransack", "~> 4.2"
+  spec.add_dependency "view_component", ">= 3.0", "< 4.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: source_monitor
 version: !ruby/object:Gem::Version
-  version: 0.11.1
+  version: 0.12.0
 platform: ruby
 authors:
 - dchuk
@@ -229,6 +229,26 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '4.2'
+- !ruby/object:Gem::Dependency
+  name: view_component
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4.0'
 description: SourceMonitor is a mountable Rails 8 engine that ingests RSS, Atom, and
   JSON feeds, scrapes full article content, and surfaces Solid Queue powered dashboards
   for monitoring and remediation.
@@ -259,6 +279,7 @@ files:
 - ".claude/agents/rails-tdd.md"
 - ".claude/agents/rails-test.md"
 - ".claude/agents/rails-view-component.md"
+- ".claude/commands/rails-audit.md"
 - ".claude/commands/release.md"
 - ".claude/hooks/block-secrets.sh"
 - ".claude/settings.json"
@@ -350,6 +371,7 @@ files:
 - Gemfile
 - Gemfile.lock
 - MIT-LICENSE
+- RAILS_AUDIT.md
 - README.md
 - Rakefile
 - VERSION
@@ -363,6 +385,7 @@ files:
 - app/assets/javascripts/source_monitor/controllers/async_submit_controller.js
 - app/assets/javascripts/source_monitor/controllers/confirm_navigation_controller.js
 - app/assets/javascripts/source_monitor/controllers/dropdown_controller.js
+- app/assets/javascripts/source_monitor/controllers/filter_submit_controller.js
 - app/assets/javascripts/source_monitor/controllers/modal_controller.js
 - app/assets/javascripts/source_monitor/controllers/notification_container_controller.js
 - app/assets/javascripts/source_monitor/controllers/notification_controller.js
@@ -370,8 +393,14 @@ files:
 - app/assets/javascripts/source_monitor/turbo_actions.js
 - app/assets/stylesheets/source_monitor/application.tailwind.css
 - app/assets/svgs/source_monitor/.keep
+- app/components/source_monitor/application_component.rb
+- app/components/source_monitor/filter_dropdown_component.rb
+- app/components/source_monitor/icon_component.rb
+- app/components/source_monitor/status_badge_component.html.erb
+- app/components/source_monitor/status_badge_component.rb
 - app/controllers/concerns/.keep
 - app/controllers/concerns/source_monitor/sanitizes_search_params.rb
+- app/controllers/concerns/source_monitor/set_source.rb
 - app/controllers/source_monitor/application_controller.rb
 - app/controllers/source_monitor/bulk_scrape_enablements_controller.rb
 - app/controllers/source_monitor/dashboard_controller.rb
@@ -383,6 +412,7 @@ files:
 - app/controllers/source_monitor/import_sessions/health_check_management.rb
 - app/controllers/source_monitor/import_sessions/opml_parser.rb
 - app/controllers/source_monitor/import_sessions_controller.rb
+- app/controllers/source_monitor/item_scrapes_controller.rb
 - app/controllers/source_monitor/items_controller.rb
 - app/controllers/source_monitor/logs_controller.rb
 - app/controllers/source_monitor/scrape_logs_controller.rb
@@ -422,6 +452,9 @@ files:
 - app/models/source_monitor/log_entry.rb
 - app/models/source_monitor/scrape_log.rb
 - app/models/source_monitor/source.rb
+- app/presenters/source_monitor/base_presenter.rb
+- app/presenters/source_monitor/source_details_presenter.rb
+- app/presenters/source_monitor/sources_filter_presenter.rb
 - app/views/layouts/source_monitor/application.html.erb
 - app/views/source_monitor/dashboard/_fetch_schedule.html.erb
 - app/views/source_monitor/dashboard/_job_metrics.html.erb
@@ -449,9 +482,11 @@ files:
 - app/views/source_monitor/items/show.html.erb
 - app/views/source_monitor/logs/index.html.erb
 - app/views/source_monitor/scrape_logs/show.html.erb
+- app/views/source_monitor/shared/_form_errors.html.erb
 - app/views/source_monitor/shared/_pagination.html.erb
 - app/views/source_monitor/shared/_toast.html.erb
 - app/views/source_monitor/source_scrape_tests/_result.html.erb
+- app/views/source_monitor/source_scrape_tests/_result_content.html.erb
 - app/views/source_monitor/source_scrape_tests/show.html.erb
 - app/views/source_monitor/sources/_bulk_scrape_enable_modal.html.erb
 - app/views/source_monitor/sources/_bulk_scrape_form.html.erb
@@ -505,6 +540,8 @@ files:
 - db/migrate/20260306233004_add_error_category_to_fetch_logs.rb
 - db/migrate/20260307120000_add_consecutive_fetch_failures_to_sources.rb
 - db/migrate/20260312120000_simplify_health_status_values.rb
+- db/migrate/20260313120000_add_composite_indexes_to_log_tables.rb
+- db/migrate/20260314120000_align_health_status_default.rb
 - docs/configuration.md
 - docs/deployment.md
 - docs/gh-cli-workflow.md
@@ -550,6 +587,7 @@ files:
 - lib/source_monitor/engine.rb
 - lib/source_monitor/events.rb
 - lib/source_monitor/favicons/discoverer.rb
+- lib/source_monitor/favicons/fetcher.rb
 - lib/source_monitor/favicons/svg_converter.rb
 - lib/source_monitor/feedjira_extensions.rb
 - lib/source_monitor/fetching/advisory_lock.rb
@@ -563,19 +601,24 @@ files:
 - lib/source_monitor/fetching/feed_fetcher/source_updater.rb
 - lib/source_monitor/fetching/fetch_error.rb
 - lib/source_monitor/fetching/fetch_runner.rb
+- lib/source_monitor/fetching/retry_orchestrator.rb
 - lib/source_monitor/fetching/retry_policy.rb
 - lib/source_monitor/fetching/stalled_fetch_reconciler.rb
 - lib/source_monitor/health.rb
 - lib/source_monitor/health/import_source_health_check.rb
 - lib/source_monitor/health/source_health_check.rb
+- lib/source_monitor/health/source_health_check_orchestrator.rb
 - lib/source_monitor/health/source_health_monitor.rb
 - lib/source_monitor/health/source_health_reset.rb
 - lib/source_monitor/http.rb
 - lib/source_monitor/http/aia_resolver.rb
 - lib/source_monitor/images/content_rewriter.rb
 - lib/source_monitor/images/downloader.rb
+- lib/source_monitor/images/processor.rb
 - lib/source_monitor/import_sessions/entry_normalizer.rb
 - lib/source_monitor/import_sessions/health_check_broadcaster.rb
+- lib/source_monitor/import_sessions/health_check_updater.rb
+- lib/source_monitor/import_sessions/opml_importer.rb
 - lib/source_monitor/instrumentation.rb
 - lib/source_monitor/items/item_creator.rb
 - lib/source_monitor/items/item_creator/content_extractor.rb
@@ -598,6 +641,8 @@ files:
 - lib/source_monitor/models/sanitizable.rb
 - lib/source_monitor/models/url_normalizable.rb
 - lib/source_monitor/pagination/paginator.rb
+- lib/source_monitor/queries.rb
+- lib/source_monitor/queries/scrape_candidates_query.rb
 - lib/source_monitor/realtime.rb
 - lib/source_monitor/realtime/adapter.rb
 - lib/source_monitor/realtime/broadcaster.rb
@@ -614,6 +659,7 @@ files:
 - lib/source_monitor/scraping/item_scraper.rb
 - lib/source_monitor/scraping/item_scraper/adapter_resolver.rb
 - lib/source_monitor/scraping/item_scraper/persistence.rb
+- lib/source_monitor/scraping/runner.rb
 - lib/source_monitor/scraping/scheduler.rb
 - lib/source_monitor/scraping/state.rb
 - lib/source_monitor/security/authentication.rb