source_monitor 0.1.1

data/lib/source_monitor/scraping/bulk_result_presenter.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  module Scraping
+    # Presenter for building flash messages from BulkSourceScraper results
+    # Extracts complex message formatting logic from the controller
+    class BulkResultPresenter
+      attr_reader :result, :pluralizer
+
+      def initialize(result:, pluralizer:)
+        @result = result
+        @pluralizer = pluralizer
+      end
+
+      def to_flash_payload
+        case result.status
+        when :success
+          build_success_payload
+        when :partial
+          build_partial_payload
+        else
+          build_error_payload
+        end
+      end
+
+      private
+
+      def build_success_payload
+        label = BulkSourceScraper.selection_label(result.selection)
+        pluralized_enqueued = pluralizer.call(result.enqueued_count, "item")
+
+        message = "Queued scraping for #{pluralized_enqueued} from the #{label}."
+
+        if result.already_enqueued_count.positive?
+          pluralized_already = pluralizer.call(result.already_enqueued_count, "item")
+          message = "#{message} #{pluralized_already.capitalize} already in progress."
+        end
+
+        { flash_key: :notice, message:, level: :success }
+      end
+
+      def build_partial_payload
+        label = BulkSourceScraper.selection_label(result.selection)
+        parts = []
+
+        if result.enqueued_count.positive?
+          pluralized_enqueued = pluralizer.call(result.enqueued_count, "item")
+          parts << "Queued #{pluralized_enqueued} from the #{label}"
+        end
+
+        if result.already_enqueued_count.positive?
+          pluralized_already = pluralizer.call(result.already_enqueued_count, "item")
+          parts << "#{pluralized_already.capitalize} already in progress"
+        end
+
+        if result.rate_limited?
+          limit = SourceMonitor.config.scraping.max_in_flight_per_source
+          parts << "Stopped after reaching the per-source limit#{" of #{limit}" if limit}"
+        end
+
+        other_failures = result.failure_details.except(:rate_limited)
+        if other_failures.values.sum.positive?
+          skipped = other_failures.map do |status, count|
+            label_key = status.to_s.tr("_", " ")
+            "#{pluralizer.call(count, label_key)}"
+          end.join(", ")
+          parts << "Skipped #{skipped}"
+        end
+
+        if parts.empty?
+          parts << "No new scrapes were queued from the #{label}"
+        end
+
+        { flash_key: :notice, message: parts.join(". ") + ".", level: :warning }
+      end
+
+      def build_error_payload
+        message = result.messages.presence&.first ||
+          "No items were queued because nothing matched the selected scope."
+
+        { flash_key: :alert, message:, level: :error }
+      end
+    end
+  end
+end

data/lib/source_monitor/scraping/bulk_source_scraper.rb

@@ -0,0 +1,233 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  module Scraping
+    # Orchestrates bulk scrape enqueues for a source based on a user-selected
+    # scope. Works alongside the single-item enqueuer to ensure we respect
+    # per-source limits and provide actionable feedback for the UI.
+    class BulkSourceScraper
+      SELECTIONS = %i[current unscraped all].freeze
+      SELECTION_LABELS = {
+        current: "current view",
+        unscraped: "unscraped items",
+        all: "all items"
+      }.freeze
+      DEFAULT_PREVIEW_LIMIT = 10
+
+      Result = Struct.new(
+        :status,
+        :selection,
+        :attempted_count,
+        :enqueued_count,
+        :already_enqueued_count,
+        :failure_count,
+        :failure_details,
+        :messages,
+        :rate_limited,
+        keyword_init: true
+      ) do
+        def success?
+          status == :success
+        end
+
+        def partial?
+          status == :partial
+        end
+
+        def error?
+          status == :error
+        end
+
+        def rate_limited?
+          !!rate_limited
+        end
+      end
+
+      def self.selection_label(selection)
+        SELECTION_LABELS[normalize_selection(selection)] || SELECTION_LABELS[:current]
+      end
+
+      def self.selection_counts(source:, preview_items:, preview_limit: 10)
+        preview_collection = Array(preview_items).compact
+        base_scope = SourceMonitor::Item.active.where(source_id: source.id)
+        {
+          current: preview_collection.size.clamp(0, preview_limit.to_i.nonzero? || preview_collection.size),
+          unscraped: base_scope.merge(unscraped_scope).count,
+          all: base_scope.count
+        }
+      end
+
+      def self.normalize_selection(selection)
+        value = selection.is_a?(String) ? selection.strip : selection
+        value = value.to_s.downcase.to_sym if value
+        value if SELECTIONS.include?(value)
+      end
+
+      def initialize(source:, selection:, preview_limit: DEFAULT_PREVIEW_LIMIT, enqueuer: SourceMonitor::Scraping::Enqueuer, config: SourceMonitor.config.scraping)
+        @source = source
+        @selection = self.class.normalize_selection(selection) || :current
+        normalized_limit = preview_limit.respond_to?(:to_i) ? preview_limit.to_i : DEFAULT_PREVIEW_LIMIT
+        @preview_limit = normalized_limit.positive? ? normalized_limit : DEFAULT_PREVIEW_LIMIT
+        @enqueuer = enqueuer
+        @config = config
+      end
+
+      def call
+        return disabled_result unless source.scraping_enabled?
+        return invalid_selection_result unless SELECTIONS.include?(selection)
+
+        items = scoped_items.to_a
+        attempted_count = items.size
+
+        return no_items_result if attempted_count.zero?
+
+        failure_details = Hash.new(0)
+        messages = []
+        enqueued_count = 0
+        already_enqueued_count = 0
+        rate_limited = false
+
+        items.each do |item|
+          enqueue_result = enqueuer.enqueue(item: item, source:, reason: :manual)
+
+          case enqueue_result.status
+          when :enqueued
+            enqueued_count += 1
+          when :already_enqueued
+            already_enqueued_count += 1
+          when :rate_limited
+            failure_details[:rate_limited] += 1
+            messages << enqueue_result.message if enqueue_result.message.present?
+            rate_limited = true
+            break
+          else
+            key = enqueue_result.status || :unknown
+            failure_details[key] += 1
+            messages << enqueue_result.message if enqueue_result.message.present?
+          end
+        end
+
+        failure_count = failure_details.values.sum
+        status = determine_status(enqueued_count:, failure_count:, already_enqueued_count:)
+
+        Result.new(
+          status:,
+          selection:,
+          attempted_count: attempted_count,
+          enqueued_count:,
+          already_enqueued_count:,
+          failure_count:,
+          failure_details: failure_details.freeze,
+          messages: messages.compact.uniq,
+          rate_limited: rate_limited
+        )
+      end
+
+      private
+
+      attr_reader :source, :selection, :preview_limit, :enqueuer, :config
+
+      def scoped_items
+        scope = case selection
+        when :current
+          base_scope.limit(preview_limit)
+        when :unscraped
+          base_scope.merge(unscraped_scope)
+        when :all
+          base_scope
+        else
+          base_scope.limit(preview_limit)
+        end
+
+        scope = without_inflight(scope)
+        apply_batch_limit(scope)
+      end
+
+      def base_scope
+        SourceMonitor::Item.active.where(source_id: source.id).order(Arel.sql("published_at DESC NULLS LAST, created_at DESC"))
+      end
+
+      def without_inflight(scope)
+        statuses = SourceMonitor::Scraping::State::IN_FLIGHT_STATUSES
+        column = SourceMonitor::Item.arel_table[:scrape_status]
+        scope.where(column.eq(nil).or(column.not_in(statuses)))
+      end
+
+      def self.unscraped_scope
+        item_table = SourceMonitor::Item.arel_table
+        failed_statuses = %w[failed partial]
+        SourceMonitor::Item.active.where(
+          item_table[:scraped_at].eq(nil)
+            .or(item_table[:scrape_status].in(failed_statuses))
+        )
+      end
+
+      def unscraped_scope
+        self.class.unscraped_scope
+      end
+
+      def apply_batch_limit(scope)
+        limit = config.max_bulk_batch_size
+        return scope unless limit
+
+        current_limit = scope.limit_value
+        effective_limit = current_limit ? [ current_limit, limit ].min : limit
+        scope.limit(effective_limit)
+      end
+
+      def determine_status(enqueued_count:, failure_count:, already_enqueued_count:)
+        if enqueued_count.positive? && failure_count.zero?
+          :success
+        elsif enqueued_count.positive?
+          :partial
+        elsif already_enqueued_count.positive?
+          :partial
+        else
+          :error
+        end
+      end
+
+      def disabled_result
+        Result.new(
+          status: :error,
+          selection:,
+          attempted_count: 0,
+          enqueued_count: 0,
+          already_enqueued_count: 0,
+          failure_count: 1,
+          failure_details: { scraping_disabled: 1 },
+          messages: [ "Scraping is disabled for this source." ],
+          rate_limited: false
+        )
+      end
+
+      def invalid_selection_result
+        Result.new(
+          status: :error,
+          selection:,
+          attempted_count: 0,
+          enqueued_count: 0,
+          already_enqueued_count: 0,
+          failure_count: 1,
+          failure_details: { invalid_selection: 1 },
+          messages: [ "Invalid selection for bulk scrape." ],
+          rate_limited: false
+        )
+      end
+
+      def no_items_result
+        Result.new(
+          status: :error,
+          selection:,
+          attempted_count: 0,
+          enqueued_count: 0,
+          already_enqueued_count: 0,
+          failure_count: 1,
+          failure_details: { no_items: 1 },
+          messages: [ "No items match the selected scope." ],
+          rate_limited: false
+        )
+      end
+    end
+  end
+end

data/lib/source_monitor/scraping/enqueuer.rb

@@ -0,0 +1,125 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  module Scraping
+    # Coordinates queuing of scraping jobs while respecting source
+    # configuration and avoiding duplicate enqueues for the same item.
+    class Enqueuer
+      Result = Struct.new(:status, :message, :item, keyword_init: true) do
+        def enqueued?
+          status == :enqueued
+        end
+
+        def already_enqueued?
+          status == :already_enqueued
+        end
+
+        def failure?
+          !enqueued? && !already_enqueued?
+        end
+      end
+
+      attr_reader :item, :source, :job_class, :reason
+
+      def self.enqueue(item:, source: nil, job_class: SourceMonitor::ScrapeItemJob, reason: :manual)
+        new(item:, source:, job_class:, reason:).enqueue
+      end
+
+      def initialize(item:, source: nil, job_class: SourceMonitor::ScrapeItemJob, reason: :manual)
+        @item = item
+        @source = source || item&.source
+        @job_class = job_class
+        @reason = reason.to_sym
+      end
+
+      def enqueue
+        log("enqueue:start", item:, source:, reason: reason)
+        return failure(:missing_item, "Item could not be found.") unless item
+        return failure(:missing_source, "Item must belong to a source.") unless source
+        return failure(:scraping_disabled, "Scraping is disabled for this source.") unless source.scraping_enabled?
+        if auto_reason? && !source.auto_scrape?
+          return failure(:auto_scrape_disabled, "Automatic scraping is disabled for this source.")
+        end
+
+        already_queued = false
+        rate_limited = false
+        rate_limit_info = nil
+
+        item.with_lock do
+          item.reload
+
+          if SourceMonitor::Scraping::State.in_flight?(item.scrape_status)
+            log("enqueue:in_flight", item:, status: item.scrape_status)
+            already_queued = true
+            next
+          end
+
+          exhausted, info = rate_limit_exhausted?
+          if exhausted
+            rate_limited = true
+            rate_limit_info = info
+            next
+          end
+
+          SourceMonitor::Scraping::State.mark_pending!(item, broadcast: false, lock: false)
+        end
+
+        if already_queued
+          log("enqueue:already_enqueued", item:, status: item.scrape_status)
+          return Result.new(status: :already_enqueued, message: "Scrape already in progress for this item.", item: item)
+        end
+
+        if rate_limited
+          message = rate_limit_message(rate_limit_info)
+          log("enqueue:rate_limited", item:, limit: rate_limit_info&.fetch(:limit, nil), in_flight: rate_limit_info&.fetch(:in_flight, nil))
+          return Result.new(status: :rate_limited, message:, item: item)
+        end
+
+        job_class.perform_later(item.id)
+        log("enqueue:job_enqueued", item:, job_class: job_class.name)
+        Result.new(status: :enqueued, message: "Scrape has been enqueued for processing.", item: item)
+      end
+
+      private
+
+      def auto_reason?
+        reason == :auto
+      end
+
+      def failure(status, message)
+        log("enqueue:failure", item:, status:, message:)
+        Result.new(status:, message:, item: item)
+      end
+
+      def log(stage, item:, **extra)
+        return unless defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger
+
+        payload = {
+          stage: "SourceMonitor::Scraping::Enqueuer##{stage}",
+          item_id: item&.id,
+          source_id: source&.id,
+          reason: reason
+        }.merge(extra.compact)
+        Rails.logger.info("[SourceMonitor::ManualScrape] #{payload.to_json}")
+      rescue StandardError
+        nil
+      end
+
+      def rate_limit_exhausted?
+        limit = SourceMonitor.config.scraping.max_in_flight_per_source
+        return [ false, nil ] unless limit
+
+        in_flight = source.items.where(scrape_status: SourceMonitor::Scraping::State::IN_FLIGHT_STATUSES).count
+        [ in_flight >= limit, { limit:, in_flight: in_flight } ]
+      end
+
+      def rate_limit_message(info)
+        return "Scraping queue is full for this source." unless info
+
+        limit = info[:limit]
+        in_flight = info[:in_flight]
+        "Unable to enqueue scrape: scraping queue is full for this source (#{in_flight}/#{limit} jobs in flight)."
+      end
+    end
+  end
+end

data/lib/source_monitor/scraping/item_scraper/adapter_resolver.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  module Scraping
+    class ItemScraper
+      # Resolves scraper adapter classes based on configuration or engine namespace.
+      class AdapterResolver
+        VALID_NAME_PATTERN = /\A[a-z0-9_]+\z/i.freeze
+
+        def initialize(name:, source:)
+          @name = name.to_s
+          @source = source
+        end
+
+        def resolve!
+          raise_unknown!("No scraper adapter configured for source") if name.blank?
+          raise_unknown!("Invalid scraper adapter: #{name}") unless VALID_NAME_PATTERN.match?(name)
+
+          configured = SourceMonitor.config.scrapers.adapter_for(name)
+          return configured if configured
+
+          constant = resolve_constant
+          return constant if constant <= SourceMonitor::Scrapers::Base
+
+          raise_unknown!("Unknown scraper adapter: #{name}")
+        rescue NameError
+          raise_unknown!("Unknown scraper adapter: #{name}")
+        end
+
+        private
+
+        attr_reader :name, :source
+
+        def resolve_constant
+          SourceMonitor::Scrapers.const_get(name.camelize)
+        end
+
+        def raise_unknown!(message)
+          raise SourceMonitor::Scraping::ItemScraper::UnknownAdapterError, message
+        end
+      end
+    end
+  end
+end

data/lib/source_monitor/scraping/item_scraper/persistence.rb

@@ -0,0 +1,189 @@
+# frozen_string_literal: true
+
+require "active_support/core_ext/object/blank"
+require "active_support/core_ext/object/deep_dup"
+require "active_support/core_ext/hash/indifferent_access"
+
+module SourceMonitor
+  module Scraping
+    class ItemScraper
+      # Persists scrape outcomes to the database and builds a Result object.
+      class Persistence
+        def initialize(item:, source:, adapter_name:)
+          @item = item
+          @source = source
+          @adapter_name = adapter_name
+        end
+
+        def persist_success(adapter_result:, started_at:)
+          completed_at = Time.current
+          success = adapter_result.status.to_s != "failed"
+          status = normalize_status(adapter_result.status, success)
+          metadata = normalize_metadata(adapter_result.metadata)
+          http_status = metadata&.[](:http_status)
+          error_info = success ? {} : extract_error_info(metadata)
+          content_length = adapter_result.html.to_s.presence && adapter_result.html.to_s.bytesize
+
+          log = nil
+          item.class.transaction do
+            apply_item_success(status:, success:, completed_at:, adapter_result:)
+            log = build_log(
+              started_at:,
+              completed_at:,
+              duration_ms: duration_ms(started_at:, completed_at:),
+              success: success,
+              http_status: http_status,
+              content_length: content_length,
+              metadata: metadata,
+              error_class: error_info[:class],
+              error_message: error_info[:message]
+            )
+          end
+
+          SourceMonitor::Scraping::ItemScraper::Result.new(
+            status: status,
+            item: item,
+            log: log,
+            message: message_for(status, success, error_info[:message], metadata)
+          )
+        end
+
+        def persist_failure(error:, started_at:, message_override: nil)
+          raise ArgumentError, "Item does not belong to a source" unless source
+
+          completed_at = Time.current
+          message = message_override.presence || error.message.presence || "Scrape failed"
+          http_status = extract_http_status(error)
+          metadata = failure_metadata(error)
+
+          log = nil
+          item.class.transaction do
+            item.update!(scrape_status: "failed", scraped_at: completed_at)
+            log = build_log(
+              started_at:,
+              completed_at: completed_at,
+              duration_ms: duration_ms(started_at:, completed_at:),
+              success: false,
+              http_status: http_status,
+              content_length: nil,
+              metadata: metadata,
+              error_class: error.class.name,
+              error_message: message
+            )
+          end
+
+          SourceMonitor::Scraping::ItemScraper::Result.new(
+            status: :failed,
+            item: item,
+            log: log,
+            message: "Scrape failed: #{message}",
+            error: error
+          )
+        end
+
+        private
+
+        attr_reader :item, :source, :adapter_name
+
+        def apply_item_success(status:, success:, completed_at:, adapter_result:)
+          attributes = {
+            scrape_status: status.to_s,
+            scraped_at: completed_at
+          }
+
+          if success
+            attributes[:scraped_html] = adapter_result.html
+            attributes[:scraped_content] = adapter_result.content
+          end
+
+          item.update!(attributes)
+        end
+
+        def build_log(started_at:, completed_at:, duration_ms:, success:, http_status:, content_length:, metadata:, error_class:, error_message:)
+          SourceMonitor::ScrapeLog.create!(
+            source: source,
+            item: item,
+            success: success,
+            scraper_adapter: adapter_name,
+            started_at: started_at,
+            completed_at: completed_at,
+            duration_ms: duration_ms,
+            http_status: http_status,
+            content_length: content_length,
+            error_class: error_class,
+            error_message: error_message,
+            metadata: metadata
+          )
+        end
+
+        def normalize_status(raw_status, success)
+          return :success if raw_status.blank? && success
+          return :failed if raw_status.blank?
+
+          raw_status.to_sym
+        end
+
+        def normalize_metadata(metadata)
+          return {} if metadata.blank?
+
+          hash = metadata.respond_to?(:to_h) ? metadata.to_h : metadata
+          hash.deep_dup.with_indifferent_access
+        rescue StandardError
+          {}
+        end
+
+        def extract_error_info(metadata)
+          {
+            class: metadata&.[](:error)&.to_s,
+            message: metadata&.[](:message)&.to_s
+          }.compact
+        end
+
+        def failure_metadata(error)
+          {
+            error: error.class.name,
+            message: error.message,
+            backtrace: Array(error.backtrace).first(5)
+          }.compact
+        end
+
+        def extract_http_status(error)
+          return error.http_status if error.respond_to?(:http_status) && error.http_status.present?
+
+          if error.respond_to?(:response)
+            response = error.response
+            if response.is_a?(Hash)
+              return response[:status] || response["status"]
+            end
+          end
+
+          if error.message && (match = error.message.match(/\b(\d{3})\b/))
+            return match[1].to_i
+          end
+
+          nil
+        end
+
+        def duration_ms(started_at:, completed_at:)
+          return nil unless started_at && completed_at
+
+          ((completed_at - started_at) * 1000).round
+        end
+
+        def message_for(status, success, error_message, metadata)
+          return "Scrape failed: #{error_message}" if !success && error_message.present?
+
+          case status.to_s
+          when "success"
+            strategy = metadata&.[](:extraction_strategy)
+            strategy.present? ? "Scrape completed via #{strategy.to_s.titleize}" : "Scrape completed successfully"
+          when "partial"
+            "Scrape completed with partial content"
+          else
+            success ? "Scrape completed" : "Scrape failed"
+          end
+        end
+      end
+    end
+  end
+end