source_monitor 0.1.1

data/lib/source_monitor/model_extensions.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+require "active_support/core_ext/string/inflections"
+
+module SourceMonitor
+  module ModelExtensions
+    class << self
+      def register(model_class, key)
+        key = key.to_sym
+        registry[key] ||= []
+        entry = registry[key].find { |registered| registered.model_class == model_class }
+
+        unless entry
+          entry = RegisteredModel.new(model_class, base_table_name_for(model_class), key)
+          registry[key] << entry
+        end
+
+        apply_to(entry)
+      end
+
+      def reload!
+        registry.each do |key, models|
+          models.each { |entry| apply_to(entry) }
+        end
+      end
+
+      private
+
+      RegisteredModel = Struct.new(:model_class, :base_table, :key)
+
+      def registry
+        @registry ||= {}
+      end
+
+      def apply_to(entry)
+        definition = SourceMonitor.config.models.for(entry.key)
+
+        assign_table_name(entry)
+        apply_concerns(entry.model_class, definition)
+        apply_validations(entry.model_class, definition)
+      end
+
+      def assign_table_name(entry)
+        model_class = entry.model_class
+        desired = "#{SourceMonitor.table_name_prefix}#{entry.base_table}"
+        model_class.table_name = desired
+      end
+
+      def apply_concerns(model_class, definition)
+        applied = model_class.instance_variable_get(:@_source_monitor_extension_concerns) || []
+
+        definition.each_concern do |signature, mod|
+          next if applied.include?(signature)
+
+          model_class.include(mod) unless model_class < mod
+          applied << signature
+        end
+
+        model_class.instance_variable_set(:@_source_monitor_extension_concerns, applied)
+      end
+
+      def apply_validations(model_class, definition)
+        remove_extension_validations(model_class)
+
+        applied_signatures = []
+        applied_filters = []
+
+        definition.validations.each do |validation|
+          signature = validation.signature
+          next if applied_signatures.include?(signature)
+
+          if validation.symbol?
+            model_class.validate(validation.handler, **validation.options)
+            applied_filters << validation.handler
+          else
+            handler = validation.handler
+            callback = proc { |record| handler.call(record) }
+            model_class.validate(**validation.options, &callback)
+            applied_filters << callback
+          end
+
+          applied_signatures << signature
+        end
+
+        model_class.instance_variable_set(:@_source_monitor_extension_validations, applied_signatures)
+        model_class.instance_variable_set(:@_source_monitor_extension_validation_filters, applied_filters)
+      end
+
+      def base_table_name_for(model_class)
+        model_class.name.demodulize.tableize
+      end
+
+      def remove_extension_validations(model_class)
+        filters = model_class.instance_variable_get(:@_source_monitor_extension_validation_filters)
+        return unless filters&.any?
+
+        callbacks = model_class._validate_callbacks
+        return unless callbacks
+
+        callbacks.to_a.each do |callback|
+          callbacks.delete(callback) if filters.include?(callback.filter)
+        end
+
+        model_class.instance_variable_set(:@_source_monitor_extension_validations, [])
+        model_class.instance_variable_set(:@_source_monitor_extension_validation_filters, [])
+      end
+    end
+  end
+end

data/lib/source_monitor/models/sanitizable.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+require "active_support/concern"
+require "active_support/hash_with_indifferent_access"
+
+module SourceMonitor
+  module Models
+    module Sanitizable
+      extend ActiveSupport::Concern
+
+      included do
+        class_attribute :sanitized_string_attributes, instance_writer: false, default: []
+        class_attribute :sanitized_hash_attributes, instance_writer: false, default: []
+      end
+
+      class_methods do
+        def sanitizes_string_attributes(*attributes)
+          configure_sanitization_callback
+          self.sanitized_string_attributes += attributes.map(&:to_sym)
+          self.sanitized_string_attributes.uniq!
+        end
+
+        def sanitizes_hash_attributes(*attributes)
+          configure_sanitization_callback
+          self.sanitized_hash_attributes += attributes.map(&:to_sym)
+          self.sanitized_hash_attributes.uniq!
+        end
+
+        private
+
+        def configure_sanitization_callback
+          return if @_source_monitor_sanitization_callback_defined
+
+          before_validation :sanitize_model_attributes
+          @_source_monitor_sanitization_callback_defined = true
+        end
+      end
+
+      private
+
+      def sanitize_model_attributes
+        sanitizer = SourceMonitor::Security::ParameterSanitizer
+
+        self.class.sanitized_string_attributes.each do |attribute|
+          value = self[attribute]
+          next unless value.is_a?(String)
+
+          self[attribute] = sanitizer.sanitize(value)
+        end
+
+        self.class.sanitized_hash_attributes.each do |attribute|
+          value = self[attribute] || {}
+          sanitized = sanitizer.sanitize(value)
+          self[attribute] = if sanitized.is_a?(Hash)
+            to_indifferent_hash(sanitized)
+          else
+            ActiveSupport::HashWithIndifferentAccess.new
+          end
+        end
+      end
+
+      def to_indifferent_hash(value)
+        case value
+        when Hash
+          value.each_with_object(ActiveSupport::HashWithIndifferentAccess.new) do |(key, val), memo|
+            memo[key] = to_indifferent_hash(val)
+          end
+        when Array
+          value.map { |element| to_indifferent_hash(element) }
+        else
+          value
+        end
+      end
+    end
+  end
+end

data/lib/source_monitor/models/url_normalizable.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+require "active_support/concern"
+require "active_support/core_ext/object/blank"
+require "uri"
+
+module SourceMonitor
+  module Models
+    module UrlNormalizable
+      extend ActiveSupport::Concern
+
+      included do
+        class_attribute :normalized_url_attributes, instance_writer: false, default: []
+      end
+
+      class_methods do
+        def normalizes_urls(*attributes)
+          return if attributes.empty?
+
+          before_validation :normalize_configured_urls
+          self.normalized_url_attributes += attributes.map(&:to_sym)
+          self.normalized_url_attributes.uniq!
+        end
+
+        def validates_url_format(*attributes)
+          attributes.each do |attribute|
+            validate_method = :"validate_#{attribute}_format"
+
+            define_method validate_method do
+              return if self[attribute].blank?
+
+              errors.add(attribute, "must be a valid HTTP(S) URL") if url_invalid?(attribute)
+            end
+
+            validate validate_method
+          end
+        end
+      end
+
+      def url_invalid?(attribute)
+        invalid_urls[attribute.to_sym]
+      end
+
+      private
+
+      def normalize_configured_urls
+        normalized_url_attributes.each do |attribute|
+          normalize_single_url(attribute)
+        end
+      end
+
+      def normalize_single_url(attribute)
+        raw_value = self[attribute]
+        invalid_urls[attribute] = false
+
+        normalized = normalize_url_value(raw_value)
+        self[attribute] = normalized
+      rescue URI::InvalidURIError
+        invalid_urls[attribute] = true
+      end
+
+      def normalize_url_value(value)
+        return nil if value.blank?
+
+        uri = URI.parse(value.to_s.strip)
+        raise URI::InvalidURIError if uri.scheme.blank? || uri.host.blank?
+
+        scheme = uri.scheme.downcase
+        raise URI::InvalidURIError unless %w[http https].include?(scheme)
+
+        uri.scheme = scheme
+        uri.host = uri.host.downcase
+        uri.path = "/" if uri.path.blank?
+        uri.fragment = nil
+
+        uri.to_s
+      end
+
+      def invalid_urls
+        @_source_monitor_invalid_urls ||= Hash.new(false)
+      end
+    end
+  end
+end

data/lib/source_monitor/pagination/paginator.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  module Pagination
+    Result = Struct.new(
+      :records,
+      :page,
+      :per_page,
+      :has_next_page,
+      :has_previous_page,
+      keyword_init: true
+    ) do
+      def has_next_page?
+        !!has_next_page
+      end
+
+      def has_previous_page?
+        !!has_previous_page
+      end
+
+      def next_page
+        return nil unless has_next_page
+
+        page + 1
+      end
+
+      def previous_page
+        return nil unless has_previous_page
+
+        [ page - 1, 1 ].max
+      end
+    end
+
+    class Paginator
+      DEFAULT_PER_PAGE = 25
+
+      def initialize(scope:, page: 1, per_page: DEFAULT_PER_PAGE)
+        @scope = scope
+        @page = normalize_page(page)
+        @per_page = normalize_per_page(per_page)
+      end
+
+      def paginate
+        paginated_records = fetch_records
+        has_next_page = paginated_records.length > per_page
+
+        Result.new(
+          records: paginated_records.first(per_page),
+          page: page,
+          per_page: per_page,
+          has_next_page: has_next_page,
+          has_previous_page: page > 1
+        )
+      end
+
+      private
+
+      attr_reader :scope, :page, :per_page
+
+      def fetch_records
+        offset = (page - 1) * per_page
+
+        relation = scope.is_a?(ActiveRecord::Relation) ? scope : Array(scope)
+
+        if relation.is_a?(Array)
+          relation.slice(offset, per_page + 1) || []
+        else
+          relation.offset(offset).limit(per_page + 1).to_a
+        end
+      end
+
+      def normalize_page(value)
+        number = value.to_i
+        number = 1 if number <= 0
+        number
+      rescue StandardError
+        1
+      end
+
+      def normalize_per_page(value)
+        number = value.to_i
+        return DEFAULT_PER_PAGE if number <= 0
+
+        [ number, 100 ].min
+      rescue StandardError
+        DEFAULT_PER_PAGE
+      end
+    end
+  end
+end

data/lib/source_monitor/realtime/adapter.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+require "active_support/core_ext/hash/indifferent_access"
+require "active_support/core_ext/hash/keys"
+
+module SourceMonitor
+  module Realtime
+    module Adapter
+      extend self
+
+      FALLBACK_ADAPTERS = %i[async test].freeze
+
+      def configure!
+        return unless action_cable_available?
+
+        desired_adapter = SourceMonitor.config.realtime.adapter
+        return unless desired_adapter
+
+        ensure_dependency!(desired_adapter)
+
+        existing_config = current_config
+        existing_adapter = extract_adapter(existing_config)
+
+        if should_replace_adapter?(existing_adapter, desired_adapter)
+          apply_configuration(SourceMonitor.config.realtime.action_cable_config)
+        elsif same_adapter?(existing_adapter, desired_adapter)
+          merge_defaults(existing_config, SourceMonitor.config.realtime.action_cable_config)
+        end
+      end
+
+      private
+
+      def action_cable_available?
+        defined?(ActionCable) && ActionCable.respond_to?(:server)
+      end
+
+      def current_config
+        ActionCable.server.config.cable || {}
+      end
+
+      def extract_adapter(config)
+        adapter = config.is_a?(Hash) ? config[:adapter] || config["adapter"] : nil
+        adapter&.to_sym
+      end
+
+      def should_replace_adapter?(existing_adapter, desired_adapter)
+        return true if existing_adapter.nil?
+        return true if FALLBACK_ADAPTERS.include?(existing_adapter)
+        return true if desired_adapter && existing_adapter != desired_adapter
+
+        false
+      end
+
+      def same_adapter?(existing_adapter, desired_adapter)
+        existing_adapter && desired_adapter && existing_adapter == desired_adapter
+      end
+
+      def apply_configuration(raw_config)
+        ActionCable.server.config.cable = normalize_config(raw_config)
+      end
+
+      def merge_defaults(existing_config, raw_config)
+        defaults = normalize_config(raw_config)
+        existing = normalize_config(existing_config)
+
+        merged = defaults.merge(existing)
+        ActionCable.server.config.cable = merged
+      end
+
+      def normalize_config(config)
+        hash = config.to_h.deep_symbolize_keys
+        hash[:adapter] = hash[:adapter].to_s if hash.key?(:adapter)
+        hash.with_indifferent_access
+      end
+
+      def ensure_dependency!(adapter)
+        case adapter.to_sym
+        when :solid_cable
+          require "solid_cable"
+        when :redis
+          require "redis"
+        end
+      rescue LoadError => error
+        raise_missing_dependency_error(adapter, error)
+      end
+
+      def raise_missing_dependency_error(adapter, error)
+        message = <<~ERROR.squish
+          SourceMonitor realtime adapter #{adapter.inspect} could not be loaded: #{error.class}: #{error.message}.
+          Ensure the corresponding gem is available or configure `SourceMonitor.config.realtime.adapter` with a supported backend.
+        ERROR
+
+        raise LoadError, message
+      end
+    end
+  end
+end

data/lib/source_monitor/realtime/broadcaster.rb

@@ -0,0 +1,237 @@
+# frozen_string_literal: true
+
+require "action_view"
+
+module SourceMonitor
+  module Realtime
+    module Broadcaster
+      extend self
+      extend ActionView::RecordIdentifier
+
+      SOURCE_INDEX_STREAM = "source_monitor_sources"
+      NOTIFICATION_STREAM = "source_monitor_notifications"
+
+      def setup!
+        return unless turbo_available?
+        return if @setup
+
+        register_callback(:after_fetch_completed, fetch_callback)
+        register_callback(:after_item_scraped, item_callback)
+
+        @setup = true
+      end
+
+      def fetch_callback
+        @fetch_callback ||= lambda { |event| handle_fetch_completed(event) }
+      end
+
+      def item_callback
+        @item_callback ||= lambda { |event| handle_item_scraped(event) }
+      end
+
+      def broadcast_source(source)
+        return unless turbo_available?
+        source = reload_record(source)
+        return unless source
+
+        broadcast_source_row(source)
+        broadcast_source_show(source)
+      end
+
+      def broadcast_item(item)
+        return unless turbo_available?
+        item = reload_record(item)
+        return unless item
+
+        Turbo::StreamsChannel.broadcast_replace_to(
+          item,
+          :details,
+          target: dom_id(item, :details),
+          html: render_html = SourceMonitor::ItemsController.render(
+            partial: "source_monitor/items/details_wrapper",
+            locals: { item: item }
+          )
+        )
+        log_info(
+          "broadcast_item",
+          item_id: item.id,
+          stream: item_stream_identifier(item),
+          status: item.scrape_status,
+          contains_scraped_label: render_html.include?("Scraped")
+        )
+      rescue StandardError => error
+        log_error("item broadcast", error)
+      end
+
+      def broadcast_toast(message:, level: :info, title: nil, delay_ms: 5000)
+        return unless turbo_available?
+        return if message.blank?
+
+        Turbo::StreamsChannel.broadcast_append_to(
+          NOTIFICATION_STREAM,
+          target: NOTIFICATION_STREAM,
+          html: SourceMonitor::ApplicationController.render(
+            partial: "source_monitor/shared/toast",
+            locals: {
+              message: message,
+              level: level,
+              title: title,
+              delay_ms: delay_ms
+            }
+          )
+        )
+      rescue StandardError => error
+        log_error("toast broadcast", error)
+      end
+
+      private
+
+      def handle_fetch_completed(event)
+        source = event&.source
+        return unless source
+
+        broadcast_source(source)
+        broadcast_fetch_toast(event)
+      end
+
+      def handle_item_scraped(event)
+        item = event&.item
+        return unless item
+
+        broadcast_item(item)
+        broadcast_source(event&.source || item.source)
+        broadcast_item_toast(event)
+      end
+
+      def broadcast_fetch_toast(event)
+        return unless event
+        source = event.source
+        status = event.status.to_s
+
+        case status
+        when "fetched"
+          processing = event.result&.item_processing
+          created = processing&.created.to_i
+          updated = processing&.updated.to_i
+          broadcast_toast(
+            message: "Fetched #{source.name} (#{created} created, #{updated} updated).",
+            level: :success
+          )
+        when "not_modified"
+          broadcast_toast(
+            message: "#{source.name} is up to date.",
+            level: :info
+          )
+        when "failed"
+          error_message = event.result&.error&.message ||
+            source.last_error ||
+            "Fetch failed"
+          broadcast_toast(
+            message: "Fetch failed for #{source.name}: #{error_message}",
+            level: :error,
+            delay_ms: 6000
+          )
+        end
+      end
+
+      def broadcast_item_toast(event)
+        return unless event
+        item = event.item
+        source = event.source
+        title = item&.title.presence || "Feed item"
+
+        if event.status.to_s == "failed"
+          message = "Scrape failed for #{title}"
+          message += " (#{source.name})" if source
+          broadcast_toast(message:, level: :error, delay_ms: 6000)
+        else
+          message = "Scrape completed for #{title}"
+          message += " (#{source.name})" if source
+          broadcast_toast(message:, level: :success)
+        end
+      end
+
+      def broadcast_source_row(source)
+        Turbo::StreamsChannel.broadcast_replace_to(
+          SOURCE_INDEX_STREAM,
+          target: dom_id(source, :row),
+          html: SourceMonitor::SourcesController.render(
+            partial: "source_monitor/sources/row",
+            locals: {
+              source: source,
+              activity_rate: SourceMonitor::Analytics::SourceActivityRates.rate_for(source)
+            }
+          )
+        )
+      rescue StandardError => error
+        log_error("source row broadcast", error)
+      end
+
+      def broadcast_source_show(source)
+        Turbo::StreamsChannel.broadcast_replace_to(
+          source,
+          :details,
+          target: dom_id(source, :details),
+          html: SourceMonitor::SourcesController.render(
+            partial: "source_monitor/sources/details_wrapper",
+            locals: { source: source }
+          )
+        )
+        log_info(
+          "broadcast_source_show",
+          source_id: source.id,
+          stream: source_stream_identifier(source),
+          status: source.fetch_status
+        )
+      rescue StandardError => error
+        log_error("source show broadcast", error)
+      end
+
+      def reload_record(record)
+        return unless record
+
+        record.reload
+      rescue StandardError
+        record
+      end
+
+      def log_info(stage, **extra)
+        return unless defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger
+
+        payload = {
+          stage: "SourceMonitor::Realtime::Broadcaster##{stage}"
+        }.merge(extra.compact)
+        Rails.logger.info("[SourceMonitor::ManualScrape] #{payload.to_json}")
+      rescue StandardError
+        nil
+      end
+
+      def item_stream_identifier(item)
+        Turbo::StreamsChannel.signed_stream_name([ item, :details ]) rescue nil
+      end
+
+      def source_stream_identifier(source)
+        Turbo::StreamsChannel.signed_stream_name([ source, :details ]) rescue nil
+      end
+
+      def turbo_available?
+        defined?(Turbo::StreamsChannel)
+      end
+
+      def register_callback(name, callback)
+        callbacks = SourceMonitor.config.events.callbacks_for(name)
+        return if callbacks.include?(callback)
+
+        SourceMonitor.config.events.public_send(name, callback)
+      end
+
+      def log_error(context, error)
+        Rails.logger.error(
+          "[SourceMonitor] Realtime #{context} failed: #{error.class}: #{error.message}"
+        ) if defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger
+      rescue StandardError
+        nil
+      end
+    end
+  end
+end

data/lib/source_monitor/realtime.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require "source_monitor/realtime/adapter"
+require "source_monitor/realtime/broadcaster"
+
+module SourceMonitor
+  module Realtime
+    class << self
+      def setup!
+        SourceMonitor::Realtime::Adapter.configure!
+        SourceMonitor::Realtime::Broadcaster.setup!
+      end
+
+      delegate :broadcast_source, :broadcast_item, :broadcast_toast, to: SourceMonitor::Realtime::Broadcaster
+    end
+  end
+end