source_monitor 0.13.0 → 0.13.1

data/lib/source_monitor/items/normalized_entry.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require "active_support/core_ext/object/blank"
+require "source_monitor/items/item_creator/entry_parser"
+require "source_monitor/items/item_creator/content_extractor"
+
+module SourceMonitor
+  module Items
+    class NormalizedEntry
+      def self.call(source:, entry:, content_extractor: nil)
+        new(source: source, entry: entry, content_extractor: content_extractor).item_attributes
+      end
+
+      def initialize(source:, entry:, content_extractor: nil)
+        @source = source
+        @entry = entry
+        @content_extractor = content_extractor || ItemCreator::ContentExtractor.new(source: source)
+      end
+
+      def attributes
+        @attributes ||= parser.parse
+      end
+
+      def item_attributes
+        attributes.merge(guid: item_guid)
+      end
+
+      def raw_guid
+        attributes[:guid]
+      end
+
+      def normalized_guid
+        raw_guid.present? ? raw_guid.downcase : nil
+      end
+
+      def raw_guid_present?
+        normalized_guid.present?
+      end
+
+      def item_guid
+        normalized_guid.presence || content_fingerprint
+      end
+
+      def content_fingerprint
+        attributes[:content_fingerprint]
+      end
+
+      private
+
+      attr_reader :source, :entry, :content_extractor
+
+      def parser
+        @parser ||= ItemCreator::EntryParser.new(
+          source: source,
+          entry: entry,
+          content_extractor: content_extractor
+        )
+      end
+    end
+  end
+end

data/lib/source_monitor/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module SourceMonitor
-  VERSION = "0.13.0"
+  VERSION = "0.13.1"
 end

data/lib/source_monitor.rb

@@ -89,6 +89,7 @@ module SourceMonitor
 
   module ImportSessions
     autoload :EntryNormalizer, "source_monitor/import_sessions/entry_normalizer"
+    autoload :Wizard, "source_monitor/import_sessions/wizard"
     autoload :HealthCheckBroadcaster, "source_monitor/import_sessions/health_check_broadcaster"
     autoload :HealthCheckUpdater, "source_monitor/import_sessions/health_check_updater"
     autoload :OPMLImporter, "source_monitor/import_sessions/opml_importer"
@@ -108,6 +109,7 @@ module SourceMonitor
 
   module Items
     autoload :ItemCreator, "source_monitor/items/item_creator"
+    autoload :NormalizedEntry, "source_monitor/items/normalized_entry"
     autoload :RetentionPruner, "source_monitor/items/retention_pruner"
     autoload :RetentionStrategies, "source_monitor/items/retention_strategies"
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: source_monitor
 version: !ruby/object:Gem::Version
-  version: 0.13.0
+  version: 0.13.1
 platform: ruby
 authors:
 - dchuk
@@ -408,9 +408,6 @@ files:
 - app/controllers/source_monitor/health_controller.rb
 - app/controllers/source_monitor/import_history_dismissals_controller.rb
 - app/controllers/source_monitor/import_sessions/bulk_configuration.rb
-- app/controllers/source_monitor/import_sessions/entry_annotation.rb
-- app/controllers/source_monitor/import_sessions/health_check_management.rb
-- app/controllers/source_monitor/import_sessions/opml_parser.rb
 - app/controllers/source_monitor/import_sessions_controller.rb
 - app/controllers/source_monitor/item_scrapes_controller.rb
 - app/controllers/source_monitor/items_controller.rb
@@ -598,7 +595,9 @@ files:
 - lib/source_monitor/fetching/feed_fetcher.rb
 - lib/source_monitor/fetching/feed_fetcher/adaptive_interval.rb
 - lib/source_monitor/fetching/feed_fetcher/entry_processor.rb
+- lib/source_monitor/fetching/feed_fetcher/failure_outcome.rb
 - lib/source_monitor/fetching/feed_fetcher/source_updater.rb
+- lib/source_monitor/fetching/feed_fetcher/success_outcome.rb
 - lib/source_monitor/fetching/fetch_error.rb
 - lib/source_monitor/fetching/fetch_runner.rb
 - lib/source_monitor/fetching/retry_orchestrator.rb
@@ -619,12 +618,14 @@ files:
 - lib/source_monitor/import_sessions/health_check_broadcaster.rb
 - lib/source_monitor/import_sessions/health_check_updater.rb
 - lib/source_monitor/import_sessions/opml_importer.rb
+- lib/source_monitor/import_sessions/wizard.rb
 - lib/source_monitor/instrumentation.rb
 - lib/source_monitor/items/batch_item_creator.rb
 - lib/source_monitor/items/item_creator.rb
 - lib/source_monitor/items/item_creator/content_extractor.rb
 - lib/source_monitor/items/item_creator/entry_parser.rb
 - lib/source_monitor/items/item_creator/entry_parser/media_extraction.rb
+- lib/source_monitor/items/normalized_entry.rb
 - lib/source_monitor/items/retention_pruner.rb
 - lib/source_monitor/items/retention_strategies.rb
 - lib/source_monitor/items/retention_strategies/destroy.rb

data/app/controllers/source_monitor/import_sessions/entry_annotation.rb

@@ -1,187 +0,0 @@
-# frozen_string_literal: true
-
-module SourceMonitor
-  module ImportSessions
-    module EntryAnnotation
-      extend ActiveSupport::Concern
-
-      private
-
-      def annotated_entries(selected_ids)
-        selected_ids ||= []
-        entries = Array(@import_session.parsed_sources)
-        return [] if entries.blank?
-
-        normalized = entries.map { |entry| normalize_entry(entry) }
-
-        feed_urls = normalized.filter_map { |entry| entry[:feed_url]&.downcase }
-        duplicate_lookup = if feed_urls.present?
-          SourceMonitor::Source.where("LOWER(feed_url) IN (?)", feed_urls).pluck(:feed_url).map(&:downcase)
-        else
-          []
-        end
-
-        normalized.map do |entry|
-          duplicate = entry[:feed_url].present? && duplicate_lookup.include?(entry[:feed_url].downcase)
-          entry.merge(
-            duplicate: duplicate,
-            selectable: entry[:status] == "valid" && !duplicate,
-            selected: selected_ids.include?(entry[:id])
-          )
-        end
-      end
-
-      def normalize_entry(entry)
-        entry = entry.to_h
-        SourceMonitor::ImportSessions::EntryNormalizer.normalize(entry)
-      end
-
-      def filter_entries(entries, filter)
-        case filter
-        when "new"
-          entries.select { |entry| entry[:selectable] }
-        when "existing"
-          entries.select { |entry| entry[:duplicate] }
-        else
-          entries
-        end
-      end
-
-      def selectable_entries_from(entries)
-        entries.select { |entry| entry[:selectable] }
-      end
-
-      def selectable_entries
-        @selectable_entries ||= annotated_entries(@selected_source_ids).select { |entry| entry[:selectable] }
-      end
-
-      def build_selection_from_params
-        @selected_source_ids ||= []
-
-        if params.dig(:import_session, :select_all) == "true"
-          return selectable_entries.map { |entry| entry[:id] }
-        end
-
-        if params.dig(:import_session, :select_none) == "true"
-          return []
-        end
-
-        ids = params.dig(:import_session, :selected_source_ids)
-        return [] unless ids
-
-        Array(ids).map { |id| id.to_s }.uniq
-      end
-
-      def health_check_selection_from_params
-        if params.dig(:import_session, :select_all) == "true"
-          return health_check_targets.dup
-        end
-
-        return [] if params.dig(:import_session, :select_none) == "true"
-
-        ids = params.dig(:import_session, :selected_source_ids)
-        return Array(@import_session.selected_source_ids).map(&:to_s) unless ids
-
-        Array(ids).map { |id| id.to_s }.uniq & health_check_targets
-      end
-
-      def advancing_from_health_check?
-        target_step != "health_check"
-      end
-
-      def advancing_from_preview?
-        target_step != "preview"
-      end
-
-      def normalize_page_param(value)
-        number = value.to_i
-        number = 1 if number <= 0
-        number
-      rescue StandardError
-        1
-      end
-
-      def permitted_filter(raw)
-        value = raw.to_s.presence
-        return unless value
-
-        %w[all new existing].find { |candidate| candidate == value }
-      end
-
-      def preview_per_page
-        25
-      end
-
-      def state_params
-        @state_params ||= begin
-          permitted = params.fetch(:import_session, {}).permit(
-            :current_step,
-            :next_step,
-            :select_all,
-            :select_none,
-            parsed_sources: [],
-            selected_source_ids: [],
-            bulk_settings: {},
-            opml_file_metadata: {}
-          )
-
-          SourceMonitor::Security::ParameterSanitizer.sanitize(permitted.to_h)
-        end
-      end
-
-      def permitted_step(value)
-        step = value.to_s.presence
-        return unless step
-
-        ImportSession::STEP_ORDER.find { |candidate| candidate == step }
-      end
-
-      def target_step
-        next_step = state_params[:next_step] || state_params["next_step"]
-        permitted_step(next_step) || @current_step || ImportSession.default_step
-      end
-
-      def session_attributes
-        attrs = state_params.except(:next_step, :current_step, "next_step", "current_step")
-        attrs[:opml_file_metadata] = build_file_metadata if uploading_file?
-        attrs[:current_step] = target_step
-        attrs
-      end
-
-      def prepare_preview_context(skip_default: false)
-        @filter = permitted_filter(params[:filter]) || "all"
-        @page = normalize_page_param(params[:page])
-        @selected_source_ids = Array(@import_session.selected_source_ids).map(&:to_s)
-
-        @preview_entries = annotated_entries(@selected_source_ids)
-
-        if !skip_default && @selected_source_ids.blank? && @preview_entries.present?
-          defaults = selectable_entries_from(@preview_entries).map { |entry| entry[:id] }
-          @selected_source_ids = defaults
-          @import_session.update_column(:selected_source_ids, defaults)
-          @preview_entries = annotated_entries(@selected_source_ids)
-        end
-
-        @filtered_entries = filter_entries(@preview_entries, @filter)
-
-        paginator = SourceMonitor::Pagination::Paginator.new(
-          scope: @filtered_entries,
-          page: @page,
-          per_page: preview_per_page
-        ).paginate
-
-        @paginated_entries = paginator.records
-        @has_next_page = paginator.has_next_page
-        @has_previous_page = paginator.has_previous_page
-        @page = paginator.page
-      end
-
-      def prepare_confirm_context
-        @selected_source_ids = Array(@import_session.selected_source_ids).map(&:to_s)
-        @selected_entries = annotated_entries(@selected_source_ids)
-          .select { |entry| @selected_source_ids.include?(entry[:id]) }
-        @bulk_settings = @import_session.bulk_settings || {}
-      end
-    end
-  end
-end

data/app/controllers/source_monitor/import_sessions/health_check_management.rb

@@ -1,112 +0,0 @@
-# frozen_string_literal: true
-
-module SourceMonitor
-  module ImportSessions
-    module HealthCheckManagement
-      extend ActiveSupport::Concern
-
-      private
-
-      def start_health_checks_if_needed
-        return unless @current_step == "health_check"
-
-        jobs_to_enqueue = []
-
-        @import_session.with_lock do
-          @import_session.reload
-          selected = Array(@import_session.selected_source_ids).map(&:to_s)
-
-          if selected.blank?
-            @import_session.update_columns(health_checks_active: false, health_check_target_ids: [])
-            next
-          end
-
-          if @import_session.health_checks_active? && @import_session.health_check_targets.sort == selected.sort
-            @health_check_target_ids = @import_session.health_check_targets
-            next
-          end
-
-          updated_entries = reset_health_results(@import_session.parsed_sources, selected)
-          @import_session.update!(
-            parsed_sources: updated_entries,
-            health_checks_active: true,
-            health_check_target_ids: selected,
-            health_check_started_at: Time.current,
-            health_check_completed_at: nil
-          )
-
-          @health_check_target_ids = selected
-          jobs_to_enqueue = selected
-        end
-
-        enqueue_health_check_jobs(@import_session, jobs_to_enqueue) if jobs_to_enqueue.any?
-      end
-
-      def reset_health_results(entries, target_ids)
-        Array(entries).map do |entry|
-          entry_hash = entry.to_h
-          entry_id = entry_hash["id"] || entry_hash[:id]
-          next entry_hash unless target_ids.include?(entry_id.to_s)
-
-          entry_hash.merge("health_status" => "pending", "health_error" => nil)
-        end
-      end
-
-      def enqueue_health_check_jobs(import_session, target_ids)
-        target_ids.each do |target_id|
-          SourceMonitor::ImportSessionHealthCheckJob.set(wait: 1.second).perform_later(import_session.id, target_id)
-        end
-      end
-
-      def deactivate_health_checks!
-        return unless @import_session.health_checks_active?
-
-        @import_session.update_columns(
-          health_checks_active: false,
-          health_check_completed_at: Time.current
-        )
-      end
-
-      def health_check_entries(selected_ids)
-        targets = health_check_targets
-        entries = Array(@import_session.parsed_sources).map { |entry| normalize_entry(entry) }
-
-        entries.select { |entry| targets.include?(entry[:id]) }.map do |entry|
-          entry.merge(selected: selected_ids.include?(entry[:id]))
-        end
-      end
-
-      def health_check_progress(entries)
-        total = health_check_targets.size
-        completed = entries.count { |entry| health_check_complete?(entry) }
-
-        {
-          completed: completed,
-          total: total,
-          pending: [ total - completed, 0 ].max,
-          active: @import_session.health_checks_active?,
-          done: total.positive? && completed >= total
-        }
-      end
-
-      def health_check_complete?(entry)
-        %w[working failing].include?(entry[:health_status].to_s)
-      end
-
-      def health_check_targets
-        targets = @import_session.health_check_targets
-        targets = Array(@import_session.selected_source_ids).map(&:to_s) if targets.blank?
-        targets
-      end
-
-      def prepare_health_check_context
-        start_health_checks_if_needed
-
-        @selected_source_ids = Array(@import_session.selected_source_ids).map(&:to_s)
-        @health_check_entries = health_check_entries(@selected_source_ids)
-        @health_check_target_ids = health_check_targets
-        @health_progress = health_check_progress(@health_check_entries)
-      end
-    end
-  end
-end

data/app/controllers/source_monitor/import_sessions/opml_parser.rb

@@ -1,130 +0,0 @@
-# frozen_string_literal: true
-
-module SourceMonitor
-  module ImportSessions
-    module OpmlParser
-      extend ActiveSupport::Concern
-
-      ALLOWED_CONTENT_TYPES = %w[text/xml application/xml text/x-opml application/opml].freeze
-      GENERIC_CONTENT_TYPES = %w[application/octet-stream binary/octet-stream].freeze
-
-      class UploadError < StandardError; end
-
-      private
-
-      def build_file_metadata
-        return {} unless params[:opml_file].respond_to?(:original_filename)
-
-        file = params[:opml_file]
-        {
-          "filename" => file.original_filename,
-          "byte_size" => file.size,
-          "content_type" => file.content_type
-        }
-      end
-
-      def uploading_file?
-        params[:opml_file].present?
-      end
-
-      def validate_upload!
-        return [ "Upload an OPML file to continue." ] unless uploading_file?
-
-        file = params[:opml_file]
-        errors = []
-
-        errors << "The uploaded file is empty. Choose another OPML file." if file.size.to_i <= 0
-
-        if file.content_type.present? && !content_type_allowed?(file.content_type) && !generic_content_type?(file.content_type)
-          errors << "Upload must be an OPML or XML file."
-        end
-
-        errors
-      end
-
-      def content_type_allowed?(content_type)
-        ALLOWED_CONTENT_TYPES.include?(content_type)
-      end
-
-      def generic_content_type?(content_type)
-        GENERIC_CONTENT_TYPES.include?(content_type)
-      end
-
-      def parse_opml_file(file)
-        content = file.read
-        file.rewind if file.respond_to?(:rewind)
-
-        raise UploadError, "The uploaded file appears to be empty." if content.blank?
-
-        document = Nokogiri::XML(content) { |config| config.strict.nonet }
-        raise UploadError, "The uploaded file is not valid XML or OPML." if document.root.nil?
-
-        outlines = document.xpath("//outline")
-
-        entries = []
-
-        outlines.each_with_index do |outline, index|
-          next unless outline.attribute_nodes.any? { |attr| attr.name.casecmp("xmlurl").zero? }
-
-          entries << build_entry(outline, index)
-        end
-
-        entries
-      rescue Nokogiri::XML::SyntaxError => error
-        raise UploadError, "We couldn't parse that OPML file: #{error.message}"
-      end
-
-      def build_entry(outline, index)
-        feed_url = outline_attribute(outline, "xmlUrl")
-        website_url = outline_attribute(outline, "htmlUrl")
-        title = outline_attribute(outline, "title") || outline_attribute(outline, "text")
-
-        if feed_url.blank?
-          return malformed_entry(index, feed_url, title, website_url, "Missing feed URL")
-        end
-
-        unless valid_feed_url?(feed_url)
-          return malformed_entry(index, feed_url, title, website_url, "Feed URL must be HTTP or HTTPS")
-        end
-
-        {
-          id: "outline-#{index}",
-          raw_outline_index: index,
-          feed_url: feed_url,
-          title: title,
-          website_url: website_url,
-          status: "valid",
-          error: nil,
-          health_status: nil,
-          health_error: nil
-        }
-      end
-
-      def malformed_entry(index, feed_url, title, website_url, error)
-        {
-          id: "outline-#{index}",
-          raw_outline_index: index,
-          feed_url: feed_url.presence,
-          title: title,
-          website_url: website_url,
-          status: "malformed",
-          error: error,
-          health_status: nil,
-          health_error: nil
-        }
-      end
-
-      def outline_attribute(outline, name)
-        attribute = outline.attribute_nodes.find { |attr| attr.name.casecmp(name).zero? }
-        attribute&.value.to_s.presence
-      end
-
-      def valid_feed_url?(url)
-        parsed = URI.parse(url)
-        parsed.is_a?(URI::HTTP) && parsed.host.present?
-      rescue URI::InvalidURIError
-        false
-      end
-    end
-  end
-end