source_monitor 0.1.3 → 0.2.0

data/app/controllers/source_monitor/sources_controller.rb

@@ -2,6 +2,7 @@
 
 require "source_monitor/sources/turbo_stream_presenter"
 require "source_monitor/scraping/bulk_result_presenter"
+require "source_monitor/sources/params"
 
 module SourceMonitor
   class SourcesController < ApplicationController
@@ -31,6 +32,8 @@ module SourceMonitor
         search_params: @search_params
       )
 
+      @recent_import_histories = SourceMonitor::ImportHistory.recent_for(source_monitor_current_user&.id).limit(5)
+
       @fetch_interval_distribution = metrics.fetch_interval_distribution
       @fetch_interval_filter = metrics.fetch_interval_filter
       @selected_fetch_interval_bucket = metrics.selected_fetch_interval_bucket
@@ -128,45 +131,11 @@ module SourceMonitor
     end
 
     def default_attributes
-      {
-        active: true,
-        scraping_enabled: false,
-        auto_scrape: false,
-        requires_javascript: false,
-        feed_content_readability_enabled: false,
-        fetch_interval_minutes: 360,
-        adaptive_fetching_enabled: true,
-        scraper_adapter: "readability"
-      }
+      SourceMonitor::Sources::Params.default_attributes
     end
 
     def source_params
-      permitted = params.require(:source).permit(
-        :name,
-        :feed_url,
-        :website_url,
-        :fetch_interval_minutes,
-        :active,
-        :auto_scrape,
-        :scraping_enabled,
-        :requires_javascript,
-        :feed_content_readability_enabled,
-        :scraper_adapter,
-        :items_retention_days,
-        :max_items,
-        :adaptive_fetching_enabled,
-        :health_auto_pause_threshold,
-        scrape_settings: [
-          :include_plain_text,
-          :timeout,
-          :javascript_enabled,
-          { selectors: %i[content title],
-            http: [],
-            readability: [] }
-        ]
-      )
-
-      SourceMonitor::Security::ParameterSanitizer.sanitize(permitted.to_h)
+      SourceMonitor::Sources::Params.sanitize(params)
     end
 
     def safe_redirect_path(raw_value)

data/app/helpers/source_monitor/application_helper.rb

@@ -303,6 +303,23 @@ module SourceMonitor
       end
     end
 
+    def formatted_setting_value(value)
+      case value
+      when TrueClass
+        "Enabled"
+      when FalseClass
+        "Disabled"
+      when Hash
+        value.to_json
+      when Array
+        value.join(", ")
+      when NilClass
+        "—"
+      else
+        value
+      end
+    end
+
     private
 
     def derive_item_scrape_status(item:, source: nil)

data/app/jobs/source_monitor/import_opml_job.rb

@@ -0,0 +1,150 @@
+# frozen_string_literal: true
+
+require "set"
+require "source_monitor/import_sessions/entry_normalizer"
+require "source_monitor/realtime/broadcaster"
+require "source_monitor/sources/params"
+
+module SourceMonitor
+  class ImportOpmlJob < ApplicationJob
+    source_monitor_queue :fetch
+
+    discard_on ActiveJob::DeserializationError
+
+    def perform(import_session_id, import_history_id)
+      @import_session = SourceMonitor::ImportSession.find_by(id: import_session_id)
+      @import_history = SourceMonitor::ImportHistory.find_by(id: import_history_id)
+      return unless import_session && import_history
+
+      import_history.update_columns(started_at: Time.current) unless import_history.started_at
+
+      processed = Set.new
+
+      selected_entries.each do |entry|
+        process_entry(entry, processed)
+      end
+
+      import_history.update!(
+        imported_sources: imported_sources,
+        failed_sources: failed_sources,
+        skipped_duplicates: skipped_duplicates,
+        bulk_settings: import_session.bulk_settings.presence || {},
+        completed_at: Time.current
+      )
+
+      broadcast_completion(import_history)
+    end
+
+    private
+
+    attr_reader :import_session, :import_history
+
+    def selected_entries
+      ids = Array(import_session.selected_source_ids).map(&:to_s)
+
+      Array(import_session.parsed_sources)
+        .map { |entry| SourceMonitor::ImportSessions::EntryNormalizer.normalize(entry) }
+        .select { |entry| ids.include?(entry[:id]) }
+    end
+
+    def process_entry(entry, processed)
+      feed_url = entry[:feed_url].to_s
+
+      if feed_url.blank?
+        failed_sources << failure_payload(feed_url, "MissingFeedURL", "Feed URL is missing")
+        return
+      end
+
+      normalized_url = feed_url.downcase
+
+      if processed.include?(normalized_url)
+        skipped_duplicates << skipped_payload(feed_url, "duplicate in import selection")
+        return
+      end
+
+      if duplicate_source?(normalized_url)
+        skipped_duplicates << skipped_payload(feed_url, "already exists")
+        processed << normalized_url
+        return
+      end
+
+      source = SourceMonitor::Source.new(build_attributes(entry))
+
+      if source.save
+        imported_sources << { id: source.id, feed_url: source.feed_url, name: source.name }
+        processed << normalized_url
+      else
+        failed_sources << failure_payload(feed_url, "ValidationFailed", source.errors.full_messages.to_sentence)
+      end
+    rescue ActiveRecord::RecordNotUnique
+      skipped_duplicates << skipped_payload(feed_url, "already exists")
+      processed << normalized_url
+    rescue StandardError => error
+      failed_sources << failure_payload(feed_url, error.class.name, error.message)
+    end
+
+    def duplicate_source?(normalized_feed_url)
+      SourceMonitor::Source.where("LOWER(feed_url) = ?", normalized_feed_url).exists?
+    end
+
+    def build_attributes(entry)
+      defaults = SourceMonitor::Sources::Params.default_attributes.deep_dup
+      settings = SourceMonitor::Security::ParameterSanitizer.sanitize(import_session.bulk_settings.presence || {})
+      settings = settings.deep_symbolize_keys
+
+      defaults.merge(settings).merge(identity_attributes(entry))
+    end
+
+    def identity_attributes(entry)
+      {
+        name: entry[:title].presence || entry[:feed_url],
+        feed_url: entry[:feed_url],
+        website_url: entry[:website_url]
+      }
+    end
+
+    def imported_sources
+      @imported_sources ||= []
+    end
+
+    def failed_sources
+      @failed_sources ||= []
+    end
+
+    def skipped_duplicates
+      @skipped_duplicates ||= []
+    end
+
+    def failure_payload(feed_url, error_class, message)
+      {
+        feed_url: feed_url,
+        error_class: error_class,
+        error_message: message
+      }
+    end
+
+    def skipped_payload(feed_url, reason)
+      {
+        feed_url: feed_url,
+        reason: reason
+      }
+    end
+
+    def broadcast_completion(history)
+      return unless defined?(Turbo::StreamsChannel)
+
+      histories = SourceMonitor::ImportHistory.recent_for(history.user_id).limit(5)
+
+      Turbo::StreamsChannel.broadcast_replace_to(
+        SourceMonitor::Realtime::Broadcaster::SOURCE_INDEX_STREAM,
+        target: "source_monitor_import_history_panel",
+        html: SourceMonitor::SourcesController.render(
+          partial: "source_monitor/sources/import_history_panel",
+          locals: { import_histories: histories }
+        )
+      )
+    rescue StandardError => error
+      Rails.logger.error("[SourceMonitor::ImportOpmlJob] broadcast failed: #{error.class}: #{error.message}") if defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger
+    end
+  end
+end

data/app/jobs/source_monitor/import_session_health_check_job.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  class ImportSessionHealthCheckJob < ApplicationJob
+    source_monitor_queue :fetch
+
+    require "source_monitor/health/import_source_health_check"
+    require "source_monitor/import_sessions/entry_normalizer"
+    require "source_monitor/import_sessions/health_check_broadcaster"
+
+    discard_on ActiveJob::DeserializationError
+
+    def perform(import_session_id, entry_id)
+      import_session = SourceMonitor::ImportSession.find_by(id: import_session_id)
+      return unless import_session
+      return unless active_for?(import_session)
+
+      result = perform_health_check(import_session, entry_id)
+      return unless result
+
+      updated_entry = nil
+
+      import_session.with_lock do
+        import_session.reload
+        return unless active_for?(import_session)
+
+        entries = Array(import_session.parsed_sources).map(&:to_h)
+        index = entries.index { |candidate| entry_id_for(candidate) == entry_id.to_s }
+        return unless index
+
+        entries[index] = entries[index].merge(
+          "health_status" => result.status,
+          "health_error" => result.error_message
+        )
+
+        selected_ids = Array(import_session.selected_source_ids).map(&:to_s)
+        selected_ids -= [ entry_id.to_s ] if result.status == "unhealthy"
+
+        attrs = {
+          parsed_sources: entries,
+          selected_source_ids: selected_ids,
+          health_check_completed_at: completion_time(entries, import_session.health_check_targets)
+        }.compact
+
+        import_session.update!(attrs)
+        normalized_entry = SourceMonitor::ImportSessions::EntryNormalizer.normalize(entries[index])
+        updated_entry = normalized_entry.merge(selected: selected_ids.include?(entry_id.to_s))
+      end
+
+      broadcaster = SourceMonitor::ImportSessions::HealthCheckBroadcaster.new(import_session)
+      broadcaster.broadcast_row(updated_entry) if updated_entry
+      broadcaster.broadcast_progress
+    rescue StandardError => error
+      Rails.logger.error(
+        "[SourceMonitor::ImportSessionHealthCheckJob] #{error.class}: #{error.message}"
+      ) if defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger
+    end
+
+    private
+
+    def active_for?(import_session)
+      import_session.current_step == "health_check" && import_session.health_checks_active?
+    end
+
+    def perform_health_check(import_session, entry_id)
+      entry = find_entry(import_session, entry_id)
+      return unless entry
+
+      SourceMonitor::Health::ImportSourceHealthCheck.new(feed_url: entry_feed_url(entry)).call
+    end
+
+    def find_entry(import_session, entry_id)
+      Array(import_session.parsed_sources).find { |entry| entry_id_for(entry) == entry_id.to_s }
+    end
+
+    def entry_id_for(entry)
+      entry.to_h["id"].presence || entry.to_h[:id].presence || entry.to_h["feed_url"].to_s
+    end
+
+    def entry_feed_url(entry)
+      entry.to_h["feed_url"] || entry.to_h[:feed_url]
+    end
+
+    def completion_time(entries, targets)
+      normalized = Array(entries).map { |entry| SourceMonitor::ImportSessions::EntryNormalizer.normalize(entry) }
+      filtered = normalized.select { |entry| targets.include?(entry[:id]) }
+      return nil if filtered.empty?
+
+      completed = filtered.count { |entry| %w[healthy unhealthy].include?(entry[:health_status].to_s) }
+      completed >= filtered.size ? Time.current : nil
+    end
+  end
+end

data/app/models/source_monitor/import_history.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  class ImportHistory < ApplicationRecord
+    validates :user_id, presence: true
+
+    scope :recent_for, lambda { |user_id|
+      scope = order(created_at: :desc)
+      scope = scope.where(user_id: user_id) if user_id
+      scope
+    }
+
+    def imported_count
+      Array(imported_sources).size
+    end
+
+    def failed_count
+      Array(failed_sources).size
+    end
+
+    def skipped_count
+      Array(skipped_duplicates).size
+    end
+
+    def completed?
+      completed_at.present?
+    end
+
+    def duration_ms
+      return unless started_at && completed_at
+
+      ((completed_at - started_at) * 1000).round
+    end
+  end
+end

data/app/models/source_monitor/import_session.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  class ImportSession < ApplicationRecord
+    STEP_ORDER = %w[upload preview health_check configure confirm].freeze
+
+    validates :current_step, inclusion: { in: STEP_ORDER }
+    validates :user_id, presence: true
+
+    class << self
+      def default_step
+        STEP_ORDER.first
+      end
+    end
+
+    def health_stream_name
+      "source_monitor_import_session_#{id}_health"
+    end
+
+    def health_check_targets
+      Array(health_check_target_ids).map(&:to_s)
+    end
+
+    def next_step
+      index = STEP_ORDER.index(current_step)
+      STEP_ORDER[index + 1] if index && index < STEP_ORDER.length - 1
+    end
+
+    def previous_step
+      index = STEP_ORDER.index(current_step)
+      STEP_ORDER[index - 1] if index&.positive?
+    end
+  end
+end

data/app/views/source_monitor/import_sessions/_header.html.erb

@@ -0,0 +1,12 @@
+<div id="import_session_header" class="flex flex-wrap items-center justify-between gap-3">
+  <div>
+    <p class="text-sm font-semibold uppercase tracking-wide text-slate-500">OPML Import</p>
+    <h1 class="text-3xl font-semibold text-slate-900">Import Wizard</h1>
+    <p class="mt-1 text-sm text-slate-500">Step <%= current_step.titleize %> · progress is saved to your session.</p>
+  </div>
+  <div class="flex items-center gap-3">
+    <%= link_to "Cancel", source_monitor.import_session_path(import_session),
+          data: { turbo_method: :delete, action: "confirm-navigation#disable" },
+          class: "inline-flex items-center rounded-md border border-slate-200 px-4 py-2 text-sm font-semibold text-slate-700 hover:bg-slate-50" %>
+  </div>
+</div>

data/app/views/source_monitor/import_sessions/_sidebar.html.erb

@@ -0,0 +1,23 @@
+<nav id="import_session_sidebar" aria-label="Import steps" class="rounded-lg border border-slate-200 bg-white shadow-sm">
+  <ol class="divide-y divide-slate-100" role="list">
+    <% wizard_steps.each do |step| %>
+      <% active = step == current_step %>
+      <li>
+        <%= link_to source_monitor.step_import_session_path(import_session, step: step),
+              class: [
+                "flex items-center justify-between px-4 py-3 text-sm font-medium transition-colors",
+                active ? "bg-blue-50 text-blue-700" : "text-slate-700 hover:bg-slate-50"
+              ].join(" "),
+              aria: { current: (active ? "page" : nil) } do %>
+          <span class="flex items-center gap-2">
+            <span class="h-2 w-2 rounded-full <%= active ? 'bg-blue-600' : 'bg-slate-300' %>" aria-hidden="true"></span>
+            <span><%= step.titleize %></span>
+          </span>
+          <% if active %>
+            <span class="text-xs font-semibold uppercase text-blue-600">Current</span>
+          <% end %>
+        <% end %>
+      </li>
+    <% end %>
+  </ol>
+</nav>

data/app/views/source_monitor/import_sessions/health_check/_progress.html.erb

@@ -0,0 +1,20 @@
+<% progress ||= { completed: 0, total: 0, pending: 0, active: false, done: false } %>
+<% progress_id = "import_session_#{import_session.id}_health_progress" %>
+
+<div id="<%= progress_id %>" class="inline-flex items-center gap-3 rounded-md border border-slate-200 px-3 py-2 text-sm font-medium text-slate-700">
+  <div class="flex items-center gap-2">
+    <% if progress[:done] %>
+      <span class="inline-block h-2 w-2 rounded-full bg-green-500" aria-hidden="true"></span>
+      <span>Health checks complete</span>
+    <% elsif progress[:active] %>
+      <span class="inline-block h-2 w-2 animate-pulse rounded-full bg-blue-500" aria-hidden="true"></span>
+      <span>Running health checks…</span>
+    <% else %>
+      <span class="inline-block h-2 w-2 rounded-full bg-slate-400" aria-hidden="true"></span>
+      <span>Health checks paused</span>
+    <% end %>
+  </div>
+  <span class="text-xs font-semibold uppercase tracking-wide text-slate-500">
+    <%= progress[:completed] %> / <%= progress[:total] %> completed
+  </span>
+</div>

data/app/views/source_monitor/import_sessions/health_check/_row.html.erb

@@ -0,0 +1,44 @@
+<% row_id = "import_session_#{import_session.id}_health_row_#{entry[:id]}" %>
+<tr id="<%= row_id %>" class="<%= entry[:health_status] == 'unhealthy' ? 'bg-rose-50/40' : '' %>">
+  <td class="px-4 py-3 text-center align-middle">
+    <%= check_box_tag "import_session[selected_source_ids][]",
+          entry[:id],
+          entry[:selected],
+          data: { select_all_target: "item", action: "select-all#toggleItem" },
+          class: "h-4 w-4 rounded border-slate-300 text-blue-600 focus:ring-blue-500" %>
+  </td>
+  <td class="px-4 py-3 align-top">
+    <div class="max-w-xl break-words font-medium text-slate-900"><%= entry[:feed_url] %></div>
+    <% if entry[:website_url].present? %>
+      <div class="mt-1 text-xs text-blue-700 truncate"><%= entry[:website_url] %></div>
+    <% end %>
+  </td>
+  <td class="px-4 py-3 align-top">
+    <div class="text-sm font-medium text-slate-900"><%= entry[:title].presence || "(No title)" %></div>
+  </td>
+  <td class="px-4 py-3 align-top">
+    <% status = entry[:health_status].presence || "pending" %>
+    <% case status %>
+    <% when "healthy" %>
+      <span class="inline-flex items-center gap-2 rounded-full bg-green-100 px-2 py-1 text-xs font-semibold text-green-800">
+        <span class="inline-block h-2 w-2 rounded-full bg-green-500" aria-hidden="true"></span>
+        Healthy
+      </span>
+    <% when "unhealthy" %>
+      <div class="space-y-1">
+        <span class="inline-flex items-center gap-2 rounded-full bg-rose-100 px-2 py-1 text-xs font-semibold text-rose-800">
+          <span class="inline-block h-2 w-2 rounded-full bg-rose-500" aria-hidden="true"></span>
+          Unhealthy
+        </span>
+        <% if entry[:health_error].present? %>
+          <div class="text-xs text-rose-700">Reason: <%= entry[:health_error] %></div>
+        <% end %>
+      </div>
+    <% else %>
+      <span class="inline-flex items-center gap-2 rounded-full bg-blue-100 px-2 py-1 text-xs font-semibold text-blue-800">
+        <span class="inline-block h-2 w-2 animate-pulse rounded-full bg-blue-500" aria-hidden="true"></span>
+        Checking…
+      </span>
+    <% end %>
+  </td>
+</tr>

data/app/views/source_monitor/import_sessions/show.html.erb

@@ -0,0 +1,15 @@
+<% content_for :title, "Import OPML" %>
+
+<div class="space-y-6" data-controller="confirm-navigation" data-confirm-navigation-message-value="You have an in-progress import. Are you sure you want to leave this wizard?">
+  <%= render "source_monitor/import_sessions/header", import_session: @import_session, current_step: @current_step %>
+
+  <div class="grid gap-6 lg:grid-cols-4">
+    <%= render "source_monitor/import_sessions/sidebar", import_session: @import_session, current_step: @current_step, wizard_steps: @wizard_steps %>
+
+    <div class="lg:col-span-3">
+      <%= turbo_frame_tag "import_session_step" do %>
+        <%= render "source_monitor/import_sessions/steps/#{@current_step}", import_session: @import_session %>
+      <% end %>
+    </div>
+  </div>
+</div>

data/app/views/source_monitor/import_sessions/show.turbo_stream.erb

@@ -0,0 +1 @@
+<%= turbo_stream.redirect_to source_monitor.step_import_session_path(@import_session, step: @current_step) %>

data/app/views/source_monitor/import_sessions/steps/_configure.html.erb

@@ -0,0 +1,53 @@
+<% bulk_source = local_assigns[:bulk_source] || @bulk_source %>
+<% selected_count = Array(import_session.selected_source_ids).size %>
+
+<div class="space-y-6">
+  <div class="rounded-lg border border-slate-200 bg-white px-4 py-3 shadow-sm">
+    <h2 class="text-lg font-semibold text-slate-900">Configure settings</h2>
+    <p class="mt-1 text-sm text-slate-600">
+      Apply the following settings to the <%= selected_count %> feeds you're importing.
+    </p>
+  </div>
+
+  <% if bulk_source&.errors&.any? %>
+    <div class="rounded border border-red-300 bg-red-50 p-4">
+      <h3 class="font-medium text-red-700">Please fix the following:</h3>
+      <ul class="mt-2 list-disc space-y-1 pl-5 text-red-700">
+        <% bulk_source.errors.full_messages.each do |message| %>
+          <li><%= message %></li>
+        <% end %>
+      </ul>
+    </div>
+  <% end %>
+
+  <div class="rounded-lg border border-slate-200 bg-white p-4 shadow-sm">
+    <%= form_with model: bulk_source,
+          url: source_monitor.step_import_session_path(import_session, step: "configure"),
+          method: :patch,
+          data: { turbo: false, action: "confirm-navigation#disable" },
+          html: { class: "space-y-6" } do |form| %>
+      <%# Hide identity fields for bulk apply; we already capture per-feed details from OPML %>
+      <%= render "source_monitor/sources/form_fields", form: form, include_identity_fields: false %>
+
+      <div class="flex items-center justify-between gap-3">
+        <div>
+          <%= button_tag "Back",
+                type: :submit,
+                name: "import_session[next_step]",
+                value: "health_check",
+                class: "inline-flex items-center rounded-md border border-slate-200 px-3 py-2 text-sm font-semibold text-slate-700 hover:bg-slate-50" %>
+        </div>
+        <div class="flex items-center gap-3">
+          <%= link_to "Cancel", source_monitor.import_session_path(import_session),
+                data: { turbo_method: :delete, action: "confirm-navigation#disable" },
+                class: "inline-flex items-center rounded-md border border-slate-200 px-3 py-2 text-sm font-semibold text-slate-700 hover:bg-slate-50" %>
+          <%= button_tag "Continue",
+                type: :submit,
+                name: "import_session[next_step]",
+                value: "confirm",
+                class: "inline-flex items-center rounded-md bg-blue-600 px-4 py-2 text-sm font-semibold text-white shadow hover:bg-blue-500" %>
+        </div>
+      </div>
+    <% end %>
+  </div>
+</div>