source_monitor 0.1.3 → 0.2.0

data/app/views/source_monitor/sources/_form_fields.html.erb

@@ -0,0 +1,142 @@
+<% source = form.object %>
+<% include_identity_fields = local_assigns.fetch(:include_identity_fields, true) %>
+
+<% if include_identity_fields %>
+  <div class="space-y-4 rounded-lg border border-slate-200 bg-white px-5 py-5 shadow-sm">
+    <div>
+      <%= form.label :name, class: "block text-sm font-medium text-slate-800" %>
+      <%= form.text_field :name, class: "mt-1 block w-full rounded border border-slate-300 px-3 py-2" %>
+    </div>
+
+    <div>
+      <%= form.label :feed_url, class: "block text-sm font-medium text-slate-800" %>
+      <%= form.url_field :feed_url, class: "mt-1 block w-full rounded border border-slate-300 px-3 py-2" %>
+    </div>
+
+    <div>
+      <%= form.label :website_url, class: "block text-sm font-medium text-slate-800" %>
+      <%= form.url_field :website_url, class: "mt-1 block w-full rounded border border-slate-300 px-3 py-2" %>
+    </div>
+
+    <label class="inline-flex items-center space-x-2">
+      <%= form.check_box :active %>
+      <span>Active</span>
+    </label>
+  </div>
+<% end %>
+
+<div class="space-y-4 rounded-lg border border-slate-200 bg-white px-5 py-5 shadow-sm">
+  <div>
+    <%= form.label :fetch_interval_minutes, "Fetch interval (minutes)", class: "block text-sm font-medium text-slate-800" %>
+    <%= form.number_field :fetch_interval_minutes, min: 1, step: 1, class: "mt-1 block w-full rounded border border-slate-300 px-3 py-2" %>
+    <p class="mt-1 text-xs text-slate-500">Minimum 5 minutes recommended.</p>
+  </div>
+
+  <label class="flex items-start space-x-3 rounded-lg border border-slate-200 bg-slate-50 px-4 py-3">
+    <%= form.check_box :adaptive_fetching_enabled, class: "mt-1" %>
+    <span>
+      <span class="block text-sm font-medium text-slate-800">Automatically adjust fetch interval</span>
+      <span class="mt-1 block text-xs text-slate-500">
+        When enabled, SourceMonitor lengthens or shortens the interval based on feed activity and failures. Disable to keep the interval fixed at the value above.
+      </span>
+    </span>
+  </label>
+
+  <div>
+    <%= form.label :health_auto_pause_threshold, "Auto-pause threshold", class: "block text-sm font-medium text-slate-800" %>
+    <%= form.number_field :health_auto_pause_threshold, min: 0, max: 1, step: 0.05, class: "mt-1 block w-full rounded border border-slate-300 px-3 py-2" %>
+    <p class="mt-1 text-xs text-slate-500">
+      Optional override for this source. When the rolling success rate falls below this fraction (defaults to <%= SourceMonitor.config.health.auto_pause_threshold %>), the source is temporarily auto-paused.
+    </p>
+  </div>
+</div>
+
+<div class="rounded-lg border border-slate-200 bg-white shadow-sm">
+  <div class="border-b border-slate-200 px-5 py-4">
+    <h2 class="text-lg font-medium">Retention</h2>
+    <p class="mt-1 text-xs text-slate-500">
+      Choose how long SourceMonitor keeps items for this source. Pruning runs after every fetch so the UI always reflects active retention rules.
+    </p>
+  </div>
+  <div class="grid gap-4 px-5 py-5 sm:grid-cols-2">
+    <div>
+      <%= form.label :items_retention_days, "Retention window (days)", class: "block text-sm font-medium text-slate-800" %>
+      <%= form.number_field :items_retention_days, min: 0, class: "mt-1 block w-full rounded border border-slate-300 px-3 py-2" %>
+      <p class="mt-1 text-xs text-slate-500">
+        Leave blank to keep historical items indefinitely. Items older than the configured window are removed automatically.
+      </p>
+    </div>
+
+    <div>
+      <%= form.label :max_items, "Maximum stored items", class: "block text-sm font-medium text-slate-800" %>
+      <%= form.number_field :max_items, min: 0, class: "mt-1 block w-full rounded border border-slate-300 px-3 py-2" %>
+      <p class="mt-1 text-xs text-slate-500">
+        Optional hard cap. SourceMonitor keeps the newest items and prunes the rest after each fetch.
+      </p>
+    </div>
+  </div>
+</div>
+
+<div class="rounded-lg border border-slate-200 bg-white shadow-sm">
+  <div class="border-b border-slate-200 px-5 py-4">
+    <h2 class="text-lg font-medium">Feed Content Processing</h2>
+    <p class="mt-1 text-xs text-slate-500">Control whether feed-supplied content is cleaned with Readability before storing items.</p>
+  </div>
+  <div class="space-y-4 px-5 py-5">
+    <label class="inline-flex items-center space-x-2">
+      <%= form.check_box :feed_content_readability_enabled %>
+      <span>Process feed content with Readability</span>
+    </label>
+    <p class="text-xs text-slate-500">
+      When enabled, the raw feed HTML is passed through Readability. Scraping configuration below remains independent, so you can mix-and-match feed content and scraping strategies.
+    </p>
+  </div>
+</div>
+
+<% scrape_settings = (source.scrape_settings || {}).with_indifferent_access %>
+<% selectors = scrape_settings[:selectors] || {} %>
+
+<div class="rounded-lg border border-slate-200 bg-white shadow-sm">
+  <div class="border-b border-slate-200 px-5 py-4">
+    <h2 class="text-lg font-medium">Scraping Configuration</h2>
+    <p class="mt-1 text-xs text-slate-500">Control how content extraction runs for this source.</p>
+  </div>
+  <div class="space-y-5 px-5 py-5">
+    <div class="grid gap-3 sm:grid-cols-2">
+      <label class="inline-flex items-center space-x-2">
+        <%= form.check_box :scraping_enabled %>
+        <span>Scraping enabled</span>
+      </label>
+
+      <label class="inline-flex items-center space-x-2">
+        <%= form.check_box :auto_scrape %>
+        <span>Auto scrape after fetch</span>
+      </label>
+
+      <label class="inline-flex items-center space-x-2">
+        <%= form.check_box :requires_javascript %>
+        <span>Requires JavaScript</span>
+      </label>
+    </div>
+
+    <div>
+      <%= form.label :scraper_adapter, "Scraper adapter", class: "block text-sm font-medium text-slate-800" %>
+      <%= form.select :scraper_adapter, [["Readability", "readability"]], {}, class: "mt-1 block w-full rounded border border-slate-300 px-3 py-2" %>
+      <p class="mt-1 text-xs text-slate-500">Additional adapters can be configured in future phases.</p>
+    </div>
+
+    <div class="grid gap-4 sm:grid-cols-2">
+      <div>
+        <label for="source_scrape_settings_selectors_content" class="block text-sm font-medium text-slate-800">Content CSS selector</label>
+        <%= text_field_tag "source[scrape_settings][selectors][content]", selectors[:content], id: "source_scrape_settings_selectors_content", class: "mt-1 block w-full rounded border border-slate-300 px-3 py-2", placeholder: ".article-body" %>
+        <p class="mt-1 text-xs text-slate-500">Optional. Overrides Readability extraction target.</p>
+      </div>
+
+      <div>
+        <label for="source_scrape_settings_selectors_title" class="block text-sm font-medium text-slate-800">Title CSS selector</label>
+        <%= text_field_tag "source[scrape_settings][selectors][title]", selectors[:title], id: "source_scrape_settings_selectors_title", class: "mt-1 block w-full rounded border border-slate-300 px-3 py-2", placeholder: "h1.article-title" %>
+        <p class="mt-1 text-xs text-slate-500">Optional. Leave blank to use feed-provided title.</p>
+      </div>
+    </div>
+  </div>
+</div>

data/app/views/source_monitor/sources/_import_history_panel.html.erb

@@ -0,0 +1,53 @@
+<% latest = import_histories&.first %>
+<div id="source_monitor_import_history_panel">
+  <% if latest.present? %>
+    <div class="rounded-lg border border-blue-100 bg-blue-50 px-4 py-3 shadow-sm">
+      <div class="flex flex-col gap-2 sm:flex-row sm:items-start sm:justify-between">
+        <div>
+          <p class="text-xs font-semibold uppercase tracking-wide text-blue-700">Recent OPML import</p>
+          <h3 class="text-lg font-semibold text-slate-900">
+            Imported <%= latest.imported_count %> • Skipped <%= latest.skipped_count %> • Failed <%= latest.failed_count %>
+          </h3>
+          <p class="text-sm text-slate-700">
+            Completed <%= time_ago_in_words(latest.completed_at || latest.created_at) %> ago.
+          </p>
+        </div>
+        <% if latest.completed? && latest.duration_ms %>
+          <div class="text-right text-sm text-slate-600">
+            <p>Duration: <%= latest.duration_ms %> ms</p>
+          </div>
+        <% end %>
+      </div>
+
+      <% if latest.failed_sources.present? %>
+        <div class="mt-3 overflow-x-auto rounded-md border border-rose-100 bg-white">
+          <table class="min-w-full divide-y divide-rose-50 text-left text-sm">
+            <thead class="bg-rose-50 text-xs font-semibold uppercase tracking-wide text-rose-700">
+              <tr>
+                <th scope="col" class="px-3 py-2">Feed URL</th>
+                <th scope="col" class="px-3 py-2">Error</th>
+              </tr>
+            </thead>
+            <tbody class="divide-y divide-rose-50 text-rose-800">
+              <% Array(latest.failed_sources).each do |failure| %>
+                <tr>
+                  <td class="px-3 py-2 align-top font-mono text-xs text-slate-800 break-all"><%= failure["feed_url"] || failure[:feed_url] %></td>
+                  <td class="px-3 py-2 align-top text-sm">
+                    <span class="font-semibold"><%= failure["error_class"] || failure[:error_class] %>:</span>
+                    <%= failure["error_message"] || failure[:error_message] %>
+                  </td>
+                </tr>
+              <% end %>
+            </tbody>
+          </table>
+        </div>
+      <% elsif latest.skipped_count.positive? %>
+        <p class="mt-2 text-sm text-slate-700">Skipped duplicates: <%= latest.skipped_count %>.</p>
+      <% end %>
+    </div>
+  <% else %>
+    <div class="rounded-lg border border-slate-200 bg-white px-4 py-3 text-sm text-slate-600">
+      No recent OPML imports.
+    </div>
+  <% end %>
+</div>

data/app/views/source_monitor/sources/index.html.erb

@@ -13,10 +13,16 @@
           <%= form.submit "Search", class: "rounded-r-md bg-blue-600 px-4 py-2 text-sm font-semibold text-white hover:bg-blue-500" %>
         </div>
       <% end %>
-      <%= link_to "New Source", source_monitor.new_source_path, class: "inline-flex items-center justify-center rounded-md bg-blue-600 px-4 py-2 text-sm font-semibold text-white shadow hover:bg-blue-500" %>
+      <div class="flex flex-col gap-2 sm:flex-row sm:items-center sm:gap-3">
+        <%= link_to "New Source", source_monitor.new_source_path, class: "inline-flex items-center justify-center rounded-md bg-blue-600 px-4 py-2 text-sm font-semibold text-white shadow hover:bg-blue-500" %>
+        <%= link_to "Import OPML", source_monitor.new_import_session_path,
+              class: "inline-flex items-center justify-center rounded-md border border-slate-200 px-4 py-2 text-sm font-semibold text-slate-700 shadow-sm hover:bg-slate-50" %>
+      </div>
     </div>
   </div>
 
+  <%= render "source_monitor/sources/import_history_panel", import_histories: @recent_import_histories %>
+
   <%= render "source_monitor/sources/fetch_interval_heatmap",
         fetch_interval_distribution: @fetch_interval_distribution,
         selected_bucket: @selected_fetch_interval_bucket,

data/config/coverage_baseline.json

@@ -9,7 +9,6 @@
     77
   ],
   "app/controllers/source_monitor/application_controller.rb": [
-    33,
     37
   ],
   "app/controllers/source_monitor/fetch_logs_controller.rb": [
@@ -21,6 +20,61 @@
     8,
     9
   ],
+  "app/controllers/source_monitor/import_sessions_controller.rb": [
+    28,
+    33,
+    50,
+    51,
+    52,
+    54,
+    76,
+    77,
+    91,
+    92,
+    93,
+    95,
+    97,
+    101,
+    102,
+    103,
+    104,
+    341,
+    347,
+    349,
+    357,
+    359,
+    360,
+    361,
+    364,
+    365,
+    366,
+    367,
+    370,
+    372,
+    376,
+    378,
+    379,
+    380,
+    382,
+    383,
+    387,
+    391,
+    393,
+    395,
+    397,
+    399,
+    401,
+    463,
+    520,
+    532,
+    536,
+    547,
+    575,
+    593,
+    594,
+    721,
+    723
+  ],
   "app/controllers/source_monitor/items_controller.rb": [
     40,
     42,
@@ -72,23 +126,23 @@
     103
   ],
   "app/controllers/source_monitor/sources_controller.rb": [
-    41,
     42,
     43,
     44,
-    48,
-    57,
-    65,
+    45,
+    49,
+    58,
     66,
-    68,
-    105,
-    111,
+    67,
+    69,
+    106,
     112,
-    114,
-    118,
+    113,
+    115,
     119,
-    121,
-    131
+    120,
+    122,
+    132
   ],
   "app/helpers/source_monitor/application_helper.rb": [
     64,
@@ -107,6 +161,9 @@
     95,
     114
   ],
+  "app/jobs/source_monitor/import_session_health_check_job.rb": [
+    56
+  ],
   "app/jobs/source_monitor/item_cleanup_job.rb": [
     40
   ],
@@ -889,12 +946,31 @@
     81
   ],
   "lib/source_monitor/health.rb": [
-    22,
-    35,
+    23,
     36,
     37,
+    38,
+    40,
+    43
+  ],
+  "lib/source_monitor/health/import_source_health_check.rb": [
+    9,
+    10,
+    14,
+    16,
+    17,
+    18,
+    20,
+    21,
+    22,
+    26,
+    34,
+    38,
     39,
-    42
+    41,
+    45,
+    49,
+    51
   ],
   "lib/source_monitor/health/source_health_check.rb": [
     43

data/config/routes.rb

@@ -5,6 +5,12 @@ SourceMonitor::Engine.routes.draw do
   resources :logs, only: :index
   resources :fetch_logs, only: :show
   resources :scrape_logs, only: :show
+  resources :import_sessions, path: "import_opml", only: %i[new create show update destroy] do
+    member do
+      get "steps/:step", action: :show, as: :step
+      patch "steps/:step", action: :update
+    end
+  end
   resources :items, only: %i[index show] do
     post :scrape, on: :member
   end

data/db/migrate/20251124090000_create_import_sessions.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+class CreateImportSessions < ActiveRecord::Migration[8.1]
+  def change
+    create_table :"#{SourceMonitor.table_name_prefix}import_sessions" do |t|
+      t.references :user, null: false, foreign_key: true
+      t.jsonb :opml_file_metadata, null: false, default: {}
+      t.jsonb :parsed_sources, null: false, default: []
+      t.jsonb :selected_source_ids, null: false, default: []
+      t.jsonb :bulk_settings, null: false, default: {}
+      t.string :current_step, null: false
+
+      t.timestamps
+    end
+
+    add_index :"#{SourceMonitor.table_name_prefix}import_sessions", :current_step
+  end
+end

data/db/migrate/20251124153000_add_health_fields_to_import_sessions.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+class AddHealthFieldsToImportSessions < ActiveRecord::Migration[8.1]
+  def change
+    change_table :"#{SourceMonitor.table_name_prefix}import_sessions" do |t|
+      t.boolean :health_checks_active, null: false, default: false
+      t.jsonb :health_check_target_ids, null: false, default: []
+      t.datetime :health_check_started_at
+      t.datetime :health_check_completed_at
+    end
+
+    add_index :"#{SourceMonitor.table_name_prefix}import_sessions", :health_checks_active
+  end
+end

data/db/migrate/20251125094500_create_import_histories.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+class CreateImportHistories < ActiveRecord::Migration[8.1]
+  def change
+    create_table :"#{SourceMonitor.table_name_prefix}import_histories" do |t|
+      t.references :user, null: false, foreign_key: true
+      t.jsonb :imported_sources, null: false, default: []
+      t.jsonb :failed_sources, null: false, default: []
+      t.jsonb :skipped_duplicates, null: false, default: []
+      t.jsonb :bulk_settings, null: false, default: {}
+      t.datetime :started_at
+      t.datetime :completed_at
+
+      t.timestamps
+    end
+
+    add_index :"#{SourceMonitor.table_name_prefix}import_histories", :created_at
+  end
+end

data/lib/source_monitor/health/import_source_health_check.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  module Health
+    class ImportSourceHealthCheck
+      Result = Struct.new(:status, :error_message, :http_status, keyword_init: true)
+
+      def initialize(feed_url:, client: nil)
+        @feed_url = feed_url
+        @client = client
+      end
+
+      def call
+        return Result.new(status: "unhealthy", error_message: "Missing feed URL", http_status: nil) if feed_url.blank?
+
+        response = connection.get(feed_url)
+        status_code = response_status(response)
+        healthy = healthy_status?(status_code)
+
+        Result.new(
+          status: healthy ? "healthy" : "unhealthy",
+          error_message: healthy ? nil : error_for_status(status_code),
+          http_status: status_code
+        )
+      rescue StandardError => error
+        Result.new(status: "unhealthy", error_message: error.message, http_status: response_status(error))
+      end
+
+      private
+
+      attr_reader :feed_url, :client
+
+      def connection
+        @connection ||= (client || SourceMonitor::HTTP.client(headers: {}, retry_requests: false))
+      end
+
+      def response_status(response)
+        return response.status if response.respond_to?(:status)
+        return response.response[:status] if response.respond_to?(:response) && response.response.is_a?(Hash)
+
+        nil
+      end
+
+      def healthy_status?(status)
+        status.present? && status.to_i.between?(200, 399)
+      end
+
+      def error_for_status(status)
+        return "Request failed" if status.blank?
+
+        "HTTP #{status}"
+      end
+    end
+  end
+end

data/lib/source_monitor/health.rb

@@ -3,6 +3,7 @@
 require "source_monitor/health/source_health_monitor"
 require "source_monitor/health/source_health_reset"
 require "source_monitor/health/source_health_check"
+require "source_monitor/health/import_source_health_check"
 
 module SourceMonitor
   module Health

data/lib/source_monitor/import_sessions/entry_normalizer.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  module ImportSessions
+    module EntryNormalizer
+      module_function
+
+      def normalize(entry)
+        entry = entry.to_h
+
+        {
+          id: string_for(entry[:id] || entry["id"] || entry[:feed_url] || entry["feed_url"]),
+          feed_url: entry[:feed_url].presence || entry["feed_url"].presence,
+          title: entry[:title].presence || entry["title"].presence,
+          website_url: entry[:website_url].presence || entry["website_url"].presence,
+          status: entry[:status].presence || entry["status"].presence || "valid",
+          error: entry[:error].presence || entry["error"].presence,
+          raw_outline_index: entry[:raw_outline_index] || entry["raw_outline_index"],
+          health_status: entry[:health_status].presence || entry["health_status"].presence,
+          health_error: entry[:health_error].presence || entry["health_error"].presence
+        }
+      end
+
+      def string_for(value)
+        value&.to_s
+      end
+      private_class_method :string_for
+    end
+  end
+end

data/lib/source_monitor/import_sessions/health_check_broadcaster.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  module ImportSessions
+    class HealthCheckBroadcaster
+      include ActionView::RecordIdentifier
+
+      require "source_monitor/import_sessions/entry_normalizer"
+
+      def initialize(import_session)
+        @import_session = import_session
+      end
+
+      def stream_name
+        import_session.health_stream_name
+      end
+
+      def broadcast_row(entry)
+        return unless turbo_available?
+        return unless entry
+
+        Turbo::StreamsChannel.broadcast_replace_to(
+          stream_name,
+          target: row_target(entry),
+          html: render_row(entry)
+        )
+      end
+
+      def broadcast_progress
+        return unless turbo_available?
+
+        Turbo::StreamsChannel.broadcast_replace_to(
+          stream_name,
+          target: progress_target,
+          html: render_progress
+        )
+      end
+
+      def progress_data
+        entries = health_entries
+        total = import_session.health_check_targets.size
+        completed = entries.count { |entry| %w[healthy unhealthy].include?(entry[:health_status].to_s) }
+
+        {
+          completed: completed,
+          total: total,
+          pending: [ total - completed, 0 ].max,
+          active: import_session.health_checks_active?,
+          done: total.positive? && completed >= total
+        }
+      end
+
+      private
+
+      attr_reader :import_session
+
+      def row_target(entry)
+        "import_session_#{import_session.id}_health_row_#{entry_id(entry)}"
+      end
+
+      def progress_target
+        "import_session_#{import_session.id}_health_progress"
+      end
+
+      def entry_id(entry)
+        entry[:id] || entry["id"]
+      end
+
+      def render_row(entry)
+        SourceMonitor::ImportSessionsController.render(
+          partial: "source_monitor/import_sessions/health_check/row",
+          locals: { import_session:, entry: entry_with_selection(entry) }
+        )
+      end
+
+      def render_progress
+        SourceMonitor::ImportSessionsController.render(
+          partial: "source_monitor/import_sessions/health_check/progress",
+          locals: { import_session:, progress: progress_data }
+        )
+      end
+
+      def entry_with_selection(entry)
+        selected_ids = Array(import_session.selected_source_ids).map(&:to_s)
+        normalized = entry.is_a?(Hash) ? entry.symbolize_keys : entry
+        normalized.merge(selected: selected_ids.include?(entry_id(entry).to_s))
+      end
+
+      def health_entries
+        targets = import_session.health_check_targets
+        selected_ids = Array(import_session.selected_source_ids).map(&:to_s)
+
+        Array(import_session.parsed_sources).map { |entry| SourceMonitor::ImportSessions::EntryNormalizer.normalize(entry) }
+          .select { |entry| targets.include?(entry[:id]) }
+          .map { |entry| entry.merge(selected: selected_ids.include?(entry[:id])) }
+      end
+
+      def turbo_available?
+        defined?(Turbo::StreamsChannel)
+      end
+    end
+  end
+end

data/lib/source_monitor/sources/params.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  module Sources
+    module Params
+      module_function
+
+      def permitted_attributes
+        [
+          :name,
+          :feed_url,
+          :website_url,
+          :fetch_interval_minutes,
+          :active,
+          :auto_scrape,
+          :scraping_enabled,
+          :requires_javascript,
+          :feed_content_readability_enabled,
+          :scraper_adapter,
+          :items_retention_days,
+          :max_items,
+          :adaptive_fetching_enabled,
+          :health_auto_pause_threshold,
+          { scrape_settings: [
+            :include_plain_text,
+            :timeout,
+            :javascript_enabled,
+            { selectors: %i[content title], http: [], readability: [] }
+          ] }
+        ]
+      end
+
+      def default_attributes
+        {
+          active: true,
+          scraping_enabled: false,
+          auto_scrape: false,
+          requires_javascript: false,
+          feed_content_readability_enabled: false,
+          fetch_interval_minutes: 360,
+          adaptive_fetching_enabled: true,
+          scraper_adapter: "readability"
+        }
+      end
+
+      def sanitize(params)
+        permitted = params.require(:source).permit(*permitted_attributes)
+        SourceMonitor::Security::ParameterSanitizer.sanitize(permitted.to_h)
+      end
+    end
+  end
+end