source_monitor 0.2.0 → 0.3.0

data/lib/source_monitor/dashboard/queries/recent_activity_query.rb

@@ -0,0 +1,138 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  module Dashboard
+    class Queries
+      class RecentActivityQuery
+        EVENT_TYPE_FETCH = "fetch_log"
+        EVENT_TYPE_SCRAPE = "scrape_log"
+        EVENT_TYPE_ITEM = "item"
+
+        def initialize(limit:)
+          @limit = limit
+        end
+
+        def call
+          rows = connection.exec_query(sanitized_sql)
+          rows.map { |row| build_event(row) }
+        end
+
+        private
+
+        attr_reader :limit
+
+        def connection
+          ActiveRecord::Base.connection
+        end
+
+        def build_event(row)
+          SourceMonitor::Dashboard::RecentActivity::Event.new(
+            type: row["resource_type"].to_sym,
+            id: row["resource_id"],
+            occurred_at: row["occurred_at"],
+            success: row["success_flag"].to_i == 1,
+            items_created: row["items_created"],
+            items_updated: row["items_updated"],
+            scraper_adapter: row["scraper_adapter"],
+            item_title: row["item_title"],
+            item_url: row["item_url"],
+            source_name: row["source_name"],
+            source_id: row["source_id"]
+          )
+        end
+
+        def sanitized_sql
+          ActiveRecord::Base.send(:sanitize_sql_array, [ unified_sql_template, limit ])
+        end
+
+        def unified_sql_template
+          <<~SQL
+            SELECT resource_type,
+                   resource_id,
+                   occurred_at,
+                   success_flag,
+                   items_created,
+                   items_updated,
+                   scraper_adapter,
+                   item_title,
+                   item_url,
+                   source_name,
+                   source_id
+            FROM (
+              #{fetch_log_sql}
+              UNION ALL
+              #{scrape_log_sql}
+              UNION ALL
+              #{item_sql}
+            ) AS dashboard_events
+            WHERE occurred_at IS NOT NULL
+            ORDER BY occurred_at DESC
+            LIMIT ?
+          SQL
+        end
+
+        def fetch_log_sql
+          <<~SQL
+            SELECT
+              '#{EVENT_TYPE_FETCH}' AS resource_type,
+              #{SourceMonitor::FetchLog.quoted_table_name}.id AS resource_id,
+              #{SourceMonitor::FetchLog.quoted_table_name}.started_at AS occurred_at,
+              CASE WHEN #{SourceMonitor::FetchLog.quoted_table_name}.success THEN 1 ELSE 0 END AS success_flag,
+              #{SourceMonitor::FetchLog.quoted_table_name}.items_created AS items_created,
+              #{SourceMonitor::FetchLog.quoted_table_name}.items_updated AS items_updated,
+              NULL AS scraper_adapter,
+              NULL AS item_title,
+              NULL AS item_url,
+              NULL AS source_name,
+              #{SourceMonitor::FetchLog.quoted_table_name}.source_id AS source_id
+            FROM #{SourceMonitor::FetchLog.quoted_table_name}
+          SQL
+        end
+
+        def scrape_log_sql
+          <<~SQL
+            SELECT
+              '#{EVENT_TYPE_SCRAPE}' AS resource_type,
+              #{SourceMonitor::ScrapeLog.quoted_table_name}.id AS resource_id,
+              #{SourceMonitor::ScrapeLog.quoted_table_name}.started_at AS occurred_at,
+              CASE WHEN #{SourceMonitor::ScrapeLog.quoted_table_name}.success THEN 1 ELSE 0 END AS success_flag,
+              NULL AS items_created,
+              NULL AS items_updated,
+              #{SourceMonitor::ScrapeLog.quoted_table_name}.scraper_adapter AS scraper_adapter,
+              NULL AS item_title,
+              NULL AS item_url,
+              #{SourceMonitor::Source.quoted_table_name}.#{quoted_source_name} AS source_name,
+              #{SourceMonitor::ScrapeLog.quoted_table_name}.source_id AS source_id
+            FROM #{SourceMonitor::ScrapeLog.quoted_table_name}
+            LEFT JOIN #{SourceMonitor::Source.quoted_table_name}
+              ON #{SourceMonitor::Source.quoted_table_name}.id = #{SourceMonitor::ScrapeLog.quoted_table_name}.source_id
+          SQL
+        end
+
+        def item_sql
+          <<~SQL
+            SELECT
+              '#{EVENT_TYPE_ITEM}' AS resource_type,
+              #{SourceMonitor::Item.quoted_table_name}.id AS resource_id,
+              #{SourceMonitor::Item.quoted_table_name}.created_at AS occurred_at,
+              1 AS success_flag,
+              NULL AS items_created,
+              NULL AS items_updated,
+              NULL AS scraper_adapter,
+              #{SourceMonitor::Item.quoted_table_name}.title AS item_title,
+              #{SourceMonitor::Item.quoted_table_name}.url AS item_url,
+              #{SourceMonitor::Source.quoted_table_name}.#{quoted_source_name} AS source_name,
+              #{SourceMonitor::Item.quoted_table_name}.source_id AS source_id
+            FROM #{SourceMonitor::Item.quoted_table_name}
+            LEFT JOIN #{SourceMonitor::Source.quoted_table_name}
+              ON #{SourceMonitor::Source.quoted_table_name}.id = #{SourceMonitor::Item.quoted_table_name}.source_id
+          SQL
+        end
+
+        def quoted_source_name
+          ActiveRecord::Base.connection.quote_column_name("name")
+        end
+      end
+    end
+  end
+end

data/lib/source_monitor/dashboard/queries/stats_query.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  module Dashboard
+    class Queries
+      class StatsQuery
+        def initialize(reference_time:)
+          @reference_time = reference_time
+        end
+
+        def call
+          {
+            total_sources: integer_value(source_counts["total_sources"]),
+            active_sources: integer_value(source_counts["active_sources"]),
+            failed_sources: integer_value(source_counts["failed_sources"]),
+            total_items: total_items_count,
+            fetches_today: fetches_today_count
+          }
+        end
+
+        private
+
+        attr_reader :reference_time
+
+        def source_counts
+          @source_counts ||= begin
+            SourceMonitor::Source.connection.exec_query(source_counts_sql).first || {}
+          end
+        end
+
+        def total_items_count
+          SourceMonitor::Item.connection.select_value(total_items_sql).to_i
+        end
+
+        def fetches_today_count
+          SourceMonitor::FetchLog.where("started_at >= ?", start_of_day).count
+        end
+
+        def source_counts_sql
+          <<~SQL.squish
+            SELECT
+              COUNT(*) AS total_sources,
+              SUM(CASE WHEN active THEN 1 ELSE 0 END) AS active_sources,
+              SUM(CASE WHEN (#{failure_condition}) THEN 1 ELSE 0 END) AS failed_sources
+            FROM #{SourceMonitor::Source.quoted_table_name}
+          SQL
+        end
+
+        def failure_condition
+          [
+            "#{SourceMonitor::Source.quoted_table_name}.failure_count > 0",
+            "#{SourceMonitor::Source.quoted_table_name}.last_error IS NOT NULL",
+            "#{SourceMonitor::Source.quoted_table_name}.last_error_at IS NOT NULL"
+          ].join(" OR ")
+        end
+
+        def total_items_sql
+          "SELECT COUNT(*) FROM #{SourceMonitor::Item.quoted_table_name}"
+        end
+
+        def start_of_day
+          reference_time.in_time_zone.beginning_of_day
+        end
+
+        def integer_value(value)
+          value.to_i
+        end
+      end
+    end
+  end
+end

data/lib/source_monitor/dashboard/queries.rb

@@ -2,6 +2,8 @@
 
 require "active_support/notifications"
 require "source_monitor/dashboard/upcoming_fetch_schedule"
+require "source_monitor/dashboard/queries/stats_query"
+require "source_monitor/dashboard/queries/recent_activity_query"
 
 module SourceMonitor
   module Dashboard
@@ -139,201 +141,6 @@ module SourceMonitor
         attr_reader :store
       end
 
-      class StatsQuery
-        def initialize(reference_time:)
-          @reference_time = reference_time
-        end
-
-        def call
-          {
-            total_sources: integer_value(source_counts["total_sources"]),
-            active_sources: integer_value(source_counts["active_sources"]),
-            failed_sources: integer_value(source_counts["failed_sources"]),
-            total_items: total_items_count,
-            fetches_today: fetches_today_count
-          }
-        end
-
-        private
-
-        attr_reader :reference_time
-
-        def source_counts
-          @source_counts ||= begin
-            SourceMonitor::Source.connection.exec_query(source_counts_sql).first || {}
-          end
-        end
-
-        def total_items_count
-          SourceMonitor::Item.connection.select_value(total_items_sql).to_i
-        end
-
-        def fetches_today_count
-          SourceMonitor::FetchLog.where("started_at >= ?", start_of_day).count
-        end
-
-        def source_counts_sql
-          <<~SQL.squish
-            SELECT
-              COUNT(*) AS total_sources,
-              SUM(CASE WHEN active THEN 1 ELSE 0 END) AS active_sources,
-              SUM(CASE WHEN (#{failure_condition}) THEN 1 ELSE 0 END) AS failed_sources
-            FROM #{SourceMonitor::Source.quoted_table_name}
-          SQL
-        end
-
-        def failure_condition
-          [
-            "#{SourceMonitor::Source.quoted_table_name}.failure_count > 0",
-            "#{SourceMonitor::Source.quoted_table_name}.last_error IS NOT NULL",
-            "#{SourceMonitor::Source.quoted_table_name}.last_error_at IS NOT NULL"
-          ].join(" OR ")
-        end
-
-        def total_items_sql
-          "SELECT COUNT(*) FROM #{SourceMonitor::Item.quoted_table_name}"
-        end
-
-        def start_of_day
-          reference_time.in_time_zone.beginning_of_day
-        end
-
-        def integer_value(value)
-          value.to_i
-        end
-      end
-
-      class RecentActivityQuery
-        EVENT_TYPE_FETCH = "fetch_log"
-        EVENT_TYPE_SCRAPE = "scrape_log"
-        EVENT_TYPE_ITEM = "item"
-
-        def initialize(limit:)
-          @limit = limit
-        end
-
-        def call
-          rows = connection.exec_query(sanitized_sql)
-          rows.map { |row| build_event(row) }
-        end
-
-        private
-
-        attr_reader :limit
-
-        def connection
-          ActiveRecord::Base.connection
-        end
-
-        def build_event(row)
-          SourceMonitor::Dashboard::RecentActivity::Event.new(
-            type: row["resource_type"].to_sym,
-            id: row["resource_id"],
-            occurred_at: row["occurred_at"],
-            success: row["success_flag"].to_i == 1,
-            items_created: row["items_created"],
-            items_updated: row["items_updated"],
-            scraper_adapter: row["scraper_adapter"],
-            item_title: row["item_title"],
-            item_url: row["item_url"],
-            source_name: row["source_name"],
-            source_id: row["source_id"]
-          )
-        end
-
-        def sanitized_sql
-          ActiveRecord::Base.send(:sanitize_sql_array, [ unified_sql_template, limit ])
-        end
-
-        def unified_sql_template
-          <<~SQL
-            SELECT resource_type,
-                   resource_id,
-                   occurred_at,
-                   success_flag,
-                   items_created,
-                   items_updated,
-                   scraper_adapter,
-                   item_title,
-                   item_url,
-                   source_name,
-                   source_id
-            FROM (
-              #{fetch_log_sql}
-              UNION ALL
-              #{scrape_log_sql}
-              UNION ALL
-              #{item_sql}
-            ) AS dashboard_events
-            WHERE occurred_at IS NOT NULL
-            ORDER BY occurred_at DESC
-            LIMIT ?
-          SQL
-        end
-
-        def fetch_log_sql
-          <<~SQL
-            SELECT
-              '#{EVENT_TYPE_FETCH}' AS resource_type,
-              #{SourceMonitor::FetchLog.quoted_table_name}.id AS resource_id,
-              #{SourceMonitor::FetchLog.quoted_table_name}.started_at AS occurred_at,
-              CASE WHEN #{SourceMonitor::FetchLog.quoted_table_name}.success THEN 1 ELSE 0 END AS success_flag,
-              #{SourceMonitor::FetchLog.quoted_table_name}.items_created AS items_created,
-              #{SourceMonitor::FetchLog.quoted_table_name}.items_updated AS items_updated,
-              NULL AS scraper_adapter,
-              NULL AS item_title,
-              NULL AS item_url,
-              NULL AS source_name,
-              #{SourceMonitor::FetchLog.quoted_table_name}.source_id AS source_id
-            FROM #{SourceMonitor::FetchLog.quoted_table_name}
-          SQL
-        end
-
-        def scrape_log_sql
-          <<~SQL
-            SELECT
-              '#{EVENT_TYPE_SCRAPE}' AS resource_type,
-              #{SourceMonitor::ScrapeLog.quoted_table_name}.id AS resource_id,
-              #{SourceMonitor::ScrapeLog.quoted_table_name}.started_at AS occurred_at,
-              CASE WHEN #{SourceMonitor::ScrapeLog.quoted_table_name}.success THEN 1 ELSE 0 END AS success_flag,
-              NULL AS items_created,
-              NULL AS items_updated,
-              #{SourceMonitor::ScrapeLog.quoted_table_name}.scraper_adapter AS scraper_adapter,
-              NULL AS item_title,
-              NULL AS item_url,
-              #{SourceMonitor::Source.quoted_table_name}.#{quoted_source_name} AS source_name,
-              #{SourceMonitor::ScrapeLog.quoted_table_name}.source_id AS source_id
-            FROM #{SourceMonitor::ScrapeLog.quoted_table_name}
-            LEFT JOIN #{SourceMonitor::Source.quoted_table_name}
-              ON #{SourceMonitor::Source.quoted_table_name}.id = #{SourceMonitor::ScrapeLog.quoted_table_name}.source_id
-          SQL
-        end
-
-        def item_sql
-          <<~SQL
-            SELECT
-              '#{EVENT_TYPE_ITEM}' AS resource_type,
-              #{SourceMonitor::Item.quoted_table_name}.id AS resource_id,
-              #{SourceMonitor::Item.quoted_table_name}.created_at AS occurred_at,
-              1 AS success_flag,
-              NULL AS items_created,
-              NULL AS items_updated,
-              NULL AS scraper_adapter,
-              #{SourceMonitor::Item.quoted_table_name}.title AS item_title,
-              #{SourceMonitor::Item.quoted_table_name}.url AS item_url,
-              #{SourceMonitor::Source.quoted_table_name}.#{quoted_source_name} AS source_name,
-              #{SourceMonitor::Item.quoted_table_name}.source_id AS source_id
-            FROM #{SourceMonitor::Item.quoted_table_name}
-            LEFT JOIN #{SourceMonitor::Source.quoted_table_name}
-              ON #{SourceMonitor::Source.quoted_table_name}.id = #{SourceMonitor::Item.quoted_table_name}.source_id
-          SQL
-        end
-
-        def quoted_source_name
-          ActiveRecord::Base.connection.quote_column_name("name")
-        end
-      end
-
       QUICK_ACTIONS = [
         SourceMonitor::Dashboard::QuickAction.new(
           label: "Add Source",

data/lib/source_monitor/engine.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module SourceMonitor
   class Engine < ::Rails::Engine
     isolate_namespace SourceMonitor

data/lib/source_monitor/fetching/feed_fetcher/adaptive_interval.rb

@@ -0,0 +1,141 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  module Fetching
+    class FeedFetcher
+      class AdaptiveInterval
+        MIN_FETCH_INTERVAL = 5.minutes.to_f
+        MAX_FETCH_INTERVAL = 24.hours.to_f
+        INCREASE_FACTOR = 1.25
+        DECREASE_FACTOR = 0.75
+        FAILURE_INCREASE_FACTOR = 1.5
+        JITTER_PERCENT = 0.1
+
+        attr_reader :source, :jitter_proc
+
+        def initialize(source:, jitter_proc: nil)
+          @source = source
+          @jitter_proc = jitter_proc
+        end
+
+        def apply_adaptive_interval!(attributes, content_changed:, failure: false)
+          if source.adaptive_fetching_enabled?
+            interval_seconds = compute_next_interval_seconds(content_changed:, failure:)
+            scheduled_time = Time.current + adjusted_interval_with_jitter(interval_seconds)
+            scheduled_time = [ scheduled_time, source.backoff_until ].compact.max if source.backoff_until.present?
+
+            attributes[:fetch_interval_minutes] = interval_minutes_for(interval_seconds)
+            attributes[:next_fetch_at] = scheduled_time
+            attributes[:backoff_until] = failure ? scheduled_time : nil
+          else
+            fixed_minutes = [ source.fetch_interval_minutes.to_i, 1 ].max
+            attributes[:next_fetch_at] = Time.current + fixed_minutes.minutes
+            attributes[:backoff_until] = nil
+          end
+        end
+
+        def compute_next_interval_seconds(content_changed:, failure:)
+          current = [ current_interval_seconds, min_fetch_interval_seconds ].max
+
+          next_interval = if failure
+                            current * failure_increase_factor_value
+          elsif content_changed
+                            current * decrease_factor_value
+          else
+                            current * increase_factor_value
+          end
+
+          next_interval = min_fetch_interval_seconds if next_interval < min_fetch_interval_seconds
+          next_interval = max_fetch_interval_seconds if next_interval > max_fetch_interval_seconds
+          next_interval.to_f
+        end
+
+        def adjusted_interval_with_jitter(interval_seconds)
+          jitter = jitter_offset(interval_seconds)
+          adjusted = interval_seconds + jitter
+          adjusted = min_fetch_interval_seconds if adjusted < min_fetch_interval_seconds
+          adjusted
+        end
+
+        def jitter_offset(interval_seconds)
+          return 0 if interval_seconds <= 0
+          return jitter_proc.call(interval_seconds) if jitter_proc.respond_to?(:call)
+
+          jitter_range = interval_seconds * jitter_percent_value
+          return 0 if jitter_range <= 0
+
+          ((rand * 2) - 1) * jitter_range
+        end
+
+        def interval_minutes_for(interval_seconds)
+          minutes = (interval_seconds / 60.0).round
+          [ minutes, 1 ].max
+        end
+
+        def configured_seconds(minutes_value, default)
+          minutes = extract_numeric(minutes_value)
+          return default unless minutes && minutes.positive?
+
+          minutes * 60.0
+        end
+
+        def configured_positive(value, default)
+          number = extract_numeric(value)
+          return default unless number && number.positive?
+
+          number
+        end
+
+        def configured_non_negative(value, default)
+          number = extract_numeric(value)
+          return default if number.nil?
+
+          number.negative? ? 0.0 : number
+        end
+
+        def extract_numeric(value)
+          return value if value.is_a?(Numeric)
+          return value.to_f if value.respond_to?(:to_f)
+
+          nil
+        rescue StandardError
+          nil
+        end
+
+        private
+
+        def current_interval_seconds
+          source.fetch_interval_minutes.to_f * 60.0
+        end
+
+        def min_fetch_interval_seconds
+          configured_seconds(fetching_config&.min_interval_minutes, MIN_FETCH_INTERVAL)
+        end
+
+        def max_fetch_interval_seconds
+          configured_seconds(fetching_config&.max_interval_minutes, MAX_FETCH_INTERVAL)
+        end
+
+        def increase_factor_value
+          configured_positive(fetching_config&.increase_factor, INCREASE_FACTOR)
+        end
+
+        def decrease_factor_value
+          configured_positive(fetching_config&.decrease_factor, DECREASE_FACTOR)
+        end
+
+        def failure_increase_factor_value
+          configured_positive(fetching_config&.failure_increase_factor, FAILURE_INCREASE_FACTOR)
+        end
+
+        def jitter_percent_value
+          configured_non_negative(fetching_config&.jitter_percent, JITTER_PERCENT)
+        end
+
+        def fetching_config
+          SourceMonitor.config.fetching
+        end
+      end
+    end
+  end
+end

data/lib/source_monitor/fetching/feed_fetcher/entry_processor.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  module Fetching
+    class FeedFetcher
+      class EntryProcessor
+        attr_reader :source
+
+        def initialize(source:)
+          @source = source
+        end
+
+        def process_feed_entries(feed)
+          return FeedFetcher::EntryProcessingResult.new(
+            created: 0,
+            updated: 0,
+            failed: 0,
+            items: [],
+            errors: [],
+            created_items: [],
+            updated_items: []
+          ) unless feed.respond_to?(:entries)
+
+          created = 0
+          updated = 0
+          failed = 0
+          items = []
+          created_items = []
+          updated_items = []
+          errors = []
+
+          Array(feed.entries).each do |entry|
+            begin
+              result = SourceMonitor::Items::ItemCreator.call(source:, entry:)
+              SourceMonitor::Events.run_item_processors(source:, entry:, result: result)
+              items << result.item
+              if result.created?
+                created += 1
+                created_items << result.item
+                SourceMonitor::Events.after_item_created(item: result.item, source:, entry:, result: result)
+              else
+                updated += 1
+                updated_items << result.item
+              end
+            rescue StandardError => error
+              failed += 1
+              errors << normalize_item_error(entry, error)
+            end
+          end
+
+          FeedFetcher::EntryProcessingResult.new(
+            created:,
+            updated:,
+            failed:,
+            items:,
+            errors: errors.compact,
+            created_items:,
+            updated_items:
+          )
+        end
+
+        private
+
+        def normalize_item_error(entry, error)
+          {
+            guid: safe_entry_guid(entry),
+            title: safe_entry_title(entry),
+            error_class: error.class.name,
+            error_message: error.message
+          }
+        rescue StandardError
+          { error_class: error.class.name, error_message: error.message }
+        end
+
+        def safe_entry_guid(entry)
+          if entry.respond_to?(:entry_id)
+            entry.entry_id
+          elsif entry.respond_to?(:id)
+            entry.id
+          end
+        end
+
+        def safe_entry_title(entry)
+          entry.title if entry.respond_to?(:title)
+        end
+      end
+    end
+  end
+end