source_monitor 0.1.1

data/config/initializers/feedjira.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require "feedjira"
+
+Feedjira.configure do |config|
+  config.parsers = [
+    Feedjira::Parser::JSONFeed,
+    Feedjira::Parser::Atom,
+    Feedjira::Parser::AtomFeedBurner,
+    Feedjira::Parser::AtomYoutube,
+    Feedjira::Parser::AtomGoogleAlerts,
+    Feedjira::Parser::GoogleDocsAtom,
+    Feedjira::Parser::ITunesRSS,
+    Feedjira::Parser::RSSFeedBurner,
+    Feedjira::Parser::RSS
+  ]
+
+  config.strip_whitespace = true
+end

data/config/routes.rb

@@ -0,0 +1,18 @@
+SourceMonitor::Engine.routes.draw do
+  get "/health", to: "health#show"
+  get "/dashboard", to: "dashboard#index", as: :dashboard
+  root to: "dashboard#index"
+  resources :logs, only: :index
+  resources :fetch_logs, only: :show
+  resources :scrape_logs, only: :show
+  resources :items, only: %i[index show] do
+    post :scrape, on: :member
+  end
+  resources :sources do
+    resource :fetch, only: :create, controller: "source_fetches"
+    resource :retry, only: :create, controller: "source_retries"
+    resource :bulk_scrape, only: :create, controller: "source_bulk_scrapes"
+    resource :health_check, only: :create, controller: "source_health_checks"
+    resource :health_reset, only: :create, controller: "source_health_resets"
+  end
+end

data/config/tailwind.config.js

@@ -0,0 +1,17 @@
+const defaultContent = [
+  "./app/views/**/*.{erb,html,html.erb}",
+  "./app/helpers/**/*.rb",
+  "./app/assets/stylesheets/source_monitor/**/*.css",
+  "./app/assets/javascripts/**/*.{js,ts,jsx,tsx}",
+  "./lib/**/*.rb",
+  "./test/dummy/app/views/**/*.{erb,html,html.erb}"
+];
+
+export default {
+  content: defaultContent,
+  important: ".fm-admin",
+  theme: {
+    extend: {}
+  },
+  plugins: []
+};

data/db/migrate/20241008120000_create_source_monitor_sources.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+class CreateSourceMonitorSources < ActiveRecord::Migration[8.0]
+  def change
+    create_table :sourcemon_sources do |t|
+      t.string :name, null: false
+      t.string :feed_url, null: false
+      t.string :website_url
+      t.boolean :active, null: false, default: true
+      t.string :feed_format
+      t.integer :fetch_interval_hours, null: false, default: 6
+      t.datetime :next_fetch_at
+      t.datetime :last_fetched_at
+      t.integer :last_fetch_duration_ms
+      t.integer :last_http_status
+      t.text :last_error
+      t.datetime :last_error_at
+      t.string :etag
+      t.datetime :last_modified
+      t.integer :failure_count, null: false, default: 0
+      t.datetime :backoff_until
+      t.integer :items_count, null: false, default: 0
+      t.boolean :scraping_enabled, null: false, default: false
+      t.boolean :auto_scrape, null: false, default: false
+      t.jsonb :scrape_settings, null: false, default: {}
+      t.string :scraper_adapter, null: false, default: "readability"
+      t.boolean :requires_javascript, null: false, default: false
+      t.jsonb :custom_headers, null: false, default: {}
+      t.integer :items_retention_days
+      t.integer :max_items
+      t.jsonb :metadata, null: false, default: {}
+
+      t.timestamps
+    end
+
+    add_index :sourcemon_sources, :feed_url, unique: true
+    add_index :sourcemon_sources, :active
+    add_index :sourcemon_sources, :next_fetch_at
+  end
+end

data/db/migrate/20241008121000_create_source_monitor_items.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+class CreateSourceMonitorItems < ActiveRecord::Migration[8.0]
+  def change
+    create_table :sourcemon_items do |t|
+      t.references :source, null: false, foreign_key: { to_table: :sourcemon_sources }
+      t.string :guid
+      t.string :content_fingerprint
+      t.string :title
+      t.string :url
+      t.string :canonical_url
+      t.string :author
+      t.jsonb :authors, null: false, default: []
+      t.text :summary
+      t.text :content
+      t.text :scraped_html
+      t.text :scraped_content
+      t.datetime :scraped_at
+      t.string :scrape_status
+      t.datetime :published_at
+      t.datetime :updated_at_source
+      t.jsonb :categories, null: false, default: []
+      t.jsonb :tags, null: false, default: []
+      t.jsonb :keywords, null: false, default: []
+      t.jsonb :enclosures, null: false, default: []
+      t.string :media_thumbnail_url
+      t.jsonb :media_content, null: false, default: []
+      t.string :language
+      t.string :copyright
+      t.string :comments_url
+      t.integer :comments_count, null: false, default: 0
+      t.jsonb :metadata, null: false, default: {}
+      t.timestamps
+    end
+
+    add_index :sourcemon_items, :guid
+    add_index :sourcemon_items, :content_fingerprint
+    add_index :sourcemon_items, :url
+    add_index :sourcemon_items, :scrape_status
+    add_index :sourcemon_items, :published_at
+    add_index :sourcemon_items, [ :source_id, :guid ], unique: true
+    add_index :sourcemon_items, [ :source_id, :content_fingerprint ], unique: true
+  end
+end

data/db/migrate/20241008122000_create_source_monitor_fetch_logs.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+class CreateSourceMonitorFetchLogs < ActiveRecord::Migration[8.0]
+  def change
+    create_table :sourcemon_fetch_logs do |t|
+      t.references :source, null: false, foreign_key: { to_table: :sourcemon_sources }
+      t.boolean :success, null: false, default: false
+      t.integer :items_created, null: false, default: 0
+      t.integer :items_updated, null: false, default: 0
+      t.integer :items_failed, null: false, default: 0
+      t.datetime :started_at, null: false
+      t.datetime :completed_at
+      t.integer :duration_ms
+      t.integer :http_status
+      t.jsonb :http_response_headers, null: false, default: {}
+      t.string :error_class
+      t.text :error_message
+      t.text :error_backtrace
+      t.integer :feed_size_bytes
+      t.integer :items_in_feed
+      t.string :job_id
+      t.jsonb :metadata, null: false, default: {}
+
+      t.timestamps
+    end
+
+    add_index :sourcemon_fetch_logs, :success
+    add_index :sourcemon_fetch_logs, :started_at
+    add_index :sourcemon_fetch_logs, :job_id
+    add_index :sourcemon_fetch_logs, :created_at
+  end
+end

data/db/migrate/20241008123000_create_source_monitor_scrape_logs.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+class CreateSourceMonitorScrapeLogs < ActiveRecord::Migration[8.0]
+  def change
+    create_table :sourcemon_scrape_logs do |t|
+      t.references :item, null: false, foreign_key: { to_table: :sourcemon_items }
+      t.references :source, null: false, foreign_key: { to_table: :sourcemon_sources }
+      t.boolean :success, null: false, default: false
+      t.datetime :started_at, null: false
+      t.datetime :completed_at
+      t.integer :duration_ms
+      t.integer :http_status
+      t.string :scraper_adapter
+      t.integer :content_length
+      t.string :error_class
+      t.text :error_message
+      t.jsonb :metadata, null: false, default: {}
+
+      t.timestamps
+    end
+
+    add_index :sourcemon_scrape_logs, :success
+    add_index :sourcemon_scrape_logs, :created_at
+  end
+end

data/db/migrate/20251008183000_change_fetch_interval_to_minutes.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+class ChangeFetchIntervalToMinutes < ActiveRecord::Migration[8.0]
+  def up
+    rename_column :sourcemon_sources, :fetch_interval_hours, :fetch_interval_minutes
+    change_column_default :sourcemon_sources, :fetch_interval_minutes, 360
+
+    execute <<~SQL
+      UPDATE sourcemon_sources
+      SET fetch_interval_minutes = fetch_interval_minutes * 60
+    SQL
+  end
+
+  def down
+    execute <<~SQL
+      UPDATE sourcemon_sources
+      SET fetch_interval_minutes = GREATEST(1, ROUND(fetch_interval_minutes::numeric / 60.0))
+    SQL
+
+    change_column_default :sourcemon_sources, :fetch_interval_minutes, 6
+    rename_column :sourcemon_sources, :fetch_interval_minutes, :fetch_interval_hours
+  end
+end

data/db/migrate/20251009090000_create_source_monitor_item_contents.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+class CreateSourceMonitorItemContents < ActiveRecord::Migration[8.0]
+  def up
+    create_table :sourcemon_item_contents do |t|
+      t.references :item, null: false, foreign_key: { to_table: :sourcemon_items }, index: { unique: true }
+      t.text :scraped_html
+      t.text :scraped_content
+
+      t.timestamps(null: false)
+    end
+
+    execute <<~SQL
+      INSERT INTO sourcemon_item_contents (item_id, scraped_html, scraped_content, created_at, updated_at)
+      SELECT id, scraped_html, scraped_content, COALESCE(updated_at, CURRENT_TIMESTAMP), COALESCE(updated_at, CURRENT_TIMESTAMP)
+      FROM sourcemon_items
+      WHERE scraped_html IS NOT NULL OR scraped_content IS NOT NULL
+    SQL
+
+    remove_column :sourcemon_items, :scraped_html, :text
+    remove_column :sourcemon_items, :scraped_content, :text
+  end
+
+  def down
+    add_column :sourcemon_items, :scraped_html, :text
+    add_column :sourcemon_items, :scraped_content, :text
+
+    execute <<~SQL
+      UPDATE sourcemon_items items
+      SET scraped_html = contents.scraped_html,
+          scraped_content = contents.scraped_content
+      FROM sourcemon_item_contents contents
+      WHERE contents.item_id = items.id
+    SQL
+
+    drop_table :sourcemon_item_contents
+  end
+end

data/db/migrate/20251009103000_add_feed_content_readability_to_sources.rb

@@ -0,0 +1,5 @@
+class AddFeedContentReadabilityToSources < ActiveRecord::Migration[8.0]
+  def change
+    add_column :sourcemon_sources, :feed_content_readability_enabled, :boolean, default: false, null: false
+  end
+end

data/db/migrate/20251010090000_add_adaptive_fetching_toggle_to_sources.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class AddAdaptiveFetchingToggleToSources < ActiveRecord::Migration[7.1]
+  def change
+    add_column :sourcemon_sources, :adaptive_fetching_enabled, :boolean, null: false, default: true
+  end
+end

data/db/migrate/20251010123000_add_deleted_at_to_source_monitor_items.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+class AddDeletedAtToSourceMonitorItems < ActiveRecord::Migration[8.0]
+  def change
+    add_column :sourcemon_items, :deleted_at, :datetime
+    add_index :sourcemon_items, :deleted_at
+  end
+end

data/db/migrate/20251010153000_add_type_to_source_monitor_sources.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+class AddTypeToSourceMonitorSources < ActiveRecord::Migration[8.0]
+  def change
+    add_column :sourcemon_sources, :type, :string
+    add_index :sourcemon_sources, :type
+  end
+end

data/db/migrate/20251010154500_add_fetch_status_to_source_monitor_sources.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+class AddFetchStatusToSourceMonitorSources < ActiveRecord::Migration[8.0]
+  def change
+    add_column :sourcemon_sources, :fetch_status, :string, default: "idle", null: false
+    add_column :sourcemon_sources, :last_fetch_started_at, :datetime
+    add_index :sourcemon_sources, :fetch_status
+  end
+end

data/db/migrate/20251010160000_create_solid_cable_messages.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+class CreateSolidCableMessages < ActiveRecord::Migration[7.1]
+  def change
+    create_table :solid_cable_messages do |t|
+      t.binary :channel, limit: 1024, null: false
+      t.binary :payload, limit: 536_870_912, null: false
+      t.datetime :created_at, null: false
+      t.integer :channel_hash, limit: 8, null: false
+
+      t.index :channel
+      t.index :channel_hash
+      t.index :created_at
+    end
+  end
+end

data/db/migrate/20251011090000_add_fetch_retry_state_to_sources.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+class AddFetchRetryStateToSources < ActiveRecord::Migration[7.2]
+  def change
+    change_table :sourcemon_sources, bulk: true do |t|
+      t.integer :fetch_retry_attempt, null: false, default: 0
+      t.datetime :fetch_circuit_opened_at
+      t.datetime :fetch_circuit_until
+    end
+
+    add_index :sourcemon_sources, :fetch_retry_attempt
+    add_index :sourcemon_sources, :fetch_circuit_until
+  end
+end

data/db/migrate/20251012090000_add_health_fields_to_sources.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+class AddHealthFieldsToSources < ActiveRecord::Migration[8.0]
+  def change
+    change_table :sourcemon_sources, bulk: true do |t|
+      t.decimal :rolling_success_rate, precision: 5, scale: 4
+      t.string :health_status, null: false, default: "healthy"
+      t.datetime :health_status_changed_at
+      t.datetime :auto_paused_at
+      t.datetime :auto_paused_until
+      t.decimal :health_auto_pause_threshold, precision: 5, scale: 4
+    end
+
+    add_index :sourcemon_sources, :health_status
+    add_index :sourcemon_sources, :auto_paused_until
+  end
+end

data/db/migrate/20251012100000_optimize_source_monitor_database_performance.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+class OptimizeSourceMonitorDatabasePerformance < ActiveRecord::Migration[8.0]
+  def change
+    add_index :sourcemon_sources, :created_at, name: "index_sourcemon_sources_on_created_at" unless index_exists?(:sourcemon_sources, :created_at)
+
+    unless index_exists?(:sourcemon_items, %i[source_id published_at created_at])
+      add_index :sourcemon_items, %i[source_id published_at created_at], name: "index_sourcemon_items_on_source_and_published_at"
+    end
+
+    add_index :sourcemon_scrape_logs, :started_at, name: "index_sourcemon_scrape_logs_on_started_at" unless index_exists?(:sourcemon_scrape_logs, :started_at)
+  end
+end

data/db/migrate/20251014064947_add_not_null_constraints_to_items.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+class AddNotNullConstraintsToItems < ActiveRecord::Migration[8.0]
+  def up
+    # First, clean up any existing invalid data
+    # For guid: use content_fingerprint or generate a UUID as fallback
+    execute <<~SQL
+      UPDATE sourcemon_items
+      SET guid = COALESCE(content_fingerprint, gen_random_uuid()::text)
+      WHERE guid IS NULL
+    SQL
+
+    # For url: use canonical_url or a placeholder as fallback
+    execute <<~SQL
+      UPDATE sourcemon_items
+      SET url = COALESCE(canonical_url, 'https://unknown.example.com')
+      WHERE url IS NULL
+    SQL
+
+    # Now add the NOT NULL constraints
+    change_column_null :sourcemon_items, :guid, false
+    change_column_null :sourcemon_items, :url, false
+  end
+
+  def down
+    # Allow NULL values again
+    change_column_null :sourcemon_items, :guid, true
+    change_column_null :sourcemon_items, :url, true
+  end
+end

data/db/migrate/20251014171659_add_performance_indexes.rb

@@ -0,0 +1,29 @@
+class AddPerformanceIndexes < ActiveRecord::Migration[8.0]
+  def change
+    # Index for activity rate calculations
+    # Query: SELECT COUNT(*) FROM items WHERE source_id IN (...) AND created_at >= ? GROUP BY source_id
+    unless index_exists?(:sourcemon_items, [ :source_id, :created_at ], name: "index_items_on_source_and_created_at_for_rates")
+      add_index :sourcemon_items,
+        [ :source_id, :created_at ],
+        name: "index_items_on_source_and_created_at_for_rates"
+    end
+
+    # Partial index for due_for_fetch queries
+    # Query: SELECT * FROM sources WHERE active = true AND (next_fetch_at IS NULL OR next_fetch_at <= ?)
+    unless index_exists?(:sourcemon_sources, [ :active, :next_fetch_at ], name: "index_sources_on_active_and_next_fetch")
+      add_index :sourcemon_sources,
+        [ :active, :next_fetch_at ],
+        where: "active = true",
+        name: "index_sources_on_active_and_next_fetch"
+    end
+
+    # Partial index for failed source queries
+    # Query: SELECT * FROM sources WHERE failure_count > 0
+    unless index_exists?(:sourcemon_sources, :failure_count, name: "index_sources_on_failures")
+      add_index :sourcemon_sources,
+        :failure_count,
+        where: "failure_count > 0",
+        name: "index_sources_on_failures"
+    end
+  end
+end

data/db/migrate/20251014172525_add_fetch_status_check_constraint.rb

@@ -0,0 +1,18 @@
+class AddFetchStatusCheckConstraint < ActiveRecord::Migration[8.0]
+  def up
+    # Add PostgreSQL CHECK constraint to enforce fetch_status enum values at database level
+    # This complements the application-level validation in the Source model
+    execute <<-SQL
+      ALTER TABLE sourcemon_sources
+      ADD CONSTRAINT check_fetch_status_values
+      CHECK (fetch_status IN ('idle', 'queued', 'fetching', 'failed'))
+    SQL
+  end
+
+  def down
+    execute <<-SQL
+      ALTER TABLE sourcemon_sources
+      DROP CONSTRAINT check_fetch_status_values
+    SQL
+  end
+end

data/db/migrate/20251015100000_create_source_monitor_log_entries.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+class CreateSourceMonitorLogEntries < ActiveRecord::Migration[7.1]
+  def change
+    create_table :sourcemon_log_entries do |t|
+      t.references :loggable, polymorphic: true, null: false, index: { name: "index_sourcemon_log_entries_on_loggable" }
+      t.references :source, null: false, foreign_key: { to_table: :sourcemon_sources }
+      t.references :item, foreign_key: { to_table: :sourcemon_items }
+      t.boolean :success, null: false, default: false
+      t.datetime :started_at, null: false
+      t.datetime :completed_at
+      t.integer :http_status
+      t.integer :duration_ms
+      t.integer :items_created
+      t.integer :items_updated
+      t.integer :items_failed
+      t.string :scraper_adapter
+      t.integer :content_length
+      t.string :error_class
+      t.text :error_message
+
+      t.timestamps
+    end
+
+    add_index :sourcemon_log_entries, :started_at
+    add_index :sourcemon_log_entries, :success
+    add_index :sourcemon_log_entries, :scraper_adapter
+
+    reversible do |direction|
+      direction.up do
+        say_with_time "Backfilling sourcemon_log_entries" do
+          fetch_log_class = Class.new(ActiveRecord::Base) do
+            self.table_name = "sourcemon_fetch_logs"
+          end
+
+          scrape_log_class = Class.new(ActiveRecord::Base) do
+            self.table_name = "sourcemon_scrape_logs"
+          end
+
+          log_entry_class = Class.new(ActiveRecord::Base) do
+            self.table_name = "sourcemon_log_entries"
+          end
+
+          fetch_log_class.find_each do |log|
+            log_entry_class.create!(
+              loggable_type: "SourceMonitor::FetchLog",
+              loggable_id: log.id,
+              source_id: log.source_id,
+              item_id: nil,
+              success: log.success,
+              started_at: log.started_at,
+              completed_at: log.completed_at,
+              http_status: log.http_status,
+              duration_ms: log.duration_ms,
+              items_created: log.items_created,
+              items_updated: log.items_updated,
+              items_failed: log.items_failed,
+              scraper_adapter: nil,
+              content_length: nil,
+              error_class: log.error_class,
+              error_message: log.error_message
+            )
+          end
+
+          scrape_log_class.find_each do |log|
+            log_entry_class.create!(
+              loggable_type: "SourceMonitor::ScrapeLog",
+              loggable_id: log.id,
+              source_id: log.source_id,
+              item_id: log.item_id,
+              success: log.success,
+              started_at: log.started_at,
+              completed_at: log.completed_at,
+              http_status: log.http_status,
+              duration_ms: log.duration_ms,
+              items_created: nil,
+              items_updated: nil,
+              items_failed: nil,
+              scraper_adapter: log.scraper_adapter,
+              content_length: log.content_length,
+              error_class: log.error_class,
+              error_message: log.error_message
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/db/migrate/20251022100000_create_source_monitor_health_check_logs.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+class CreateSourceMonitorHealthCheckLogs < ActiveRecord::Migration[7.2]
+  def change
+    create_table :sourcemon_health_check_logs do |t|
+      t.references :source, null: false, foreign_key: { to_table: :sourcemon_sources }
+      t.boolean :success, null: false, default: false
+      t.datetime :started_at, null: false
+      t.datetime :completed_at
+      t.integer :duration_ms
+      t.integer :http_status
+      t.jsonb :http_response_headers, null: false, default: {}
+      t.string :error_class
+      t.text :error_message
+
+      t.timestamps
+    end
+
+    add_index :sourcemon_health_check_logs, :started_at
+    add_index :sourcemon_health_check_logs, :success
+  end
+end

data/db/migrate/20251108120116_refresh_fetch_status_constraint.rb

@@ -0,0 +1,29 @@
+class RefreshFetchStatusConstraint < ActiveRecord::Migration[8.0]
+  ALLOWED_STATUSES = %w[idle queued fetching failed invalid].freeze
+  PREVIOUS_STATUSES = %w[idle queued fetching failed].freeze
+
+  def up
+    replace_constraint(ALLOWED_STATUSES)
+  end
+
+  def down
+    replace_constraint(PREVIOUS_STATUSES)
+  end
+
+  private
+
+  def replace_constraint(statuses)
+    quoted_values = statuses.map { |status| ActiveRecord::Base.connection.quote(status) }.join(", ")
+
+    execute <<~SQL
+      ALTER TABLE sourcemon_sources
+      DROP CONSTRAINT IF EXISTS check_fetch_status_values
+    SQL
+
+    execute <<~SQL
+      ALTER TABLE sourcemon_sources
+      ADD CONSTRAINT check_fetch_status_values
+      CHECK (fetch_status IN (#{quoted_values}))
+    SQL
+  end
+end