source_monitor 0.10.0 → 0.10.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 303d253e46391a54167ab1396f8f855228fb4cd867dbcf22614c9aa75b9b2e30
-  data.tar.gz: 19b54173bc76cb68615b44dd93fe1ac525e9260da83e4dbfa5311e9c71ccb73a
+  metadata.gz: ff36371d856b56cbafa201913319fda084fa7083f1483c7c3051bb595055efbe
+  data.tar.gz: 54dfba3bb65b0577d09ecda2d2bba42caa27e0f2c8abfaad35e8a7e122042cef
 SHA512:
-  metadata.gz: 2ff0ad53a04b7685490ec6d0ae39d48906dcc92a9b16062a7cc056316dcef88c38a039372551f8a9ef2e0f3c9236a3d0a66aa150127be4d902d85c4d35a42230
-  data.tar.gz: 11c424108aece6ae5b5866bebc79df7d1972f5df2c0091a3fd4720e4779284cefbb7222ff7fa31deae90b5777650ef5593ade5f0140aa9699e314a0b36e082a2
+  metadata.gz: 18ff88e88b9d3d6db95f1f5048b3d4d05eef1f93d7620432681dc12f64dbc4343df1400595fa81841ab5f7a2e294e41c378f4c6c989c9d209062fa99f19038f3
+  data.tar.gz: a29a25f83a159a634eb68871396695b3309f4ca7a50d23a37b0d775ac28c5f2443dcf3201de917e5c5d6eb0357cbb00688cc359e52661c86a822bdbc0aece365

data/CHANGELOG.md

@@ -15,6 +15,16 @@ All notable changes to this project are documented below. The format follows [Ke
 
 - No unreleased changes yet.
 
+## [0.10.1] - 2026-02-25
+
+### Fixed
+
+- **Backfill word counts rake task optimized for large datasets.** Replaced row-by-row saves with `insert_all` (Phase 1) and `upsert_all` (Phase 2), eliminating N+1 queries and `touch` cascades. ~1000x query reduction for large datasets.
+- **ActiveRecord::Deadlocked no longer silently swallowed in jobs.** `DownloadContentImagesJob` and `FaviconFetchJob` previously caught all `StandardError` including database deadlocks, causing Active Storage operations to fail silently during concurrent access. Deadlocks now propagate so the job framework can retry.
+- **Thread-safe configuration access.** `SourceMonitor.configure`, `.config`, and `.reset_configuration!` now synchronize via `Monitor` to prevent race conditions during parallel test execution.
+- **Flaky seed-dependent test failures resolved.** Added `clean_source_monitor_tables!` to `StaggerFetchTimesTaskTest` to prevent cross-test database contamination from `setup_once` records leaking via test-prof with thread-based parallelism.
+- **Suppressed spurious DeprecationRegistry warning in test output.** The "http.timeout already exists" warning from the deprecation skip-path test no longer leaks to stderr.
+
 ## [0.10.0] - 2026-02-24
 
 ### Added

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    source_monitor (0.10.0)
+    source_monitor (0.10.1)
       cssbundling-rails (~> 1.4)
       faraday (~> 2.9)
       faraday-follow_redirects (~> 0.4)

data/VERSION

@@ -1 +1 @@
-0.10.0
+0.10.1

data/app/jobs/source_monitor/download_content_images_job.rb

@@ -60,6 +60,8 @@ module SourceMonitor
 
         # Generate a serving URL for the blob
         url_mapping[image_url] = Rails.application.routes.url_helpers.rails_blob_path(blob, only_path: true)
+      rescue ActiveRecord::Deadlocked
+        raise # let job framework retry on database deadlock
       rescue StandardError
         # Individual image failure should not block others.
         # Original URL will be preserved (graceful fallback).

data/app/jobs/source_monitor/favicon_fetch_job.rb

@@ -23,6 +23,8 @@ module SourceMonitor
       else
         record_failed_attempt(source)
       end
+    rescue ActiveRecord::Deadlocked
+      raise # let job framework retry on database deadlock
     rescue StandardError => error
       record_failed_attempt(source) if source
       log_error(source, error)

data/lib/source_monitor/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module SourceMonitor
-  VERSION = "0.10.0"
+  VERSION = "0.10.1"
 end

data/lib/source_monitor.rb

@@ -24,6 +24,7 @@ rescue LoadError
   # Ransack powers search forms when available.
 end
 
+require "monitor"
 require "source_monitor/version"
 require "active_support/core_ext/module/redefine_method"
 
@@ -198,15 +199,20 @@ module SourceMonitor
     autoload :StreamResponder, "source_monitor/turbo_streams/stream_responder"
   end
 
+  CONFIG_MONITOR = Monitor.new
+  private_constant :CONFIG_MONITOR
+
   class << self
     def configure
-      yield config
-      config.check_deprecations!
-      SourceMonitor::ModelExtensions.reload!
+      CONFIG_MONITOR.synchronize do
+        yield config
+        config.check_deprecations!
+        SourceMonitor::ModelExtensions.reload!
+      end
     end
 
     def config
-      @config ||= Configuration.new
+      @config ||= CONFIG_MONITOR.synchronize { @config ||= Configuration.new }
     end
 
     def events
@@ -214,11 +220,13 @@ module SourceMonitor
     end
 
     def reset_configuration!
-      @config = Configuration.new
-      SourceMonitor::ModelExtensions.reload!
-      SourceMonitor::Health.setup!
-      SourceMonitor::Realtime.setup!
-      SourceMonitor::Dashboard::TurboBroadcaster.setup!
+      CONFIG_MONITOR.synchronize do
+        @config = Configuration.new
+        SourceMonitor::ModelExtensions.reload!
+        SourceMonitor::Health.setup!
+        SourceMonitor::Realtime.setup!
+        SourceMonitor::Dashboard::TurboBroadcaster.setup!
+      end
     end
 
     def queue_name(role)

data/lib/tasks/source_monitor_tasks.rake

@@ -1,32 +1,84 @@
 # frozen_string_literal: true
 
+def backfill_resolve_source_scope
+  if ENV["SOURCE_IDS"].present?
+    ENV["SOURCE_IDS"].split(",").map(&:strip).map(&:to_i).reject(&:zero?)
+  elsif ENV["SOURCE_ID"].present?
+    id = ENV["SOURCE_ID"].to_i
+    id.positive? ? [ id ] : nil
+  end
+end
+
 namespace :source_monitor do
-  desc "Backfill word counts for existing item_content records."
+  desc "Backfill word counts for existing item_content records. " \
+       "Env vars: SOURCE_ID, SOURCE_IDS (comma-separated), BATCH_SIZE (default 500)."
   task backfill_word_counts: :environment do
-    # Phase 1: Create ItemContent for items with feed content but no ItemContent
-    items_needing_content = SourceMonitor::Item
+    batch_size = (ENV["BATCH_SIZE"] || 500).to_i
+    source_ids = backfill_resolve_source_scope
+    sanitizer = ActionView::Base.full_sanitizer
+
+    # Phase 1: Batch-create ItemContent for items with feed content but no ItemContent
+    items_scope = SourceMonitor::Item
       .where.not(content: [ nil, "" ])
       .where.missing(:item_content)
+    items_scope = items_scope.where(source_id: source_ids) if source_ids
 
     created = 0
-    items_needing_content.find_each do |item|
-      item.ensure_feed_content_record
-      created += 1
-      puts "Created #{created} missing ItemContent records..." if (created % 100).zero?
+    now = Time.current
+
+    items_scope.select(:id, :content).find_in_batches(batch_size: batch_size) do |batch|
+      records = batch.filter_map do |item|
+        next if item.content.blank?
+
+        stripped = sanitizer.sanitize(item.content)
+        word_count = stripped.present? ? stripped.split.size : nil
+
+        { item_id: item.id, feed_word_count: word_count, created_at: now, updated_at: now }
+      end
+
+      SourceMonitor::ItemContent.insert_all(records) if records.any?
+      created += records.size
+      puts "Phase 1: created #{created} ItemContent records..."
     end
-    puts "Created #{created} ItemContent records for feed-only items." if created > 0
+    puts "Phase 1 complete: #{created} records created." if created > 0
 
-    # Phase 2: Recompute word counts for all existing ItemContent
-    total = SourceMonitor::ItemContent.count
+    # Phase 2: Batch-recompute word counts for existing ItemContent
+    contents_scope = SourceMonitor::ItemContent
+    if source_ids
+      contents_scope = contents_scope.joins(:item).where(SourceMonitor::Item.table_name => { source_id: source_ids })
+    end
+
+    total = contents_scope.count
     processed = 0
+    updated = 0
+
+    contents_scope.includes(:item).find_in_batches(batch_size: batch_size) do |batch|
+      updates = batch.filter_map do |content|
+        feed_content = content.item&.content
+        scraped = content.scraped_content
+
+        feed_wc = if feed_content.present?
+          stripped = sanitizer.sanitize(feed_content)
+          stripped.present? ? stripped.split.size : nil
+        end
+
+        scraped_wc = scraped.present? ? scraped.split.size : nil
+
+        next if content.feed_word_count == feed_wc && content.scraped_word_count == scraped_wc
+
+        { id: content.id, item_id: content.item_id, feed_word_count: feed_wc, scraped_word_count: scraped_wc }
+      end
+
+      if updates.any?
+        SourceMonitor::ItemContent.upsert_all(updates, unique_by: :id, update_only: %i[feed_word_count scraped_word_count])
+        updated += updates.size
+      end
 
-    SourceMonitor::ItemContent.find_each do |content|
-      content.save!
-      processed += 1
-      puts "Processed #{processed}/#{total} records..." if (processed % 100).zero?
+      processed += batch.size
+      puts "Phase 2: #{processed}/#{total} checked, #{updated} updated..."
     end
 
-    puts "Done. Backfilled word counts for #{processed} records (#{created} newly created)."
+    puts "Done. Phase 1: #{created} created. Phase 2: #{updated}/#{total} updated."
   end
 
   namespace :cleanup do

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: source_monitor
 version: !ruby/object:Gem::Version
-  version: 0.10.0
+  version: 0.10.1
 platform: ruby
 authors:
 - dchuk