source_monitor 0.12.4 → 0.13.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c9f29bb075dfefc28a290e718ca9e983b489e7b62e5c8654778bb3c4c0c8df91
-  data.tar.gz: e404576eb8a66613e653257216039a4b67eef5b1d8614bc22742396b0b5014cb
+  metadata.gz: ba447fce3d49e4605a01154bfbf1a28179ac1833ba1f240add5f1b9adae3ecb3
+  data.tar.gz: 5c90d5148475fd74aa53568df2df60e7143613496df6b985d98acfb5fd84b4c6
 SHA512:
-  metadata.gz: b91d1d3f7d0b9f5f12612e8764b423b2be5a164aa3a52abe7c3ce48e587891b2f45b37ed281a5dd70e14ca3735c0abc2c89427c1f450900f9fa3650a052b08f1
-  data.tar.gz: cbe4936aa544c0a345b803ef8ffb11ff69b3237676f4ce34cdddf06259855c0150569cef5ca400f939044387466f36ba5bc0c41b18d712ebd4d7e54afd95cbd4
+  metadata.gz: 4427aaa63507229534a56998290cc6175a18116bba7881c80ce16ef97c9b2dee9b64a4fd37f4d30cac2155805fa19b53258dbb7831f28581ffba56f1a67d814e
+  data.tar.gz: 1b072b1041d24be68a54f2c3a282101c29129fa0e5b34aff15a1ef07f2665ed494760cf37b7827b19cc399b1332cd720696519b2ab46e0c17df0d1a850e28fd0

data/.claude/skills/sm-architecture/reference/module-map.md

@@ -66,6 +66,7 @@ Complete module tree with each module's responsibility.
 | Module | File | Responsibility |
 |--------|------|----------------|
 | `ItemCreator` | `items/item_creator.rb` | Create or update Item from feed entry |
+| `BatchItemCreator` | `items/batch_item_creator.rb` | Pre-fetch lookup index of existing items for batch entry processing |
 | `ItemCreator::EntryParser` | `items/item_creator/entry_parser.rb` | Parse Feedjira entry into attribute hash |
 | `ItemCreator::ContentExtractor` | `items/item_creator/content_extractor.rb` | Process content through readability parser |
 | `RetentionPruner` | `items/retention_pruner.rb` | Prune items by age/count per source |

data/.claude/skills/sm-upgrade/reference/version-history.md

@@ -2,6 +2,20 @@
 
 Version-specific migration notes for each major/minor version transition. Agents should reference this file when guiding users through multi-version upgrades.
 
+## 0.12.4 to 0.13.0
+
+**Key changes:**
+- Performance: GUID normalization to lowercase on write; plain btree index used instead of LOWER(guid) sequential scans
+- New `AdvisoryLock#acquire!` and `AdvisoryLock#release!` methods alongside existing `with_lock` block API
+- New `BatchItemCreator` class for bulk item lookups; reduces per-fetch DB queries from ~N*2 to 2
+- `ItemCreator` accepts optional `existing_items_index` parameter for batch-mode deduplication
+- `FetchRunner` restructured into 3-phase execution (lock/fetch/write) so DB connections are not held idle during HTTP requests
+- `EntryProcessor` integrates batch index from `BatchItemCreator`
+
+**Action items:**
+1. `bundle update source_monitor`
+2. No migrations, config changes, or breaking changes.
+
 ## 0.12.3 to 0.12.4
 
 **Key changes:**

data/CHANGELOG.md

@@ -15,6 +15,14 @@ All notable changes to this project are documented below. The format follows [Ke
 
 - No unreleased changes yet.
 
+## [0.13.0] - 2026-03-24
+
+### Changed
+- **Fetch pipeline performance overhaul** — resolves production DB overload on small servers (2-core/4GB) where 569 fetch jobs backed up and PostgreSQL hit 150% CPU
+  - Normalize GUIDs to lowercase on write, replacing `LOWER(guid)` queries that forced sequential scans — existing btree indexes now used correctly
+  - Restructure advisory lock in FetchRunner into 3 phases (lock/fetch/write) so DB connections are not held idle during HTTP requests
+  - Add BatchItemCreator for bulk item lookups — reduces per-fetch queries from ~N*2 to 2 regardless of entry count
+
 ## [0.12.4] - 2026-03-17
 
 ### Fixed

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    source_monitor (0.12.4)
+    source_monitor (0.13.0)
       cssbundling-rails (~> 1.4)
       faraday (~> 2.9)
       faraday-follow_redirects (~> 0.4)

data/README.md

@@ -9,8 +9,8 @@ SourceMonitor is a production-ready Rails 8 mountable engine for ingesting, norm
 In your host Rails app:
 
 ```bash
-bundle add source_monitor --version "~> 0.12.4"
-# or add `gem "source_monitor", "~> 0.12.4"` manually, then run:
+bundle add source_monitor --version "~> 0.13.0"
+# or add `gem "source_monitor", "~> 0.13.0"` manually, then run:
 bundle install
 ```
 
@@ -46,7 +46,7 @@ This exposes `bin/source_monitor` (via Bundler binstubs) so you can run the guid
 Before running any SourceMonitor commands inside your host app, add the gem and install dependencies:
 
 ```bash
-bundle add source_monitor --version "~> 0.12.4"
+bundle add source_monitor --version "~> 0.13.0"
 # or edit your Gemfile, then run
 bundle install
 ```

data/VERSION

@@ -1 +1 @@
-0.12.4
+0.13.0

data/app/assets/builds/source_monitor/application.css

@@ -1275,14 +1275,14 @@ video {
   border-color: transparent;
 }
 
-.fm-admin .border-violet-200 {
+.fm-admin .border-violet-100 {
   --tw-border-opacity: 1;
-  border-color: rgb(221 214 254 / var(--tw-border-opacity, 1));
+  border-color: rgb(237 233 254 / var(--tw-border-opacity, 1));
 }
 
-.fm-admin .border-violet-100 {
+.fm-admin .border-violet-200 {
   --tw-border-opacity: 1;
-  border-color: rgb(237 233 254 / var(--tw-border-opacity, 1));
+  border-color: rgb(221 214 254 / var(--tw-border-opacity, 1));
 }
 
 .fm-admin .bg-amber-100 {
@@ -1673,6 +1673,10 @@ video {
   text-transform: uppercase;
 }
 
+.fm-admin .lowercase {
+  text-transform: lowercase;
+}
+
 .fm-admin .capitalize {
   text-transform: capitalize;
 }
@@ -1856,6 +1860,11 @@ video {
   color: rgb(109 40 217 / var(--tw-text-opacity, 1));
 }
 
+.fm-admin .text-violet-900 {
+  --tw-text-opacity: 1;
+  color: rgb(76 29 149 / var(--tw-text-opacity, 1));
+}
+
 .fm-admin .text-white {
   --tw-text-opacity: 1;
   color: rgb(255 255 255 / var(--tw-text-opacity, 1));
@@ -1866,11 +1875,6 @@ video {
   color: rgb(161 98 7 / var(--tw-text-opacity, 1));
 }
 
-.fm-admin .text-violet-900 {
-  --tw-text-opacity: 1;
-  color: rgb(76 29 149 / var(--tw-text-opacity, 1));
-}
-
 .fm-admin .underline {
   text-decoration-line: underline;
 }
@@ -2064,14 +2068,14 @@ video {
   color: rgb(15 23 42 / var(--tw-text-opacity, 1));
 }
 
-.fm-admin .hover\:text-white:hover {
+.fm-admin .hover\:text-violet-600:hover {
   --tw-text-opacity: 1;
-  color: rgb(255 255 255 / var(--tw-text-opacity, 1));
+  color: rgb(124 58 237 / var(--tw-text-opacity, 1));
 }
 
-.fm-admin .hover\:text-violet-600:hover {
+.fm-admin .hover\:text-white:hover {
   --tw-text-opacity: 1;
-  color: rgb(124 58 237 / var(--tw-text-opacity, 1));
+  color: rgb(255 255 255 / var(--tw-text-opacity, 1));
 }
 
 .fm-admin .hover\:underline:hover {

data/docs/setup.md

@@ -18,8 +18,8 @@ This guide consolidates the new guided installer, verification commands, and rol
 Run these commands inside your host Rails application before invoking the guided workflow:
 
 ```bash
-bundle add source_monitor --version "~> 0.12.4"
-# or add gem "source_monitor", "~> 0.12.4" to Gemfile manually
+bundle add source_monitor --version "~> 0.13.0"
+# or add gem "source_monitor", "~> 0.13.0" to Gemfile manually
 bundle install
 ```
 

data/docs/upgrade.md

@@ -46,6 +46,21 @@ If a removed option raises an error (`SourceMonitor::DeprecatedOptionError`), yo
 
 ## Version-Specific Notes
 
+### Upgrading to 0.13.0
+
+**What changed:**
+- **Performance:** GUID normalization to lowercase on write; plain btree index used instead of LOWER(guid) sequential scans
+- **Fetch pipeline:** Restructured FetchRunner into 3-phase execution (lock/fetch/write) so DB connections not held idle during HTTP requests
+- **Batch processing:** New BatchItemCreator for bulk item lookups reduces per-fetch queries from ~N*2 to 2
+
+**Upgrade steps:**
+```bash
+bundle update source_monitor
+bin/rails source_monitor:upgrade
+```
+
+No migrations, configuration changes, or breaking changes required.
+
 ### Upgrading to 0.12.0
 
 **What changed:**

data/lib/source_monitor/fetching/advisory_lock.rb

@@ -13,6 +13,8 @@ module SourceMonitor
         @connection_pool = connection_pool
       end
 
+      # Block-based API: acquires lock, yields, releases. Holds a DB connection
+      # for the entire duration of the block.
       def with_lock
         connection_pool.with_connection do |connection|
           locked = try_lock(connection)
@@ -26,6 +28,31 @@ module SourceMonitor
         end
       end
 
+      # Non-blocking acquire: tries to get the advisory lock. Returns true if
+      # acquired, false otherwise. Raises NotAcquiredError when raise_on_failure
+      # is true (default). The lock is session-scoped -- it stays held until
+      # release! is called on the same DB connection, or the connection is closed.
+      def acquire!(raise_on_failure: true)
+        locked = false
+        connection_pool.with_connection do |connection|
+          locked = try_lock(connection)
+        end
+        raise NotAcquiredError, "advisory lock #{namespace}/#{key} busy" if !locked && raise_on_failure
+
+        locked
+      end
+
+      # Releases the advisory lock. Safe to call even if the lock is not held.
+      # Because advisory locks are session-scoped, this must run on the same
+      # connection that acquired the lock. In a connection pool the pool returns
+      # the same connection to the same thread, so this works correctly as long
+      # as acquire! and release! are called from the same thread.
+      def release!
+        connection_pool.with_connection do |connection|
+          release(connection)
+        end
+      end
+
       private
 
       attr_reader :namespace, :key, :connection_pool

data/lib/source_monitor/fetching/feed_fetcher/entry_processor.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "source_monitor/items/batch_item_creator"
+
 module SourceMonitor
   module Fetching
     class FeedFetcher
@@ -22,6 +24,34 @@ module SourceMonitor
             updated_items: []
           ) unless feed.respond_to?(:entries)
 
+          entries = Array(feed.entries)
+          return empty_result if entries.empty?
+
+          # Pre-fetch existing items in bulk (2 SELECTs instead of N per-entry).
+          # If the batch index build fails, fall back to per-entry lookups.
+          existing_items_index = begin
+            SourceMonitor::Items::BatchItemCreator.build_index(source: source, entries: entries)
+          rescue StandardError
+            nil
+          end
+
+          process_entries_with_index(entries, existing_items_index)
+        end
+
+        private
+
+        def empty_result
+          FeedFetcher::EntryProcessingResult.new(
+            created: 0, updated: 0, unchanged: 0, failed: 0,
+            items: [], errors: [], created_items: [], updated_items: []
+          )
+        end
+
+        # Processes entries one at a time through ItemCreator.call.
+        # When existing_items_index is provided, ItemCreator skips per-entry
+        # SELECT queries and uses the pre-fetched index instead.
+        # When nil, ItemCreator falls back to individual DB lookups.
+        def process_entries_with_index(entries, existing_items_index)
           created = 0
           updated = 0
           unchanged = 0
@@ -31,9 +61,12 @@ module SourceMonitor
           updated_items = []
           errors = []
 
-          Array(feed.entries).each do |entry|
+          entries.each do |entry|
             begin
-              result = SourceMonitor::Items::ItemCreator.call(source:, entry:)
+              result = SourceMonitor::Items::ItemCreator.call(
+                source: source, entry: entry,
+                existing_items_index: existing_items_index
+              )
               SourceMonitor::Events.run_item_processors(source:, entry:, result: result)
               items << result.item
               if result.created?
@@ -54,19 +87,11 @@ module SourceMonitor
           end
 
           FeedFetcher::EntryProcessingResult.new(
-            created:,
-            updated:,
-            unchanged:,
-            failed:,
-            items:,
-            errors: errors.compact,
-            created_items:,
-            updated_items:
+            created:, updated:, unchanged:, failed:,
+            items:, errors: errors.compact, created_items:, updated_items:
           )
         end
 
-        private
-
         def enqueue_image_download(item)
           return unless SourceMonitor.config.images.download_enabled?
           return if item.content.blank?

data/lib/source_monitor/fetching/fetch_runner.rb

@@ -56,13 +56,35 @@ module SourceMonitor
         @retry_scheduled = false
         result = nil
 
-        lock.with_lock do
+        # Phase 1: Acquire advisory lock and mark source as fetching.
+        # Uses a DB connection briefly, then releases it.
+        lock.acquire!
+        begin
           mark_fetching!
+        rescue StandardError
+          lock.release!
+          raise
+        end
+
+        # Phase 2: HTTP fetch -- no DB connection held during network I/O.
+        # This is the key optimization: on slow feeds (up to 30s timeout),
+        # we no longer hold a DB connection idle while waiting for HTTP.
+        begin
           result = fetcher_class.new(source: source).call
+        rescue StandardError => fetch_error
+          # Ensure lock is released before propagating
+          lock.release!
+          raise fetch_error
+        end
+
+        # Phase 3: Post-fetch DB writes under the advisory lock (still held).
+        begin
           log_handler_result("RetentionHandler", retention_handler.call(source:, result:))
           log_handler_result("FollowUpHandler", follow_up_handler.call(source:, result:))
           schedule_retry_if_needed(result)
           mark_complete!(result)
+        ensure
+          lock.release!
         end
 
         log_handler_result("EventPublisher", event_publisher.call(source:, result:))

data/lib/source_monitor/items/batch_item_creator.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+require "source_monitor/items/item_creator"
+
+module SourceMonitor
+  module Items
+    # Builds a pre-fetched lookup index of existing items for a batch of entries.
+    #
+    # Instead of N individual SELECT queries (one per feed entry) to check for
+    # existing items, this class:
+    #   1. Pre-parses all entries to collect GUIDs + fingerprints
+    #   2. Does a single WHERE guid IN (...) query to find existing items by GUID
+    #   3. Does a single WHERE content_fingerprint IN (...) for remaining entries
+    #   4. Returns an index hash that ItemCreator can use to skip per-entry SELECTs
+    #
+    # The actual item creation/update is still done by ItemCreator.call, which
+    # accepts the index via the existing_items_index parameter.
+    class BatchItemCreator
+      # Builds a lookup index from a batch of feed entries.
+      # Returns a Hash with :by_guid and :by_fingerprint keys.
+      def self.build_index(source:, entries:)
+        new(source: source, entries: entries).build_index
+      end
+
+      def initialize(source:, entries:)
+        @source = source
+        @entries = Array(entries)
+      end
+
+      def build_index
+        return { by_guid: {}, by_fingerprint: {} } if @entries.empty?
+
+        # Step 1: Pre-parse entries to extract GUIDs and fingerprints for bulk lookup.
+        entry_identifiers = @entries.map do |entry|
+          parser = ItemCreator::EntryParser.new(
+            source: @source,
+            entry: entry,
+            content_extractor: content_extractor
+          )
+          attrs = parser.parse
+          raw_guid = attrs[:guid]
+          normalized_guid = raw_guid.present? ? raw_guid.downcase : nil
+          guid = normalized_guid.presence || attrs[:content_fingerprint]
+
+          { guid: guid, fingerprint: attrs[:content_fingerprint], raw_guid_present: normalized_guid.present? }
+        end
+
+        # Step 2: Batch-fetch existing items by GUID (single query)
+        guids = entry_identifiers
+          .select { |ei| ei[:raw_guid_present] }
+          .filter_map { |ei| ei[:guid] }
+          .uniq
+
+        existing_by_guid = if guids.any?
+          @source.all_items.where(guid: guids).index_by(&:guid)
+        else
+          {}
+        end
+
+        # Step 3: For entries without a GUID match, batch-fetch by fingerprint
+        unmatched_fingerprints = entry_identifiers.filter_map do |ei|
+          guid = ei[:guid]
+          next if ei[:raw_guid_present] && existing_by_guid.key?(guid)
+
+          ei[:fingerprint].presence
+        end.uniq
+
+        existing_by_fingerprint = if unmatched_fingerprints.any?
+          @source.all_items
+            .where(content_fingerprint: unmatched_fingerprints)
+            .index_by(&:content_fingerprint)
+        else
+          {}
+        end
+
+        { by_guid: existing_by_guid, by_fingerprint: existing_by_fingerprint }
+      end
+
+      private
+
+      def content_extractor
+        @content_extractor ||= ItemCreator::ContentExtractor.new(source: @source)
+      end
+    end
+  end
+end

data/lib/source_monitor/items/item_creator.rb

@@ -33,21 +33,30 @@ module SourceMonitor
       KEYWORD_SEPARATORS = /[,;]+/.freeze
       METADATA_ROOT_KEY = "feedjira_entry".freeze
 
-      def self.call(source:, entry:)
-        new(source:, entry:).call
+      # Process a single feed entry, creating or updating the corresponding item.
+      #
+      # @param existing_items_index [Hash, nil] Optional pre-fetched lookup of
+      #   existing items keyed by guid and content_fingerprint. When provided,
+      #   skips per-entry SELECT queries (used by BatchItemCreator).
+      def self.call(source:, entry:, existing_items_index: nil)
+        new(source:, entry:, existing_items_index: existing_items_index).call
       end
 
-      def initialize(source:, entry:)
+      def initialize(source:, entry:, existing_items_index: nil)
         @source = source
         @entry = entry
+        @existing_items_index = existing_items_index
       end
 
       def call
         attributes = build_attributes
         raw_guid = attributes[:guid]
-        attributes[:guid] = raw_guid.presence || attributes[:content_fingerprint]
+        # Normalize GUID to lowercase so the plain btree index on guid is used
+        # for lookups instead of LOWER(guid) which forces sequential scans.
+        normalized_guid = raw_guid.present? ? raw_guid.downcase : nil
+        attributes[:guid] = normalized_guid.presence || attributes[:content_fingerprint]
 
-        existing_item, matched_by = existing_item_for(attributes, raw_guid_present: raw_guid.present?)
+        existing_item, matched_by = existing_item_for(attributes, raw_guid_present: normalized_guid.present?)
 
         if existing_item
           apply_attributes(existing_item, attributes)
@@ -61,34 +70,54 @@ module SourceMonitor
           end
         end
 
-        create_new_item(attributes, raw_guid_present: raw_guid.present?)
+        create_new_item(attributes, raw_guid_present: normalized_guid.present?)
       end
 
       private
 
-      attr_reader :source, :entry
+      attr_reader :source, :entry, :existing_items_index
 
       def existing_item_for(attributes, raw_guid_present:)
         guid = attributes[:guid]
         fingerprint = attributes[:content_fingerprint]
 
         if raw_guid_present
-          existing = find_item_by_guid(guid)
+          existing = lookup_by_guid(guid)
           return [ existing, :guid ] if existing
         end
 
         if fingerprint.present?
-          existing = find_item_by_fingerprint(fingerprint)
+          existing = lookup_by_fingerprint(fingerprint)
           return [ existing, :fingerprint ] if existing
         end
 
         [ nil, nil ]
       end
 
+      # When a pre-fetched index is available (batch mode), look up from it
+      # instead of issuing a per-entry SELECT query.
+      def lookup_by_guid(guid)
+        if existing_items_index
+          existing_items_index[:by_guid]&.dig(guid)
+        else
+          find_item_by_guid(guid)
+        end
+      end
+
+      def lookup_by_fingerprint(fingerprint)
+        if existing_items_index
+          existing_items_index[:by_fingerprint]&.dig(fingerprint)
+        else
+          find_item_by_fingerprint(fingerprint)
+        end
+      end
+
       def find_item_by_guid(guid)
         return if guid.blank?
 
-        source.all_items.where("LOWER(guid) = ?", guid.downcase).first
+        # GUIDs are normalized to lowercase on write, so we can use a plain
+        # equality check that hits the btree index on (source_id, guid).
+        source.all_items.find_by(guid: guid.downcase)
       end
 
       def find_item_by_fingerprint(fingerprint)

data/lib/source_monitor/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module SourceMonitor
-  VERSION = "0.12.4"
+  VERSION = "0.13.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: source_monitor
 version: !ruby/object:Gem::Version
-  version: 0.12.4
+  version: 0.13.0
 platform: ruby
 authors:
 - dchuk
@@ -620,6 +620,7 @@ files:
 - lib/source_monitor/import_sessions/health_check_updater.rb
 - lib/source_monitor/import_sessions/opml_importer.rb
 - lib/source_monitor/instrumentation.rb
+- lib/source_monitor/items/batch_item_creator.rb
 - lib/source_monitor/items/item_creator.rb
 - lib/source_monitor/items/item_creator/content_extractor.rb
 - lib/source_monitor/items/item_creator/entry_parser.rb