search-engine-for-typesense 30.1.8.10 → 30.1.8.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 384ef1b163431fc5ed44e94b039ed6c63475d03eac7beb021895f3707c2ed68b
-  data.tar.gz: 8a138c55d4de91d43d4d99ccff25c3b0be6c514660b5afd8d94d04384e7a815a
+  metadata.gz: e12c8943b56f8054dbb1cfff9c52f17f0a926664eeb77436ee711cfba06e9d0c
+  data.tar.gz: 833df3fbf7c7ff98576c29e218a15dea996230e1a1a624f63fdc896a3728d2ed
 SHA512:
-  metadata.gz: af0f76bc2e13c1a14685a5cd9f50efa6a1493c57a697c14534a053668549393906f2ad22b87e2efe27ccf7c1b45cf7b8d0bf2d7298280bddad53048239618ce6
-  data.tar.gz: abafcd224584bb4ac206de5e95959c3a22102560f97bd6d3ea6122045377e2b0e3c397a699a813ec83d428df1d80003b8eb490581af98e01b8090067f78e08cb
+  metadata.gz: 65dcbf97c8b91ae1f85836dcb69a955fa6cf63a52bf71063b8ebd53a8ec7c82eac0a8df211fc9e2e4fc8da0f488b5c8f3ee442dbb870d9002987c959e990f689
+  data.tar.gz: 78fbe84b31b3fa457144ce7c81466a1b074a25c4218705f19d4ce3875f0e7cd6fedfbd32f72658396eca255db0aee5554693c2df078590d65bd04765489a274f

data/lib/search_engine/hydration/materializers.rb

@@ -294,6 +294,8 @@ module SearchEngine
       end
 
       def exists?(relation)
+        return count(relation).positive? if relation.send(:curation_filter_curated_hits?)
+
         loaded = relation.instance_variable_get(:@__loaded)
         memo = relation.instance_variable_get(:@__result_memo)
         return memo.found.to_i.positive? if loaded && memo
@@ -303,8 +305,8 @@ module SearchEngine
 
       def count(relation)
         if relation.send(:curation_filter_curated_hits?)
-          to_a(relation)
-          return relation.send(:curated_indices_for_current_result).size
+          result = execute(relation)
+          return curated_total_count(result)
         end
 
         loaded = relation.instance_variable_get(:@__loaded)
@@ -364,6 +366,21 @@ module SearchEngine
       end
       module_function :fetch_found_only
 
+      def curated_total_count(result)
+        raw = result.raw || {}
+        base_count = raw['found_docs'] || raw[:found_docs] || result.found
+
+        base_count.to_i + curated_hits_count(result)
+      end
+      module_function :curated_total_count
+
+      def curated_hits_count(result)
+        result.to_a.count do |obj|
+          obj.respond_to?(:curated_hit?) && obj.curated_hit?
+        end
+      end
+      module_function :curated_hits_count
+
       # Detect Typesense 400 errors caused by missing infix/prefix configuration
       # e.g., "Could not find `name` in the infix index. Make sure to enable infix search by specifying `infix: true` in the schema."
       def infix_missing_error?(error)

data/lib/search_engine/postgres_outbox/drain_enqueuer.rb

@@ -27,7 +27,7 @@ module SearchEngine
         targets = delivery_targets
         return enqueue_legacy(limit: limit) if targets.empty?
 
-        repository.materialize_deliveries!
+        materialize_deliveries(limit: limit)
         targets.each { |target| enqueue_target(target, limit: limit) }
         nil
       end
@@ -36,6 +36,12 @@ module SearchEngine
 
       attr_reader :repository, :targets_resolver
 
+      def materialize_deliveries(limit:)
+        return repository.materialize_deliveries! if limit.nil?
+
+        repository.materialize_deliveries!(limit: limit)
+      end
+
       def enqueue_legacy(limit:)
         return drain_job.perform_later if limit.nil?
 

data/lib/search_engine/postgres_outbox/repository.rb

@@ -111,11 +111,78 @@ module SearchEngine
 
       # Create missing delivery rows for all configured delivery targets.
       # @return [void]
-      def materialize_deliveries!
+      def materialize_deliveries!(limit: SearchEngine.config.postgres_outbox.batch_size)
         targets = delivery_targets
         return if targets.empty?
 
-        execute(<<~SQL)
+        rows = []
+        connection.transaction do
+          rows = select_rows(delivery_materialization_select_sql(limit.to_i, targets))
+          next if rows.empty?
+
+          execute(materialization_supersede_older_deliveries_sql(rows, targets))
+          execute(supersede_older_pending_sql(rows))
+          execute(delivery_materialization_insert_sql(rows, targets))
+        end
+
+        rows
+      end
+
+      private
+
+      attr_reader :target_key
+
+      def connection
+        @connection ||= begin
+          require 'active_record'
+          ActiveRecord::Base.connection
+        end
+      end
+
+      def claim_pending_deliveries(limit:, worker_id:)
+        reset_stale_delivery_processing!
+        rows = claim_pending_delivery_rows(limit: limit, worker_id: worker_id)
+
+        if rows.empty?
+          materialize_deliveries!(limit: limit)
+          rows = claim_pending_delivery_rows(limit: limit, worker_id: worker_id)
+        end
+
+        rows.map { |row| Event.new(row) }
+      end
+
+      def delivery_materialization_select_sql(limit, targets)
+        <<~SQL
+          WITH target(target_key, queue_name) AS (
+            VALUES #{delivery_target_values_sql(targets)}
+          ),
+          candidate_events AS (
+            SELECT outbox.*
+            FROM #{quoted_table} outbox
+            WHERE outbox.status IN ('pending', 'processing', 'failed')
+              AND (outbox.next_attempt_at IS NULL OR outbox.next_attempt_at <= CURRENT_TIMESTAMP)
+              AND EXISTS (
+                SELECT 1
+                FROM target
+                WHERE NOT EXISTS (
+                  SELECT 1
+                  FROM #{quoted_delivery_table} deliveries
+                  WHERE deliveries.event_id = outbox.id
+                    AND deliveries.target_key = target.target_key
+                )
+              )
+            ORDER BY outbox.id ASC
+            LIMIT #{limit}
+            FOR UPDATE SKIP LOCKED
+          )
+          SELECT DISTINCT ON (collection, document_id) *
+          FROM candidate_events
+          ORDER BY collection, document_id, id DESC
+        SQL
+      end
+
+      def delivery_materialization_insert_sql(rows, targets)
+        <<~SQL
           INSERT INTO #{quoted_delivery_table} (
             event_id,
             target_key,
@@ -132,29 +199,19 @@ module SearchEngine
                  0,
                  CURRENT_TIMESTAMP,
                  CURRENT_TIMESTAMP
-          FROM #{quoted_table} outbox
+          FROM (
+            VALUES #{materialization_event_values_sql(rows)}
+          ) AS selected_events(event_id)
+          INNER JOIN #{quoted_table} outbox
+            ON outbox.id = selected_events.event_id
           CROSS JOIN (
             VALUES #{delivery_target_values_sql(targets)}
           ) AS target(target_key, queue_name)
-          WHERE outbox.status IN ('pending', 'processing', 'failed')
           ON CONFLICT (event_id, target_key) DO NOTHING
         SQL
       end
 
-      private
-
-      attr_reader :target_key
-
-      def connection
-        @connection ||= begin
-          require 'active_record'
-          ActiveRecord::Base.connection
-        end
-      end
-
-      def claim_pending_deliveries(limit:, worker_id:)
-        materialize_deliveries!
-        reset_stale_delivery_processing!
+      def claim_pending_delivery_rows(limit:, worker_id:)
         rows = []
 
         connection.transaction do
@@ -164,7 +221,7 @@ module SearchEngine
           execute(delivery_claim_update_sql(delivery_ids, worker_id)) unless delivery_ids.empty?
         end
 
-        rows.map { |row| Event.new(row) }
+        rows
       end
 
       def reset_stale_delivery_processing!
@@ -271,6 +328,46 @@ module SearchEngine
         SQL
       end
 
+      def materialization_supersede_older_deliveries_sql(rows, targets)
+        <<~SQL
+          WITH updated_deliveries AS (
+            UPDATE #{quoted_delivery_table} older_deliveries
+            SET status = 'superseded',
+                processed_at = CURRENT_TIMESTAMP,
+                locked_at = NULL,
+                locked_by = NULL,
+                updated_at = CURRENT_TIMESTAMP
+            FROM #{quoted_table} older_events,
+                 (
+                   VALUES #{coalesce_values_sql(rows)}
+                 ) AS latest(collection, document_id, id),
+                 (
+                   VALUES #{delivery_target_values_sql(targets)}
+                 ) AS target(target_key, queue_name)
+            WHERE older_deliveries.event_id = older_events.id
+              AND older_deliveries.status = 'pending'
+              AND older_deliveries.target_key = target.target_key
+              AND older_events.collection = latest.collection
+              AND older_events.document_id = latest.document_id
+              AND older_events.id < latest.id
+            RETURNING older_deliveries.event_id
+          ),
+          aggregate AS (
+            #{event_status_aggregate_sql('SELECT event_id FROM updated_deliveries')}
+          )
+          UPDATE #{quoted_table} events
+          SET status = aggregate.status,
+              processed_at = CASE
+                WHEN aggregate.status IN ('processed', 'superseded') THEN CURRENT_TIMESTAMP
+                ELSE NULL
+              END,
+              last_error = aggregate.last_error,
+              updated_at = CURRENT_TIMESTAMP
+          FROM aggregate
+          WHERE events.id = aggregate.event_id
+        SQL
+      end
+
       def supersede_older_pending_sql(rows)
         <<~SQL
           UPDATE #{quoted_table} older
@@ -468,6 +565,12 @@ module SearchEngine
         end.join(', ')
       end
 
+      def materialization_event_values_sql(rows)
+        rows.map do |row|
+          "(#{quote(row_value(row, :id))})"
+        end.join(', ')
+      end
+
       def quoted_table
         connection.quote_table_name(SearchEngine.config.postgres_outbox.table_name)
       end

data/lib/search_engine/relation.rb

@@ -563,12 +563,6 @@ module SearchEngine
 
     # pluck helpers reside in Materializers
 
-    def curated_indices_for_current_result
-      @__result_memo.to_a.each_with_index.select do |obj, _idx|
-        obj.respond_to?(:curated_hit?) && obj.curated_hit?
-      end.map(&:last)
-    end
-
     def curation_filter_curated_hits?
       @state[:curation] && @state[:curation][:filter_curated_hits]
     end

data/lib/search_engine/version.rb

@@ -3,5 +3,5 @@
 module SearchEngine
   # Current gem version.
   # @return [String]
-  VERSION = '30.1.8.10'
+  VERSION = '30.1.8.12'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: search-engine-for-typesense
 version: !ruby/object:Gem::Version
-  version: 30.1.8.10
+  version: 30.1.8.12
 platform: ruby
 authors:
 - Nikita Shkoda