full_search 0.4.5 → 0.4.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fccc79c0dbfc8eeb24424e156c9be822470e2bbc3ecfdb75539f8a70010b3d38
-  data.tar.gz: 3e50db78f978cb095834dd9f0d58802dfb1de69f9c80e04fc6c18cbdf1d9ccef
+  metadata.gz: a47cd95db7244715db988cdce15e8466dc2131d3a995769071ca7b513f7b91e4
+  data.tar.gz: 35303e2f06de22ccfcd1e20018d8f1a83cd1d32bd2d883cdd7ba60c1919d3541
 SHA512:
-  metadata.gz: 239eb848fe5631b1a8212840246ccd6be8387642b8f269687ae5052dc765afd25ccb49bd55c395b760aa772eb2da696bf786a2155f451326dbd8908da4f95532
-  data.tar.gz: bd01ae3b672adc80f97c5ee70699444d773cb2a66c8ab2b98133f3528cba5099b3750fea9682cd6cdb4315e0fac17b7781b720a60f6c79519bf1857b78324aa4
+  metadata.gz: 1316924b824ef6814d2899ee165426b8eee7b021fc7bde0ee358f6a9066f2668561f27fb1de12f12bd0d3b836f91a2d428eb1c9da9e0db7dc494cf037ea707e5
+  data.tar.gz: 3c359e74e2898302ef51ada7f864c22c1b2dd340fb27e5e097b59baa78bae97b800d74ad020f172f202c2da85d3094925aba3fbad9cdc00dc5929fd990803dab

data/lib/full_search/bulk_import.rb

@@ -19,6 +19,19 @@ module FullSearch
         models_in_bulk_import.delete(model)
         FullSearch::Index.create_triggers!(model)
         FullSearch::BackfillJob.perform_later(model.name)
+      rescue => e
+        # Triggers are still dropped at this point; the FTS index will rot on
+        # every subsequent write. Retry once, then raise loudly so the caller
+        # knows the index is compromised instead of failing silent.
+        retry_count = (Thread.current[:_fs_bulk_import_retry] ||= 0)
+        if retry_count < 1
+          Thread.current[:_fs_bulk_import_retry] = retry_count + 1
+          retry
+        end
+        raise FullSearch::TriggerRestoreError,
+          "Failed to restore FTS triggers for #{model.table_name} after bulk import: #{e.message}"
+      ensure
+        Thread.current[:_fs_bulk_import_retry] = nil
       end
 
       def bulk_importing?(model)

data/lib/full_search/errors.rb

@@ -10,4 +10,5 @@ module FullSearch
   class UnsupportedDatabaseError < Error; end
   class MissingTableError < Error; end
   class InvalidQueryError < Error; end
+  class TriggerRestoreError < Error; end
 end

data/lib/full_search/index.rb

@@ -167,8 +167,8 @@ module FullSearch
 
         conn = connection
         begin
-          conn.execute("DELETE FROM #{qt(fts_table_name(model))};")
-          conn.execute("DELETE FROM #{qt(trigram_table_name(model))};") if trigram_table_needed?(model) && trigram_table_exists?(model)
+          with_busy_retry { conn.execute("DELETE FROM #{qt(fts_table_name(model))};") }
+          with_busy_retry { conn.execute("DELETE FROM #{qt(trigram_table_name(model))};") } if trigram_table_needed?(model) && trigram_table_exists?(model)
         rescue ActiveRecord::StatementInvalid => e
           raise unless e.message.match?(/fts5: corruption|invalid fts5 file format|malformed database schema/i)
 
@@ -191,12 +191,12 @@ module FullSearch
       # drop the shadow tables directly, then drop the (now-empty) virtual
       # table wrapper, so the caller can recreate a clean index.
       def drop_fts_table_safely!(table_name)
-        connection.execute("DROP TABLE IF EXISTS #{qt(table_name)};")
+        with_busy_retry { connection.execute("DROP TABLE IF EXISTS #{qt(table_name)};") }
       rescue ActiveRecord::StatementInvalid => e
         raise unless e.message.match?(/fts5: corruption|invalid fts5 file format|malformed database schema/i)
 
         drop_shadow_tables!(table_name)
-        connection.execute("DROP TABLE IF EXISTS #{qt(table_name)};")
+        with_busy_retry { connection.execute("DROP TABLE IF EXISTS #{qt(table_name)};") }
       end
 
       # FTS5 stores its index data in a fixed set of shadow tables named
@@ -249,12 +249,14 @@ module FullSearch
       def healthy?(model)
         return false unless table_exists?(model)
 
-        connection.execute("SELECT count(*) FROM #{qt(fts_table_name(model))}")
-        true
-      rescue ActiveRecord::StatementInvalid => e
-        raise unless e.message.match?(/fts5: corruption|invalid fts5 file format|malformed database schema/i)
+        begin
+          with_busy_retry { connection.execute("SELECT count(*) FROM #{qt(fts_table_name(model))}") }
+          true
+        rescue ActiveRecord::StatementInvalid => e
+          raise unless e.message.match?(/fts5: corruption|invalid fts5 file format|malformed database schema/i)
 
-        false
+          false
+        end
       end
 
       def trigram_table_needed?(model)
@@ -267,8 +269,40 @@ module FullSearch
         ).any?
       end
 
+      def missing_triggers?(model)
+        return false if FullSearch.bulk_importing?(model)
+        return false unless model_table_exists?(model)
+        return false unless table_exists?(model)
+
+        IndexCache.fetch("missing_triggers:#{model.table_name}") do
+          existing = connection.execute(
+            "SELECT name FROM sqlite_master WHERE type='trigger' AND tbl_name=#{q(model.table_name)}"
+          ).map { |r| r["name"] }
+
+          expected = trigger_names(model)
+          expected += trigram_trigger_names(model) if trigram_table_needed?(model)
+          (expected - existing).any?
+        end
+      end
+
       private
 
+      BUSY_RETRIES = 3
+      BUSY_BACKOFF = 0.1
+
+      def with_busy_retry
+        attempts = 0
+        begin
+          yield
+        rescue ActiveRecord::StatementInvalid => e
+          raise unless e.message.match?(/database is locked/i) && attempts < BUSY_RETRIES
+          attempts += 1
+          sleep(BUSY_BACKOFF * attempts)
+          IndexCache.clear!
+          retry
+        end
+      end
+
       def connection
         ActiveRecord::Base.connection
       end
@@ -290,9 +324,11 @@ module FullSearch
       end
 
       def model_table_exists?(model)
-        connection.execute(
-          "SELECT name FROM sqlite_master WHERE type='table' AND name=#{q(model.table_name)} LIMIT 1"
-        ).any?
+        IndexCache.fetch("model_table_exists:#{model.table_name}") do
+          connection.execute(
+            "SELECT name FROM sqlite_master WHERE type='table' AND name=#{q(model.table_name)} LIMIT 1"
+          ).any?
+        end
       end
 
       def table_exists?(model)
@@ -354,16 +390,7 @@ module FullSearch
 
       def ensure_triggers!(model)
         return if FullSearch.bulk_importing?(model)
-
-        existing = connection.execute(
-          "SELECT name FROM sqlite_master WHERE type='trigger' AND tbl_name=#{q(model.table_name)}"
-        ).map { |r| r["name"] }
-
-        expected = trigger_names(model)
-        expected += trigram_trigger_names(model) if trigram_table_needed?(model)
-        return if (expected - existing).empty? && (existing - expected).empty?
-
-        rebuild!(model)
+        rebuild!(model) if missing_triggers?(model)
       end
 
       def trigger_names(model)
@@ -378,6 +405,8 @@ module FullSearch
         %W[#{base}_ai #{base}_ad #{base}_au]
       end
 
+      private
+
       def insert_trigger_sql(model)
         dsl = model.full_search_dsl
         cols = dsl.fields + dsl.filters + extra_columns(model)

data/lib/full_search/multi_search.rb

@@ -12,40 +12,42 @@ module FullSearch
     end
 
     def call
-      searched = groups.map do |group|
-        model = fetch(group, :model)
-        raise FullSearch::NotConfiguredError, "#{model} is not full_search configured" unless model.full_search_dsl
+      IndexCache.with_cache do
+        searched = groups.map do |group|
+          model = fetch(group, :model)
+          raise FullSearch::NotConfiguredError, "#{model} is not full_search configured" unless model.full_search_dsl
 
-        filters = group[:filters] || {}
-        limit = positive_integer(group[:limit], 8)
-        offset = positive_integer(group[:offset], 0)
-        raw_limit = limit + 1
+          filters = group[:filters] || {}
+          limit = positive_integer(group[:limit], 8)
+          offset = positive_integer(group[:offset], 0)
+          raw_limit = limit + 1
 
-        relation = model.full_search(
-          query,
-          filters: filters,
-          limit: raw_limit,
-          offset: offset,
-          highlight: group[:highlight],
-          highlight_fields: group[:highlight_fields],
-          matching_strategy: group[:matching_strategy],
-          per_strategy_limit: group[:per_strategy_limit],
-          scope: group[:scope],
-          includes: group[:includes]
-        )
+          relation = model.full_search(
+            query,
+            filters: filters,
+            limit: raw_limit,
+            offset: offset,
+            highlight: group[:highlight],
+            highlight_fields: group[:highlight_fields],
+            matching_strategy: group[:matching_strategy],
+            per_strategy_limit: group[:per_strategy_limit],
+            scope: group[:scope],
+            includes: group[:includes]
+          )
 
-        records = relation.to_a
-        has_more = records.size > limit
-        records = records.first(limit) if has_more
+          records = relation.to_a
+          has_more = records.size > limit
+          records = records.first(limit) if has_more
 
-        group.slice(:key, :label, :icon, :model).merge(
-          results: records,
-          has_more: has_more,
-          total_count: records.size
-        )
-      end
+          group.slice(:key, :label, :icon, :model).merge(
+            results: records,
+            has_more: has_more,
+            total_count: records.size
+          )
+        end
 
-      {groups: searched, total_count: searched.sum { |g| g[:total_count] }}
+        {groups: searched, total_count: searched.sum { |g| g[:total_count] }}
+      end
     end
 
     private

data/lib/full_search/search.rb

@@ -92,6 +92,21 @@ module FullSearch
         raise MissingTableError, "FTS table `#{FullSearch::Index.fts_table_name(model)}` does not exist. Run `bin/rails full_search:prepare` to create it."
       end
 
+      if FullSearch::Index.missing_triggers?(model)
+        if FullSearch.config.auto_rebuild_on_stale_query
+          FullSearch::Index.rebuild!(model)
+          return
+        end
+
+        case FullSearch.config.stale_query_behavior
+        when :raise
+          raise ConfigChangedError, "FTS triggers for #{model.table_name} are missing; run full_search:rebuild"
+        when :log_and_fallback
+          Rails.logger.warn("[full_search] FTS triggers for #{model.table_name} are missing; results may be incomplete")
+        end
+        return
+      end
+
       stored = begin
         FullSearch::Index.stored_config_hash(model)
       rescue ActiveRecord::StatementInvalid

data/lib/full_search/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module FullSearch
-  VERSION = "0.4.5"
+  VERSION = "0.4.7"
 end

data/lib/tasks/full_search.rake

@@ -105,7 +105,9 @@ namespace :full_search do
         ""
       end
 
-      puts "#{model.table_name}: #{status}#{drift_info}"
+      trigger_info = FullSearch::Index.missing_triggers?(model) ? " | MISSING TRIGGERS" : ""
+
+      puts "#{model.table_name}: #{status}#{drift_info}#{trigger_info}"
     end
   end
 
@@ -120,6 +122,10 @@ namespace :full_search do
         next
       end
 
+      if FullSearch::Index.missing_triggers?(model)
+        unhealthy << "#{model.table_name}: missing FTS triggers"
+      end
+
       stored = FullSearch::Index.stored_config_hash(model)
       current = model.full_search_dsl.config_hash
       if stored != current

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: full_search
 version: !ruby/object:Gem::Version
-  version: 0.4.5
+  version: 0.4.7
 platform: ruby
 authors:
 - Ben D'Angelo
@@ -189,7 +189,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '2.0'
 requirements: []
-rubygems_version: 4.0.15
+rubygems_version: 4.0.10
 specification_version: 4
 summary: SQLite FTS5 full-text search for Rails/ActiveRecord
 test_files: []