full_search 0.3.7 → 0.3.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d25d24145bf1aef1508210eb9d6879b21218ed8760ef6a61812706720a12ac17
-  data.tar.gz: 94b71b2b86db367ef8fde640550ab7b5945b50829d618c5e4eb41a4d9a6e0d65
+  metadata.gz: a86070d05b32fc24ce6ac007a33c673c3831760ce76554bbb9d636e6e0c6be0f
+  data.tar.gz: 293b0be9931c7ffa5de9971aa52b2bf9014c2b0816d1364f839ccba57af8f814
 SHA512:
-  metadata.gz: 4074fd91514f5f209cbf6d5f4b2fdd36eed20ccfd894e0ccaac330e25bf7e0cda0c2c689833f2fcf947df768cf0282e88d717f1a001797e901ee0ebb18ab6e1a
-  data.tar.gz: f16291785a67d4e9cec7e48fe407a3c4735e21711752feea5ff2ad88573ef8010732feb8eaa9a1f1c40bfb14c14f476dd5f242fb9e624ba68f1a4ec18ca077ba
+  metadata.gz: cd9854cc22d14bd739723e5011a831fd34d0e523ef6a6df09bf45fa1f3a7880c466f4ba121eb07d9d92daf65449edf74b5e0ddc42dc15b521ea34e78b496be3a
+  data.tar.gz: f704e822cee03ad98f2ed96dbc802e3e5b777df659ecdc3d8187ace6088494ab318b6f1598c74b840697909b465e539a76ee9f758d6a709041d9e0b198d45129

data/README.md

@@ -193,6 +193,20 @@ end
 
 ### In production
 
+**`auto_rebuild_schema`** — Must be `false` in production. The generated initializer defaults to `Rails.env.local?`, so it's already off. If enabled, every process (web, worker, console) attempts schema rebuilds on boot, which is unnecessary and can cause issues during zero-downtime deploys where old and new processes overlap.
+
+**`auto_rebuild_on_stale_query`** — Must be `false` in production. A query-time rebuild is dangerous — it can cause timeouts or race conditions under load. The generated initializer defaults to `Rails.env.local?`, keeping it off in production.
+
+**`lock_rebuilds`** — This option uses a Ruby `Mutex` and only prevents concurrent rebuilds within the same process. It does **not** coordinate across processes or hosts. Multi-process or multi-host deployments must run `full_search:rebuild` from a single deployment step.
+
+**Docker entrypoint** — For containerized environments, add `full_search:prepare` after `db:prepare:with_data`:
+
+```bash
+./bin/rails full_search:prepare
+```
+
+`full_search:prepare` is idempotent — it only creates missing FTS tables and installs triggers, making it safe to run on every deploy.
+
 Auto-rebuild runs on every Rails process boot (web, worker, console). For zero-downtime deploys where old processes still serve traffic, or if you prefer explicit control, set `auto_rebuild_schema` to `false` and run the rebuild task manually:
 
 ```bash

data/lib/full_search/backfill_job.rb

@@ -8,6 +8,7 @@ module FullSearch
 
     def perform(model_name)
       model = model_name.to_s.constantize
+      FullSearch::Index.sqlite!(model)
       FullSearch::Index.rebuild!(model)
     end
   end

data/lib/full_search/callbacks.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "full_search/errors"
+
 module FullSearch
   module Callbacks
     def self.install!(model)
@@ -64,34 +66,35 @@ module FullSearch
 
       value = FullSearch::Model.evaluate_source(record, field)
       table = qt(FullSearch::Index.fts_table_name(record.class))
-      conn = ActiveRecord::Base.connection
+      conn = connection_for(record.class)
       conn.execute(
         "UPDATE #{table} SET #{qc(field.as || field.name)} = #{q(value.to_s)} WHERE rowid = #{q(record.id)}"
       )
-    rescue => e
-      raise unless e.message.include?("no such table")
+    rescue ActiveRecord::StatementInvalid => e
+      raise_missing_table_or_original(e, record.class)
     end
 
     def self.remove_record!(record)
       table = qt(FullSearch::Index.fts_table_name(record.class))
-      conn = ActiveRecord::Base.connection
+      conn = connection_for(record.class)
       conn.execute("DELETE FROM #{table} WHERE rowid = #{q(record.id)}")
-    rescue => e
-      raise unless e.message.include?("no such table")
+    rescue ActiveRecord::StatementInvalid => e
+      raise_missing_table_or_original(e, record.class)
     end
 
     def self.reindex_dependents!(parent_record, dependent_model, field)
       fk = association_key(dependent_model, field.reindex_on)
-      conn = ActiveRecord::Base.connection
+      conn = connection_for(dependent_model)
       sql = "SELECT id FROM #{qt(dependent_model.table_name)} WHERE #{qc(fk)} = #{q(parent_record.id)}"
       dependent_ids = conn.execute(sql).map { |r| r["id"] }
 
-      dependent_ids.each do |dep_id|
-        if field.async
-          FullSearch::ReindexJob.perform_later(dependent_model.name, dep_id, field.name)
-        else
-          dependent = dependent_model.find_by(id: dep_id)
-          reindex_field!(dependent, field.name) if dependent
+      return if dependent_ids.empty?
+
+      if field.async
+        dependent_ids.each { |id| FullSearch::ReindexJob.perform_later(dependent_model.name, id, field.name) }
+      else
+        dependent_model.where(id: dependent_ids).find_each(batch_size: 500) do |record|
+          reindex_field!(record, field.name)
         end
       end
     end
@@ -113,5 +116,20 @@ module FullSearch
       reflection = model.reflect_on_association(association_name.to_sym)
       reflection&.foreign_key&.to_s || "#{association_name}_id"
     end
+
+    def self.connection_for(klass)
+      klass.connection
+    rescue NoMethodError
+      ActiveRecord::Base.connection
+    end
+
+    def self.raise_missing_table_or_original(error, model_class)
+      if error.message.include?("no such table")
+        FullSearch::Instrumentation.instrument("missing_table", model: model_class.name, error: error.message)
+        return
+      end
+      raise error
+    end
+    private_class_method :raise_missing_table_or_original
   end
 end

data/lib/full_search/config.rb

@@ -12,9 +12,9 @@ module FullSearch
       @lock_rebuilds = true
       @default_async_reindex = true
       @default_async_source_reindex = true
-      @default_tokenizer = "unicode61"
+      @default_tokenizer = FullSearch::Constants::DEFAULT_TOKENIZER
       @auto_rebuild_on_stale_query = false
-      @min_like_prefix_length = 3
+      @min_like_prefix_length = FullSearch::Constants::DEFAULT_MIN_LIKE_PREFIX_LENGTH
     end
   end
 

data/lib/full_search/constants.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module FullSearch
+  module Constants
+    MIN_TERM_LENGTH = 3
+    TWO_TYPO_MIN_LENGTH = 9
+    DEFAULT_MIN_LIKE_PREFIX_LENGTH = 3
+    REBUILDING_HASH = "__rebuilding__"
+    DEFAULT_TOKENIZER = "unicode61"
+    MAX_EXACT_MATCH_BOOST_IDS = 100
+  end
+end

data/lib/full_search/dsl.rb

@@ -45,7 +45,9 @@ module FullSearch
       end
       str = name.to_s
       raise InvalidFieldError, "#{model_class.name}: duplicate exact_match name #{name.inspect}" if exact_matches.any? { |e| e.name == str }
-      @exact_matches << ExactMatch.new(name: str, source: source, sql: sql&.to_s, normalize: normalize, version: version)
+      sql_str = sql&.to_s
+      validate_sql_fragment!(sql_str, "exact_match sql:") if sql_str
+      @exact_matches << ExactMatch.new(name: str, source: source, sql: sql_str, normalize: normalize, version: version)
     end
 
     def rank_by(column, direction = :desc)
@@ -92,7 +94,9 @@ module FullSearch
     end
 
     def index_if(sql: nil)
-      @index_if_sql = sql&.to_s
+      sql_str = sql&.to_s
+      validate_sql_fragment!(sql_str, "index_if sql:") if sql_str
+      @index_if_sql = sql_str
     end
 
     def conditional_index?
@@ -109,7 +113,7 @@ module FullSearch
 
     def typo_tolerance(enabled = true, min_term_length: nil)
       @typo_tolerance = enabled
-      @typo_tolerance_min_term_length = min_term_length || 3
+      @typo_tolerance_min_term_length = min_term_length || FullSearch::Constants::MIN_TERM_LENGTH
     end
 
     def typo_tolerance?
@@ -117,7 +121,7 @@ module FullSearch
     end
 
     def typo_tolerance_min_term_length
-      @typo_tolerance_min_term_length || 3
+      @typo_tolerance_min_term_length || FullSearch::Constants::MIN_TERM_LENGTH
     end
 
     def config_hash
@@ -139,6 +143,12 @@ module FullSearch
 
     private
 
+    def validate_sql_fragment!(sql, option_name)
+      if sql.match?(/;|--|\/\*|\*\//)
+        raise InvalidFieldError, "#{model_class.name}: #{option_name} contains unsafe characters"
+      end
+    end
+
     def valid_name?(name)
       name.to_s.match?(/\A[a-zA-Z_]\w*\z/)
     end

data/lib/full_search/highlighter.rb

@@ -59,7 +59,7 @@ module FullSearch
         length = query.length
         min_length = dsl.typo_tolerance_min_term_length.to_i
         return nil if length < min_length
-        return 2 if length >= 9
+        return 2 if length >= FullSearch::Constants::TWO_TYPO_MIN_LENGTH
         1
       end
 
@@ -170,8 +170,8 @@ module FullSearch
       end
 
       def max_allowed_typos(length)
-        return -1 if length < 3
-        return 2 if length >= 9
+        return -1 if length < FullSearch::Constants::MIN_TERM_LENGTH
+        return 2 if length >= FullSearch::Constants::TWO_TYPO_MIN_LENGTH
         1
       end
 

data/lib/full_search/index.rb

@@ -65,21 +65,23 @@ module FullSearch
         create_metadata_table!
 
         with_rebuild_lock(model) do
-          drop_triggers!(model)
-          conn.execute("DROP TABLE IF EXISTS #{qt(fts_table_name(model))};")
-          conn.execute("DROP TABLE IF EXISTS #{qt(trigram_table_name(model))};")
-          conn.execute(create_virtual_table_sql(model))
-          if dsl.typo_tolerance?
-            FullSearch::Typo.warn_unsupported!(model) unless FullSearch::Typo.supported?(model)
-            conn.execute(create_trigram_virtual_table_sql(model))
+          FullSearch::Instrumentation.instrument("rebuild", model: model.name) do
+            drop_triggers!(model)
+            conn.execute("DROP TABLE IF EXISTS #{qt(fts_table_name(model))};")
+            conn.execute("DROP TABLE IF EXISTS #{qt(trigram_table_name(model))};")
+            conn.execute(create_virtual_table_sql(model))
+            if dsl.typo_tolerance?
+              FullSearch::Typo.warn_unsupported!(model) unless FullSearch::Typo.supported?(model)
+              conn.execute(create_trigram_virtual_table_sql(model))
+            end
+            conn.execute(backfill_sql(model))
+            conn.execute(backfill_trigram_sql(model)) if dsl.typo_tolerance?
+            reindex_source_fields!(model) if dsl.fields.any?(&:source)
+            create_triggers!(model)
+            optimize!(model)
+            store_config_hash!(model, rebuilt_at: Time.current)
+            verified_tables.add(model.table_name)
           end
-          conn.execute(backfill_sql(model))
-          conn.execute(backfill_trigram_sql(model)) if dsl.typo_tolerance?
-          reindex_source_fields!(model) if dsl.fields.any?(&:source)
-          create_triggers!(model)
-          optimize!(model)
-          store_config_hash!(model, rebuilt_at: Time.current)
-          verified_tables.add(model.table_name)
         end
       end
 
@@ -103,21 +105,35 @@ module FullSearch
       end
 
       def optimize!(model)
-        sqlite!(model)
-        connection.execute("INSERT INTO #{qt(fts_table_name(model))}(#{qt(fts_table_name(model))}) VALUES('optimize');")
+        FullSearch::Instrumentation.instrument("optimize", model: model.name) do
+          sqlite!(model)
+          connection.execute("INSERT INTO #{qt(fts_table_name(model))}(#{qt(fts_table_name(model))}) VALUES('optimize');")
+        end
       end
 
       def reindex_source_fields!(model)
         dsl = model.full_search_dsl
         return unless dsl
 
-        model.find_each do |record|
-          dsl.fields.each do |field|
-            FullSearch::Callbacks.reindex_field!(record, field.name) if field.source
-          end
+        source_fields = dsl.fields.select(&:source)
+        return if source_fields.empty?
+
+        model.find_each(batch_size: 500) do |record|
+          pairs = source_fields.to_h { |f| [column_name(f), FullSearch::Model.evaluate_source(record, f).to_s] }
+          next if pairs.empty?
+          upsert_source_row!(model, record.id, pairs)
         end
       end
 
+      def upsert_source_row!(model, rowid, values)
+        table = qt(fts_table_name(model))
+        sets = values.map { |name, value| "#{qc(name)} = #{q(value)}" }.join(", ")
+
+        connection.execute(<<~SQL)
+          UPDATE #{table} SET #{sets} WHERE rowid = #{q(rowid)};
+        SQL
+      end
+
       def create_triggers!(model)
         connection.execute(insert_trigger_sql(model))
         connection.execute(delete_trigger_sql(model))
@@ -501,15 +517,13 @@ module FullSearch
       # For multi-process or multi-host deployments, run full_search:rebuild from a single
       # deployment step. See README.
       def with_rebuild_lock(model)
-        if FullSearch.config.lock_rebuilds
-          connection.transaction do
+        connection.transaction do
+          if FullSearch.config.lock_rebuilds
             connection.execute(
-              "INSERT INTO full_search_index_versions (table_name, config_hash, rebuilt_at) VALUES (#{q(model.table_name)}, #{q("__rebuilding__")}, datetime('now'))
+              "INSERT INTO full_search_index_versions (table_name, config_hash, rebuilt_at) VALUES (#{q(model.table_name)}, #{q(FullSearch::Constants::REBUILDING_HASH)}, datetime('now'))
                ON CONFLICT(table_name) DO UPDATE SET config_hash=excluded.config_hash;"
             )
-            yield
           end
-        else
           yield
         end
       end

data/lib/full_search/instrumentation.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module FullSearch
+  module Instrumentation
+    def self.instrument(name, payload = {}, &block)
+      ActiveSupport::Notifications.instrument("full_search.#{name}", payload, &block)
+    end
+  end
+end

data/lib/full_search/search.rb

@@ -19,42 +19,44 @@ module FullSearch
       @includes = includes
     end
 
-    MIN_TERM_LENGTH = 3
+    MIN_TERM_LENGTH = FullSearch::Constants::MIN_TERM_LENGTH
 
     def relation
-      validate_filter_keys!
-      validate_required_filters!
-      check_stale_config!
-
-      return model.none if dsl.tokenize == "trigram" && query.length < MIN_TERM_LENGTH && !dsl.typo_tolerance?
-
-      parsed = QueryParser.parse(query)
-      exact_ids = ExactMatch.ids_for(model, query, filters)
-      primary_ids = fts_match_ids(parsed)
-      fallback_ids = (dsl.typo_tolerance? && matching_strategy != "all") ? trigram_match_ids(parsed, primary_ids, candidate_limit: per_strategy_limit) : []
-      fuzzy_ids = (dsl.typo_tolerance? && matching_strategy != "all" && primary_ids.empty? && fallback_ids.empty?) ? fuzzy_match_ids(parsed, candidate_limit: per_strategy_limit) : []
-
-      all_ids = (exact_ids + primary_ids + fallback_ids + fuzzy_ids).uniq
-      return model.none if all_ids.empty?
-
-      rel = model.where(id: all_ids)
-      rel = rel.where(model.arel_table[dsl.soft_delete_column].eq(nil)) if dsl.soft_delete_column && !include_soft_deleted
-      rel = apply_ranking(rel, all_ids, exact_ids)
-      rel = scope.call(rel) if scope
-      rel = rel.limit(limit) if limit
-      rel = rel.offset(offset) if offset
-      rel = rel.includes(includes) if includes
-
-      if highlight
-        records = rel.to_a
-        Highlighter.apply!(records, model, query, record_ids: records.map(&:id))
-        records
-      elsif highlight_fields
-        records = rel.to_a
-        Highlighter.apply_fields!(records, model, query, record_ids: records.map(&:id))
-        records
-      else
-        rel
+      FullSearch::Instrumentation.instrument("query", model: model.name, query: query) do
+        validate_filter_keys!
+        validate_required_filters!
+        check_stale_config!
+
+        return model.none if dsl.tokenize == "trigram" && query.length < MIN_TERM_LENGTH && !dsl.typo_tolerance?
+
+        parsed = QueryParser.parse(query)
+        exact_ids = ExactMatch.ids_for(model, query, filters)
+        primary_ids = fts_match_ids(parsed)
+        fallback_ids = (dsl.typo_tolerance? && matching_strategy != "all") ? trigram_match_ids(parsed, primary_ids, candidate_limit: per_strategy_limit) : []
+        fuzzy_ids = (dsl.typo_tolerance? && matching_strategy != "all" && primary_ids.empty? && fallback_ids.empty?) ? fuzzy_match_ids(parsed, candidate_limit: per_strategy_limit) : []
+
+        all_ids = (exact_ids + primary_ids + fallback_ids + fuzzy_ids).uniq
+        return model.none if all_ids.empty?
+
+        rel = model.where(id: all_ids)
+        rel = rel.where(model.arel_table[dsl.soft_delete_column].eq(nil)) if dsl.soft_delete_column && !include_soft_deleted
+        rel = apply_ranking(rel, all_ids, exact_ids)
+        rel = scope.call(rel) if scope
+        rel = rel.limit(limit) if limit
+        rel = rel.offset(offset) if offset
+        rel = rel.includes(includes) if includes
+
+        if highlight
+          records = rel.to_a
+          Highlighter.apply!(records, model, query, record_ids: records.map(&:id))
+          records
+        elsif highlight_fields
+          records = rel.to_a
+          Highlighter.apply_fields!(records, model, query, record_ids: records.map(&:id))
+          records
+        else
+          rel
+        end
       end
     end
 
@@ -279,7 +281,7 @@ module FullSearch
     def max_allowed_typos(length)
       min_length = dsl.typo_tolerance_min_term_length.to_i
       return -1 if length < min_length
-      return 2 if length >= 9
+      return 2 if length >= FullSearch::Constants::TWO_TYPO_MIN_LENGTH
       1
     end
 
@@ -305,7 +307,8 @@ module FullSearch
       tbl = qt(model.table_name)
 
       if exact_ids.any?
-        order_parts << "CASE #{tbl}.id #{exact_ids.map { |id| "WHEN #{q(id)} THEN 0" }.join(" ")} ELSE 1 END"
+        boost_ids = exact_ids.first(FullSearch::Constants::MAX_EXACT_MATCH_BOOST_IDS)
+        order_parts << "CASE #{tbl}.id #{boost_ids.map { |id| "WHEN #{q(id)} THEN 0" }.join(" ")} ELSE 1 END"
       end
 
       fts_table = qt(FullSearch::Index.fts_table_name(model))

data/lib/full_search/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module FullSearch
-  VERSION = "0.3.7"
+  VERSION = "0.3.8"
 end

data/lib/full_search.rb

@@ -4,7 +4,9 @@ require "active_record"
 require "active_support"
 require "active_support/concern"
 
+require "full_search/constants"
 require "full_search/index_cache"
+require "full_search/instrumentation"
 require "full_search/version"
 require "full_search/config"
 require "full_search/distance"
@@ -39,6 +41,10 @@ module FullSearch
       @models ||= Set.new
     end
 
+    def sorted_models
+      models.sort_by { |m| m.table_name.to_s }
+    end
+
     def register_model(model)
       models << model
     end
@@ -48,11 +54,11 @@ module FullSearch
     end
 
     def optimize!
-      models.each { |model| Index.optimize!(model) }
+      sorted_models.each { |model| Index.optimize!(model) }
     end
 
     def setup!
-      models.each do |model|
+      sorted_models.each do |model|
         Index.ensure_table!(model)
         Callbacks.install!(model)
       end

data/lib/tasks/full_search.rake

@@ -12,7 +12,7 @@ def resolve_full_search_models(args)
       klass || FullSearch.models.find { |m| m.table_name == name }
     end.compact
   else
-    FullSearch.models
+    FullSearch.sorted_models
   end
 end
 
@@ -31,7 +31,7 @@ namespace :full_search do
         klass
       end.compact
     else
-      FullSearch.models
+      FullSearch.sorted_models
     end
 
     FullSearch.setup!
@@ -77,7 +77,7 @@ namespace :full_search do
   task optimize: :environment do
     Rails.application.eager_load!
     FullSearch.optimize!
-    FullSearch.models.each do |model|
+    FullSearch.sorted_models.each do |model|
       puts "Optimized #{FullSearch::Index.fts_table_name(model)}"
     end
   end
@@ -85,7 +85,7 @@ namespace :full_search do
   desc "Show full_search index status"
   task status: :environment do
     Rails.application.eager_load!
-    FullSearch.models.each do |model|
+    FullSearch.sorted_models.each do |model|
       stored = FullSearch::Index.stored_config_hash(model)
       current = model.full_search_dsl.config_hash
       status = (stored == current) ? "ok" : "stale"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: full_search
 version: !ruby/object:Gem::Version
-  version: 0.3.7
+  version: 0.3.8
 platform: ruby
 authors:
 - Ben D'Angelo
@@ -144,6 +144,7 @@ files:
 - lib/full_search/bulk_import.rb
 - lib/full_search/callbacks.rb
 - lib/full_search/config.rb
+- lib/full_search/constants.rb
 - lib/full_search/distance.rb
 - lib/full_search/dsl.rb
 - lib/full_search/errors.rb
@@ -151,6 +152,7 @@ files:
 - lib/full_search/highlighter.rb
 - lib/full_search/index.rb
 - lib/full_search/index_cache.rb
+- lib/full_search/instrumentation.rb
 - lib/full_search/model.rb
 - lib/full_search/multi_search.rb
 - lib/full_search/optimize_job.rb