full_search 0.3.3 → 0.3.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8c3e974821a38f57a0d71c6a2ad0260942773836c3404c9b3665bb9cc0a6d305
-  data.tar.gz: 841b5bb5c5086942f47bbf7626455d4a27d4261778dfc845e22d2d5262d0ff41
+  metadata.gz: ed7b0998cc6d4014ab815a6152ba07562da34c98b3cca3aa3886e63fb45acc5b
+  data.tar.gz: f29430f7ac4d5a3175e1eb46c803aa3a9b9ce68e3e930f2f75ecfd8ad03242d7
 SHA512:
-  metadata.gz: 88f20b95e78a0de07571c06560df3c9e45ac5927106dc600e6d8f4a9702611ee776d0f2c2edec9c9cfd5a26aacc5e91c727ae156043c46767ca995c4228f37b5
-  data.tar.gz: 8ba992243ac6ae15fb755d0817c75e6f8e41ec1cb4bad856c75cc9b7fe9ee3f760da92b97fcdbd1093677eaf83e93701c1f917ac82f26c650274e3891de74f6f
+  metadata.gz: cce0189e77ec25883ad52dae321dea4f99cb8f1f245f331b5cdeb94f6c3554c675e84b0214e4036ca1c01fc53f22e41594324729d39688c8e48d0bcd791356f3
+  data.tar.gz: a9d3f31295d082892dc5cf797f2d38a9cb8151629f9e6d2c0ac0ad1b8dcfeb5e732b740fb91d138d0b3a7f1db513c72c36aa61a1410d2b052bd57bda0790c5dd

data/README.md

@@ -57,6 +57,24 @@ Customer.search("sam", filters: { account_id: 1 }).page(params[:page])
 - FTS5 `highlight()` support for result snippets
 - Opt-in trigram typo/substring fallback (requires SQLite >= 3.34)
 
+### Conditional indexing
+
+Use `index_if` to restrict which records are included in the FTS index. Records that don't match the SQL condition are excluded from search results entirely.
+
+```ruby
+class Customer < ApplicationRecord
+  full_search do
+    field :first_name, weight: 5
+    field :last_name,  weight: 5
+    filter :account_id, required: true
+
+    index_if sql: "active = 1 AND deleted_at IS NULL"
+  end
+end
+```
+
+This is enforced at the database level — the FTS virtual table only contains rows matching the condition, and triggers skip inserts/updates for non-matching records.
+
 ### Per-model operations
 
 Once a model declares `full_search`, you can call these class methods:
@@ -126,6 +144,7 @@ bin/rails full_search:reset
 | `full_search:rebuild` | Drops and recreates the FTS virtual table only when the DSL config hash has changed (safe for production). Pass model names to target specific tables. |
 | `full_search:reset` | Force a full rebuild — drops and recreates all FTS tables regardless of config hash. Use when data may be out of sync. |
 | `full_search:optimize` | Run FTS5 [`optimize`](https://www.sqlite.org/fts5.html#the_optimize_command) to merge b-tree segments. Useful after bulk updates. |
+| `full_search:backfill` | Force-rebuild FTS indexes for specified models (or all). Useful for recovery after bulk operations. |
 | `full_search:status` | Show each model's index status (`ok` / `stale`) and count of empty sourced fields. |
 
 ## Background jobs
@@ -150,6 +169,27 @@ If you're not on Solid Queue, most job frameworks support recurring schedules vi
 
 Every FTS table stores a SHA256 digest of its DSL configuration in the `full_search_index_versions` table. When the DSL changes (e.g., adding a field), the stored hash no longer matches, and `rebuild!` is triggered automatically (if `auto_rebuild_schema` is true) to bring the index in line with the new definition. If `auto_rebuild_schema` is false, queries raise `ConfigChangedError` when the hash doesn't match (or log a warning and fall back if configured with `:log_and_fallback`).
 
+### Bulk imports
+
+When importing thousands of records, disable synchronous FTS maintenance and rebuild the index afterward:
+
+```ruby
+FullSearch.bulk_import(Customer) do
+  Customer.insert_all(records)
+end
+# automatically enqueues FullSearch::BackfillJob for Customer
+```
+
+While inside the block, database triggers are dropped so raw SQL operations (including `insert_all!`, `update_all`, and `delete_all`) bypass FTS table updates entirely. On exit, triggers are re-created and a `FullSearch::BackfillJob` is enqueued to rebuild the index from scratch. Reindexing of computed `source:` blocks within the block is also skipped.
+
+You can also call `Customer.bulk_import { ... }` directly on the model.
+
+For manual recovery outside a bulk-import block:
+
+```bash
+bin/rails 'full_search:backfill[customers]'
+```
+
 ### Query-time auto-rebuild in local environments
 
 By default, the generated initializer also enables `auto_rebuild_on_stale_query` in local environments:

data/lib/full_search/backfill_job.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+require "active_job"
+
+module FullSearch
+  class BackfillJob < ActiveJob::Base
+    queue_as :low
+
+    def perform(model_name)
+      model = model_name.to_s.constantize
+      FullSearch::Index.rebuild!(model)
+    end
+  end
+end

data/lib/full_search/bulk_import.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module FullSearch
+  module BulkImport
+    class << self
+      def bulk_import(model)
+        start_bulk_import(model)
+        yield
+      ensure
+        end_bulk_import(model)
+      end
+
+      def start_bulk_import(model)
+        models_in_bulk_import << model
+        FullSearch::Index.drop_triggers!(model)
+      end
+
+      def end_bulk_import(model)
+        models_in_bulk_import.delete(model)
+        FullSearch::Index.create_triggers!(model)
+        FullSearch::BackfillJob.perform_later(model.name)
+      end
+
+      def bulk_importing?(model)
+        models_in_bulk_import.include?(model)
+      end
+
+      private
+
+      def models_in_bulk_import
+        Thread.current[:full_search_bulk_import_models] ||= Set.new
+      end
+    end
+  end
+end

data/lib/full_search/callbacks.rb

@@ -10,11 +10,13 @@ module FullSearch
       source_fields = dsl.fields.select(&:source)
       return if source_fields.empty?
 
-      model.after_save do
+      model.after_save_commit do
+        next if FullSearch.bulk_importing?(self.class)
         FullSearch::Callbacks.reindex_record!(self)
       end
 
-      model.after_destroy do
+      model.after_destroy_commit do
+        next if FullSearch.bulk_importing?(self.class)
         FullSearch::Callbacks.remove_record!(self)
       end
 
@@ -22,10 +24,12 @@ module FullSearch
         next unless field.reindex_on
 
         assoc_class = associated_class(model, field.reindex_on)
-        assoc_class&.after_save do |record|
+        assoc_class&.after_save_commit do |record|
+          next if FullSearch.bulk_importing?(record.class)
           FullSearch::Callbacks.reindex_dependents!(record, model, field)
         end
-        assoc_class&.after_destroy do |record|
+        assoc_class&.after_destroy_commit do |record|
+          next if FullSearch.bulk_importing?(record.class)
           FullSearch::Callbacks.reindex_dependents!(record, model, field)
         end
       end
@@ -41,10 +45,15 @@ module FullSearch
       dsl = record.class.full_search_dsl
       return unless dsl
 
+      model_name = record.class.name
       dsl.fields.each do |field|
         next unless field.source
 
-        reindex_field!(record, field.name)
+        if model_name && field.async_source
+          FullSearch::ReindexJob.perform_later(model_name, record.id, field.name)
+        else
+          reindex_field!(record, field.name)
+        end
       end
     end
 

data/lib/full_search/config.rb

@@ -2,13 +2,16 @@
 
 module FullSearch
   class Config
-    attr_accessor :auto_rebuild_schema, :stale_query_behavior, :lock_rebuilds, :default_async_reindex, :default_tokenizer, :auto_rebuild_on_stale_query
+    attr_accessor :auto_rebuild_schema, :stale_query_behavior, :lock_rebuilds,
+                  :default_async_reindex, :default_async_source_reindex,
+                  :default_tokenizer, :auto_rebuild_on_stale_query
 
     def initialize
       @auto_rebuild_schema = false
       @stale_query_behavior = :raise
       @lock_rebuilds = true
       @default_async_reindex = true
+      @default_async_source_reindex = true
       @default_tokenizer = "unicode61"
       @auto_rebuild_on_stale_query = false
     end

data/lib/full_search/dsl.rb

@@ -2,10 +2,10 @@
 
 module FullSearch
   class Dsl
-    attr_reader :fields, :exact_matches, :filters, :model_class, :highlight_config, :rank_bys
+    attr_reader :fields, :exact_matches, :filters, :model_class, :highlight_config, :rank_bys, :index_if_sql
 
-    Field = Data.define(:name, :weight, :source, :reindex_on, :async, :as, :version)
-    ExactMatch = Data.define(:name, :source, :version)
+    Field = Data.define(:name, :weight, :source, :reindex_on, :async, :async_source, :as, :version)
+    ExactMatch = Data.define(:name, :source, :sql, :version)
     Filter = Data.define(:name, :required)
     RankBy = Data.define(:column, :direction)
 
@@ -19,7 +19,10 @@ module FullSearch
       @soft_delete_column = nil
     end
 
-    def field(name, weight: 1, source: nil, reindex_on: nil, async: FullSearch.config.default_async_reindex, as: nil, version: nil)
+    def field(name, weight: 1, source: nil, reindex_on: nil,
+              async: FullSearch.config.default_async_reindex,
+              async_source: FullSearch.config.default_async_source_reindex,
+              as: nil, version: nil)
       unless valid_name?(name)
         raise InvalidFieldError, "#{model_class.name}: invalid field name #{name.inspect}"
       end
@@ -29,16 +32,20 @@ module FullSearch
       str = name.to_s
       raise InvalidFieldError, "#{model_class.name}: duplicate field name #{name.inspect}" if fields.any? { |f| f.name == str }
       raise InvalidFieldError, "#{model_class.name}: field name #{name.inspect} conflicts with existing filter" if filters.any? { |f| f.name == str }
-      @fields << Field.new(name: str, weight: weight.to_i, source: source, reindex_on: reindex_on&.to_s, async: async, as: as&.to_s, version: version)
+      @fields << Field.new(
+        name: str, weight: weight.to_i, source: source,
+        reindex_on: reindex_on&.to_s, async: async,
+        async_source: async_source, as: as&.to_s, version: version
+      )
     end
 
-    def exact_match(name, source: -> { public_send(name) }, version: nil)
+    def exact_match(name, source: -> { public_send(name) }, sql: nil, version: nil)
       unless valid_name?(name)
         raise InvalidFieldError, "#{model_class.name}: invalid exact_match name #{name.inspect}"
       end
       str = name.to_s
       raise InvalidFieldError, "#{model_class.name}: duplicate exact_match name #{name.inspect}" if exact_matches.any? { |e| e.name == str }
-      @exact_matches << ExactMatch.new(name: str, source: source, version: version)
+      @exact_matches << ExactMatch.new(name: str, source: source, sql: sql&.to_s, version: version)
     end
 
     def rank_by(column, direction = :desc)
@@ -84,6 +91,14 @@ module FullSearch
       @highlight_config = {open_tag: open_tag, close_tag: close_tag}
     end
 
+    def index_if(sql: nil)
+      @index_if_sql = sql&.to_s
+    end
+
+    def conditional_index?
+      @index_if_sql.present?
+    end
+
     def typo_tolerance(enabled = true, min_term_length: nil)
       @typo_tolerance = enabled
       @typo_tolerance_min_term_length = min_term_length || 3
@@ -105,8 +120,9 @@ module FullSearch
         soft_delete_column,
         typo_tolerance?,
         typo_tolerance_min_term_length,
-        fields.map { |f| [f.name, f.weight, f.source.nil? ? "column" : "proc:#{f.version}", f.reindex_on, f.async, f.as] },
-        exact_matches.map { |e| [e.name, "proc:#{e.version}"] },
+        index_if_sql,
+        fields.map { |f| [f.name, f.weight, f.source.nil? ? "column" : "proc:#{f.version}", f.reindex_on, f.async, f.async_source, f.as] },
+        exact_matches.map { |e| [e.name, "proc:#{e.version}", e.sql] },
         filters.map { |f| [f.name, f.required] },
         rank_bys.map { |r| [r.column, r.direction] }
       ].inspect)

data/lib/full_search/exact_match.rb

@@ -9,16 +9,34 @@ module FullSearch
       cleaned = query.to_s.strip
       return [] if cleaned.empty?
 
-      relation = model.all
-      filters.each { |name, value| relation = relation.where(name => value) }
+      sql_matches = dsl.exact_matches.select(&:sql)
+      ruby_matches = dsl.exact_matches.reject(&:sql)
 
       ids = []
-      relation.find_each do |record|
-        dsl.exact_matches.each do |em|
-          value = record.instance_exec(&em.source)
-          ids << record.id if value.to_s.casecmp?(cleaned)
+
+      if sql_matches.any?
+        base = model.all
+        filters.each { |name, value| base = base.where(name => value) }
+
+        conditions = sql_matches.map do |em|
+          "(#{em.sql}) = #{model.connection.quote(cleaned)}"
+        end.join(" OR ")
+
+        ids += base.where(conditions).pluck(:id)
+      end
+
+      if ruby_matches.any?
+        base = model.all
+        filters.each { |name, value| base = base.where(name => value) }
+
+        base.find_each do |record|
+          ruby_matches.each do |em|
+            value = record.instance_exec(&em.source)
+            ids << record.id if value.to_s.casecmp?(cleaned)
+          end
         end
       end
+
       ids.uniq
     end
   end

data/lib/full_search/highlighter.rb

@@ -3,8 +3,8 @@
 module FullSearch
   class Highlighter
     class << self
-      def apply!(records, model, query)
-        snippets = build_snippets(model, query)
+      def apply!(records, model, query, record_ids: nil)
+        snippets = build_snippets(model, query, record_ids: record_ids)
         if snippets.values.all?(&:nil?) && records.any?
           snippets = manual_snippets(records, model, query)
         end
@@ -12,8 +12,8 @@ module FullSearch
         records
       end
 
-      def apply_fields!(records, model, query)
-        fields = build_field_snippets(model, query)
+      def apply_fields!(records, model, query, record_ids: nil)
+        fields = build_field_snippets(model, query, record_ids: record_ids)
         if fields.values.all?(&:empty?) && records.any?
           fields = manual_field_snippets(records, model, query)
         end
@@ -28,8 +28,8 @@ module FullSearch
 
       private
 
-      def build_snippets(model, query)
-        rows = highlight_rows(model, query)
+      def build_snippets(model, query, record_ids: nil)
+        rows = highlight_rows(model, query, record_ids: record_ids)
         cols = model.full_search_dsl.fields.map(&:name)
 
         rows.to_h do |row|
@@ -38,8 +38,8 @@ module FullSearch
         end
       end
 
-      def build_field_snippets(model, query)
-        rows = highlight_rows(model, query)
+      def build_field_snippets(model, query, record_ids: nil)
+        rows = highlight_rows(model, query, record_ids: record_ids)
         dsl = model.full_search_dsl
         fields = dsl.fields
         open_tag = (dsl.highlight_config || {open_tag: "<mark>"})[:open_tag]
@@ -175,7 +175,7 @@ module FullSearch
         1
       end
 
-      def highlight_rows(model, query)
+      def highlight_rows(model, query, record_ids: nil)
         dsl = model.full_search_dsl
         config = dsl.highlight_config || {open_tag: "<mark>", close_tag: "</mark>"}
         match_expr = QueryParser.to_match_expression(QueryParser.parse(query))
@@ -195,6 +195,10 @@ module FullSearch
           WHERE #{table} MATCH #{q(match_expr)}
         SQL
 
+        if record_ids.present?
+          sql += " AND rowid IN (#{record_ids.map { |id| connection.quote(id) }.join(", ")})"
+        end
+
         connection.execute(sql)
       end
 

data/lib/full_search/index.rb

@@ -106,8 +106,33 @@ module FullSearch
       end
 
       def reindex_source_fields!(model)
+        dsl = model.full_search_dsl
+        return unless dsl
+
         model.find_each do |record|
-          FullSearch::Callbacks.reindex_record!(record)
+          dsl.fields.each do |field|
+            FullSearch::Callbacks.reindex_field!(record, field.name) if field.source
+          end
+        end
+      end
+
+      def create_triggers!(model)
+        connection.execute(insert_trigger_sql(model))
+        connection.execute(delete_trigger_sql(model))
+        connection.execute(update_trigger_sql(model))
+        if model.full_search_dsl.soft_delete_column
+          connection.execute(soft_delete_removal_trigger_sql(model))
+        end
+        if model.full_search_dsl.typo_tolerance?
+          connection.execute(insert_trigram_trigger_sql(model))
+          connection.execute(delete_trigram_trigger_sql(model))
+          connection.execute(update_trigram_trigger_sql(model))
+        end
+      end
+
+      def drop_triggers!(model)
+        (trigger_names(model) + trigram_trigger_names(model)).each do |name|
+          connection.execute("DROP TRIGGER IF EXISTS #{qt(name)};")
         end
       end
 
@@ -197,7 +222,7 @@ module FullSearch
 
       def create_virtual_table_sql(model)
         dsl = model.full_search_dsl
-        columns = (dsl.fields + dsl.filters.map { |f| FilterColumnPlaceholder.new(name: f.name) })
+        columns = (dsl.fields + dsl.filters.map { |f| FilterColumnPlaceholder.new(name: f.name) } + extra_columns(model))
         column_list = columns.map { |c| (c.respond_to?(:unindexed?) && c.unindexed?) ? "#{qc(c.name)} UNINDEXED" : qc(c.name) }.join(", ")
 
         "CREATE VIRTUAL TABLE #{qt(fts_table_name(model))} USING fts5(#{column_list}, tokenize='#{dsl.tokenize}');"
@@ -205,18 +230,22 @@ module FullSearch
 
       def backfill_sql(model)
         dsl = model.full_search_dsl
-        cols = (dsl.fields + dsl.filters)
+        cols = (dsl.fields + dsl.filters + extra_columns(model))
         select = cols.map do |c|
-          if c.respond_to?(:source) && c.source
+          if c.respond_to?(:unindexed?) && c.name == "indexed"
+            "'1'"
+          elsif c.respond_to?(:source) && c.source
             source_value_sql(c.source)
           else
             "#{qt(model.table_name)}.#{qc(c.name)}"
           end
         end.join(", ")
 
+        where_clause = dsl.conditional_index? ? " WHERE (#{dsl.index_if_sql})" : ""
+
         <<~SQL
           INSERT INTO #{qt(fts_table_name(model))}(rowid, #{cols.map { |c| qc(c.name) }.join(", ")})
-          SELECT #{qt(model.table_name)}.id, #{select} FROM #{qt(model.table_name)};
+          SELECT #{qt(model.table_name)}.id, #{select} FROM #{qt(model.table_name)}#{where_clause};
         SQL
       end
 
@@ -225,6 +254,8 @@ module FullSearch
       end
 
       def ensure_triggers!(model)
+        return if FullSearch.bulk_importing?(model)
+
         existing = connection.execute(
           "SELECT name FROM sqlite_master WHERE type='trigger' AND tbl_name=#{q(model.table_name)}"
         ).map { |r| r["name"] }
@@ -236,26 +267,6 @@ module FullSearch
         rebuild!(model)
       end
 
-      def create_triggers!(model)
-        connection.execute(insert_trigger_sql(model))
-        connection.execute(delete_trigger_sql(model))
-        connection.execute(update_trigger_sql(model))
-        if model.full_search_dsl.soft_delete_column
-          connection.execute(soft_delete_removal_trigger_sql(model))
-        end
-        if model.full_search_dsl.typo_tolerance?
-          connection.execute(insert_trigram_trigger_sql(model))
-          connection.execute(delete_trigram_trigger_sql(model))
-          connection.execute(update_trigram_trigger_sql(model))
-        end
-      end
-
-      def drop_triggers!(model)
-        (trigger_names(model) + trigram_trigger_names(model)).each do |name|
-          connection.execute("DROP TRIGGER IF EXISTS #{qt(name)};")
-        end
-      end
-
       def trigger_names(model)
         base = fts_table_name(model)
         names = %W[#{base}_ai #{base}_ad #{base}_au]
@@ -270,15 +281,30 @@ module FullSearch
 
       def insert_trigger_sql(model)
         dsl = model.full_search_dsl
-        cols = dsl.fields + dsl.filters
-        values = cols.map { |c| column_ref(c, prefix: "new") }.join(", ")
-
-        <<~SQL
-          CREATE TRIGGER #{qt(trigger_names(model).first)} AFTER INSERT ON #{qt(model.table_name)} BEGIN
-            INSERT INTO #{qt(fts_table_name(model))}(rowid, #{col_names(cols)})
-            VALUES (new.id, #{values});
-          END;
-        SQL
+        cols = dsl.fields + dsl.filters + extra_columns(model)
+        cols_str = col_names(cols)
+        fts_table = qt(fts_table_name(model))
+        tbl = qt(model.table_name)
+
+        if dsl.conditional_index?
+          select_parts = cols.map { |c| column_ref(c, prefix: "new") }.join(", ")
+          <<~SQL
+            CREATE TRIGGER #{qt(trigger_names(model).first)} AFTER INSERT ON #{tbl}
+            BEGIN
+              INSERT INTO #{fts_table}(rowid, #{cols_str})
+              SELECT new.id, #{select_parts} FROM #{tbl} WHERE #{tbl}.id = new.id AND (#{dsl.index_if_sql});
+            END;
+          SQL
+        else
+          values = cols.map { |c| column_ref(c, prefix: "new") }.join(", ")
+          <<~SQL
+            CREATE TRIGGER #{qt(trigger_names(model).first)} AFTER INSERT ON #{tbl}
+            BEGIN
+              INSERT INTO #{fts_table}(rowid, #{col_names(cols)})
+              VALUES (new.id, #{values});
+            END;
+          SQL
+        end
       end
 
       def delete_trigger_sql(model)
@@ -291,21 +317,34 @@ module FullSearch
 
       def update_trigger_sql(model)
         dsl = model.full_search_dsl
-        cols = dsl.fields + dsl.filters
-        values = cols.map { |c| column_ref(c, prefix: "new") }.join(", ")
+        cols = dsl.fields + dsl.filters + extra_columns(model)
         cols_str = col_names(cols)
         fts_table = qt(fts_table_name(model))
+        tbl = qt(model.table_name)
 
         when_clause = SoftDelete.active_update_clause(model)
 
-        <<~SQL
-          CREATE TRIGGER #{qt(trigger_names(model)[2])} AFTER UPDATE ON #{qt(model.table_name)} #{when_clause}
-          BEGIN
-            DELETE FROM #{fts_table} WHERE rowid = old.id;
-            INSERT INTO #{fts_table}(rowid, #{cols_str})
-            VALUES (new.id, #{values});
-          END;
-        SQL
+        if dsl.conditional_index?
+          select_parts = cols.map { |c| column_ref(c, prefix: "new") }.join(", ")
+          <<~SQL
+            CREATE TRIGGER #{qt(trigger_names(model)[2])} AFTER UPDATE ON #{qt(model.table_name)} #{when_clause}
+            BEGIN
+              DELETE FROM #{fts_table} WHERE rowid = old.id;
+              INSERT INTO #{fts_table}(rowid, #{cols_str})
+              SELECT new.id, #{select_parts} FROM #{tbl} WHERE #{tbl}.id = new.id AND (#{dsl.index_if_sql});
+            END;
+          SQL
+        else
+          values = cols.map { |c| column_ref(c, prefix: "new") }.join(", ")
+          <<~SQL
+            CREATE TRIGGER #{qt(trigger_names(model)[2])} AFTER UPDATE ON #{qt(model.table_name)} #{when_clause}
+            BEGIN
+              DELETE FROM #{fts_table} WHERE rowid = old.id;
+              INSERT INTO #{fts_table}(rowid, #{cols_str})
+              VALUES (new.id, #{values});
+            END;
+          SQL
+        end
       end
 
       def soft_delete_removal_trigger_sql(model)
@@ -329,7 +368,7 @@ module FullSearch
 
       def create_trigram_virtual_table_sql(model)
         dsl = model.full_search_dsl
-        columns = (dsl.fields + dsl.filters.map { |f| FilterColumnPlaceholder.new(name: f.name) })
+        columns = (dsl.fields + dsl.filters.map { |f| FilterColumnPlaceholder.new(name: f.name) } + extra_columns(model))
         column_list = columns.map { |c| (c.respond_to?(:unindexed?) && c.unindexed?) ? "#{qc(c.name)} UNINDEXED" : qc(c.name) }.join(", ")
 
         "CREATE VIRTUAL TABLE #{qt(trigram_table_name(model))} USING fts5(#{column_list}, tokenize='trigram');"
@@ -337,32 +376,51 @@ module FullSearch
 
       def backfill_trigram_sql(model)
         dsl = model.full_search_dsl
-        cols = (dsl.fields + dsl.filters)
+        cols = (dsl.fields + dsl.filters + extra_columns(model))
         select = cols.map do |c|
-          if c.respond_to?(:source) && c.source
+          if c.respond_to?(:unindexed?) && c.name == "indexed"
+            "'1'"
+          elsif c.respond_to?(:source) && c.source
             source_value_sql(c.source)
           else
             "#{qt(model.table_name)}.#{qc(c.name)}"
           end
         end.join(", ")
 
+        where_clause = dsl.conditional_index? ? " WHERE (#{dsl.index_if_sql})" : ""
+
         <<~SQL
           INSERT INTO #{qt(trigram_table_name(model))}(rowid, #{cols.map { |c| qc(c.name) }.join(", ")})
-          SELECT #{qt(model.table_name)}.id, #{select} FROM #{qt(model.table_name)};
+          SELECT #{qt(model.table_name)}.id, #{select} FROM #{qt(model.table_name)}#{where_clause};
         SQL
       end
 
       def insert_trigram_trigger_sql(model)
         dsl = model.full_search_dsl
-        cols = dsl.fields + dsl.filters
-        values = cols.map { |c| column_ref(c, prefix: "new") }.join(", ")
-
-        <<~SQL
-          CREATE TRIGGER #{qt(trigram_trigger_names(model).first)} AFTER INSERT ON #{qt(model.table_name)} BEGIN
-            INSERT INTO #{qt(trigram_table_name(model))}(rowid, #{col_names(cols)})
-            VALUES (new.id, #{values});
-          END;
-        SQL
+        cols = dsl.fields + dsl.filters + extra_columns(model)
+        cols_str = col_names(cols)
+        trigram_table = qt(trigram_table_name(model))
+        tbl = qt(model.table_name)
+
+        if dsl.conditional_index?
+          select_parts = cols.map { |c| column_ref(c, prefix: "new") }.join(", ")
+          <<~SQL
+            CREATE TRIGGER #{qt(trigram_trigger_names(model).first)} AFTER INSERT ON #{tbl}
+            BEGIN
+              INSERT INTO #{trigram_table}(rowid, #{cols_str})
+              SELECT new.id, #{select_parts} FROM #{tbl} WHERE #{tbl}.id = new.id AND (#{dsl.index_if_sql});
+            END;
+          SQL
+        else
+          values = cols.map { |c| column_ref(c, prefix: "new") }.join(", ")
+          <<~SQL
+            CREATE TRIGGER #{qt(trigram_trigger_names(model).first)} AFTER INSERT ON #{tbl}
+            BEGIN
+              INSERT INTO #{trigram_table}(rowid, #{col_names(cols)})
+              VALUES (new.id, #{values});
+            END;
+          SQL
+        end
       end
 
       def delete_trigram_trigger_sql(model)
@@ -375,21 +433,39 @@ module FullSearch
 
       def update_trigram_trigger_sql(model)
         dsl = model.full_search_dsl
-        cols = dsl.fields + dsl.filters
-        values = cols.map { |c| column_ref(c, prefix: "new") }.join(", ")
+        cols = dsl.fields + dsl.filters + extra_columns(model)
         cols_str = col_names(cols)
         trigram_table = qt(trigram_table_name(model))
+        tbl = qt(model.table_name)
 
         when_clause = SoftDelete.active_update_clause(model)
 
-        <<~SQL
-          CREATE TRIGGER #{qt(trigram_trigger_names(model)[2])} AFTER UPDATE ON #{qt(model.table_name)} #{when_clause}
-          BEGIN
-            DELETE FROM #{trigram_table} WHERE rowid = old.id;
-            INSERT INTO #{trigram_table}(rowid, #{cols_str})
-            VALUES (new.id, #{values});
-          END;
-        SQL
+        if dsl.conditional_index?
+          select_parts = cols.map { |c| column_ref(c, prefix: "new") }.join(", ")
+          <<~SQL
+            CREATE TRIGGER #{qt(trigram_trigger_names(model)[2])} AFTER UPDATE ON #{qt(model.table_name)} #{when_clause}
+            BEGIN
+              DELETE FROM #{trigram_table} WHERE rowid = old.id;
+              INSERT INTO #{trigram_table}(rowid, #{cols_str})
+              SELECT new.id, #{select_parts} FROM #{tbl} WHERE #{tbl}.id = new.id AND (#{dsl.index_if_sql});
+            END;
+          SQL
+        else
+          values = cols.map { |c| column_ref(c, prefix: "new") }.join(", ")
+          <<~SQL
+            CREATE TRIGGER #{qt(trigram_trigger_names(model)[2])} AFTER UPDATE ON #{qt(model.table_name)} #{when_clause}
+            BEGIN
+              DELETE FROM #{trigram_table} WHERE rowid = old.id;
+              INSERT INTO #{trigram_table}(rowid, #{cols_str})
+              VALUES (new.id, #{values});
+            END;
+          SQL
+        end
+      end
+
+      def extra_columns(model)
+        dsl = model.full_search_dsl
+        dsl&.conditional_index? ? [FilterColumnPlaceholder.new(name: "indexed")] : []
       end
 
       def col_names(cols)
@@ -397,7 +473,9 @@ module FullSearch
       end
 
       def column_ref(col, prefix:)
-        if col.respond_to?(:source) && col.source
+        if col.respond_to?(:unindexed?) && col.name == "indexed"
+          "'1'"
+        elsif col.respond_to?(:source) && col.source
           "''"
         else
           "#{prefix}.#{qc(col.name)}"

data/lib/full_search/model.rb

@@ -5,7 +5,7 @@ module FullSearch
     extend ActiveSupport::Concern
 
     class_methods do
-      def full_search(query_or_options = nil, filters: {}, include_soft_deleted: false, limit: nil, offset: nil, highlight: false, highlight_fields: false, matching_strategy: nil, &block)
+      def full_search(query_or_options = nil, filters: {}, include_soft_deleted: false, limit: nil, offset: nil, highlight: false, highlight_fields: false, matching_strategy: nil, per_strategy_limit: nil, &block)
         if block_given? || query_or_options.is_a?(Hash)
           @full_search_dsl ||= FullSearch::Dsl.new(self)
           @full_search_dsl.tokenize(query_or_options[:tokenize]) if query_or_options.is_a?(Hash) && query_or_options.key?(:tokenize)
@@ -16,7 +16,7 @@ module FullSearch
           FullSearch.register_model(self)
           @full_search_dsl
         else
-          FullSearch::Search.new(self, query_or_options, filters: filters, include_soft_deleted: include_soft_deleted, limit: limit, offset: offset, highlight: highlight, highlight_fields: highlight_fields, matching_strategy: matching_strategy).relation
+          FullSearch::Search.new(self, query_or_options, filters: filters, include_soft_deleted: include_soft_deleted, limit: limit, offset: offset, highlight: highlight, highlight_fields: highlight_fields, matching_strategy: matching_strategy, per_strategy_limit: per_strategy_limit).relation
         end
       end
 
@@ -39,6 +39,14 @@ module FullSearch
       def reindex!
         FullSearch::Index.reindex_source_fields!(self)
       end
+
+      def bulk_import(&block)
+        FullSearch.bulk_import(self, &block)
+      end
+
+      def backfill_search_index!
+        FullSearch::BackfillJob.perform_later(name)
+      end
     end
 
     included do |base|

data/lib/full_search/multi_search.rb

@@ -27,7 +27,9 @@ module FullSearch
           limit: raw_limit,
           offset: offset,
           highlight: group[:highlight],
-          highlight_fields: group[:highlight_fields]
+          highlight_fields: group[:highlight_fields],
+          matching_strategy: group[:matching_strategy],
+          per_strategy_limit: group[:per_strategy_limit]
         )
 
         relation = group[:scope].call(relation) if group[:scope]

data/lib/full_search/reindex_job.rb

@@ -4,12 +4,18 @@ require "active_job"
 
 module FullSearch
   class ReindexJob < ActiveJob::Base
-    def perform(model_name, record_id, field_name)
-      model = model_name.constantize
+    queue_as :low
+
+    def perform(model_name, record_id, field_name = nil)
+      model = model_name.is_a?(Class) ? model_name : model_name.to_s.constantize
       record = model.find_by(id: record_id)
       return unless record
 
-      FullSearch::Callbacks.reindex_field!(record, field_name)
+      if field_name
+        FullSearch::Callbacks.reindex_field!(record, field_name)
+      else
+        FullSearch::Callbacks.reindex_record!(record)
+      end
     end
   end
 end

data/lib/full_search/search.rb

@@ -2,9 +2,9 @@
 
 module FullSearch
   class Search
-    attr_reader :model, :query, :filters, :include_soft_deleted, :limit, :offset, :highlight, :highlight_fields, :matching_strategy
+    attr_reader :model, :query, :filters, :include_soft_deleted, :limit, :offset, :highlight, :highlight_fields, :matching_strategy, :per_strategy_limit
 
-    def initialize(model, query, filters:, include_soft_deleted:, limit:, offset:, highlight: false, highlight_fields: false, matching_strategy: nil)
+    def initialize(model, query, filters:, include_soft_deleted:, limit:, offset:, highlight: false, highlight_fields: false, matching_strategy: nil, per_strategy_limit: nil)
       @model = model
       @query = query.to_s.strip
       @filters = filters
@@ -14,6 +14,7 @@ module FullSearch
       @highlight = highlight
       @highlight_fields = highlight_fields
       @matching_strategy = matching_strategy
+      @per_strategy_limit = per_strategy_limit
     end
 
     MIN_TERM_LENGTH = 3
@@ -28,8 +29,8 @@ module FullSearch
       parsed = QueryParser.parse(query)
       exact_ids = ExactMatch.ids_for(model, query, filters)
       primary_ids = fts_match_ids(parsed)
-      fallback_ids = (dsl.typo_tolerance? && matching_strategy != "all") ? trigram_match_ids(parsed, primary_ids) : []
-      fuzzy_ids = (dsl.typo_tolerance? && matching_strategy != "all" && primary_ids.empty? && fallback_ids.empty?) ? fuzzy_match_ids(parsed) : []
+      fallback_ids = (dsl.typo_tolerance? && matching_strategy != "all") ? trigram_match_ids(parsed, primary_ids, candidate_limit: per_strategy_limit) : []
+      fuzzy_ids = (dsl.typo_tolerance? && matching_strategy != "all" && primary_ids.empty? && fallback_ids.empty?) ? fuzzy_match_ids(parsed, candidate_limit: per_strategy_limit) : []
 
       all_ids = (exact_ids + primary_ids + fallback_ids + fuzzy_ids).uniq
       return model.none if all_ids.empty?
@@ -42,9 +43,13 @@ module FullSearch
       rel = apply_ranking(rel, all_ids, exact_ids)
 
       if highlight
-        Highlighter.apply!(rel.to_a, model, query)
+        records = rel.to_a
+        Highlighter.apply!(records, model, query, record_ids: records.map(&:id))
+        records
       elsif highlight_fields
-        Highlighter.apply_fields!(rel.to_a, model, query)
+        records = rel.to_a
+        Highlighter.apply_fields!(records, model, query, record_ids: records.map(&:id))
+        records
       else
         rel
       end
@@ -109,10 +114,12 @@ module FullSearch
         "AND #{fts_table}.#{qc(name)} = #{q(value)}"
       end.join(" ")
 
-      connection.execute("#{sql} #{filter_conditions}").map { |r| r["id"] }
+      indexed_filter = dsl.conditional_index? ? " AND #{fts_table}.indexed = '1'" : ""
+
+      connection.execute("#{sql} #{filter_conditions}#{indexed_filter}").map { |r| r["id"] }
     end
 
-    def trigram_match_ids(parsed, primary_ids)
+    def trigram_match_ids(parsed, primary_ids, candidate_limit: nil)
       return [] if primary_ids.any?
 
       match_expr = QueryParser.to_match_expression(parsed)
@@ -122,7 +129,7 @@ module FullSearch
       return [] if term.nil?
 
       if term.length < dsl.typo_tolerance_min_term_length.to_i
-        return like_prefix_ids(term)
+        return like_prefix_ids(term, candidate_limit: candidate_limit)
       end
 
       trigram_table = qt(FullSearch::Index.trigram_table_name(model))
@@ -139,10 +146,14 @@ module FullSearch
         "AND #{trigram_table}.#{qc(name)} = #{q(value)}"
       end.join(" ")
 
-      connection.execute("#{sql} #{filter_conditions}").map { |r| r["id"] }
+      indexed_filter = dsl.conditional_index? ? " AND #{trigram_table}.indexed = '1'" : ""
+
+      sql += filter_conditions + indexed_filter
+      order_and_limit!(sql, tbl, candidate_limit)
+      connection.execute(sql).map { |r| r["id"] }
     end
 
-    def like_prefix_ids(term)
+    def like_prefix_ids(term, candidate_limit: nil)
       column_fields = dsl.fields.select { |f| f.source.nil? }
       source_fields = dsl.fields.select { |f| f.source }
       tbl = qt(model.table_name)
@@ -152,6 +163,8 @@ module FullSearch
         soft_delete_clause = "AND #{tbl}.#{qc(dsl.soft_delete_column)} IS NULL"
       end
 
+      indexed_clause = dsl.conditional_index? ? "AND (#{dsl.index_if_sql}) " : ""
+
       ids = []
 
       if column_fields.any?
@@ -166,9 +179,10 @@ module FullSearch
         sql = <<~SQL
           SELECT #{tbl}.id
           FROM #{tbl}
-          WHERE (#{like_conditions}) #{filter_conditions} #{soft_delete_clause}
+          WHERE (#{like_conditions}) #{filter_conditions} #{soft_delete_clause} #{indexed_clause}
         SQL
 
+        order_and_limit!(sql, tbl, candidate_limit)
         ids = connection.execute(sql).map { |r| r["id"] }
         return ids if ids.any?
       end
@@ -183,20 +197,23 @@ module FullSearch
           "AND #{fts_table}.#{qc(name)} = #{q(value)}"
         end.join(" ")
 
+        fts_indexed = dsl.conditional_index? ? " AND #{fts_table}.indexed = '1'" : ""
+
         sql = <<~SQL
           SELECT #{tbl}.id
           FROM #{fts_table}
           JOIN #{tbl} ON #{tbl}.id = #{fts_table}.rowid
-          WHERE (#{like_conditions}) #{filter_conditions} #{soft_delete_clause}
+          WHERE (#{like_conditions}) #{filter_conditions} #{soft_delete_clause}#{fts_indexed}
         SQL
 
+        order_and_limit!(sql, tbl, candidate_limit)
         ids = connection.execute(sql).map { |r| r["id"] }
       end
 
       ids
     end
 
-    def fuzzy_match_ids(parsed)
+    def fuzzy_match_ids(parsed, candidate_limit: nil)
       term = parsed.last
       return [] if term.nil?
 
@@ -217,6 +234,8 @@ module FullSearch
         soft_delete_clause = "AND #{tbl}.#{qc(dsl.soft_delete_column)} IS NULL"
       end
 
+      indexed_clause = dsl.conditional_index? ? "AND (#{dsl.index_if_sql}) " : ""
+
       filter_conditions = filters.map do |name, value|
         "AND #{tbl}.#{qc(name)} = #{q(value)}"
       end.join(" ")
@@ -228,12 +247,21 @@ module FullSearch
       sql = <<~SQL
         SELECT #{tbl}.id
         FROM #{tbl}
-        WHERE (#{conditions}) #{filter_conditions} #{soft_delete_clause}
+        WHERE (#{conditions}) #{filter_conditions} #{soft_delete_clause} #{indexed_clause}
       SQL
 
+      order_and_limit!(sql, tbl, candidate_limit)
       connection.execute(sql).map { |r| r["id"] }
     end
 
+    def order_and_limit!(sql, tbl, candidate_limit)
+      order_parts = dsl.rank_bys.map do |rank_by|
+        "#{tbl}.#{qc(rank_by.column)} #{rank_by.direction.to_s.upcase} NULLS LAST"
+      end
+      sql << " ORDER BY #{order_parts.join(", ")}" if order_parts.any?
+      sql << " LIMIT #{candidate_limit.to_i}" if candidate_limit
+    end
+
     def max_allowed_typos(length)
       min_length = dsl.typo_tolerance_min_term_length.to_i
       return -1 if length < min_length

data/lib/full_search/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module FullSearch
-  VERSION = "0.3.3"
+  VERSION = "0.3.4"
 end

data/lib/full_search.rb

@@ -22,6 +22,8 @@ require "full_search/soft_delete"
 require "full_search/callbacks"
 require "full_search/reindex_job"
 require "full_search/optimize_job"
+require "full_search/bulk_import"
+require "full_search/backfill_job"
 require "full_search/test_helpers"
 require "full_search/multi_search"
 require "full_search/schema_dumper_patch"
@@ -54,6 +56,14 @@ module FullSearch
         Callbacks.install!(model)
       end
     end
+
+    def bulk_import(model, &block)
+      BulkImport.bulk_import(model, &block)
+    end
+
+    def bulk_importing?(model)
+      BulkImport.bulk_importing?(model)
+    end
   end
 end
 

data/lib/generators/full_search/install/templates/full_search.rb

@@ -8,4 +8,8 @@ FullSearch.configure do |config|
   config.stale_query_behavior = Rails.env.production? ? :log_and_fallback : :raise
 
   config.lock_rebuilds = true
+
+  # Reindex computed source: fields synchronously (false) or via background job (true).
+  # Bulk imports should use FullSearch.bulk_import(Model) { ... } to defer reindexing.
+  config.default_async_source_reindex = true
 end

data/lib/tasks/full_search.rake

@@ -40,6 +40,16 @@ namespace :full_search do
     end
   end
 
+  desc "Backfill FTS indexes for given models (or all)"
+  task :backfill, [:models] => :environment do |_t, args|
+    Rails.application.eager_load!
+
+    resolve_full_search_models(args).each do |model|
+      FullSearch::BackfillJob.perform_now(model.name)
+      puts "Backfilled #{FullSearch::Index.fts_table_name(model)}"
+    end
+  end
+
   desc "Optimize full_search indexes"
   task optimize: :environment do
     Rails.application.eager_load!

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: full_search
 version: !ruby/object:Gem::Version
-  version: 0.3.3
+  version: 0.3.4
 platform: ruby
 authors:
 - Ben D'Angelo
@@ -140,6 +140,8 @@ extra_rdoc_files: []
 files:
 - README.md
 - lib/full_search.rb
+- lib/full_search/backfill_job.rb
+- lib/full_search/bulk_import.rb
 - lib/full_search/callbacks.rb
 - lib/full_search/config.rb
 - lib/full_search/distance.rb