full_search 0.1.3 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 90ca9a454b8cacf2619c28ced3b07420fbf26ab34ddb55b79f93ce82ce69c06e
-  data.tar.gz: e37851d2e693989a7585fe40460af0bab1ae91c9b171faca76114394b3c97799
+  metadata.gz: 50bb1c53fe9f326612a2413a441a37acfe48f25bf4730786de3631c7230b2572
+  data.tar.gz: ab53bbd57f8fd9cb464af29d37aa70cd0efb18a2835eff1b501d61a14d0de43f
 SHA512:
-  metadata.gz: 76f1883ee4d70669bd85bfac929b2dfaf820efb40400cfaabe98fd0d7dc68ad867e551cb0cb7d292f51694e74fa0349080497adb6a6dd52154b1e80b259540e1
-  data.tar.gz: f5e1fa063bcf8ff247ff52506f98fcacd98d47de827601a48d2070f8da58d380dc72d628089d69f63a89bc0a88dacec123c68566ec82e502defeaa76a5546958
+  metadata.gz: ab98601c786c02808246cf4657ef7c3912e36e0fbc429417c0ff564a3ec1e2575216fda8d64ee2ab22db32c707bcbce0915fcc3d57038059f1dfe05bde48963e
+  data.tar.gz: 52564a1ce083d5aa0f6cb624f8f9aa4a0bb0ac13a85216f14a857a5c2c2c744d37fa4370272679c3bd13c76eb6ec732814463cfa143bb714654243968c6c800c

data/README.md

@@ -1,7 +1,19 @@
 # full_search
 
+[![CI](https://github.com/bendangelo/full_search/actions/workflows/ci.yml/badge.svg)](https://github.com/bendangelo/full_search/actions/workflows/ci.yml)
+
 SQLite FTS5 full-text search for Rails/ActiveRecord. A lightweight, self-contained alternative to `pg_search` for apps already running on SQLite.
 
+## Requirements
+
+- Ruby 3.1+
+- Rails 8.0+
+- SQLite 3.34+ (if using typo tolerance)
+
+## When to use
+
+`full_search` is designed for apps with **100,000 records or fewer per table** that want full-text search without running a separate service. If you're on SQLite and need keyword search, phrase matching, typo-tolerant substring queries, or result highlighting, this gem gives you full-text search suitable for small to medium SQLite-backed apps with zero infrastructure — no Elasticsearch, no Meilisearch, no Sidekiq queue.
+
 ## Installation
 
 Add to your Gemfile:
@@ -30,12 +42,12 @@ end
 ```
 
 ```ruby
-Customer.full_search("sam", filters: { account_id: 1 }).page(params[:page])
+Customer.search("sam", filters: { account_id: 1 }).page(params[:page])
 ```
 
 ## Features
 
-- Declarative `full_search` DSL
+- Declarative `full_search` DSL with `search` / `full_search` query methods
 - SQLite FTS5 backed
 - Required-filter enforcement for multi-tenant apps
 - Exact-match queries for encrypted identifiers
@@ -44,14 +56,102 @@ Customer.full_search("sam", filters: { account_id: 1 }).page(params[:page])
 - Phrase, exclusion, and OR query operators
 - FTS5 `highlight()` support for result snippets
 - Opt-in trigram typo/substring fallback (requires SQLite >= 3.34)
-- Rake tasks: `full_search:rebuild`, `full_search:optimize`, `full_search:status`
+
+### Per-model operations
+
+Once a model declares `full_search`, you can call these class methods:
+
+| Method | What it does |
+|--------|-------------|
+| `Customer.rebuild!` | Force-drop and recreate the model's FTS table, backfill from the source table, and reinstall triggers. |
+| `Customer.reindex!` | Re-evaluates computed `source:` fields for every record and updates the FTS table in-place. Table structure is untouched. |
+| `Customer.optimize!` | Runs FTS5 `optimize` on the model's index to merge b-tree segments. |
+
+## Index management
+
+FTS indexes are SQLite virtual tables (`customers_fts`, `vehicles_fts`, etc.) that mirror your model tables. They stay in sync via database triggers on `INSERT` / `UPDATE` / `DELETE`.
+
+### Rebuild vs reindex
+
+These two operations are often confused:
+
+- **Rebuild** (`full_search:rebuild` / `Customer.rebuild!`) — drops and recreates the FTS virtual table. Needed when the DSL changes (fields added/removed, tokenizer changed, etc.). The table is re-created from scratch, backfilled, triggers re-installed, and the index is optimized.
+- **Reindex** (`Customer.reindex!` / `FullSearch::Index.reindex_source_fields!`) — updates existing FTS rows with fresh values from computed `source:` fields only. The table structure is untouched. Database triggers cover regular column changes automatically; only Ruby-evaluated `source:` blocks need an explicit reindex.
+
+> **Note on rebuild locking:** The `lock_rebuilds` option prevents concurrent rebuilds
+> within the same process/connection. For multi-process or multi-host deployments,
+> run `full_search:rebuild` from a single deployment step.
+
+### Auto-rebuild on app load
+
+When `auto_rebuild_schema` is enabled (default in the generated initializer), the railtie hooks into Rails `after_initialize` and:
+
+1. Creates missing FTS tables for every model using `full_search`
+2. Compares each table's stored config hash against the current DSL
+3. Rebuilds the index automatically when the DSL changes (new fields, different tokenizer, etc.)
+
+```ruby
+# config/initializers/full_search.rb
+FullSearch.configure do |config|
+  config.auto_rebuild_schema = true
+end
+```
+
+### In production
+
+Auto-rebuild runs on every Rails process boot (web, worker, console). For zero-downtime deploys where old processes still serve traffic, or if you prefer explicit control, set `auto_rebuild_schema` to `false` and run the rebuild task manually:
+
+```bash
+# Rebuild only indexes whose DSL has changed (fast, safe for production)
+bin/rails full_search:rebuild
+
+# Target specific models by table name
+bin/rails 'full_search:rebuild[customers,vehicles]'
+```
+
+The rebuild task checks each model's stored config hash against the current DSL and only drops/recreates indexes when they differ. For a full forced reset (drops and recreates all FTS tables regardless):
+
+```bash
+bin/rails full_search:reset
+```
+
+### Rake tasks
+
+| Task | Description |
+|------|-------------|
+| `full_search:rebuild` | Drops and recreates the FTS virtual table only when the DSL config hash has changed (safe for production). Pass model names to target specific tables. |
+| `full_search:reset` | Force a full rebuild — drops and recreates all FTS tables regardless of config hash. Use when data may be out of sync. |
+| `full_search:optimize` | Run FTS5 [`optimize`](https://www.sqlite.org/fts5.html#the_optimize_command) to merge b-tree segments. Useful after bulk updates. |
+| `full_search:status` | Show each model's index status (`ok` / `stale`) and count of empty sourced fields. |
+
+## Background jobs
+
+### Scheduled optimization
+
+FTS5 b-tree segments accumulate over time as rows are inserted, updated, and deleted. Running `optimize` periodically merges these segments, keeping queries fast. For most apps, once a day during a low-traffic window is sufficient. Apps with heavy write volume may benefit from every few hours.
+
+The gem ships `FullSearch::OptimizeJob` ready to use. Schedule it with Solid Queue's `recurring.yml`:
+
+```yaml
+# config/recurring.yml
+full_search_optimize:
+  class: FullSearch::OptimizeJob
+  schedule: daily at 4am
+  description: "Merge FTS5 b-tree segments for full_search indexes"
+```
+
+If you're not on Solid Queue, most job frameworks support recurring schedules via gems like `sidekiq-cron` or `whenever`.
+
+### Config hash drift detection
+
+Every FTS table stores a SHA256 digest of its DSL configuration in the `full_search_index_versions` table. When the DSL changes (e.g., adding a field), the stored hash no longer matches, and `rebuild!` is triggered automatically (if `auto_rebuild_schema` is true) to bring the index in line with the new definition. If `auto_rebuild_schema` is false, queries raise `ConfigChangedError` when the hash doesn't match (or log a warning and fall back if configured with `:log_and_fallback`).
 
 ## Query operators
 
 ```ruby
-Customer.full_search('"Sam Smith"', filters: { account_id: 1 })      # phrase
-Customer.full_search("honda -civic", filters: { account_id: 1 })     # exclusion
-Customer.full_search("honda OR toyota", filters: { account_id: 1 })  # OR
+Customer.search('"Sam Smith"', filters: { account_id: 1 })      # phrase
+Customer.search("honda -civic", filters: { account_id: 1 })     # exclusion
+Customer.search("honda OR toyota", filters: { account_id: 1 })  # OR
 ```
 
 ## Highlighting
@@ -66,7 +166,7 @@ end
 ```
 
 ```ruby
-Customer.full_search("sam", filters: { account_id: 1 }, highlight: true)
+Customer.search("sam", filters: { account_id: 1 }, highlight: true)
 # each result has #full_search_snippet
 ```
 
@@ -81,9 +181,15 @@ class Customer < ApplicationRecord
 end
 ```
 
-`typo_tolerance` uses an FTS5 trigram shadow table as a fallback when the primary index returns no results. It is substring matching, not edit-distance correction, and requires SQLite >= 3.34.
+`typo_tolerance` uses an FTS5 trigram shadow table as a fallback when the primary index returns no results. It is substring/trigram fallback, not edit-distance correction, and requires SQLite >= 3.34.
 
 ## Known limitations
 
 - Queries run with `highlight: true` return an Array of records, not an `ActiveRecord::Relation`. No further chaining (`.where`, `.order`, `.limit`) is possible after highlighting is applied.
-- Per-model only; no built-in multi-model aggregator
+- Per-model only; `multi_search` returns grouped results, not a unified ranked list
+- No built-in multi-model aggregator
+
+## Security
+
+If you discover a security vulnerability, please email the maintainer directly at
+the address listed in the gemspec. Do not file a public issue.

data/lib/full_search/callbacks.rb

@@ -3,6 +3,8 @@
 module FullSearch
   module Callbacks
     def self.install!(model)
+      return if model.instance_variable_get(:@__full_search_callbacks_installed)
+
       dsl = model.full_search_dsl
 
       source_fields = dsl.fields.select(&:source)
@@ -27,6 +29,12 @@ module FullSearch
           FullSearch::Callbacks.reindex_dependents!(record, model, field)
         end
       end
+
+      model.instance_variable_set(:@__full_search_callbacks_installed, true)
+    end
+
+    def self.reset_installed_flag!(model)
+      model.instance_variable_set(:@__full_search_callbacks_installed, false)
     end
 
     def self.reindex_record!(record)
@@ -46,22 +54,24 @@ module FullSearch
       return unless field&.source
 
       value = record.instance_exec(&field.source)
-      table = FullSearch::Index.fts_table_name(record.class)
-      quoted_value = ActiveRecord::Base.connection.quote(value.to_s)
-      ActiveRecord::Base.connection.execute(
-        "UPDATE #{table} SET #{field.name} = #{quoted_value} WHERE rowid = #{record.id}"
+      table = qt(FullSearch::Index.fts_table_name(record.class))
+      conn = ActiveRecord::Base.connection
+      conn.execute(
+        "UPDATE #{table} SET #{qc(field.name)} = #{q(value.to_s)} WHERE rowid = #{q(record.id)}"
       )
     end
 
     def self.remove_record!(record)
-      table = FullSearch::Index.fts_table_name(record.class)
-      ActiveRecord::Base.connection.execute("DELETE FROM #{table} WHERE rowid = #{record.id}")
+      table = qt(FullSearch::Index.fts_table_name(record.class))
+      conn = ActiveRecord::Base.connection
+      conn.execute("DELETE FROM #{table} WHERE rowid = #{q(record.id)}")
     end
 
     def self.reindex_dependents!(parent_record, dependent_model, field)
       fk = association_key(dependent_model, field.reindex_on)
-      sql = "SELECT id FROM #{dependent_model.table_name} WHERE #{fk} = #{parent_record.id}"
-      dependent_ids = ActiveRecord::Base.connection.execute(sql).map { |r| r["id"] }
+      conn = ActiveRecord::Base.connection
+      sql = "SELECT id FROM #{qt(dependent_model.table_name)} WHERE #{qc(fk)} = #{q(parent_record.id)}"
+      dependent_ids = conn.execute(sql).map { |r| r["id"] }
 
       dependent_ids.each do |dep_id|
         if field.async
@@ -73,6 +83,14 @@ module FullSearch
       end
     end
 
+    class << self
+      include FullSearch::Quoting
+    end
+
+    def self.connection
+      ActiveRecord::Base.connection
+    end
+
     def self.associated_class(model, association_name)
       reflection = model.reflect_on_association(association_name.to_sym)
       reflection&.klass

data/lib/full_search/config.rb

@@ -2,10 +2,10 @@
 
 module FullSearch
   class Config
-    attr_accessor :auto_manage_schema, :stale_query_behavior, :lock_rebuilds, :default_async_reindex, :default_tokenizer
+    attr_accessor :auto_rebuild_schema, :stale_query_behavior, :lock_rebuilds, :default_async_reindex, :default_tokenizer
 
     def initialize
-      @auto_manage_schema = false
+      @auto_rebuild_schema = false
       @stale_query_behavior = :raise
       @lock_rebuilds = true
       @default_async_reindex = true

data/lib/full_search/distance.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module FullSearch
+  module Distance
+    def self.damerau_levenshtein(a, b)
+      a_len = a.length
+      b_len = b.length
+      return a_len if b_len == 0
+      return b_len if a_len == 0
+
+      d = Array.new(a_len + 1) { Array.new(b_len + 1, 0) }
+      (0..a_len).each { |i| d[i][0] = i }
+      (0..b_len).each { |j| d[0][j] = j }
+
+      (1..a_len).each do |i|
+        (1..b_len).each do |j|
+          cost = (a[i - 1] == b[j - 1]) ? 0 : 1
+          d[i][j] = [
+            d[i - 1][j] + 1,
+            d[i][j - 1] + 1,
+            d[i - 1][j - 1] + cost
+          ].min
+
+          if i > 1 && j > 1 && a[i - 1] == b[j - 2] && a[i - 2] == b[j - 1]
+            d[i][j] = [d[i][j], d[i - 2][j - 2] + 1].min
+          end
+        end
+      end
+
+      d[a_len][b_len]
+    end
+  end
+end

data/lib/full_search/dsl.rb

@@ -2,10 +2,10 @@
 
 module FullSearch
   class Dsl
-    attr_reader :fields, :exact_matches, :filters, :model_class, :tokenize, :highlight_config, :rank_bys
+    attr_reader :fields, :exact_matches, :filters, :model_class, :highlight_config, :rank_bys
 
-    Field = Data.define(:name, :weight, :source, :reindex_on, :async)
-    ExactMatch = Data.define(:name, :source)
+    Field = Data.define(:name, :weight, :source, :reindex_on, :async, :as, :version)
+    ExactMatch = Data.define(:name, :source, :version)
     Filter = Data.define(:name, :required)
     RankBy = Data.define(:column, :direction)
 
@@ -19,32 +19,49 @@ module FullSearch
       @soft_delete_column = nil
     end
 
-    def field(name, weight: 1, source: nil, reindex_on: nil, async: FullSearch.config.default_async_reindex)
+    def field(name, weight: 1, source: nil, reindex_on: nil, async: FullSearch.config.default_async_reindex, as: nil, version: nil)
       unless valid_name?(name)
         raise InvalidFieldError, "Invalid field name: #{name.inspect}"
       end
-      @fields << Field.new(name: name.to_s, weight: weight.to_i, source: source, reindex_on: reindex_on&.to_s, async: async)
+      if as && !valid_name?(as)
+        raise InvalidFieldError, "Invalid field alias (as): #{as.inspect}"
+      end
+      str = name.to_s
+      raise InvalidFieldError, "Duplicate field name: #{name.inspect}" if fields.any? { |f| f.name == str }
+      raise InvalidFieldError, "Field name conflicts with filter: #{name.inspect}" if filters.any? { |f| f.name == str }
+      @fields << Field.new(name: str, weight: weight.to_i, source: source, reindex_on: reindex_on&.to_s, async: async, as: as&.to_s, version: version)
     end
 
-    def exact_match(name, source: -> { public_send(name) })
+    def exact_match(name, source: -> { public_send(name) }, version: nil)
       unless valid_name?(name)
         raise InvalidFieldError, "Invalid exact_match name: #{name.inspect}"
       end
-      @exact_matches << ExactMatch.new(name: name.to_s, source: source)
+      str = name.to_s
+      raise InvalidFieldError, "Duplicate exact_match name: #{name.inspect}" if exact_matches.any? { |e| e.name == str }
+      @exact_matches << ExactMatch.new(name: str, source: source, version: version)
     end
 
     def rank_by(column, direction = :desc)
       unless valid_name?(column)
         raise InvalidFieldError, "Invalid rank_by column: #{column.inspect}"
       end
-      @rank_bys << RankBy.new(column: column.to_s, direction: direction.to_sym)
+      dir = direction.to_s.downcase
+      unless %w[asc desc].include?(dir)
+        raise InvalidFieldError, "Invalid rank_by direction: #{direction.inspect}. Use :asc or :desc."
+      end
+      str = column.to_s
+      raise InvalidFieldError, "Duplicate rank_by column: #{column.inspect}" if rank_bys.any? { |r| r.column == str }
+      @rank_bys << RankBy.new(column: str, direction: dir.to_sym)
     end
 
     def filter(name, required: false)
       unless valid_name?(name)
         raise InvalidFieldError, "Invalid filter name: #{name.inspect}"
       end
-      @filters << Filter.new(name: name.to_s, required: required)
+      str = name.to_s
+      raise InvalidFieldError, "Duplicate filter name: #{name.inspect}" if filters.any? { |f| f.name == str }
+      raise InvalidFieldError, "Filter name conflicts with field: #{name.inspect}" if fields.any? { |f| f.name == str }
+      @filters << Filter.new(name: str, required: required)
     end
 
     def soft_delete_column(name = :_no_arg_)
@@ -64,7 +81,7 @@ module FullSearch
     end
 
     def highlight(open_tag: "<mark>", close_tag: "</mark>")
-      @highlight_config = { open_tag: open_tag, close_tag: close_tag }
+      @highlight_config = {open_tag: open_tag, close_tag: close_tag}
     end
 
     def typo_tolerance(enabled = true, min_term_length: nil)
@@ -88,8 +105,8 @@ module FullSearch
         soft_delete_column,
         typo_tolerance?,
         typo_tolerance_min_term_length,
-        fields.map { |f| [f.name, f.weight, f.source.nil? ? "column" : "source", f.reindex_on, f.async] },
-        exact_matches.map { |e| [e.name] },
+        fields.map { |f| [f.name, f.weight, f.source.nil? ? "column" : "proc:#{f.version}", f.reindex_on, f.async, f.as] },
+        exact_matches.map { |e| [e.name, "proc:#{e.version}"] },
         filters.map { |f| [f.name, f.required] },
         rank_bys.map { |r| [r.column, r.direction] }
       ].inspect)

data/lib/full_search/errors.rb

@@ -3,8 +3,10 @@
 module FullSearch
   class Error < StandardError; end
   class MissingRequiredFilterError < Error; end
+  class UnknownFilterError < Error; end
   class ConfigChangedError < Error; end
   class InvalidFieldError < Error; end
   class NotConfiguredError < Error; end
   class UnsupportedDatabaseError < Error; end
+  class InvalidQueryError < Error; end
 end

data/lib/full_search/exact_match.rb

@@ -12,11 +12,14 @@ module FullSearch
       relation = model.all
       filters.each { |name, value| relation = relation.where(name => value) }
 
-      conditions = dsl.exact_matches.map do |em|
-        relation.model.arel_table[em.name].eq(cleaned)
+      ids = []
+      relation.find_each do |record|
+        dsl.exact_matches.each do |em|
+          value = record.instance_exec(&em.source)
+          ids << record.id if value.to_s.casecmp?(cleaned)
+        end
       end
-
-      relation.where(conditions.reduce(:or)).pluck(:id)
+      ids.uniq
     end
   end
 end