full_search 0.3.8 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a86070d05b32fc24ce6ac007a33c673c3831760ce76554bbb9d636e6e0c6be0f
-  data.tar.gz: 293b0be9931c7ffa5de9971aa52b2bf9014c2b0816d1364f839ccba57af8f814
+  metadata.gz: 8175568a013e8a29d107a5b3bceebc7e9be6b9c85f56d8ef178d3c2c72a73678
+  data.tar.gz: d77fe210358656fe1b471baeb374c27d2f493544739d4ec326c06aee3dddadf1
 SHA512:
-  metadata.gz: cd9854cc22d14bd739723e5011a831fd34d0e523ef6a6df09bf45fa1f3a7880c466f4ba121eb07d9d92daf65449edf74b5e0ddc42dc15b521ea34e78b496be3a
-  data.tar.gz: f704e822cee03ad98f2ed96dbc802e3e5b777df659ecdc3d8187ace6088494ab318b6f1598c74b840697909b465e539a76ee9f758d6a709041d9e0b198d45129
+  metadata.gz: 6e786b098bc3cd046151835f11bbf98b4a5213feb46e70a0ae6872a4f24bba6eb22ec5e8a17ebe045328eaab1d2016682fda67e47c5582fb73ae72b262c7bf08
+  data.tar.gz: 4d5284fe9b059ee7010ff099cd88e494beffe1804084f29be86921bd895059cc98dd567fdf9c09ef20a2ec9af99badef3416f6085bb3f85e019f44657028244a

data/README.md

@@ -223,6 +223,18 @@ The rebuild task checks each model's stored config hash against the current DSL
 bin/rails full_search:reset
 ```
 
+### Schema dump
+
+By default, `full_search` includes its FTS virtual tables in `db/schema.rb` so that `db:schema:load` produces a complete, loadable database. To exclude them:
+
+```ruby
+FullSearch.configure do |config|
+  config.dump_schema_virtual_tables = false
+end
+```
+
+When disabled, you must run `bin/rails full_search:prepare` after every `db:schema:load` to recreate the indexes.
+
 ### Rake tasks
 
 | Task | Description |
@@ -233,6 +245,7 @@ bin/rails full_search:reset
 | `full_search:optimize` | Run FTS5 [`optimize`](https://www.sqlite.org/fts5.html#the_optimize_command) to merge b-tree segments. Useful after bulk updates. |
 | `full_search:backfill` | Force-rebuild FTS indexes for specified models (or all). Useful for recovery after bulk operations. |
 | `full_search:status` | Show each model's index status (`ok` / `stale`) and count of empty sourced fields. |
+| `full_search:health_check` | Verify all indexes are present and current; exits `1` if any table is missing or stale. Useful for deploy gates or Docker healthchecks. |
 
 ## Background jobs
 
@@ -375,6 +388,202 @@ FullSearch.multi_search(
 )
 ```
 
+## Testing
+
+`full_search` ships with `FullSearch::TestHelpers`, a framework-agnostic module you can include in Minitest or RSpec. It provides helpers to rebuild, reindex, and reset FTS tables from your tests.
+
+### Setup
+
+Call `setup_for_tests!` once in your test boot file. It applies safe defaults:
+
+- Disables rebuild locking (`lock_rebuilds = false`)
+- Enables query-time auto-rebuild for stale indexes (`auto_rebuild_on_stale_query = true`)
+- Keeps stale-query behaviour as `raise` so real config drift fails fast
+- Forces Active Job to run inline so background reindex jobs execute synchronously
+
+#### Minitest (`test/test_helper.rb`)
+
+```ruby
+require "minitest/autorun"
+require "full_search"
+require "full_search/test_helpers"
+
+class ActiveSupport::TestCase
+  include FullSearch::TestHelpers
+end
+
+FullSearch::TestHelpers.setup_for_tests!
+```
+
+#### RSpec (`spec/rails_helper.rb`)
+
+```ruby
+require "full_search"
+require "full_search/test_helpers"
+
+RSpec.configure do |config|
+  config.include FullSearch::TestHelpers, type: :model
+end
+
+FullSearch::TestHelpers.setup_for_tests!
+```
+
+### Why you must rebuild after creating data
+
+FTS tables are updated by database triggers on normal inserts/updates, but computed `source:` fields are evaluated by Ruby during a rebuild/reindex. The safest pattern is to create your records, then call `rebuild_full_search_index(model)` before searching.
+
+### Available helpers
+
+```ruby
+# Rebuild a single model's FTS table from scratch (drops and recreates it).
+# Accepts a model class, symbol, or string class name.
+rebuild_full_search_index(Customer)
+rebuild_full_search_index(:customer)
+rebuild_full_search_index("Customer")
+
+# Re-evaluate computed source: fields only. Leaves table structure untouched.
+reindex_full_search(Customer)
+
+# Rebuild every registered search model, or only the ones passed in.
+reset_full_search!
+reset_full_search!(Customer, Vehicle)
+
+# Idempotently create any missing FTS tables/triggers for registered models.
+ensure_full_search_tables
+
+# Run a block with inline Active Job, restoring the adapter afterwards.
+with_full_search_async_jobs_inline do
+  # ReindexJob / BackfillJob execute synchronously here
+end
+
+# Run a block inside a rebuild, dropping the table at the end.
+with_full_search_rebuild(Customer) do
+  # search and assert here
+end
+
+# Scope anonymous searchable models to a block so they don't leak into
+# the global registry and affect later tests or Rake tasks.
+with_full_search_models_registered do
+  model = Class.new(Customer) do
+    full_search { field :first_name, weight: 5 }
+  end
+  model.table_name = "customers"
+  rebuild_full_search_index(model)
+  # ... search and assert
+end
+```
+
+### Minitest example
+
+```ruby
+class CustomerSearchTest < ActiveSupport::TestCase
+  def setup
+    @account = Account.create!(name: "Acme")
+    @customer = Customer.create!(account: @account, first_name: "Sam")
+    rebuild_full_search_index(Customer)
+  end
+
+  def test_finds_by_first_name
+    results = Customer.search("Sam", filters: {account_id: @account.id})
+    assert_includes results.to_a, @customer
+  end
+end
+```
+
+### RSpec example
+
+```ruby
+RSpec.describe Customer, type: :model do
+  let(:account) { create(:account) }
+  let(:customer) { create(:customer, account: account, first_name: "Sam") }
+
+  before { rebuild_full_search_index(Customer) }
+
+  it "finds by first name" do
+    results = Customer.search("Sam", filters: {account_id: account.id})
+    expect(results).to include(customer)
+  end
+end
+```
+
+### Advanced configuration
+
+If you need to override the defaults set by `setup_for_tests!`, use `FullSearch::TestHelpers.configure`:
+
+```ruby
+FullSearch::TestHelpers.configure do |config|
+  config.lock_rebuilds = true
+  config.auto_rebuild_on_stale_query = false
+  config.stale_query_behavior = :log_and_fallback
+end
+```
+
+## Production readiness
+
+`full_search` is designed for small to medium SQLite-backed Rails apps. Before running in production, review this checklist.
+
+### Initial deploy
+
+1. Run your normal database setup so application tables exist:
+   ```bash
+   bin/rails db:prepare:with_data
+   ```
+2. Create the FTS virtual tables and triggers:
+   ```bash
+   bin/rails full_search:prepare
+   ```
+3. For containerized deploys, add step 2 to your entrypoint after `db:prepare:with_data`.
+
+### Configuration
+
+The generated initializer defaults to production-safe values. Do **not** change these in production:
+
+- `auto_rebuild_schema` must be `false`. If enabled, every web/worker/console process tries to rebuild indexes on boot.
+- `auto_rebuild_on_stale_query` must be `false`. A query-time rebuild under load can cause timeouts and race conditions.
+
+If you need to change the search DSL, ship the change and then run:
+
+```bash
+bin/rails full_search:rebuild
+```
+
+from a single deployment step. The gem checks each model's stored config hash and only rebuilds indexes whose DSL has changed.
+
+### Monitoring and maintenance
+
+- **Health check** — use the built-in Rake task for deploy gates or container healthchecks:
+  ```bash
+  bin/rails full_search:health_check
+  ```
+  It exits `0` when every registered model has a current FTS table, or `1` if any table is missing or stale.
+
+- **Status overview** — `bin/rails full_search:status` prints `ok` / `stale` and the count of empty sourced fields per model.
+
+- **Scheduled optimize** — queue `FullSearch::OptimizeJob` once a day during a low-traffic window to merge FTS5 b-tree segments:
+  ```yaml
+  # config/recurring.yml
+  full_search_optimize:
+    class: FullSearch::OptimizeJob
+    schedule: daily at 4am
+    description: "Merge FTS5 b-tree segments for full_search indexes"
+  ```
+
+- **Recovery after bulk operations** — if a bulk operation bypassed triggers, rebuild the affected index:
+  ```bash
+  bin/rails 'full_search:backfill[customers]'
+  ```
+
+### Common errors
+
+| Error | Meaning | Fix |
+|-------|---------|-----|
+| `FullSearch::MissingTableError` | The FTS table does not exist yet. | Run `bin/rails full_search:prepare`. |
+| `FullSearch::ConfigChangedError` | The DSL has changed and the index is stale. | Run `bin/rails full_search:rebuild`. |
+
+### Locking note
+
+`lock_rebuilds` uses a Ruby `Mutex` and only prevents concurrent rebuilds within the same process/connection. It does not coordinate across processes or hosts. Always run `full_search:rebuild` from a single deployment step.
+
 ## Known limitations
 
 - Queries run with `highlight: true` return an Array of records, not an `ActiveRecord::Relation`. No further chaining (`.where`, `.order`, `.limit`) is possible after highlighting is applied.

data/lib/full_search/config.rb

@@ -4,7 +4,8 @@ module FullSearch
   class Config
     attr_accessor :auto_rebuild_schema, :stale_query_behavior, :lock_rebuilds,
       :default_async_reindex, :default_async_source_reindex,
-      :default_tokenizer, :auto_rebuild_on_stale_query, :min_like_prefix_length
+      :default_tokenizer, :auto_rebuild_on_stale_query, :min_like_prefix_length,
+      :dump_schema_virtual_tables
 
     def initialize
       @auto_rebuild_schema = false
@@ -15,6 +16,7 @@ module FullSearch
       @default_tokenizer = FullSearch::Constants::DEFAULT_TOKENIZER
       @auto_rebuild_on_stale_query = false
       @min_like_prefix_length = FullSearch::Constants::DEFAULT_MIN_LIKE_PREFIX_LENGTH
+      @dump_schema_virtual_tables = true
     end
   end
 

data/lib/full_search/constants.rb

@@ -6,7 +6,7 @@ module FullSearch
     TWO_TYPO_MIN_LENGTH = 9
     DEFAULT_MIN_LIKE_PREFIX_LENGTH = 3
     REBUILDING_HASH = "__rebuilding__"
-    DEFAULT_TOKENIZER = "unicode61"
+    DEFAULT_TOKENIZER = "porter"
     MAX_EXACT_MATCH_BOOST_IDS = 100
   end
 end

data/lib/full_search/index.rb

@@ -36,7 +36,7 @@ module FullSearch
         end
 
         trigram_was_created = false
-        if dsl.typo_tolerance? && !trigram_table_exists?(model)
+        if trigram_table_needed?(model) && !trigram_table_exists?(model)
           FullSearch::Typo.warn_unsupported!(model) unless FullSearch::Typo.supported?(model)
           conn.execute(create_trigram_virtual_table_sql(model))
           trigram_was_created = true
@@ -70,12 +70,12 @@ module FullSearch
             conn.execute("DROP TABLE IF EXISTS #{qt(fts_table_name(model))};")
             conn.execute("DROP TABLE IF EXISTS #{qt(trigram_table_name(model))};")
             conn.execute(create_virtual_table_sql(model))
-            if dsl.typo_tolerance?
+            if trigram_table_needed?(model)
               FullSearch::Typo.warn_unsupported!(model) unless FullSearch::Typo.supported?(model)
               conn.execute(create_trigram_virtual_table_sql(model))
             end
             conn.execute(backfill_sql(model))
-            conn.execute(backfill_trigram_sql(model)) if dsl.typo_tolerance?
+            conn.execute(backfill_trigram_sql(model)) if trigram_table_needed?(model)
             reindex_source_fields!(model) if dsl.fields.any?(&:source)
             create_triggers!(model)
             optimize!(model)
@@ -141,7 +141,7 @@ module FullSearch
         if model.full_search_dsl.soft_delete_column
           connection.execute(soft_delete_removal_trigger_sql(model))
         end
-        if model.full_search_dsl.typo_tolerance?
+        if trigram_table_needed?(model)
           connection.execute(insert_trigram_trigger_sql(model))
           connection.execute(delete_trigram_trigger_sql(model))
           connection.execute(update_trigram_trigger_sql(model))
@@ -177,7 +177,7 @@ module FullSearch
         else
           connection.adapter_name.downcase.include?("sqlite")
         end
-      rescue
+      rescue ActiveRecord::ConnectionNotEstablished, NoMethodError
         connection.adapter_name.downcase.include?("sqlite")
       end
 
@@ -199,6 +199,16 @@ module FullSearch
         !table_exists?(model)
       end
 
+      def trigram_table_needed?(model)
+        model.full_search_dsl&.typo_tolerance? && model.full_search_dsl.tokenize != "trigram"
+      end
+
+      def trigram_table_exists?(model)
+        connection.execute(
+          "SELECT name FROM sqlite_master WHERE type='table' AND name=#{q(trigram_table_name(model))} LIMIT 1"
+        ).any?
+      end
+
       private
 
       def connection
@@ -292,7 +302,7 @@ module FullSearch
         ).map { |r| r["name"] }
 
         expected = trigger_names(model)
-        expected += trigram_trigger_names(model) if model.full_search_dsl.typo_tolerance?
+        expected += trigram_trigger_names(model) if trigram_table_needed?(model)
         return if (expected - existing).empty? && (existing - expected).empty?
 
         rebuild!(model)
@@ -391,12 +401,6 @@ module FullSearch
         SQL
       end
 
-      def trigram_table_exists?(model)
-        connection.execute(
-          "SELECT name FROM sqlite_master WHERE type='table' AND name=#{q(trigram_table_name(model))} LIMIT 1"
-        ).any?
-      end
-
       def create_trigram_virtual_table_sql(model)
         dsl = model.full_search_dsl
         columns = (dsl.fields + dsl.filters.map { |f| FilterColumnPlaceholder.new(name: f.name) } + extra_columns(model))

data/lib/full_search/schema_dumper_patch.rb

@@ -11,6 +11,8 @@ module FullSearch
     private
 
     def virtual_tables(stream)
+      return unless FullSearch.config.dump_schema_virtual_tables
+
       virtual_tables = @connection.virtual_tables.reject { |name, _| ignored?(name) }
 
       if virtual_tables.any?

data/lib/full_search/search.rb

@@ -89,7 +89,7 @@ module FullSearch
 
       stored = begin
         FullSearch::Index.stored_config_hash(model)
-      rescue
+      rescue ActiveRecord::StatementInvalid
         nil
       end
       return unless stored
@@ -146,6 +146,8 @@ module FullSearch
         return like_prefix_ids(term, candidate_limit: candidate_limit)
       end
 
+      return [] unless FullSearch::Index.trigram_table_exists?(model)
+
       trigram_table = qt(FullSearch::Index.trigram_table_name(model))
       tbl = qt(model.table_name)
 

data/lib/full_search/test_helpers.rb

@@ -2,16 +2,82 @@
 
 module FullSearch
   module TestHelpers
-    def rebuild_full_search_index(model_name)
-      model = model_name.is_a?(Class) ? model_name : model_name.to_s.camelize.constantize
+    class << self
+      def setup_for_tests!
+        configure do |config|
+          config.lock_rebuilds = false
+          config.auto_rebuild_on_stale_query = true
+          config.stale_query_behavior = :raise
+        end
+
+        inline_active_job_if_configured
+      end
+
+      def configure
+        yield FullSearch.config
+      end
+
+      private
+
+      def inline_active_job_if_configured
+        return unless defined?(ActiveJob::Base)
+
+        ActiveJob::Base.queue_adapter = :inline
+      end
+    end
+
+    def rebuild_full_search_index(model)
+      model = resolve_full_search_model(model)
       FullSearch::Index.drop!(model)
       FullSearch::Index.rebuild!(model)
     end
 
+    def reindex_full_search(model)
+      model = resolve_full_search_model(model)
+      FullSearch::Index.reindex_source_fields!(model)
+    end
+
+    def reset_full_search!(*models)
+      models = FullSearch.models.to_a if models.empty?
+      models.each { |model| rebuild_full_search_index(model) }
+    end
+
     def ensure_full_search_tables
       FullSearch.models.each do |model|
         FullSearch::Index.ensure_table!(model)
       end
     end
+
+    def with_full_search_rebuild(model)
+      model = resolve_full_search_model(model)
+      rebuild_full_search_index(model)
+      yield
+    ensure
+      FullSearch::Index.drop!(model)
+    end
+
+    def with_full_search_async_jobs_inline
+      original_adapter = ActiveJob::Base.queue_adapter if defined?(ActiveJob::Base)
+      ActiveJob::Base.queue_adapter = :inline if defined?(ActiveJob::Base)
+      yield
+    ensure
+      ActiveJob::Base.queue_adapter = original_adapter if defined?(ActiveJob::Base)
+    end
+
+    def with_full_search_models_registered(*models)
+      previous_registry = FullSearch.models.dup
+      models.each { |model| FullSearch.register_model(model) }
+      yield
+    ensure
+      FullSearch.models.replace(previous_registry)
+    end
+
+    private
+
+    def resolve_full_search_model(model)
+      return model if model.is_a?(Class)
+
+      model.to_s.camelize.constantize
+    end
   end
 end

data/lib/full_search/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module FullSearch
-  VERSION = "0.3.8"
+  VERSION = "0.4.0"
 end

data/lib/generators/full_search/install/install_generator.rb

@@ -19,7 +19,7 @@ module FullSearch
 
         say "Running full_search:prepare to create FTS tables..."
         rake("full_search:prepare")
-      rescue => e
+      rescue RuntimeError => e
         say "Skipping full_search:prepare — #{e.message}", :yellow
         say "Run `bin/rails full_search:prepare` after your database is ready.", :yellow
       end

data/lib/generators/full_search/install/templates/full_search.rb

@@ -12,4 +12,8 @@ FullSearch.configure do |config|
   # Reindex computed source: fields synchronously (false) or via background job (true).
   # Bulk imports should use FullSearch.bulk_import(Model) { ... } to defer reindexing.
   config.default_async_source_reindex = true
+
+  # Uncomment to exclude full_search virtual tables from db/schema.rb.
+  # When false, you must run `bin/rails full_search:prepare` after db:schema:load.
+  # config.dump_schema_virtual_tables = false
 end

data/lib/tasks/full_search.rake

@@ -108,4 +108,30 @@ namespace :full_search do
       puts "#{model.table_name}: #{status}#{drift_info}"
     end
   end
+
+  desc "Verify full_search indexes are present and up to date (exit 1 if unhealthy)"
+  task health_check: :environment do
+    Rails.application.eager_load!
+    unhealthy = []
+
+    FullSearch.sorted_models.each do |model|
+      if FullSearch::Index.missing_table?(model)
+        unhealthy << "#{model.table_name}: missing FTS table"
+        next
+      end
+
+      stored = FullSearch::Index.stored_config_hash(model)
+      current = model.full_search_dsl.config_hash
+      if stored != current
+        unhealthy << "#{model.table_name}: stale config hash"
+      end
+    end
+
+    if unhealthy.any?
+      unhealthy.each { |message| puts "[FAIL] #{message}" }
+      exit 1
+    else
+      puts "[OK] All full_search indexes are healthy"
+    end
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: full_search
 version: !ruby/object:Gem::Version
-  version: 0.3.8
+  version: 0.4.0
 platform: ruby
 authors:
 - Ben D'Angelo