full_search 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: a18ce04c25c9a2670d1ad9394594a277b78201e74e11585c63e1848d6d6cf343
+  data.tar.gz: 4d170670fd73af2e350ee427fce766658769612ea592772b29089ac395bc7b0c
+SHA512:
+  metadata.gz: d0276bef97a441548574747ddd7aeb67609b390bad8a8864618231cfac30d41f24e2d01cb55205b070d5c0f2249afbf4b7f0b133d51acb396e2a61a528923b18
+  data.tar.gz: 6e7808b9b6f3e8ccb8e3964c3494f42336c18975f6c79c335a397927aaf6ef3d920f84dd5548000182ddac652b376f71c1751c090f0c65191ccb62e4627dd5a7

data/README.md

@@ -0,0 +1,89 @@
+# full_search
+
+SQLite FTS5 full-text search for Rails/ActiveRecord. A lightweight, self-contained alternative to `pg_search` for apps already running on SQLite.
+
+## Installation
+
+Add to your Gemfile:
+
+```ruby
+gem "full_search"
+```
+
+Run:
+
+```bash
+bundle install
+bin/rails generate full_search:install
+```
+
+## Usage
+
+```ruby
+class Customer < ApplicationRecord
+  full_search do
+    field :first_name, weight: 5
+    field :last_name,  weight: 5
+    filter :account_id, required: true
+  end
+end
+```
+
+```ruby
+Customer.full_search("sam", filters: { account_id: 1 }).page(params[:page])
+```
+
+## Features
+
+- Declarative `full_search` DSL
+- SQLite FTS5 backed
+- Required-filter enforcement for multi-tenant apps
+- Exact-match queries for encrypted identifiers
+- Soft-delete awareness
+- Automatic table creation and schema-drift detection
+- Phrase, exclusion, and OR query operators
+- FTS5 `highlight()` support for result snippets
+- Opt-in trigram typo/substring fallback (requires SQLite >= 3.34)
+- Rake tasks: `full_search:rebuild`, `full_search:optimize`, `full_search:status`
+
+## Query operators
+
+```ruby
+Customer.full_search('"Sam Smith"', filters: { account_id: 1 })      # phrase
+Customer.full_search("honda -civic", filters: { account_id: 1 })     # exclusion
+Customer.full_search("honda OR toyota", filters: { account_id: 1 })  # OR
+```
+
+## Highlighting
+
+```ruby
+class Customer < ApplicationRecord
+  full_search do
+    field :first_name, weight: 5
+    highlight open_tag: "<mark>", close_tag: "</mark>"
+  end
+end
+```
+
+```ruby
+Customer.full_search("sam", filters: { account_id: 1 }, highlight: true)
+# each result has #full_search_snippet
+```
+
+## Typo tolerance
+
+```ruby
+class Customer < ApplicationRecord
+  full_search do
+    field :first_name, weight: 5
+    typo_tolerance
+  end
+end
+```
+
+`typo_tolerance` uses an FTS5 trigram shadow table as a fallback when the primary index returns no results. It is substring matching, not edit-distance correction, and requires SQLite >= 3.34.
+
+## Known limitations
+
+- Queries run with `highlight: true` return an Array of records, not an `ActiveRecord::Relation`. No further chaining (`.where`, `.order`, `.limit`) is possible after highlighting is applied.
+- Per-model only; no built-in multi-model aggregator

data/lib/full_search/callbacks.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+module FullSearch
+  module Callbacks
+    def self.install!(model)
+      dsl = model.full_search_dsl
+
+      source_fields = dsl.fields.select(&:source)
+      return if source_fields.empty?
+
+      model.after_save do
+        FullSearch::Callbacks.reindex_record!(self)
+      end
+
+      model.after_destroy do
+        FullSearch::Callbacks.remove_record!(self)
+      end
+
+      dsl.fields.each do |field|
+        next unless field.reindex_on
+
+        assoc_class = associated_class(model, field.reindex_on)
+        assoc_class&.after_save do |record|
+          FullSearch::Callbacks.reindex_dependents!(record, model, field)
+        end
+        assoc_class&.after_destroy do |record|
+          FullSearch::Callbacks.reindex_dependents!(record, model, field)
+        end
+      end
+    end
+
+    def self.reindex_record!(record)
+      dsl = record.class.full_search_dsl
+      return unless dsl
+
+      dsl.fields.each do |field|
+        next unless field.source
+
+        reindex_field!(record, field.name)
+      end
+    end
+
+    def self.reindex_field!(record, field_name)
+      dsl = record.class.full_search_dsl
+      field = dsl.fields.find { |f| f.name == field_name }
+      return unless field&.source
+
+      value = record.instance_exec(&field.source)
+      table = FullSearch::Index.fts_table_name(record.class)
+      quoted_value = ActiveRecord::Base.connection.quote(value.to_s)
+      ActiveRecord::Base.connection.execute(
+        "UPDATE #{table} SET #{field.name} = #{quoted_value} WHERE rowid = #{record.id}"
+      )
+    end
+
+    def self.remove_record!(record)
+      table = FullSearch::Index.fts_table_name(record.class)
+      ActiveRecord::Base.connection.execute("DELETE FROM #{table} WHERE rowid = #{record.id}")
+    end
+
+    def self.reindex_dependents!(parent_record, dependent_model, field)
+      fk = association_key(dependent_model, field.reindex_on)
+      sql = "SELECT id FROM #{dependent_model.table_name} WHERE #{fk} = #{parent_record.id}"
+      dependent_ids = ActiveRecord::Base.connection.execute(sql).map { |r| r["id"] }
+
+      dependent_ids.each do |dep_id|
+        if field.async
+          FullSearch::ReindexJob.perform_later(dependent_model.name, dep_id, field.name)
+        else
+          dependent = dependent_model.find_by(id: dep_id)
+          reindex_field!(dependent, field.name) if dependent
+        end
+      end
+    end
+
+    def self.associated_class(model, association_name)
+      reflection = model.reflect_on_association(association_name.to_sym)
+      reflection&.klass
+    end
+
+    def self.association_key(model, association_name)
+      reflection = model.reflect_on_association(association_name.to_sym)
+      reflection&.foreign_key&.to_s || "#{association_name}_id"
+    end
+  end
+end

data/lib/full_search/config.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module FullSearch
+  class Config
+    attr_accessor :auto_manage_schema, :stale_query_behavior, :lock_rebuilds, :default_async_reindex, :default_tokenizer
+
+    def initialize
+      @auto_manage_schema = false
+      @stale_query_behavior = :raise
+      @lock_rebuilds = true
+      @default_async_reindex = true
+      @default_tokenizer = "unicode61"
+    end
+  end
+
+  class << self
+    def config
+      @config ||= Config.new
+    end
+
+    def configure
+      yield config
+    end
+  end
+end

data/lib/full_search/dsl.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+module FullSearch
+  class Dsl
+    attr_reader :fields, :exact_matches, :filters, :model_class, :tokenize, :highlight_config
+
+    Field = Data.define(:name, :weight, :source, :reindex_on, :async)
+    ExactMatch = Data.define(:name, :source)
+    Filter = Data.define(:name, :required)
+
+    def initialize(model_class)
+      @model_class = model_class
+      @fields = []
+      @exact_matches = []
+      @filters = []
+      @tokenize = FullSearch.config.default_tokenizer
+      @soft_delete_column = nil
+    end
+
+    def field(name, weight: 1, source: nil, reindex_on: nil, async: FullSearch.config.default_async_reindex)
+      unless valid_name?(name)
+        raise InvalidFieldError, "Invalid field name: #{name.inspect}"
+      end
+      @fields << Field.new(name: name.to_s, weight: weight.to_i, source: source, reindex_on: reindex_on&.to_s, async: async)
+    end
+
+    def exact_match(name, source: -> { public_send(name) })
+      unless valid_name?(name)
+        raise InvalidFieldError, "Invalid exact_match name: #{name.inspect}"
+      end
+      @exact_matches << ExactMatch.new(name: name.to_s, source: source)
+    end
+
+    def filter(name, required: false)
+      unless valid_name?(name)
+        raise InvalidFieldError, "Invalid filter name: #{name.inspect}"
+      end
+      @filters << Filter.new(name: name.to_s, required: required)
+    end
+
+    def soft_delete_column(name = :_no_arg_)
+      if name == :_no_arg_
+        @soft_delete_column
+      else
+        @soft_delete_column = name.to_s
+      end
+    end
+
+    def tokenize(value = :_no_arg_)
+      if value == :_no_arg_
+        @tokenize
+      else
+        @tokenize = Tokenizer.validate!(value)
+      end
+    end
+
+    def highlight(open_tag: "<mark>", close_tag: "</mark>")
+      @highlight_config = { open_tag: open_tag, close_tag: close_tag }
+    end
+
+    def typo_tolerance(enabled = true, min_term_length: nil)
+      @typo_tolerance = enabled
+      @typo_tolerance_min_term_length = min_term_length || 3
+    end
+
+    def typo_tolerance?
+      !!@typo_tolerance
+    end
+
+    def typo_tolerance_min_term_length
+      @typo_tolerance_min_term_length || 3
+    end
+
+    def config_hash
+      require "digest"
+      Digest::SHA256.hexdigest([
+        model_class.table_name,
+        tokenize,
+        soft_delete_column,
+        typo_tolerance?,
+        typo_tolerance_min_term_length,
+        fields.map { |f| [f.name, f.weight, f.source.nil? ? "column" : "source", f.reindex_on, f.async] },
+        exact_matches.map { |e| [e.name] },
+        filters.map { |f| [f.name, f.required] }
+      ].inspect)
+    end
+
+    private
+
+    def valid_name?(name)
+      name.to_s.match?(/\A[a-zA-Z_]\w*\z/)
+    end
+  end
+end

data/lib/full_search/errors.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module FullSearch
+  class Error < StandardError; end
+  class MissingRequiredFilterError < Error; end
+  class ConfigChangedError < Error; end
+  class InvalidFieldError < Error; end
+  class NotConfiguredError < Error; end
+  class UnsupportedDatabaseError < Error; end
+end

data/lib/full_search/exact_match.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module FullSearch
+  class ExactMatch
+    def self.ids_for(model, query, filters)
+      dsl = model.full_search_dsl
+      return [] if dsl.exact_matches.empty?
+
+      cleaned = query.to_s.strip
+      return [] if cleaned.empty?
+
+      relation = model.all
+      filters.each { |name, value| relation = relation.where(name => value) }
+
+      conditions = dsl.exact_matches.map do |em|
+        relation.model.arel_table[em.name].eq(cleaned)
+      end
+
+      relation.where(conditions.reduce(:or)).pluck(:id)
+    end
+  end
+end

data/lib/full_search/highlighter.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module FullSearch
+  class Highlighter
+    def self.apply!(records, model, query)
+      dsl = model.full_search_dsl
+      config = dsl.highlight_config
+      return records unless config
+
+      match_expr = QueryParser.to_match_expression(QueryParser.parse(query))
+      return records if match_expr.empty?
+
+      table = FullSearch::Index.fts_table_name(model)
+
+      content_cols = dsl.fields.map(&:name)
+      return records if content_cols.empty?
+
+      highlight_parts = content_cols.each_with_index.map do |col, idx|
+        "highlight(#{table}, #{idx}, #{quote(config[:open_tag])}, #{quote(config[:close_tag])}) AS #{col}_snippet"
+      end.join(", ")
+
+      sql = <<~SQL
+        SELECT rowid, #{highlight_parts}
+        FROM #{table}
+        WHERE #{table} MATCH #{quote(match_expr)}
+      SQL
+
+      rows = connection.execute(sql)
+
+      snippets = rows.map do |r|
+        [r["rowid"], content_cols.map { |col| r["#{col}_snippet"] }.compact.join(" ").strip]
+      end.to_h
+
+      records.each do |record|
+        record.full_search_snippet = snippets[record.id]
+      end
+
+      records
+    end
+
+    private
+
+    def self.connection
+      ActiveRecord::Base.connection
+    end
+
+    def self.quote(value)
+      connection.quote(value)
+    end
+  end
+end