full_search 0.1.3 → 0.3.0

data/lib/full_search/model.rb

@@ -5,7 +5,7 @@ module FullSearch
     extend ActiveSupport::Concern
 
     class_methods do
-      def full_search(query_or_options = nil, filters: {}, include_soft_deleted: false, limit: nil, offset: nil, highlight: false, highlight_fields: false, &block)
+      def full_search(query_or_options = nil, filters: {}, include_soft_deleted: false, limit: nil, offset: nil, highlight: false, highlight_fields: false, matching_strategy: nil, &block)
         if block_given? || query_or_options.is_a?(Hash)
           @full_search_dsl ||= FullSearch::Dsl.new(self)
           @full_search_dsl.tokenize(query_or_options[:tokenize]) if query_or_options.is_a?(Hash) && query_or_options.key?(:tokenize)
@@ -13,10 +13,11 @@ module FullSearch
           FullSearch::Index.ensure_table!(self)
           FullSearch::Callbacks.install!(self)
           include InstanceMethods
-          FullSearch.models << self unless FullSearch.models.include?(self)
+
+          FullSearch.register_model(self)
           @full_search_dsl
         else
-          FullSearch::Search.new(self, query_or_options, filters: filters, include_soft_deleted: include_soft_deleted, limit: limit, offset: offset, highlight: highlight, highlight_fields: highlight_fields).relation
+          FullSearch::Search.new(self, query_or_options, filters: filters, include_soft_deleted: include_soft_deleted, limit: limit, offset: offset, highlight: highlight, highlight_fields: highlight_fields, matching_strategy: matching_strategy).relation
         end
       end
 
@@ -27,6 +28,24 @@ module FullSearch
       def full_search_ids(query, filters: {}, include_soft_deleted: false, limit: 1000)
         full_search(query, filters: filters, include_soft_deleted: include_soft_deleted, limit: limit).pluck(:id)
       end
+
+      def rebuild!
+        FullSearch::Index.rebuild!(self)
+      end
+
+      def optimize!
+        FullSearch::Index.optimize!(self)
+      end
+
+      def reindex!
+        FullSearch::Index.reindex_source_fields!(self)
+      end
+    end
+
+    included do |base|
+      unless base.respond_to?(:search)
+        base.singleton_class.alias_method :search, :full_search
+      end
     end
 
     module InstanceMethods
@@ -34,7 +53,7 @@ module FullSearch
 
       def full_search_text_for(field_name)
         dsl = self.class.full_search_dsl
-        field = dsl.fields.find { |f| f.name == field_name.to_s }
+        field = dsl.fields.find { |f| f.name == field_name.to_s || f.as == field_name.to_s }
         return nil unless field
 
         field.source ? instance_exec(&field.source) : public_send(field.name)

data/lib/full_search/multi_search.rb

@@ -14,6 +14,8 @@ module FullSearch
     def call
       searched = groups.map do |group|
         model = fetch(group, :model)
+        raise FullSearch::NotConfiguredError, "#{model} is not full_search configured" unless model.full_search_dsl
+
         filters = group[:filters] || {}
         limit = positive_integer(group[:limit], 8)
         offset = positive_integer(group[:offset], 0)
@@ -24,6 +26,7 @@ module FullSearch
           filters: filters,
           limit: raw_limit,
           offset: offset,
+          highlight: group[:highlight],
           highlight_fields: group[:highlight_fields]
         )
 
@@ -40,7 +43,7 @@ module FullSearch
         )
       end
 
-      { groups: searched, total_count: searched.sum { |g| g[:total_count] } }
+      {groups: searched, total_count: searched.sum { |g| g[:total_count] }}
     end
 
     private

data/lib/full_search/optimize_job.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require "active_job"
+
+module FullSearch
+  class OptimizeJob < ActiveJob::Base
+    queue_as :low
+
+    def perform
+      FullSearch.optimize!
+    end
+  end
+end

data/lib/full_search/query_parser.rb

@@ -6,121 +6,134 @@ module FullSearch
   module QueryParser
     Token = Data.define(:type, :value)
 
-    def self.parse(query)
-      tokens = tokenize(query.to_s.strip)
-      return [] if tokens.empty?
+    MAX_QUERY_LENGTH = 255
 
-      or_clauses = split_or(tokens)
+    class << self
+      def parse(query)
+        query = query.to_s.strip
+        validate!(query)
+        tokens = tokenize(query)
+        return [] if tokens.empty?
 
-      if or_clauses.size == 1
-        parse_and_clause(or_clauses.first)
-      else
-        [:or, or_clauses.map { |clause| parse_and_clause(clause) }]
+        or_clauses = split_or(tokens)
+
+        if or_clauses.size == 1
+          parse_and_clause(or_clauses.first)
+        else
+          [:or, or_clauses.map { |clause| parse_and_clause(clause) }]
+        end
       end
-    end
 
-    def self.to_match_expression(parsed)
-      return '""' if parsed.empty?
-
-      type, value = parsed
-      case type
-      when :and
-        build_and_expression(value)
-      when :or
-        value.map { |sub| to_match_expression(sub) }.join(" OR ")
-      else
-        node_to_match(parsed)
+      def to_match_expression(parsed)
+        return '""' if parsed.empty?
+
+        type, value = parsed
+        case type
+        when :and
+          build_and_expression(value)
+        when :or
+          value.map { |sub| to_match_expression(sub) }.join(" OR ")
+        else
+          node_to_match(parsed)
+        end
       end
-    end
 
-    private
+      private
 
-    def self.build_and_expression(nodes)
-      non_excludes = nodes.reject { |n| n.first == :exclude }
-      excludes = nodes.select { |n| n.first == :exclude }
+      def build_and_expression(nodes)
+        non_excludes = nodes.reject { |n| n.first == :exclude }
+        excludes = nodes.select { |n| n.first == :exclude }
 
-      parts = non_excludes.map { |n| node_to_match(n) }
-      result = parts.size == 1 ? parts.first : "(#{parts.join(' AND ')})"
+        parts = non_excludes.map { |n| node_to_match(n) }
+        result = (parts.size == 1) ? parts.first : "(#{parts.join(" AND ")})"
 
-      excludes.each do |node|
-        _, value = node
-        result = "#{result} NOT \"#{escape(value)}\""
-      end
+        excludes.each do |node|
+          _, value = node
+          result = "#{result} NOT \"#{escape(value)}\""
+        end
 
-      result
-    end
+        result
+      end
 
-    def self.node_to_match(node)
-      type, value = node
-      case type
-      when :term
-        '"' + escape(value) + '"*'
-      when :phrase
-        '"' + escape(value) + '"'
-      when :exclude
-        nil
+      def node_to_match(node)
+        type, value = node
+        case type
+        when :term
+          '"' + escape(value) + '"*'
+        when :phrase
+          '"' + escape(value) + '"'
+        when :exclude
+          nil
+        end
       end
-    end
 
-    def self.tokenize(query)
-      tokens = []
-      scanner = StringScanner.new(query)
+      def validate!(query)
+        if query.bytesize > MAX_QUERY_LENGTH
+          raise InvalidQueryError, "Query too long (max #{MAX_QUERY_LENGTH} chars)"
+        end
+        raise InvalidQueryError, "Query contains invalid characters" if query.include?("\x00")
+      end
 
-      until scanner.eos?
-        scanner.skip(/\s+/)
-        break if scanner.eos?
+      def tokenize(query)
+        tokens = []
+        scanner = StringScanner.new(query)
 
-        if scanner.scan(/"/)
-          phrase = scanner.scan_until(/"/)
-          if phrase
-            tokens << Token.new(:phrase, phrase.chomp('"'))
+        until scanner.eos?
+          scanner.skip(/\s+/)
+          break if scanner.eos?
+
+          if scanner.scan('"')
+            phrase = scanner.scan_until(/"/)
+            if phrase
+              tokens << Token.new(:phrase, phrase.chomp('"'))
+            else
+              tokens << Token.new(:term, scanner.rest)
+              scanner.terminate
+            end
+          elsif scanner.scan("-")
+            scanner.skip(/\s+/)
+            tokens << Token.new(:exclude, scanner.scan(/\S+/))
+          elsif scanner.scan(/OR|or/i)
+            tokens << Token.new(:or, nil)
           else
-            tokens << Token.new(:term, scanner.rest)
-            scanner.terminate
+            tokens << Token.new(:term, scanner.scan(/\S+/))
           end
-        elsif scanner.scan(/-/)
-          scanner.skip(/\s+/)
-          tokens << Token.new(:exclude, scanner.scan(/\S+/))
-        elsif scanner.scan(/OR|or/i)
-          tokens << Token.new(:or, nil)
-        else
-          tokens << Token.new(:term, scanner.scan(/\S+/))
         end
-      end
 
-      tokens
-    end
+        tokens
+      end
 
-    def self.split_or(tokens)
-      clauses = []
-      current = []
+      def split_or(tokens)
+        clauses = []
+        current = []
 
-      tokens.each do |token|
-        if token.type == :or
-          clauses << current unless current.empty?
-          current = []
-        else
-          current << token
+        tokens.each do |token|
+          if token.type == :or
+            clauses << current unless current.empty?
+            current = []
+          else
+            current << token
+          end
         end
+        clauses << current unless current.empty?
+        clauses
       end
-      clauses << current unless current.empty?
-      clauses
-    end
 
-    def self.parse_and_clause(tokens)
-      nodes = tokens.map do |token|
-        case token.type
-        when :term then [:term, token.value]
-        when :phrase then [:phrase, token.value]
-        when :exclude then [:exclude, token.value]
+      def parse_and_clause(tokens)
+        nodes = tokens.map do |token|
+          case token.type
+          when :term then [:term, token.value]
+          when :phrase then [:phrase, token.value]
+          when :exclude then [:exclude, token.value]
+          end
         end
-      end
 
-      nodes.size == 1 ? nodes.first : [:and, nodes]
-    end
+        (nodes.size == 1) ? nodes.first : [:and, nodes]
+      end
 
-    def self.escape(value)
-      value.to_s.gsub('"', '""')
+      def escape(value)
+        value.to_s.gsub('"', '""')
+      end
     end
   end
 end

data/lib/full_search/quoting.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module FullSearch
+  module Quoting
+    private
+
+    def q(value) = connection.quote(value)
+    def qt(name) = connection.quote_table_name(name)
+    def qc(name) = connection.quote_column_name(name)
+  end
+end

data/lib/full_search/railtie.rb

@@ -10,16 +10,10 @@ module FullSearch
 
     initializer "full_search.ensure_tables" do
       config.after_initialize do
-        next unless FullSearch.config.auto_manage_schema
+        next unless FullSearch.config.auto_rebuild_schema
         ActiveSupport.on_load(:active_record) do
           FullSearch.models.each do |model|
-            FullSearch::Index.ensure_table!(model)
-            next unless FullSearch.config.auto_manage_schema == true
-
-            stored = FullSearch::Index.stored_config_hash(model)
-            if stored && stored != model.full_search_dsl.config_hash
-              FullSearch::Index.rebuild!(model)
-            end
+            FullSearch::Index.rebuild_if_needed!(model)
           end
         end
       end

data/lib/full_search/search.rb

@@ -2,9 +2,9 @@
 
 module FullSearch
   class Search
-    attr_reader :model, :query, :filters, :include_soft_deleted, :limit, :offset, :highlight, :highlight_fields
+    attr_reader :model, :query, :filters, :include_soft_deleted, :limit, :offset, :highlight, :highlight_fields, :matching_strategy
 
-    def initialize(model, query, filters:, include_soft_deleted:, limit:, offset:, highlight: false, highlight_fields: false)
+    def initialize(model, query, filters:, include_soft_deleted:, limit:, offset:, highlight: false, highlight_fields: false, matching_strategy: nil)
       @model = model
       @query = query.to_s.strip
       @filters = filters
@@ -13,17 +13,25 @@ module FullSearch
       @offset = offset
       @highlight = highlight
       @highlight_fields = highlight_fields
+      @matching_strategy = matching_strategy
     end
 
+    MIN_TERM_LENGTH = 3
+
     def relation
+      validate_filter_keys!
       validate_required_filters!
+      check_stale_config!
+
+      return model.none if dsl.tokenize == "trigram" && query.length < MIN_TERM_LENGTH && !dsl.typo_tolerance?
 
       parsed = QueryParser.parse(query)
       exact_ids = ExactMatch.ids_for(model, query, filters)
       primary_ids = fts_match_ids(parsed)
-      fallback_ids = dsl.typo_tolerance? ? trigram_match_ids(parsed, primary_ids) : []
+      fallback_ids = (dsl.typo_tolerance? && matching_strategy != "all") ? trigram_match_ids(parsed, primary_ids) : []
+      fuzzy_ids = (dsl.typo_tolerance? && matching_strategy != "all" && primary_ids.empty? && fallback_ids.empty?) ? fuzzy_match_ids(parsed) : []
 
-      all_ids = (exact_ids + primary_ids + fallback_ids).uniq
+      all_ids = (exact_ids + primary_ids + fallback_ids + fuzzy_ids).uniq
       return model.none if all_ids.empty?
 
       rel = model.where(id: all_ids)
@@ -48,6 +56,15 @@ module FullSearch
       model.full_search_dsl
     end
 
+    def validate_filter_keys!
+      allowed = dsl.filters.map(&:name).to_set
+      filters.each_key do |key|
+        unless allowed.include?(key.to_s)
+          raise UnknownFilterError, "Unknown filter: #{key}"
+        end
+      end
+    end
+
     def validate_required_filters!
       dsl.filters.each do |filter|
         next unless filter.required
@@ -55,21 +72,36 @@ module FullSearch
       end
     end
 
+    def check_stale_config!
+      stored = FullSearch::Index.stored_config_hash(model)
+      return unless stored
+
+      if stored != dsl.config_hash
+        case FullSearch.config.stale_query_behavior
+        when :raise
+          raise ConfigChangedError, "FTS index for #{model.table_name} is stale; run full_search:rebuild"
+        when :log_and_fallback
+          Rails.logger.warn("[full_search] FTS index for #{model.table_name} is stale; results may be incomplete")
+        end
+      end
+    end
+
     def fts_match_ids(parsed)
       return [] if query.empty?
 
       match_expr = QueryParser.to_match_expression(parsed)
-      fts_table = FullSearch::Index.fts_table_name(model)
+      fts_table = qt(FullSearch::Index.fts_table_name(model))
+      tbl = qt(model.table_name)
 
       sql = <<~SQL
-        SELECT #{model.table_name}.id
+        SELECT #{tbl}.id
         FROM #{fts_table}
-        JOIN #{model.table_name} ON #{model.table_name}.id = #{fts_table}.rowid
-        WHERE #{fts_table} MATCH #{connection.quote(match_expr)}
+        JOIN #{tbl} ON #{tbl}.id = #{fts_table}.rowid
+        WHERE #{fts_table} MATCH #{q(match_expr)}
       SQL
 
       filter_conditions = filters.map do |name, value|
-        "AND #{fts_table}.#{name} = #{connection.quote(value)}"
+        "AND #{fts_table}.#{qc(name)} = #{q(value)}"
       end.join(" ")
 
       connection.execute("#{sql} #{filter_conditions}").map { |r| r["id"] }
@@ -81,76 +113,171 @@ module FullSearch
       match_expr = QueryParser.to_match_expression(parsed)
       return [] if match_expr.empty?
 
-      term = parsed.last rescue nil
+      term = parsed.last
       return [] if term.nil?
 
       if term.length < dsl.typo_tolerance_min_term_length.to_i
         return like_prefix_ids(term)
       end
 
-      trigram_table = FullSearch::Index.trigram_table_name(model)
+      trigram_table = qt(FullSearch::Index.trigram_table_name(model))
+      tbl = qt(model.table_name)
 
       sql = <<~SQL
-        SELECT #{model.table_name}.id
+        SELECT #{tbl}.id
         FROM #{trigram_table}
-        JOIN #{model.table_name} ON #{model.table_name}.id = #{trigram_table}.rowid
-        WHERE #{trigram_table} MATCH #{connection.quote(match_expr)}
+        JOIN #{tbl} ON #{tbl}.id = #{trigram_table}.rowid
+        WHERE #{trigram_table} MATCH #{q(match_expr)}
       SQL
 
       filter_conditions = filters.map do |name, value|
-        "AND #{trigram_table}.#{name} = #{connection.quote(value)}"
+        "AND #{trigram_table}.#{qc(name)} = #{q(value)}"
       end.join(" ")
 
       connection.execute("#{sql} #{filter_conditions}").map { |r| r["id"] }
     end
 
     def like_prefix_ids(term)
+      column_fields = dsl.fields.select { |f| f.source.nil? }
+      source_fields = dsl.fields.select { |f| f.source }
+      tbl = qt(model.table_name)
+
+      soft_delete_clause = ""
+      if dsl.soft_delete_column && !include_soft_deleted
+        soft_delete_clause = "AND #{tbl}.#{qc(dsl.soft_delete_column)} IS NULL"
+      end
+
+      ids = []
+
+      if column_fields.any?
+        like_conditions = column_fields.map do |field|
+          "#{tbl}.#{qc(field.name)} LIKE #{q("#{term}%")}"
+        end.join(" OR ")
+
+        filter_conditions = filters.map do |name, value|
+          "AND #{tbl}.#{qc(name)} = #{q(value)}"
+        end.join(" ")
+
+        sql = <<~SQL
+          SELECT #{tbl}.id
+          FROM #{tbl}
+          WHERE (#{like_conditions}) #{filter_conditions} #{soft_delete_clause}
+        SQL
+
+        ids = connection.execute(sql).map { |r| r["id"] }
+        return ids if ids.any?
+      end
+
+      if source_fields.any?
+        fts_table = qt(FullSearch::Index.fts_table_name(model))
+        like_conditions = source_fields.map do |field|
+          "#{fts_table}.#{qc(field.name)} LIKE #{q("#{term}%")}"
+        end.join(" OR ")
+
+        filter_conditions = filters.map do |name, value|
+          "AND #{fts_table}.#{qc(name)} = #{q(value)}"
+        end.join(" ")
+
+        sql = <<~SQL
+          SELECT #{tbl}.id
+          FROM #{fts_table}
+          JOIN #{tbl} ON #{tbl}.id = #{fts_table}.rowid
+          WHERE (#{like_conditions}) #{filter_conditions} #{soft_delete_clause}
+        SQL
+
+        ids = connection.execute(sql).map { |r| r["id"] }
+      end
+
+      ids
+    end
+
+    def fuzzy_match_ids(parsed)
+      term = parsed.last
+      return [] if term.nil?
+
+      term_str = term.is_a?(Array) ? extract_last_term_string(term) : term.to_s
+      return [] if term_str.empty?
+
+      max_typos = max_allowed_typos(term_str.length)
+      return [] if max_typos < 0
+
       column_fields = dsl.fields.select { |f| f.source.nil? }
       return [] if column_fields.empty?
 
-      like_conditions = column_fields.map do |field|
-        "#{connection.quote_table_name(model.table_name)}.#{connection.quote_column_name(field.name)} LIKE #{connection.quote("#{term}%")}"
+      register_levenshtein!
+      tbl = qt(model.table_name)
+
+      soft_delete_clause = ""
+      if dsl.soft_delete_column && !include_soft_deleted
+        soft_delete_clause = "AND #{tbl}.#{qc(dsl.soft_delete_column)} IS NULL"
+      end
+
+      filter_conditions = filters.map do |name, value|
+        "AND #{tbl}.#{qc(name)} = #{q(value)}"
+      end.join(" ")
+
+      conditions = column_fields.map do |field|
+        "levenshtein(LOWER(#{tbl}.#{qc(field.name)}), #{q(term_str.downcase)}) <= #{max_typos}"
       end.join(" OR ")
 
       sql = <<~SQL
-        SELECT #{model.table_name}.id
-        FROM #{model.table_name}
-        WHERE (#{like_conditions})
+        SELECT #{tbl}.id
+        FROM #{tbl}
+        WHERE (#{conditions}) #{filter_conditions} #{soft_delete_clause}
       SQL
 
-      filter_conditions = filters.map do |name, value|
-        "AND #{model.table_name}.#{name} = #{connection.quote(value)}"
-      end.join(" ")
+      connection.execute(sql).map { |r| r["id"] }
+    end
 
-      connection.execute("#{sql} #{filter_conditions}").map { |r| r["id"] }
+    def max_allowed_typos(length)
+      min_length = dsl.typo_tolerance_min_term_length.to_i
+      return -1 if length < min_length
+      return 2 if length >= 9
+      1
+    end
+
+    def extract_last_term_string(terms)
+      last = terms.last
+      last.is_a?(Array) ? last.last.to_s : last.to_s
+    end
+
+    def register_levenshtein!
+      return if @levenshtein_registered
+
+      raw = connection.raw_connection
+      raw.create_function("levenshtein", 2) do |func, s1, s2|
+        func.result = Distance.damerau_levenshtein(s1.to_s, s2.to_s)
+      end
+      @levenshtein_registered = true
     end
 
     def apply_ranking(rel, all_ids, exact_ids)
       return rel if all_ids.empty?
 
       order_parts = []
+      tbl = qt(model.table_name)
 
       if exact_ids.any?
-        order_parts << "CASE #{model.table_name}.id #{exact_ids.map { |id| "WHEN #{id} THEN 0" }.join(" ")} ELSE 1 END"
+        order_parts << "CASE #{tbl}.id #{exact_ids.map { |id| "WHEN #{q(id)} THEN 0" }.join(" ")} ELSE 1 END"
       end
 
-      fts_table = FullSearch::Index.fts_table_name(model)
+      fts_table = qt(FullSearch::Index.fts_table_name(model))
       match_expr = QueryParser.to_match_expression(QueryParser.parse(query))
 
       rank_subquery = <<~SQL
         SELECT rowid, rank
         FROM #{fts_table}
-        WHERE #{fts_table} MATCH #{connection.quote(match_expr)}
+        WHERE #{fts_table} MATCH #{q(match_expr)}
       SQL
 
       rel = rel
-        .select("#{model.table_name}.*, fts_rank.rank AS full_search_rank")
-        .joins("LEFT JOIN (#{rank_subquery}) AS fts_rank ON fts_rank.rowid = #{model.table_name}.id")
+        .select("#{tbl}.*, fts_rank.rank AS full_search_rank")
+        .joins("LEFT JOIN (#{rank_subquery}) AS fts_rank ON fts_rank.rowid = #{tbl}.id")
 
       order_parts << "COALESCE(fts_rank.rank, 1)"
 
       dsl.rank_bys.each do |rank_by|
-        col = "#{connection.quote_table_name(model.table_name)}.#{connection.quote_column_name(rank_by.column)}"
+        col = "#{tbl}.#{qc(rank_by.column)}"
         order_parts << "#{col} #{rank_by.direction.to_s.upcase} NULLS LAST"
       end
 
@@ -158,7 +285,11 @@ module FullSearch
     end
 
     def connection
+      model.connection
+    rescue NoMethodError
       ActiveRecord::Base.connection
     end
+
+    include Quoting
   end
 end

data/lib/full_search/soft_delete.rb

@@ -2,11 +2,24 @@
 
 module FullSearch
   module SoftDelete
-    def self.delete_transition_sql(model)
+    def self.active_update_clause(model)
       dsl = model.full_search_dsl
       return nil unless dsl&.soft_delete_column
 
-      "WHEN new.#{dsl.soft_delete_column} IS NULL"
+      col = model.connection.quote_column_name(dsl.soft_delete_column)
+      "WHEN new.#{col} IS NULL"
+    end
+
+    def self.soft_delete_remove_clause(model)
+      dsl = model.full_search_dsl
+      return nil unless dsl&.soft_delete_column
+
+      col = model.connection.quote_column_name(dsl.soft_delete_column)
+      "WHEN old.#{col} IS NULL AND new.#{col} IS NOT NULL"
+    end
+
+    def self.delete_transition_sql(model)
+      active_update_clause(model)
     end
   end
 end