full_search 0.2.0 → 0.3.1

data/lib/full_search/schema_dumper_patch.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module FullSearch
+  module VirtualTablesPatch
+    def virtual_tables
+      super.reject { |_, options| options.first.nil? }
+    end
+  end
+
+  module SchemaDumperPatch
+    private
+
+    def virtual_tables(stream)
+      virtual_tables = @connection.virtual_tables.reject { |name, _| ignored?(name) }
+
+      if virtual_tables.any?
+        stream.puts
+        stream.puts "  # Virtual tables defined in this database."
+        stream.puts "  # Note that virtual tables may not work with other database engines. Be careful if changing database."
+        virtual_tables.sort.each do |table_name, options|
+          module_name, arguments = options
+          next if module_name.nil? || arguments.nil?
+
+          stream.puts "  create_virtual_table #{table_name.inspect}, #{module_name.inspect}, #{arguments.split(", ").inspect}"
+        end
+      end
+    end
+  end
+end
+
+ActiveSupport.on_load(:active_record) do
+  begin
+    require "active_record/connection_adapters/sqlite3_adapter"
+  rescue LoadError
+    next
+  end
+
+  if defined?(ActiveRecord::ConnectionAdapters::SQLite3Adapter)
+    ActiveRecord::ConnectionAdapters::SQLite3Adapter.prepend(FullSearch::VirtualTablesPatch)
+  end
+
+  if defined?(ActiveRecord::ConnectionAdapters::SQLite3::SchemaDumper)
+    ActiveRecord::ConnectionAdapters::SQLite3::SchemaDumper.prepend(FullSearch::SchemaDumperPatch)
+  end
+end

data/lib/full_search/search.rb

@@ -16,14 +16,20 @@ module FullSearch
       @matching_strategy = matching_strategy
     end
 
+    MIN_TERM_LENGTH = 3
+
     def relation
+      validate_filter_keys!
       validate_required_filters!
+      check_stale_config!
+
+      return model.none if dsl.tokenize == "trigram" && query.length < MIN_TERM_LENGTH && !dsl.typo_tolerance?
 
       parsed = QueryParser.parse(query)
       exact_ids = ExactMatch.ids_for(model, query, filters)
       primary_ids = fts_match_ids(parsed)
-      fallback_ids = dsl.typo_tolerance? && matching_strategy != "all" ? trigram_match_ids(parsed, primary_ids) : []
-      fuzzy_ids = dsl.typo_tolerance? && matching_strategy != "all" && primary_ids.empty? && fallback_ids.empty? ? fuzzy_match_ids(parsed) : []
+      fallback_ids = (dsl.typo_tolerance? && matching_strategy != "all") ? trigram_match_ids(parsed, primary_ids) : []
+      fuzzy_ids = (dsl.typo_tolerance? && matching_strategy != "all" && primary_ids.empty? && fallback_ids.empty?) ? fuzzy_match_ids(parsed) : []
 
       all_ids = (exact_ids + primary_ids + fallback_ids + fuzzy_ids).uniq
       return model.none if all_ids.empty?
@@ -50,6 +56,15 @@ module FullSearch
       model.full_search_dsl
     end
 
+    def validate_filter_keys!
+      allowed = dsl.filters.map(&:name).to_set
+      filters.each_key do |key|
+        unless allowed.include?(key.to_s)
+          raise UnknownFilterError, "Unknown filter: #{key}"
+        end
+      end
+    end
+
     def validate_required_filters!
       dsl.filters.each do |filter|
         next unless filter.required
@@ -57,21 +72,36 @@ module FullSearch
       end
     end
 
+    def check_stale_config!
+      stored = FullSearch::Index.stored_config_hash(model)
+      return unless stored
+
+      if stored != dsl.config_hash
+        case FullSearch.config.stale_query_behavior
+        when :raise
+          raise ConfigChangedError, "FTS index for #{model.table_name} is stale; run full_search:rebuild"
+        when :log_and_fallback
+          Rails.logger.warn("[full_search] FTS index for #{model.table_name} is stale; results may be incomplete")
+        end
+      end
+    end
+
     def fts_match_ids(parsed)
       return [] if query.empty?
 
       match_expr = QueryParser.to_match_expression(parsed)
-      fts_table = FullSearch::Index.fts_table_name(model)
+      fts_table = qt(FullSearch::Index.fts_table_name(model))
+      tbl = qt(model.table_name)
 
       sql = <<~SQL
-        SELECT #{model.table_name}.id
+        SELECT #{tbl}.id
         FROM #{fts_table}
-        JOIN #{model.table_name} ON #{model.table_name}.id = #{fts_table}.rowid
-        WHERE #{fts_table} MATCH #{connection.quote(match_expr)}
+        JOIN #{tbl} ON #{tbl}.id = #{fts_table}.rowid
+        WHERE #{fts_table} MATCH #{q(match_expr)}
       SQL
 
       filter_conditions = filters.map do |name, value|
-        "AND #{fts_table}.#{name} = #{connection.quote(value)}"
+        "AND #{fts_table}.#{qc(name)} = #{q(value)}"
       end.join(" ")
 
       connection.execute("#{sql} #{filter_conditions}").map { |r| r["id"] }
@@ -83,24 +113,25 @@ module FullSearch
       match_expr = QueryParser.to_match_expression(parsed)
       return [] if match_expr.empty?
 
-      term = parsed.last rescue nil
+      term = parsed.last
       return [] if term.nil?
 
       if term.length < dsl.typo_tolerance_min_term_length.to_i
         return like_prefix_ids(term)
       end
 
-      trigram_table = FullSearch::Index.trigram_table_name(model)
+      trigram_table = qt(FullSearch::Index.trigram_table_name(model))
+      tbl = qt(model.table_name)
 
       sql = <<~SQL
-        SELECT #{model.table_name}.id
+        SELECT #{tbl}.id
         FROM #{trigram_table}
-        JOIN #{model.table_name} ON #{model.table_name}.id = #{trigram_table}.rowid
-        WHERE #{trigram_table} MATCH #{connection.quote(match_expr)}
+        JOIN #{tbl} ON #{tbl}.id = #{trigram_table}.rowid
+        WHERE #{trigram_table} MATCH #{q(match_expr)}
       SQL
 
       filter_conditions = filters.map do |name, value|
-        "AND #{trigram_table}.#{name} = #{connection.quote(value)}"
+        "AND #{trigram_table}.#{qc(name)} = #{q(value)}"
       end.join(" ")
 
       connection.execute("#{sql} #{filter_conditions}").map { |r| r["id"] }
@@ -109,53 +140,59 @@ module FullSearch
     def like_prefix_ids(term)
       column_fields = dsl.fields.select { |f| f.source.nil? }
       source_fields = dsl.fields.select { |f| f.source }
+      tbl = qt(model.table_name)
+
+      soft_delete_clause = ""
+      if dsl.soft_delete_column && !include_soft_deleted
+        soft_delete_clause = "AND #{tbl}.#{qc(dsl.soft_delete_column)} IS NULL"
+      end
 
       ids = []
 
       if column_fields.any?
         like_conditions = column_fields.map do |field|
-          "#{connection.quote_table_name(model.table_name)}.#{connection.quote_column_name(field.name)} LIKE #{connection.quote("#{term}%")}"
+          "#{tbl}.#{qc(field.name)} LIKE #{q("#{term}%")}"
         end.join(" OR ")
 
-        sql = <<~SQL
-          SELECT #{model.table_name}.id
-          FROM #{model.table_name}
-          WHERE (#{like_conditions})
-        SQL
-
         filter_conditions = filters.map do |name, value|
-          "AND #{model.table_name}.#{name} = #{connection.quote(value)}"
+          "AND #{tbl}.#{qc(name)} = #{q(value)}"
         end.join(" ")
 
-        ids = connection.execute("#{sql} #{filter_conditions}").map { |r| r["id"] }
+        sql = <<~SQL
+          SELECT #{tbl}.id
+          FROM #{tbl}
+          WHERE (#{like_conditions}) #{filter_conditions} #{soft_delete_clause}
+        SQL
+
+        ids = connection.execute(sql).map { |r| r["id"] }
         return ids if ids.any?
       end
 
       if source_fields.any?
-        fts_table = FullSearch::Index.fts_table_name(model)
+        fts_table = qt(FullSearch::Index.fts_table_name(model))
         like_conditions = source_fields.map do |field|
-          "#{fts_table}.#{field.name} LIKE #{connection.quote("#{term}%")}"
+          "#{fts_table}.#{qc(field.name)} LIKE #{q("#{term}%")}"
         end.join(" OR ")
 
+        filter_conditions = filters.map do |name, value|
+          "AND #{fts_table}.#{qc(name)} = #{q(value)}"
+        end.join(" ")
+
         sql = <<~SQL
-          SELECT #{model.table_name}.id
+          SELECT #{tbl}.id
           FROM #{fts_table}
-          JOIN #{model.table_name} ON #{model.table_name}.id = #{fts_table}.rowid
-          WHERE (#{like_conditions})
+          JOIN #{tbl} ON #{tbl}.id = #{fts_table}.rowid
+          WHERE (#{like_conditions}) #{filter_conditions} #{soft_delete_clause}
         SQL
 
-        filter_conditions = filters.map do |name, value|
-          "AND #{fts_table}.#{name} = #{connection.quote(value)}"
-        end.join(" ")
-
-        ids = connection.execute("#{sql} #{filter_conditions}").map { |r| r["id"] }
+        ids = connection.execute(sql).map { |r| r["id"] }
       end
 
       ids
     end
 
     def fuzzy_match_ids(parsed)
-      term = parsed.last rescue nil
+      term = parsed.last
       return [] if term.nil?
 
       term_str = term.is_a?(Array) ? extract_last_term_string(term) : term.to_s
@@ -168,22 +205,28 @@ module FullSearch
       return [] if column_fields.empty?
 
       register_levenshtein!
+      tbl = qt(model.table_name)
+
+      soft_delete_clause = ""
+      if dsl.soft_delete_column && !include_soft_deleted
+        soft_delete_clause = "AND #{tbl}.#{qc(dsl.soft_delete_column)} IS NULL"
+      end
+
+      filter_conditions = filters.map do |name, value|
+        "AND #{tbl}.#{qc(name)} = #{q(value)}"
+      end.join(" ")
 
       conditions = column_fields.map do |field|
-        "levenshtein(LOWER(#{connection.quote_table_name(model.table_name)}.#{connection.quote_column_name(field.name)}), #{connection.quote(term_str.downcase)}) <= #{max_typos}"
+        "levenshtein(LOWER(#{tbl}.#{qc(field.name)}), #{q(term_str.downcase)}) <= #{max_typos}"
       end.join(" OR ")
 
       sql = <<~SQL
-        SELECT #{model.table_name}.id
-        FROM #{model.table_name}
-        WHERE (#{conditions})
+        SELECT #{tbl}.id
+        FROM #{tbl}
+        WHERE (#{conditions}) #{filter_conditions} #{soft_delete_clause}
       SQL
 
-      filter_conditions = filters.map do |name, value|
-        "AND #{model.table_name}.#{name} = #{connection.quote(value)}"
-      end.join(" ")
-
-      connection.execute("#{sql} #{filter_conditions}").map { |r| r["id"] }
+      connection.execute(sql).map { |r| r["id"] }
     end
 
     def max_allowed_typos(length)
@@ -203,65 +246,38 @@ module FullSearch
 
       raw = connection.raw_connection
       raw.create_function("levenshtein", 2) do |func, s1, s2|
-        func.result = damerau_levenshtein(s1.to_s, s2.to_s)
+        func.result = Distance.damerau_levenshtein(s1.to_s, s2.to_s)
       end
       @levenshtein_registered = true
     end
 
-    def damerau_levenshtein(a, b)
-      a_len = a.length
-      b_len = b.length
-      return a_len if b_len == 0
-      return b_len if a_len == 0
-
-      d = Array.new(a_len + 1) { Array.new(b_len + 1, 0) }
-      (0..a_len).each { |i| d[i][0] = i }
-      (0..b_len).each { |j| d[0][j] = j }
-
-      (1..a_len).each do |i|
-        (1..b_len).each do |j|
-          cost = a[i - 1] == b[j - 1] ? 0 : 1
-          d[i][j] = [
-            d[i - 1][j] + 1,
-            d[i][j - 1] + 1,
-            d[i - 1][j - 1] + cost
-          ].min
-
-          if i > 1 && j > 1 && a[i - 1] == b[j - 2] && a[i - 2] == b[j - 1]
-            d[i][j] = [d[i][j], d[i - 2][j - 2] + 1].min
-          end
-        end
-      end
-
-      d[a_len][b_len]
-    end
-
     def apply_ranking(rel, all_ids, exact_ids)
       return rel if all_ids.empty?
 
       order_parts = []
+      tbl = qt(model.table_name)
 
       if exact_ids.any?
-        order_parts << "CASE #{model.table_name}.id #{exact_ids.map { |id| "WHEN #{id} THEN 0" }.join(" ")} ELSE 1 END"
+        order_parts << "CASE #{tbl}.id #{exact_ids.map { |id| "WHEN #{q(id)} THEN 0" }.join(" ")} ELSE 1 END"
       end
 
-      fts_table = FullSearch::Index.fts_table_name(model)
+      fts_table = qt(FullSearch::Index.fts_table_name(model))
       match_expr = QueryParser.to_match_expression(QueryParser.parse(query))
 
       rank_subquery = <<~SQL
         SELECT rowid, rank
         FROM #{fts_table}
-        WHERE #{fts_table} MATCH #{connection.quote(match_expr)}
+        WHERE #{fts_table} MATCH #{q(match_expr)}
       SQL
 
       rel = rel
-        .select("#{model.table_name}.*, fts_rank.rank AS full_search_rank")
-        .joins("LEFT JOIN (#{rank_subquery}) AS fts_rank ON fts_rank.rowid = #{model.table_name}.id")
+        .select("#{tbl}.*, fts_rank.rank AS full_search_rank")
+        .joins("LEFT JOIN (#{rank_subquery}) AS fts_rank ON fts_rank.rowid = #{tbl}.id")
 
       order_parts << "COALESCE(fts_rank.rank, 1)"
 
       dsl.rank_bys.each do |rank_by|
-        col = "#{connection.quote_table_name(model.table_name)}.#{connection.quote_column_name(rank_by.column)}"
+        col = "#{tbl}.#{qc(rank_by.column)}"
         order_parts << "#{col} #{rank_by.direction.to_s.upcase} NULLS LAST"
       end
 
@@ -269,7 +285,11 @@ module FullSearch
     end
 
     def connection
+      model.connection
+    rescue NoMethodError
       ActiveRecord::Base.connection
     end
+
+    include Quoting
   end
 end

data/lib/full_search/soft_delete.rb

@@ -2,11 +2,24 @@
 
 module FullSearch
   module SoftDelete
-    def self.delete_transition_sql(model)
+    def self.active_update_clause(model)
       dsl = model.full_search_dsl
       return nil unless dsl&.soft_delete_column
 
-      "WHEN new.#{dsl.soft_delete_column} IS NULL"
+      col = model.connection.quote_column_name(dsl.soft_delete_column)
+      "WHEN new.#{col} IS NULL"
+    end
+
+    def self.soft_delete_remove_clause(model)
+      dsl = model.full_search_dsl
+      return nil unless dsl&.soft_delete_column
+
+      col = model.connection.quote_column_name(dsl.soft_delete_column)
+      "WHEN old.#{col} IS NULL AND new.#{col} IS NOT NULL"
+    end
+
+    def self.delete_transition_sql(model)
+      active_update_clause(model)
     end
   end
 end

data/lib/full_search/typo.rb

@@ -5,23 +5,25 @@ module FullSearch
     MIN_SQLITE_VERSION = [3, 34].freeze
 
     class << self
-      def supported?
-        major, minor = sqlite_version_parts
+      def supported?(model = nil)
+        conn = model&.connection || ActiveRecord::Base.connection
+        major, minor = sqlite_version_parts(conn)
         major > MIN_SQLITE_VERSION.first || (major == MIN_SQLITE_VERSION.first && minor >= MIN_SQLITE_VERSION.last)
       end
 
-      def warn_unsupported!
-        warn "[full_search] SQLite #{sqlite_version} does not support the trigram tokenizer. typo_tolerance requires SQLite >= 3.34."
+      def warn_unsupported!(model = nil)
+        version = sqlite_version(model&.connection || ActiveRecord::Base.connection)
+        warn "[full_search] SQLite #{version} does not support the trigram tokenizer. typo_tolerance requires SQLite >= 3.34."
       end
 
       private
 
-      def sqlite_version_parts
-        sqlite_version.split(".").first(2).map(&:to_i)
+      def sqlite_version_parts(conn)
+        sqlite_version(conn).split(".").first(2).map(&:to_i)
       end
 
-      def sqlite_version
-        ActiveRecord::Base.connection.execute("SELECT sqlite_version() AS v").first["v"]
+      def sqlite_version(conn)
+        conn.execute("SELECT sqlite_version() AS v").first["v"]
       end
     end
   end

data/lib/full_search/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module FullSearch
-  VERSION = "0.2.0"
+  VERSION = "0.3.1"
 end

data/lib/full_search.rb

@@ -6,6 +6,8 @@ require "active_support/concern"
 
 require "full_search/version"
 require "full_search/config"
+require "full_search/distance"
+require "full_search/quoting"
 require "full_search/errors"
 require "full_search/tokenizer"
 require "full_search/dsl"
@@ -22,6 +24,7 @@ require "full_search/reindex_job"
 require "full_search/optimize_job"
 require "full_search/test_helpers"
 require "full_search/multi_search"
+require "full_search/schema_dumper_patch"
 
 ActiveSupport.on_load(:active_record) do
   include FullSearch::Model
@@ -30,7 +33,15 @@ end
 module FullSearch
   class << self
     def models
-      @models ||= []
+      @models ||= Set.new
+    end
+
+    def register_model(model)
+      models << model
+    end
+
+    def deregister_model(model)
+      models.delete(model)
     end
 
     def optimize!

data/lib/tasks/full_search.rake

@@ -3,7 +3,12 @@
 def resolve_full_search_models(args)
   if args[:models]
     args[:models].split(",").map do |name|
-      klass = name.singularize.camelize.constantize rescue nil
+      klass = begin
+        name.singularize.camelize.constantize
+      rescue NameError, LoadError => e
+        warn "[full_search] Could not resolve model '#{name}': #{e.message}"
+        nil
+      end
       klass || FullSearch.models.find { |m| m.table_name == name }
     end.compact
   else
@@ -50,14 +55,17 @@ namespace :full_search do
     FullSearch.models.each do |model|
       stored = FullSearch::Index.stored_config_hash(model)
       current = model.full_search_dsl.config_hash
-      status = stored == current ? "ok" : "stale"
+      status = (stored == current) ? "ok" : "stale"
       source_fields = model.full_search_dsl.fields.select(&:source).map(&:name)
 
       drift_info = if source_fields.any?
+        conn = ActiveRecord::Base.connection
+        qc = ->(name) { conn.quote_column_name(name) }
+        qt = ->(name) { conn.quote_table_name(name) }
         empty_count = ActiveRecord::Base.connection.execute(<<~SQL).first["c"]
           SELECT COUNT(*) AS c
-          FROM #{FullSearch::Index.fts_table_name(model)}
-          WHERE #{source_fields.map { |c| "#{c} = ''" }.join(' OR ')}
+          FROM #{qt.call(FullSearch::Index.fts_table_name(model))}
+          WHERE #{source_fields.map { |c| "#{qc.call(c)} = ''" }.join(" OR ")}
         SQL
         " | empty source fields: #{empty_count}"
       else

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: full_search
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.1
 platform: ruby
 authors:
 - Ben D'Angelo
@@ -16,6 +16,9 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '8.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '9'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -23,6 +26,9 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '8.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '9'
 - !ruby/object:Gem::Dependency
   name: railties
   requirement: !ruby/object:Gem::Requirement
@@ -30,6 +36,9 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '8.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '9'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -37,6 +46,9 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '8.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '9'
 - !ruby/object:Gem::Dependency
   name: sqlite3
   requirement: !ruby/object:Gem::Requirement
@@ -51,20 +63,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '2.0'
-- !ruby/object:Gem::Dependency
-  name: ostruct
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0.6'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0.6'
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
@@ -93,6 +91,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '13.0'
+- !ruby/object:Gem::Dependency
+  name: ostruct
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.6'
 - !ruby/object:Gem::Dependency
   name: irb
   requirement: !ruby/object:Gem::Requirement
@@ -107,6 +119,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: standard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.40'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.40'
 description: Declarative full-text search for Rails apps backed by SQLite FTS5
 executables: []
 extensions: []
@@ -116,6 +142,7 @@ files:
 - lib/full_search.rb
 - lib/full_search/callbacks.rb
 - lib/full_search/config.rb
+- lib/full_search/distance.rb
 - lib/full_search/dsl.rb
 - lib/full_search/errors.rb
 - lib/full_search/exact_match.rb
@@ -125,8 +152,10 @@ files:
 - lib/full_search/multi_search.rb
 - lib/full_search/optimize_job.rb
 - lib/full_search/query_parser.rb
+- lib/full_search/quoting.rb
 - lib/full_search/railtie.rb
 - lib/full_search/reindex_job.rb
+- lib/full_search/schema_dumper_patch.rb
 - lib/full_search/search.rb
 - lib/full_search/soft_delete.rb
 - lib/full_search/test_helpers.rb
@@ -147,12 +176,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 4.0.0
+      version: 3.1.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '2.0'
 requirements: []
 rubygems_version: 4.0.15
 specification_version: 4