full_search 0.2.0 → 0.3.0

data/lib/full_search/index.rb

@@ -1,12 +1,21 @@
 # frozen_string_literal: true
 
-require "ostruct"
+class FilterColumnPlaceholder
+  attr_reader :name
+  def initialize(name:)
+    @name = name.to_s
+  end
+
+  def unindexed?
+    true
+  end
+end
 
 module FullSearch
   class Index
     class << self
       def ensure_table!(model)
-        return unless FullSearch::Index.sqlite?
+        sqlite!(model)
 
         conn = connection
         dsl = model.full_search_dsl
@@ -14,30 +23,31 @@ module FullSearch
 
         create_metadata_table!
 
-        table_was_created = false
+        fts_was_created = false
         unless table_exists?(model)
           conn.execute(create_virtual_table_sql(model))
-          table_was_created = true
+          fts_was_created = true
         end
 
+        trigram_was_created = false
         if dsl.typo_tolerance? && !trigram_table_exists?(model)
-          FullSearch::Typo.warn_unsupported! unless FullSearch::Typo.supported?
+          FullSearch::Typo.warn_unsupported!(model) unless FullSearch::Typo.supported?(model)
           conn.execute(create_trigram_virtual_table_sql(model))
-          table_was_created = true
+          trigram_was_created = true
         end
 
-        if table_was_created
-          conn.execute(backfill_sql(model))
-          conn.execute(backfill_trigram_sql(model)) if dsl.typo_tolerance?
+        if fts_was_created || trigram_was_created
+          conn.execute(backfill_sql(model)) if fts_was_created
+          conn.execute(backfill_trigram_sql(model)) if trigram_was_created
           reindex_source_fields!(model) if dsl.fields.any?(&:source)
-          store_config_hash!(model)
         end
 
+        store_config_hash!(model)
         ensure_triggers!(model) if model_table_exists?(model)
       end
 
       def rebuild!(model)
-        return unless FullSearch::Index.sqlite?
+        sqlite!(model)
 
         dsl = model.full_search_dsl
         return unless dsl
@@ -47,11 +57,11 @@ module FullSearch
 
         with_rebuild_lock(model) do
           drop_triggers!(model)
-          conn.execute("DROP TABLE IF EXISTS #{fts_table_name(model)};")
-          conn.execute("DROP TABLE IF EXISTS #{trigram_table_name(model)};")
+          conn.execute("DROP TABLE IF EXISTS #{qt(fts_table_name(model))};")
+          conn.execute("DROP TABLE IF EXISTS #{qt(trigram_table_name(model))};")
           conn.execute(create_virtual_table_sql(model))
           if dsl.typo_tolerance?
-            FullSearch::Typo.warn_unsupported! unless FullSearch::Typo.supported?
+            FullSearch::Typo.warn_unsupported!(model) unless FullSearch::Typo.supported?(model)
             conn.execute(create_trigram_virtual_table_sql(model))
           end
           conn.execute(backfill_sql(model))
@@ -64,7 +74,7 @@ module FullSearch
       end
 
       def rebuild_if_needed!(model)
-        return false unless FullSearch::Index.sqlite?
+        sqlite!(model)
 
         dsl = model.full_search_dsl
         return false unless dsl
@@ -82,7 +92,8 @@ module FullSearch
       end
 
       def optimize!(model)
-        connection.execute("INSERT INTO #{fts_table_name(model)}(#{fts_table_name(model)}) VALUES('optimize');")
+        sqlite!(model)
+        connection.execute("INSERT INTO #{qt(fts_table_name(model))}(#{qt(fts_table_name(model))}) VALUES('optimize');")
       end
 
       def reindex_source_fields!(model)
@@ -92,9 +103,10 @@ module FullSearch
       end
 
       def drop!(model)
+        sqlite!(model)
         drop_triggers!(model)
-        connection.execute("DROP TABLE IF EXISTS #{fts_table_name(model)};")
-        connection.execute("DROP TABLE IF EXISTS #{trigram_table_name(model)};")
+        connection.execute("DROP TABLE IF EXISTS #{qt(fts_table_name(model))};")
+        connection.execute("DROP TABLE IF EXISTS #{qt(trigram_table_name(model))};")
       end
 
       def fts_table_name(model)
@@ -105,10 +117,21 @@ module FullSearch
         "#{fts_table_name(model)}_trigram"
       end
 
-      def sqlite?
+      def sqlite?(model = nil)
+        if model
+          model.connection.adapter_name.downcase.include?("sqlite")
+        else
+          connection.adapter_name.downcase.include?("sqlite")
+        end
+      rescue
         connection.adapter_name.downcase.include?("sqlite")
       end
 
+      def sqlite!(model)
+        adapter = model ? model.connection.adapter_name : connection.adapter_name
+        raise UnsupportedDatabaseError, "full_search requires SQLite, but #{adapter} is configured" unless sqlite?(model)
+      end
+
       def stored_config_hash(model)
         row = connection.execute(
           "SELECT config_hash FROM full_search_index_versions WHERE table_name=#{q(model.table_name)}"
@@ -126,6 +149,14 @@ module FullSearch
         connection.quote(value)
       end
 
+      def qt(name)
+        connection.quote_table_name(name)
+      end
+
+      def qc(name)
+        connection.quote_column_name(name)
+      end
+
       def model_table_exists?(model)
         connection.execute(
           "SELECT name FROM sqlite_master WHERE type='table' AND name=#{q(model.table_name)} LIMIT 1"
@@ -156,11 +187,11 @@ module FullSearch
 
       def create_virtual_table_sql(model)
         dsl = model.full_search_dsl
-        columns = (dsl.fields + dsl.filters.map { |f| OpenStruct.new(name: f.name, unindexed?: true) })
-        column_list = columns.map { |c| c.respond_to?(:unindexed?) && c.unindexed? ? "#{c.name} UNINDEXED" : c.name }.join(", ")
+        columns = (dsl.fields + dsl.filters.map { |f| FilterColumnPlaceholder.new(name: f.name) })
+        column_list = columns.map { |c| (c.respond_to?(:unindexed?) && c.unindexed?) ? "#{qc(c.name)} UNINDEXED" : qc(c.name) }.join(", ")
 
         <<~SQL
-          CREATE VIRTUAL TABLE #{fts_table_name(model)} USING fts5(
+          CREATE VIRTUAL TABLE #{qt(fts_table_name(model))} USING fts5(
             #{column_list},
             tokenize='#{dsl.tokenize}'
           );
@@ -174,13 +205,13 @@ module FullSearch
           if c.respond_to?(:source) && c.source
             source_value_sql(c.source)
           else
-            "#{model.table_name}.#{c.name}"
+            "#{qt(model.table_name)}.#{qc(c.name)}"
           end
         end.join(", ")
 
         <<~SQL
-          INSERT INTO #{fts_table_name(model)}(rowid, #{cols.map(&:name).join(", ")})
-          SELECT #{model.table_name}.id, #{select} FROM #{model.table_name};
+          INSERT INTO #{qt(fts_table_name(model))}(rowid, #{cols.map { |c| qc(c.name) }.join(", ")})
+          SELECT #{qt(model.table_name)}.id, #{select} FROM #{qt(model.table_name)};
         SQL
       end
 
@@ -193,13 +224,20 @@ module FullSearch
           "SELECT name FROM sqlite_master WHERE type='trigger' AND tbl_name=#{q(model.table_name)}"
         ).map { |r| r["name"] }
 
-        create_triggers!(model) unless (trigger_names(model) - existing).empty?
+        expected = trigger_names(model)
+        expected += trigram_trigger_names(model) if model.full_search_dsl.typo_tolerance?
+        return if (expected - existing).empty? && (existing - expected).empty?
+
+        rebuild!(model)
       end
 
       def create_triggers!(model)
         connection.execute(insert_trigger_sql(model))
         connection.execute(delete_trigger_sql(model))
         connection.execute(update_trigger_sql(model))
+        if model.full_search_dsl.soft_delete_column
+          connection.execute(soft_delete_removal_trigger_sql(model))
+        end
         if model.full_search_dsl.typo_tolerance?
           connection.execute(insert_trigram_trigger_sql(model))
           connection.execute(delete_trigram_trigger_sql(model))
@@ -209,13 +247,15 @@ module FullSearch
 
       def drop_triggers!(model)
         (trigger_names(model) + trigram_trigger_names(model)).each do |name|
-          connection.execute("DROP TRIGGER IF EXISTS #{name};")
+          connection.execute("DROP TRIGGER IF EXISTS #{qt(name)};")
         end
       end
 
       def trigger_names(model)
         base = fts_table_name(model)
-        %W[#{base}_ai #{base}_ad #{base}_au]
+        names = %W[#{base}_ai #{base}_ad #{base}_au]
+        names << "#{base}_au_soft_delete" if model.full_search_dsl.soft_delete_column
+        names
       end
 
       def trigram_trigger_names(model)
@@ -229,8 +269,8 @@ module FullSearch
         values = cols.map { |c| column_ref(c, prefix: "new") }.join(", ")
 
         <<~SQL
-          CREATE TRIGGER #{trigger_names(model).first} AFTER INSERT ON #{model.table_name} BEGIN
-            INSERT INTO #{fts_table_name(model)}(rowid, #{col_names(cols)})
+          CREATE TRIGGER #{qt(trigger_names(model).first)} AFTER INSERT ON #{qt(model.table_name)} BEGIN
+            INSERT INTO #{qt(fts_table_name(model))}(rowid, #{col_names(cols)})
             VALUES (new.id, #{values});
           END;
         SQL
@@ -238,8 +278,8 @@ module FullSearch
 
       def delete_trigger_sql(model)
         <<~SQL
-          CREATE TRIGGER #{trigger_names(model)[1]} AFTER DELETE ON #{model.table_name} BEGIN
-            DELETE FROM #{fts_table_name(model)} WHERE rowid = old.id;
+          CREATE TRIGGER #{qt(trigger_names(model)[1])} AFTER DELETE ON #{qt(model.table_name)} BEGIN
+            DELETE FROM #{qt(fts_table_name(model))} WHERE rowid = old.id;
           END;
         SQL
       end
@@ -249,12 +289,12 @@ module FullSearch
         cols = dsl.fields + dsl.filters
         values = cols.map { |c| column_ref(c, prefix: "new") }.join(", ")
         cols_str = col_names(cols)
-        fts_table = fts_table_name(model)
+        fts_table = qt(fts_table_name(model))
 
-        when_clause = SoftDelete.delete_transition_sql(model)
+        when_clause = SoftDelete.active_update_clause(model)
 
         <<~SQL
-          CREATE TRIGGER #{trigger_names(model)[2]} AFTER UPDATE ON #{model.table_name} #{when_clause}
+          CREATE TRIGGER #{qt(trigger_names(model)[2])} AFTER UPDATE ON #{qt(model.table_name)} #{when_clause}
           BEGIN
             DELETE FROM #{fts_table} WHERE rowid = old.id;
             INSERT INTO #{fts_table}(rowid, #{cols_str})
@@ -263,6 +303,19 @@ module FullSearch
         SQL
       end
 
+      def soft_delete_removal_trigger_sql(model)
+        model.full_search_dsl
+        fts_table = qt(fts_table_name(model))
+        when_clause = SoftDelete.soft_delete_remove_clause(model)
+
+        <<~SQL
+          CREATE TRIGGER #{qt("#{trigger_names(model)[2]}_soft_delete")} AFTER UPDATE ON #{qt(model.table_name)} #{when_clause}
+          BEGIN
+            DELETE FROM #{fts_table} WHERE rowid = old.id;
+          END;
+        SQL
+      end
+
       def trigram_table_exists?(model)
         connection.execute(
           "SELECT name FROM sqlite_master WHERE type='table' AND name=#{q(trigram_table_name(model))} LIMIT 1"
@@ -271,11 +324,11 @@ module FullSearch
 
       def create_trigram_virtual_table_sql(model)
         dsl = model.full_search_dsl
-        columns = (dsl.fields + dsl.filters.map { |f| OpenStruct.new(name: f.name, unindexed?: true) })
-        column_list = columns.map { |c| c.respond_to?(:unindexed?) && c.unindexed? ? "#{c.name} UNINDEXED" : c.name }.join(", ")
+        columns = (dsl.fields + dsl.filters.map { |f| FilterColumnPlaceholder.new(name: f.name) })
+        column_list = columns.map { |c| (c.respond_to?(:unindexed?) && c.unindexed?) ? "#{qc(c.name)} UNINDEXED" : qc(c.name) }.join(", ")
 
         <<~SQL
-          CREATE VIRTUAL TABLE #{trigram_table_name(model)} USING fts5(
+          CREATE VIRTUAL TABLE #{qt(trigram_table_name(model))} USING fts5(
             #{column_list},
             tokenize='trigram'
           );
@@ -289,13 +342,13 @@ module FullSearch
           if c.respond_to?(:source) && c.source
             source_value_sql(c.source)
           else
-            "#{model.table_name}.#{c.name}"
+            "#{qt(model.table_name)}.#{qc(c.name)}"
           end
         end.join(", ")
 
         <<~SQL
-          INSERT INTO #{trigram_table_name(model)}(rowid, #{cols.map(&:name).join(", ")})
-          SELECT #{model.table_name}.id, #{select} FROM #{model.table_name};
+          INSERT INTO #{qt(trigram_table_name(model))}(rowid, #{cols.map { |c| qc(c.name) }.join(", ")})
+          SELECT #{qt(model.table_name)}.id, #{select} FROM #{qt(model.table_name)};
         SQL
       end
 
@@ -305,8 +358,8 @@ module FullSearch
         values = cols.map { |c| column_ref(c, prefix: "new") }.join(", ")
 
         <<~SQL
-          CREATE TRIGGER #{trigram_trigger_names(model).first} AFTER INSERT ON #{model.table_name} BEGIN
-            INSERT INTO #{trigram_table_name(model)}(rowid, #{col_names(cols)})
+          CREATE TRIGGER #{qt(trigram_trigger_names(model).first)} AFTER INSERT ON #{qt(model.table_name)} BEGIN
+            INSERT INTO #{qt(trigram_table_name(model))}(rowid, #{col_names(cols)})
             VALUES (new.id, #{values});
           END;
         SQL
@@ -314,8 +367,8 @@ module FullSearch
 
       def delete_trigram_trigger_sql(model)
         <<~SQL
-          CREATE TRIGGER #{trigram_trigger_names(model)[1]} AFTER DELETE ON #{model.table_name} BEGIN
-            DELETE FROM #{trigram_table_name(model)} WHERE rowid = old.id;
+          CREATE TRIGGER #{qt(trigram_trigger_names(model)[1])} AFTER DELETE ON #{qt(model.table_name)} BEGIN
+            DELETE FROM #{qt(trigram_table_name(model))} WHERE rowid = old.id;
           END;
         SQL
       end
@@ -325,12 +378,12 @@ module FullSearch
         cols = dsl.fields + dsl.filters
         values = cols.map { |c| column_ref(c, prefix: "new") }.join(", ")
         cols_str = col_names(cols)
-        trigram_table = trigram_table_name(model)
+        trigram_table = qt(trigram_table_name(model))
 
-        when_clause = SoftDelete.delete_transition_sql(model)
+        when_clause = SoftDelete.active_update_clause(model)
 
         <<~SQL
-          CREATE TRIGGER #{trigram_trigger_names(model)[2]} AFTER UPDATE ON #{model.table_name} #{when_clause}
+          CREATE TRIGGER #{qt(trigram_trigger_names(model)[2])} AFTER UPDATE ON #{qt(model.table_name)} #{when_clause}
           BEGIN
             DELETE FROM #{trigram_table} WHERE rowid = old.id;
             INSERT INTO #{trigram_table}(rowid, #{cols_str})
@@ -340,22 +393,25 @@ module FullSearch
       end
 
       def col_names(cols)
-        cols.map(&:name).join(", ")
+        cols.map { |c| qc(c.name) }.join(", ")
       end
 
       def column_ref(col, prefix:)
         if col.respond_to?(:source) && col.source
           "''"
         else
-          "#{prefix}.#{col.name}"
+          "#{prefix}.#{qc(col.name)}"
         end
       end
 
+      # NOTE: This lock prevents concurrent rebuilds within the same process/connection only.
+      # For multi-process or multi-host deployments, run full_search:rebuild from a single
+      # deployment step. See README.
       def with_rebuild_lock(model)
         if FullSearch.config.lock_rebuilds
           connection.transaction do
             connection.execute(
-              "INSERT INTO full_search_index_versions (table_name, config_hash, rebuilt_at) VALUES (#{q(model.table_name)}, #{q("")}, datetime('now'))
+              "INSERT INTO full_search_index_versions (table_name, config_hash, rebuilt_at) VALUES (#{q(model.table_name)}, #{q("__rebuilding__")}, datetime('now'))
                ON CONFLICT(table_name) DO UPDATE SET config_hash=excluded.config_hash;"
             )
             yield

data/lib/full_search/model.rb

@@ -13,7 +13,8 @@ module FullSearch
           FullSearch::Index.ensure_table!(self)
           FullSearch::Callbacks.install!(self)
           include InstanceMethods
-          FullSearch.models << self unless FullSearch.models.include?(self)
+
+          FullSearch.register_model(self)
           @full_search_dsl
         else
           FullSearch::Search.new(self, query_or_options, filters: filters, include_soft_deleted: include_soft_deleted, limit: limit, offset: offset, highlight: highlight, highlight_fields: highlight_fields, matching_strategy: matching_strategy).relation

data/lib/full_search/multi_search.rb

@@ -14,6 +14,8 @@ module FullSearch
     def call
       searched = groups.map do |group|
         model = fetch(group, :model)
+        raise FullSearch::NotConfiguredError, "#{model} is not full_search configured" unless model.full_search_dsl
+
         filters = group[:filters] || {}
         limit = positive_integer(group[:limit], 8)
         offset = positive_integer(group[:offset], 0)
@@ -24,6 +26,7 @@ module FullSearch
           filters: filters,
           limit: raw_limit,
           offset: offset,
+          highlight: group[:highlight],
           highlight_fields: group[:highlight_fields]
         )
 
@@ -40,7 +43,7 @@ module FullSearch
         )
       end
 
-      { groups: searched, total_count: searched.sum { |g| g[:total_count] } }
+      {groups: searched, total_count: searched.sum { |g| g[:total_count] }}
     end
 
     private

data/lib/full_search/query_parser.rb

@@ -6,121 +6,134 @@ module FullSearch
   module QueryParser
     Token = Data.define(:type, :value)
 
-    def self.parse(query)
-      tokens = tokenize(query.to_s.strip)
-      return [] if tokens.empty?
+    MAX_QUERY_LENGTH = 255
 
-      or_clauses = split_or(tokens)
+    class << self
+      def parse(query)
+        query = query.to_s.strip
+        validate!(query)
+        tokens = tokenize(query)
+        return [] if tokens.empty?
 
-      if or_clauses.size == 1
-        parse_and_clause(or_clauses.first)
-      else
-        [:or, or_clauses.map { |clause| parse_and_clause(clause) }]
+        or_clauses = split_or(tokens)
+
+        if or_clauses.size == 1
+          parse_and_clause(or_clauses.first)
+        else
+          [:or, or_clauses.map { |clause| parse_and_clause(clause) }]
+        end
       end
-    end
 
-    def self.to_match_expression(parsed)
-      return '""' if parsed.empty?
-
-      type, value = parsed
-      case type
-      when :and
-        build_and_expression(value)
-      when :or
-        value.map { |sub| to_match_expression(sub) }.join(" OR ")
-      else
-        node_to_match(parsed)
+      def to_match_expression(parsed)
+        return '""' if parsed.empty?
+
+        type, value = parsed
+        case type
+        when :and
+          build_and_expression(value)
+        when :or
+          value.map { |sub| to_match_expression(sub) }.join(" OR ")
+        else
+          node_to_match(parsed)
+        end
       end
-    end
 
-    private
+      private
 
-    def self.build_and_expression(nodes)
-      non_excludes = nodes.reject { |n| n.first == :exclude }
-      excludes = nodes.select { |n| n.first == :exclude }
+      def build_and_expression(nodes)
+        non_excludes = nodes.reject { |n| n.first == :exclude }
+        excludes = nodes.select { |n| n.first == :exclude }
 
-      parts = non_excludes.map { |n| node_to_match(n) }
-      result = parts.size == 1 ? parts.first : "(#{parts.join(' AND ')})"
+        parts = non_excludes.map { |n| node_to_match(n) }
+        result = (parts.size == 1) ? parts.first : "(#{parts.join(" AND ")})"
 
-      excludes.each do |node|
-        _, value = node
-        result = "#{result} NOT \"#{escape(value)}\""
-      end
+        excludes.each do |node|
+          _, value = node
+          result = "#{result} NOT \"#{escape(value)}\""
+        end
 
-      result
-    end
+        result
+      end
 
-    def self.node_to_match(node)
-      type, value = node
-      case type
-      when :term
-        '"' + escape(value) + '"*'
-      when :phrase
-        '"' + escape(value) + '"'
-      when :exclude
-        nil
+      def node_to_match(node)
+        type, value = node
+        case type
+        when :term
+          '"' + escape(value) + '"*'
+        when :phrase
+          '"' + escape(value) + '"'
+        when :exclude
+          nil
+        end
       end
-    end
 
-    def self.tokenize(query)
-      tokens = []
-      scanner = StringScanner.new(query)
+      def validate!(query)
+        if query.bytesize > MAX_QUERY_LENGTH
+          raise InvalidQueryError, "Query too long (max #{MAX_QUERY_LENGTH} chars)"
+        end
+        raise InvalidQueryError, "Query contains invalid characters" if query.include?("\x00")
+      end
 
-      until scanner.eos?
-        scanner.skip(/\s+/)
-        break if scanner.eos?
+      def tokenize(query)
+        tokens = []
+        scanner = StringScanner.new(query)
 
-        if scanner.scan(/"/)
-          phrase = scanner.scan_until(/"/)
-          if phrase
-            tokens << Token.new(:phrase, phrase.chomp('"'))
+        until scanner.eos?
+          scanner.skip(/\s+/)
+          break if scanner.eos?
+
+          if scanner.scan('"')
+            phrase = scanner.scan_until(/"/)
+            if phrase
+              tokens << Token.new(:phrase, phrase.chomp('"'))
+            else
+              tokens << Token.new(:term, scanner.rest)
+              scanner.terminate
+            end
+          elsif scanner.scan("-")
+            scanner.skip(/\s+/)
+            tokens << Token.new(:exclude, scanner.scan(/\S+/))
+          elsif scanner.scan(/OR|or/i)
+            tokens << Token.new(:or, nil)
           else
-            tokens << Token.new(:term, scanner.rest)
-            scanner.terminate
+            tokens << Token.new(:term, scanner.scan(/\S+/))
           end
-        elsif scanner.scan(/-/)
-          scanner.skip(/\s+/)
-          tokens << Token.new(:exclude, scanner.scan(/\S+/))
-        elsif scanner.scan(/OR|or/i)
-          tokens << Token.new(:or, nil)
-        else
-          tokens << Token.new(:term, scanner.scan(/\S+/))
         end
-      end
 
-      tokens
-    end
+        tokens
+      end
 
-    def self.split_or(tokens)
-      clauses = []
-      current = []
+      def split_or(tokens)
+        clauses = []
+        current = []
 
-      tokens.each do |token|
-        if token.type == :or
-          clauses << current unless current.empty?
-          current = []
-        else
-          current << token
+        tokens.each do |token|
+          if token.type == :or
+            clauses << current unless current.empty?
+            current = []
+          else
+            current << token
+          end
         end
+        clauses << current unless current.empty?
+        clauses
       end
-      clauses << current unless current.empty?
-      clauses
-    end
 
-    def self.parse_and_clause(tokens)
-      nodes = tokens.map do |token|
-        case token.type
-        when :term then [:term, token.value]
-        when :phrase then [:phrase, token.value]
-        when :exclude then [:exclude, token.value]
+      def parse_and_clause(tokens)
+        nodes = tokens.map do |token|
+          case token.type
+          when :term then [:term, token.value]
+          when :phrase then [:phrase, token.value]
+          when :exclude then [:exclude, token.value]
+          end
         end
-      end
 
-      nodes.size == 1 ? nodes.first : [:and, nodes]
-    end
+        (nodes.size == 1) ? nodes.first : [:and, nodes]
+      end
 
-    def self.escape(value)
-      value.to_s.gsub('"', '""')
+      def escape(value)
+        value.to_s.gsub('"', '""')
+      end
     end
   end
 end

data/lib/full_search/quoting.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module FullSearch
+  module Quoting
+    private
+
+    def q(value) = connection.quote(value)
+    def qt(name) = connection.quote_table_name(name)
+    def qc(name) = connection.quote_column_name(name)
+  end
+end