full_search 0.1.0

data/lib/full_search/index.rb

@@ -0,0 +1,335 @@
+# frozen_string_literal: true
+
+require "ostruct"
+
+module FullSearch
+  class Index
+    class << self
+      def ensure_table!(model)
+        return unless FullSearch::Index.sqlite?
+
+        conn = connection
+        dsl = model.full_search_dsl
+        return unless dsl
+
+        create_metadata_table!
+
+        unless table_exists?(model)
+          conn.execute(create_virtual_table_sql(model))
+        end
+
+        if dsl.typo_tolerance? && !trigram_table_exists?(model)
+          FullSearch::Typo.warn_unsupported! unless FullSearch::Typo.supported?
+          conn.execute(create_trigram_virtual_table_sql(model))
+        end
+
+        ensure_triggers!(model) if model_table_exists?(model)
+        store_config_hash!(model)
+      end
+
+      def rebuild!(model)
+        return unless FullSearch::Index.sqlite?
+
+        dsl = model.full_search_dsl
+        return unless dsl
+        conn = connection
+
+        create_metadata_table!
+
+        with_rebuild_lock(model) do
+          drop_triggers!(model)
+          conn.execute("DROP TABLE IF EXISTS #{fts_table_name(model)};")
+          conn.execute("DROP TABLE IF EXISTS #{trigram_table_name(model)};")
+          conn.execute(create_virtual_table_sql(model))
+          if dsl.typo_tolerance?
+            FullSearch::Typo.warn_unsupported! unless FullSearch::Typo.supported?
+            conn.execute(create_trigram_virtual_table_sql(model))
+          end
+          conn.execute(backfill_sql(model))
+          conn.execute(backfill_trigram_sql(model)) if dsl.typo_tolerance?
+          create_triggers!(model)
+          optimize!(model)
+          store_config_hash!(model, rebuilt_at: Time.current)
+        end
+      end
+
+      def optimize!(model)
+        connection.execute("INSERT INTO #{fts_table_name(model)}(#{fts_table_name(model)}) VALUES('optimize');")
+      end
+
+      def drop!(model)
+        drop_triggers!(model)
+        connection.execute("DROP TABLE IF EXISTS #{fts_table_name(model)};")
+        connection.execute("DROP TABLE IF EXISTS #{trigram_table_name(model)};")
+      end
+
+      def fts_table_name(model)
+        "#{model.table_name}_fts"
+      end
+
+      def trigram_table_name(model)
+        "#{fts_table_name(model)}_trigram"
+      end
+
+      def sqlite?
+        connection.adapter_name.downcase.include?("sqlite")
+      end
+
+      def stored_config_hash(model)
+        row = connection.execute(
+          "SELECT config_hash FROM full_search_index_versions WHERE table_name=#{q(model.table_name)}"
+        ).first
+        row&.[]("config_hash")
+      end
+
+      private
+
+      def connection
+        ActiveRecord::Base.connection
+      end
+
+      def q(value)
+        connection.quote(value)
+      end
+
+      def model_table_exists?(model)
+        connection.execute(
+          "SELECT name FROM sqlite_master WHERE type='table' AND name=#{q(model.table_name)} LIMIT 1"
+        ).any?
+      end
+
+      def table_exists?(model)
+        connection.execute(
+          "SELECT name FROM sqlite_master WHERE type='table' AND name=#{q(fts_table_name(model))} LIMIT 1"
+        ).any?
+      end
+
+      def create_metadata_table!
+        connection.execute(<<~SQL)
+          CREATE TABLE IF NOT EXISTS full_search_index_versions (
+            table_name TEXT PRIMARY KEY,
+            config_hash TEXT NOT NULL,
+            rebuilt_at DATETIME NOT NULL
+          );
+        SQL
+      end
+
+      def store_config_hash!(model, rebuilt_at: Time.current)
+        sql = "INSERT INTO full_search_index_versions (table_name, config_hash, rebuilt_at) VALUES (#{q(model.table_name)}, #{q(model.full_search_dsl.config_hash)}, #{q(rebuilt_at)})
+               ON CONFLICT(table_name) DO UPDATE SET config_hash=excluded.config_hash, rebuilt_at=excluded.rebuilt_at;"
+        connection.execute(sql)
+      end
+
+      def create_virtual_table_sql(model)
+        dsl = model.full_search_dsl
+        columns = (dsl.fields + dsl.filters.map { |f| OpenStruct.new(name: f.name, unindexed?: true) })
+        column_list = columns.map { |c| c.respond_to?(:unindexed?) && c.unindexed? ? "#{c.name} UNINDEXED" : c.name }.join(", ")
+
+        <<~SQL
+          CREATE VIRTUAL TABLE #{fts_table_name(model)} USING fts5(
+            #{column_list},
+            tokenize='#{dsl.tokenize}'
+          );
+        SQL
+      end
+
+      def backfill_sql(model)
+        dsl = model.full_search_dsl
+        cols = (dsl.fields + dsl.filters)
+        select = cols.map do |c|
+          if c.respond_to?(:source) && c.source
+            source_value_sql(c.source)
+          else
+            "#{model.table_name}.#{c.name}"
+          end
+        end.join(", ")
+
+        <<~SQL
+          INSERT INTO #{fts_table_name(model)}(rowid, #{cols.map(&:name).join(", ")})
+          SELECT #{model.table_name}.id, #{select} FROM #{model.table_name};
+        SQL
+      end
+
+      def source_value_sql(_source_proc)
+        "''"
+      end
+
+      def ensure_triggers!(model)
+        existing = connection.execute(
+          "SELECT name FROM sqlite_master WHERE type='trigger' AND tbl_name=#{q(model.table_name)}"
+        ).map { |r| r["name"] }
+
+        create_triggers!(model) unless (trigger_names(model) - existing).empty?
+      end
+
+      def create_triggers!(model)
+        connection.execute(insert_trigger_sql(model))
+        connection.execute(delete_trigger_sql(model))
+        connection.execute(update_trigger_sql(model))
+        if model.full_search_dsl.typo_tolerance?
+          connection.execute(insert_trigram_trigger_sql(model))
+          connection.execute(delete_trigram_trigger_sql(model))
+          connection.execute(update_trigram_trigger_sql(model))
+        end
+      end
+
+      def drop_triggers!(model)
+        (trigger_names(model) + trigram_trigger_names(model)).each do |name|
+          connection.execute("DROP TRIGGER IF EXISTS #{name};")
+        end
+      end
+
+      def trigger_names(model)
+        base = fts_table_name(model)
+        %W[#{base}_ai #{base}_ad #{base}_au]
+      end
+
+      def trigram_trigger_names(model)
+        base = trigram_table_name(model)
+        %W[#{base}_ai #{base}_ad #{base}_au]
+      end
+
+      def insert_trigger_sql(model)
+        dsl = model.full_search_dsl
+        cols = dsl.fields + dsl.filters
+        values = cols.map { |c| column_ref(c, prefix: "new") }.join(", ")
+
+        <<~SQL
+          CREATE TRIGGER #{trigger_names(model).first} AFTER INSERT ON #{model.table_name} BEGIN
+            INSERT INTO #{fts_table_name(model)}(rowid, #{col_names(cols)})
+            VALUES (new.id, #{values});
+          END;
+        SQL
+      end
+
+      def delete_trigger_sql(model)
+        <<~SQL
+          CREATE TRIGGER #{trigger_names(model)[1]} AFTER DELETE ON #{model.table_name} BEGIN
+            DELETE FROM #{fts_table_name(model)} WHERE rowid = old.id;
+          END;
+        SQL
+      end
+
+      def update_trigger_sql(model)
+        dsl = model.full_search_dsl
+        cols = dsl.fields + dsl.filters
+        values = cols.map { |c| column_ref(c, prefix: "new") }.join(", ")
+        cols_str = col_names(cols)
+        fts_table = fts_table_name(model)
+
+        when_clause = SoftDelete.delete_transition_sql(model)
+
+        <<~SQL
+          CREATE TRIGGER #{trigger_names(model)[2]} AFTER UPDATE ON #{model.table_name} #{when_clause}
+          BEGIN
+            DELETE FROM #{fts_table} WHERE rowid = old.id;
+            INSERT INTO #{fts_table}(rowid, #{cols_str})
+            VALUES (new.id, #{values});
+          END;
+        SQL
+      end
+
+      def trigram_table_exists?(model)
+        connection.execute(
+          "SELECT name FROM sqlite_master WHERE type='table' AND name=#{q(trigram_table_name(model))} LIMIT 1"
+        ).any?
+      end
+
+      def create_trigram_virtual_table_sql(model)
+        dsl = model.full_search_dsl
+        columns = (dsl.fields + dsl.filters.map { |f| OpenStruct.new(name: f.name, unindexed?: true) })
+        column_list = columns.map { |c| c.respond_to?(:unindexed?) && c.unindexed? ? "#{c.name} UNINDEXED" : c.name }.join(", ")
+
+        <<~SQL
+          CREATE VIRTUAL TABLE #{trigram_table_name(model)} USING fts5(
+            #{column_list},
+            tokenize='trigram'
+          );
+        SQL
+      end
+
+      def backfill_trigram_sql(model)
+        dsl = model.full_search_dsl
+        cols = (dsl.fields + dsl.filters)
+        select = cols.map do |c|
+          if c.respond_to?(:source) && c.source
+            source_value_sql(c.source)
+          else
+            "#{model.table_name}.#{c.name}"
+          end
+        end.join(", ")
+
+        <<~SQL
+          INSERT INTO #{trigram_table_name(model)}(rowid, #{cols.map(&:name).join(", ")})
+          SELECT #{model.table_name}.id, #{select} FROM #{model.table_name};
+        SQL
+      end
+
+      def insert_trigram_trigger_sql(model)
+        dsl = model.full_search_dsl
+        cols = dsl.fields + dsl.filters
+        values = cols.map { |c| column_ref(c, prefix: "new") }.join(", ")
+
+        <<~SQL
+          CREATE TRIGGER #{trigram_trigger_names(model).first} AFTER INSERT ON #{model.table_name} BEGIN
+            INSERT INTO #{trigram_table_name(model)}(rowid, #{col_names(cols)})
+            VALUES (new.id, #{values});
+          END;
+        SQL
+      end
+
+      def delete_trigram_trigger_sql(model)
+        <<~SQL
+          CREATE TRIGGER #{trigram_trigger_names(model)[1]} AFTER DELETE ON #{model.table_name} BEGIN
+            DELETE FROM #{trigram_table_name(model)} WHERE rowid = old.id;
+          END;
+        SQL
+      end
+
+      def update_trigram_trigger_sql(model)
+        dsl = model.full_search_dsl
+        cols = dsl.fields + dsl.filters
+        values = cols.map { |c| column_ref(c, prefix: "new") }.join(", ")
+        cols_str = col_names(cols)
+        trigram_table = trigram_table_name(model)
+
+        when_clause = SoftDelete.delete_transition_sql(model)
+
+        <<~SQL
+          CREATE TRIGGER #{trigram_trigger_names(model)[2]} AFTER UPDATE ON #{model.table_name} #{when_clause}
+          BEGIN
+            DELETE FROM #{trigram_table} WHERE rowid = old.id;
+            INSERT INTO #{trigram_table}(rowid, #{cols_str})
+            VALUES (new.id, #{values});
+          END;
+        SQL
+      end
+
+      def col_names(cols)
+        cols.map(&:name).join(", ")
+      end
+
+      def column_ref(col, prefix:)
+        if col.respond_to?(:source) && col.source
+          "''"
+        else
+          "#{prefix}.#{col.name}"
+        end
+      end
+
+      def with_rebuild_lock(model)
+        if FullSearch.config.lock_rebuilds
+          connection.transaction do
+            connection.execute(
+              "INSERT INTO full_search_index_versions (table_name, config_hash, rebuilt_at) VALUES (#{q(model.table_name)}, #{q("")}, datetime('now'))
+               ON CONFLICT(table_name) DO UPDATE SET config_hash=excluded.config_hash;"
+            )
+            yield
+          end
+        else
+          yield
+        end
+      end
+    end
+  end
+end

data/lib/full_search/model.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module FullSearch
+  module Model
+    extend ActiveSupport::Concern
+
+    class_methods do
+      def full_search(query_or_options = nil, filters: {}, include_soft_deleted: false, limit: nil, offset: nil, highlight: false, &block)
+        if block_given? || query_or_options.is_a?(Hash)
+          @full_search_dsl ||= FullSearch::Dsl.new(self)
+          @full_search_dsl.tokenize(query_or_options[:tokenize]) if query_or_options.is_a?(Hash) && query_or_options.key?(:tokenize)
+          @full_search_dsl.instance_eval(&block) if block_given?
+          FullSearch::Index.ensure_table!(self)
+          FullSearch::Callbacks.install!(self)
+          include InstanceMethods
+          FullSearch.models << self unless FullSearch.models.include?(self)
+          @full_search_dsl
+        else
+          FullSearch::Search.new(self, query_or_options, filters: filters, include_soft_deleted: include_soft_deleted, limit: limit, offset: offset, highlight: highlight).relation
+        end
+      end
+
+      def full_search_dsl
+        @full_search_dsl
+      end
+
+      def full_search_ids(query, filters: {}, include_soft_deleted: false, limit: 1000)
+        full_search(query, filters: filters, include_soft_deleted: include_soft_deleted, limit: limit).pluck(:id)
+      end
+    end
+
+    module InstanceMethods
+      attr_accessor :full_search_snippet
+
+      def full_search_text_for(field_name)
+        dsl = self.class.full_search_dsl
+        field = dsl.fields.find { |f| f.name == field_name.to_s }
+        return nil unless field
+
+        field.source ? instance_exec(&field.source) : public_send(field.name)
+      end
+    end
+  end
+end

data/lib/full_search/query_parser.rb

@@ -0,0 +1,126 @@
+# frozen_string_literal: true
+
+require "strscan"
+
+module FullSearch
+  module QueryParser
+    Token = Data.define(:type, :value)
+
+    def self.parse(query)
+      tokens = tokenize(query.to_s.strip)
+      return [] if tokens.empty?
+
+      or_clauses = split_or(tokens)
+
+      if or_clauses.size == 1
+        parse_and_clause(or_clauses.first)
+      else
+        [:or, or_clauses.map { |clause| parse_and_clause(clause) }]
+      end
+    end
+
+    def self.to_match_expression(parsed)
+      return '""' if parsed.empty?
+
+      type, value = parsed
+      case type
+      when :and
+        build_and_expression(value)
+      when :or
+        value.map { |sub| to_match_expression(sub) }.join(" OR ")
+      else
+        node_to_match(parsed)
+      end
+    end
+
+    private
+
+    def self.build_and_expression(nodes)
+      non_excludes = nodes.reject { |n| n.first == :exclude }
+      excludes = nodes.select { |n| n.first == :exclude }
+
+      parts = non_excludes.map { |n| node_to_match(n) }
+      result = parts.size == 1 ? parts.first : "(#{parts.join(' AND ')})"
+
+      excludes.each do |node|
+        _, value = node
+        result = "#{result} NOT \"#{escape(value)}\""
+      end
+
+      result
+    end
+
+    def self.node_to_match(node)
+      type, value = node
+      case type
+      when :term
+        '"' + escape(value) + '"*'
+      when :phrase
+        '"' + escape(value) + '"'
+      when :exclude
+        nil
+      end
+    end
+
+    def self.tokenize(query)
+      tokens = []
+      scanner = StringScanner.new(query)
+
+      until scanner.eos?
+        scanner.skip(/\s+/)
+        break if scanner.eos?
+
+        if scanner.scan(/"/)
+          phrase = scanner.scan_until(/"/)
+          if phrase
+            tokens << Token.new(:phrase, phrase.chomp('"'))
+          else
+            tokens << Token.new(:term, scanner.rest)
+            scanner.terminate
+          end
+        elsif scanner.scan(/-/)
+          scanner.skip(/\s+/)
+          tokens << Token.new(:exclude, scanner.scan(/\S+/))
+        elsif scanner.scan(/OR|or/i)
+          tokens << Token.new(:or, nil)
+        else
+          tokens << Token.new(:term, scanner.scan(/\S+/))
+        end
+      end
+
+      tokens
+    end
+
+    def self.split_or(tokens)
+      clauses = []
+      current = []
+
+      tokens.each do |token|
+        if token.type == :or
+          clauses << current unless current.empty?
+          current = []
+        else
+          current << token
+        end
+      end
+      clauses << current unless current.empty?
+      clauses
+    end
+
+    def self.parse_and_clause(tokens)
+      nodes = tokens.map do |token|
+        case token.type
+        when :term then [:term, token.value]
+        when :phrase then [:phrase, token.value]
+        when :exclude then [:exclude, token.value]
+        end
+      end
+
+      nodes.size == 1 ? nodes.first : [:and, nodes]
+    end
+
+    def self.escape(value)
+      value.to_s.gsub('"', '""')
+    end
+  end
+end

data/lib/full_search/railtie.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require "rails"
+
+module FullSearch
+  class Railtie < Rails::Railtie
+    rake_tasks do
+      load File.expand_path("../tasks/full_search.rake", __dir__)
+    end
+
+    initializer "full_search.ensure_tables" do
+      config.after_initialize do
+        next unless FullSearch.config.auto_manage_schema
+        ActiveSupport.on_load(:active_record) do
+          FullSearch.models.each do |model|
+            FullSearch::Index.ensure_table!(model)
+            next unless FullSearch.config.auto_manage_schema == true
+
+            stored = FullSearch::Index.stored_config_hash(model)
+            if stored && stored != model.full_search_dsl.config_hash
+              FullSearch::Index.rebuild!(model)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/full_search/reindex_job.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+require "active_job"
+
+module FullSearch
+  class ReindexJob < ActiveJob::Base
+    def perform(model_name, record_id, field_name)
+      model = model_name.constantize
+      record = model.find_by(id: record_id)
+      return unless record
+
+      FullSearch::Callbacks.reindex_field!(record, field_name)
+    end
+  end
+end

data/lib/full_search/search.rb

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+module FullSearch
+  class Search
+    attr_reader :model, :query, :filters, :include_soft_deleted, :limit, :offset, :highlight
+
+    def initialize(model, query, filters:, include_soft_deleted:, limit:, offset:, highlight: false)
+      @model = model
+      @query = query.to_s.strip
+      @filters = filters
+      @include_soft_deleted = include_soft_deleted
+      @limit = limit
+      @offset = offset
+      @highlight = highlight
+    end
+
+    def relation
+      validate_required_filters!
+
+      parsed = QueryParser.parse(query)
+      exact_ids = ExactMatch.ids_for(model, query, filters)
+      primary_ids = fts_match_ids(parsed)
+      fallback_ids = dsl.typo_tolerance? ? trigram_match_ids(parsed, primary_ids) : []
+
+      all_ids = (exact_ids + primary_ids + fallback_ids).uniq
+      return model.none if all_ids.empty?
+
+      rel = model.where(id: all_ids)
+      rel = rel.where(model.arel_table[dsl.soft_delete_column].eq(nil)) if dsl.soft_delete_column && !include_soft_deleted
+      rel = rel.limit(limit) if limit
+      rel = rel.offset(offset) if offset
+
+      order_sql = case_clause(all_ids)
+      rel = rel.order(Arel.sql(order_sql))
+
+      if highlight
+        Highlighter.apply!(rel.to_a, model, query)
+      else
+        rel
+      end
+    end
+
+    private
+
+    def dsl
+      model.full_search_dsl
+    end
+
+    def validate_required_filters!
+      dsl.filters.each do |filter|
+        next unless filter.required
+        raise MissingRequiredFilterError, "Missing required filter: #{filter.name}" unless filters.key?(filter.name.to_sym) || filters.key?(filter.name)
+      end
+    end
+
+    def fts_match_ids(parsed)
+      return [] if query.empty?
+
+      match_expr = QueryParser.to_match_expression(parsed)
+      fts_table = FullSearch::Index.fts_table_name(model)
+
+      sql = <<~SQL
+        SELECT #{model.table_name}.id
+        FROM #{fts_table}
+        JOIN #{model.table_name} ON #{model.table_name}.id = #{fts_table}.rowid
+        WHERE #{fts_table} MATCH #{connection.quote(match_expr)}
+      SQL
+
+      filter_conditions = filters.map do |name, value|
+        "AND #{fts_table}.#{name} = #{connection.quote(value)}"
+      end.join(" ")
+
+      connection.execute("#{sql} #{filter_conditions}").map { |r| r["id"] }
+    end
+
+    def trigram_match_ids(parsed, primary_ids)
+      return [] if primary_ids.any?
+
+      match_expr = QueryParser.to_match_expression(parsed)
+      return [] if match_expr.empty?
+
+      term = parsed.last rescue nil
+      return [] if term.nil? || term.length < dsl.typo_tolerance_min_term_length.to_i
+
+      trigram_table = FullSearch::Index.trigram_table_name(model)
+
+      sql = <<~SQL
+        SELECT #{model.table_name}.id
+        FROM #{trigram_table}
+        JOIN #{model.table_name} ON #{model.table_name}.id = #{trigram_table}.rowid
+        WHERE #{trigram_table} MATCH #{connection.quote(match_expr)}
+      SQL
+
+      filter_conditions = filters.map do |name, value|
+        "AND #{trigram_table}.#{name} = #{connection.quote(value)}"
+      end.join(" ")
+
+      connection.execute("#{sql} #{filter_conditions}").map { |r| r["id"] }
+    end
+
+    def bare_term?(parsed)
+      parsed.is_a?(Array) && parsed.size == 2 && parsed.first == :term
+    end
+
+    def case_clause(ids)
+      "CASE id #{ids.map.with_index { |id, i| "WHEN #{id} THEN #{i}" }.join(" ")} END"
+    end
+
+    def connection
+      ActiveRecord::Base.connection
+    end
+  end
+end

data/lib/full_search/soft_delete.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module FullSearch
+  module SoftDelete
+    def self.delete_transition_sql(model)
+      dsl = model.full_search_dsl
+      return nil unless dsl&.soft_delete_column
+
+      "WHEN new.#{dsl.soft_delete_column} IS NULL"
+    end
+  end
+end

data/lib/full_search/test_helpers.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module FullSearch
+  module TestHelpers
+    def rebuild_full_search_index(model_name)
+      model = model_name.is_a?(Class) ? model_name : model_name.to_s.camelize.constantize
+      FullSearch::Index.drop!(model)
+      FullSearch::Index.rebuild!(model)
+    end
+
+    def ensure_full_search_tables
+      FullSearch.models.each do |model|
+        FullSearch::Index.ensure_table!(model)
+      end
+    end
+  end
+end