knitsearch 0.1.0

data/lib/generators/knitsearch/install/install_generator.rb

@@ -0,0 +1,208 @@
+# frozen_string_literal: true
+
+require "rails/generators"
+require "rails/generators/migration"
+require "rails/generators/active_record"
+require "active_record"
+
+module Knitsearch
+  class InstallGenerator < ::Rails::Generators::Base
+    include ::Rails::Generators::Migration
+
+    desc "Create FTS5 search index for a model. Usage: " \
+         "bin/rails generate knitsearch:install Article title body"
+
+    argument :model_name, type: :string, banner: "MODEL"
+    argument :columns, type: :array, default: [], banner: "column1 column2 ..."
+
+    class_option :dictionary, type: :string, default: "simple",
+                 desc: "Dictionary for stemming: simple (none), english (default: simple)"
+    class_option :tokenizer, type: :string, default: nil,
+                 desc: "[DEPRECATED] Use --dictionary instead"
+    class_option :associated, type: :array, default: [],
+                 desc: "Associated fields to index, format: assoc:column. Repeat for multiple. Example: --associated user:display_name --associated tags:name"
+
+    source_root File.expand_path("templates", __dir__)
+
+    def self.next_migration_number(dirname)
+      ::ActiveRecord::Generators::Base.next_migration_number(dirname)
+    end
+
+    def verify_sqlite_adapter
+      adapter = primary_adapter_from_configuration
+      return if adapter == "sqlite3"
+
+      raise ::Thor::Error,
+            "knitsearch requires SQLite. Detected adapter: #{adapter.inspect}. " \
+            "Use pg_search or your database's native FTS instead."
+    end
+
+    def verify_columns_provided
+      return unless columns.empty?
+
+      raise ::Thor::Error,
+            "knitsearch:install requires at least one column to index. " \
+            "Example: bin/rails generate knitsearch:install Article title body"
+    end
+
+    def verify_table_exists
+      return if table_exists?(source_table_name)
+
+      raise ::Thor::Error,
+            "Table `#{source_table_name}` does not exist. " \
+            "Run `bin/rails db:migrate` first to create the #{model_name} table."
+    end
+
+    def verify_column_names
+      invalid = columns.reject { |c| c.match?(/\A[a-z_][a-z0-9_]*\z/) }
+      return if invalid.empty?
+
+      raise ::Thor::Error,
+            "Column names must be lowercase identifiers (letters, digits, underscores). " \
+            "Invalid: #{invalid.inspect}"
+    end
+
+    def verify_migration_not_exists
+      pattern = File.join(destination_root, "db/migrate/*_create_#{fts_table_name}.rb")
+      existing = Dir.glob(pattern)
+      return if existing.empty?
+
+      raise ::Thor::Error,
+            "A migration for #{fts_table_name} already exists: #{File.basename(existing.first)}. " \
+            "Delete it first if you want to regenerate."
+    end
+
+    def create_migration
+      timestamp = self.class.next_migration_number("db/migrate")
+      migration_filename = "#{timestamp}_create_#{fts_table_name}.rb"
+      migration_path = File.join(destination_root, "db/migrate", migration_filename)
+
+      if options[:tokenizer].present?
+        puts "WARNING: --tokenizer is deprecated. Use --dictionary instead."
+      end
+
+      rich_text_cols = detect_rich_text_columns
+      dictionary = options[:dictionary] || "simple"
+      associated = parse_associated_against
+
+      associated_clause = associated.any? ? ",\n        associated_against: #{associated.inspect}" : ""
+
+      migration_content = <<~RUBY
+        class Create#{fts_table_name.camelize} < ActiveRecord::Migration#{migration_version}
+          include Knitsearch::Migration
+
+          def up
+            create_searchable_table #{source_table_name.inspect},
+              columns: #{columns.inspect},
+              dictionary: #{dictionary.inspect},
+              rich_text_columns: #{rich_text_cols.inspect}#{associated_clause}
+          end
+
+          def down
+            drop_searchable_table #{source_table_name.inspect}
+          end
+        end
+      RUBY
+
+      create_file migration_path, migration_content
+
+      puts "\nMigration created: #{migration_path}"
+      puts "\nNext steps:"
+      puts "  1. bin/rails db:migrate"
+      if associated.any?
+        puts "  2. Add to #{model_name} model:"
+        puts "       searchable_by("
+        puts "         against: { #{columns.map { |c| "#{c}: \"A\"" }.join(", ")} },"
+        puts "         associated_against: #{associated.inspect}"
+        puts "       )"
+        puts "  3. bin/rails knitsearch:backfill[#{model_name}]"
+      else
+        puts "  2. Add to #{model_name} model:"
+        puts "       searchable_by(against: { #{columns.map { |c| "#{c}: \"A\"" }.join(", ")} })"
+        puts "  3. bin/rails knitsearch:backfill[#{model_name}]"
+      end
+    end
+
+    no_tasks do
+      def model_class
+        @model_class ||= model_name.classify.constantize
+      rescue ::NameError
+        raise ::Thor::Error,
+              "Could not find model #{model_name}. " \
+              "Generate it first: bin/rails generate model #{model_name}"
+      end
+
+      def source_table_name
+        model_class.table_name
+      end
+
+      def fts_table_name
+        "#{source_table_name}_fts"
+      end
+
+      def migration_class_name
+        "Create#{fts_table_name.camelize}"
+      end
+
+      def migration_version
+        "[#{::Rails::VERSION::MAJOR}.#{::Rails::VERSION::MINOR}]"
+      end
+
+      def table_exists?(table_name)
+        ::ActiveRecord::Base.connection.table_exists?(table_name)
+      end
+
+      def detect_rich_text_columns
+        columns.select do |col|
+          model_class.respond_to?(:rich_text_attributes) &&
+            model_class.rich_text_attributes.include?(col.to_sym)
+        end
+      end
+
+      def parse_associated_against
+        return {} if options[:associated].empty?
+
+        result = {}
+
+        options[:associated].each do |item|
+          parts = item.split(":")
+          if parts.size < 2
+            raise ::Thor::Error,
+                  "Invalid --associated format: #{item.inspect}. " \
+                  "Expected: association:column (e.g., user:name) or association:column:weight (e.g., tags:name:C)"
+          end
+
+          assoc_name = parts[0].to_sym
+          column_name = parts[1].to_sym
+          weight = parts[2]&.upcase || "C"
+
+          unless [:A, :B, :C, :D].include?(weight.to_sym)
+            raise ::Thor::Error,
+                  "Invalid weight for #{assoc_name}:#{column_name}: #{weight.inspect}. " \
+                  "Must be A, B, C, or D."
+          end
+
+          unless model_class.reflect_on_association(assoc_name)
+            raise ::Thor::Error,
+                  "#{model_name} does not have association #{assoc_name.inspect}. " \
+                  "Check your model's association declarations."
+          end
+
+          result[assoc_name] = [column_name] unless result[assoc_name]
+        end
+
+        result
+      end
+    end
+
+    private
+      def primary_adapter_from_configuration
+        env_name = (defined?(::Rails) && ::Rails.respond_to?(:env)) ? ::Rails.env.to_s : (ENV["RAILS_ENV"] || "development")
+        configs  = ::ActiveRecord::Base.configurations.configs_for(env_name: env_name)
+        return nil if configs.empty?
+
+        primary = configs.find { |c| c.name == "primary" } || configs.first
+        primary.adapter
+      end
+  end
+end

data/lib/generators/knitsearch/install/templates/migration.rb.tt

@@ -0,0 +1,7 @@
+class <%= migration_class_name %> < ActiveRecord::Migration<%= migration_version %>
+  include Knitsearch::Migration
+
+  def up
+    create_searchable_table <%= source_table_name.inspect %>, columns: <%= columns.inspect %>, tokenizer: <%= options[:tokenizer].to_sym.inspect %>
+  end
+end

data/lib/generators/knitsearch/multisearch_install/multisearch_install_generator.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+require "rails/generators"
+require "rails/generators/migration"
+require "rails/generators/active_record"
+require "active_record"
+
+module Knitsearch
+  class MultisearchInstallGenerator < ::Rails::Generators::Base
+    include ::Rails::Generators::Migration
+
+    desc "Create FTS5 multi-model search index. Usage: bin/rails generate knitsearch:multisearch_install"
+
+    class_option :force, type: :boolean, default: false, desc: "Overwrite if migration already exists"
+
+    source_root File.expand_path("templates", __dir__)
+
+    def self.next_migration_number(dirname)
+      ::ActiveRecord::Generators::Base.next_migration_number(dirname)
+    end
+
+    def verify_sqlite_adapter
+      adapter = primary_adapter_from_configuration
+      return if adapter == "sqlite3"
+
+      raise ::Thor::Error,
+            "knitsearch multi-model search requires SQLite. Detected adapter: #{adapter.inspect}. " \
+            "Use a different full-text search solution for your database."
+    end
+
+    def verify_table_not_exists
+      return if !table_exists?("knitsearches")
+
+      return if options[:force]
+
+      raise ::Thor::Error,
+            "Table `knitsearches` already exists. " \
+            "Run `bin/rails generate knitsearch:multisearch_install --force` to overwrite."
+    end
+
+    def create_migration
+      timestamp = self.class.next_migration_number("db/migrate")
+      migration_filename = "#{timestamp}_create_knitsearches_multisearch.rb"
+      migration_path = File.join(destination_root, "db/migrate", migration_filename)
+
+      migration_content = <<~RUBY
+        class CreateKnitsearchesMultisearch < ActiveRecord::Migration#{migration_version}
+          include Knitsearch::Migration
+
+          def up
+            create_multisearch_table
+          end
+
+          def down
+            drop_multisearch_table
+          end
+        end
+      RUBY
+
+      create_file migration_path, migration_content
+
+      puts "\nMigration created: #{migration_path}"
+      puts "\nNext steps:"
+      puts "  1. bin/rails db:migrate"
+      puts "  2. Add `multisearchable against: [:column1, :column2]` to your models"
+      puts "  3. Run backfill for existing records: Model.knitsearch_multisearch_backfill!"
+    end
+
+    no_tasks do
+      def migration_version
+        "[#{::Rails::VERSION::MAJOR}.#{::Rails::VERSION::MINOR}]"
+      end
+
+      def table_exists?(table_name)
+        ::ActiveRecord::Base.connection.table_exists?(table_name)
+      end
+    end
+
+    private
+      def primary_adapter_from_configuration
+        env_name = (defined?(::Rails) && ::Rails.respond_to?(:env)) ? ::Rails.env.to_s : (ENV["RAILS_ENV"] || "development")
+        configs  = ::ActiveRecord::Base.configurations.configs_for(env_name: env_name)
+        return nil if configs.empty?
+
+        primary = configs.find { |c| c.name == "primary" } || configs.first
+        primary.adapter
+      end
+  end
+end

data/lib/knitsearch/document.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module Knitsearch
+  class Document < ActiveRecord::Base
+    self.table_name = "knitsearches"
+    belongs_to :searchable, polymorphic: true
+
+    def self.backfill!(model_class)
+      model_class.find_each(&:knitsearch_sync_document)
+    end
+  end
+end

data/lib/knitsearch/engine.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require "rails/engine"
+
+module Knitsearch
+  class Engine < ::Rails::Engine
+    initializer "knitsearch.schema_dumper_ignore_tables" do
+      ActiveSupport.on_load(:active_record) do
+        pattern = /(_fts|_fts_data|_fts_idx|_fts_content|_fts_docsize|_fts_config|_fts_vocab)$/
+        unless ActiveRecord::SchemaDumper.ignore_tables.include?(pattern)
+          ActiveRecord::SchemaDumper.ignore_tables << pattern
+        end
+      end
+    end
+
+    initializer "knitsearch.hook_multisearchable" do
+      ActiveSupport.on_load(:active_record) do
+        include Knitsearch::Multisearchable
+      end
+    end
+  end
+end

data/lib/knitsearch/fuzzy_corrector.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+module Knitsearch
+  module FuzzyCorrector
+    extend self
+
+    PREFIX_LENGTH = 3
+
+    def correct(query, vocab_table:, connection:, threshold:, skip_last: false)
+      return query if query.nil?
+
+      str = query.to_s
+      return query if str.strip.empty?
+      return query unless vocab_table_available?(connection, vocab_table)
+
+      tokens = str.split(/\s+/)
+      last_index = tokens.length - 1
+
+      corrected = tokens.each_with_index.map do |token, i|
+        if skip_last && i == last_index
+          token
+        else
+          correct_token(token, vocab_table: vocab_table, connection: connection, threshold: threshold)
+        end
+      end
+
+      corrected.join(" ")
+    end
+
+    private
+      def correct_token(token, vocab_table:, connection:, threshold:)
+        lowered = token.downcase
+        return token if lowered.length < PREFIX_LENGTH
+
+        prefix = lowered[0, PREFIX_LENGTH]
+        candidates = fetch_candidates(connection, vocab_table, prefix)
+        return token if candidates.empty?
+
+        scored = candidates
+          .map { |term, cnt| [term, cnt.to_i, Knitsearch::Levenshtein.distance(lowered, term)] }
+          .select { |_, _, d| d <= threshold }
+        return token if scored.empty?
+
+        exact = scored.find { |_, _, d| d == 0 }
+        if exact
+          max_freq = scored.map { |_, c, _| c }.max
+          # Keep the user's word unless it's >1 order of magnitude less common than alternatives.
+          # Log-scale comparison is corpus-independent: works equally on small and huge indexes.
+          if Math.log10(max_freq.to_f) - Math.log10(exact[1].to_f) < 1.0
+            return exact[0]
+          end
+        end
+
+        best = scored.min_by { |term, cnt, d| [-cnt, d, term] }
+        best ? best[0] : token
+      end
+
+      def fetch_candidates(connection, vocab_table, prefix)
+        binds = [ActiveRecord::Relation::QueryAttribute.new(
+          "prefix", "#{prefix}%", ActiveRecord::Type::Value.new
+        )]
+        result = connection.exec_query(
+          "SELECT term, cnt FROM #{connection.quote_table_name(vocab_table)} WHERE term LIKE ?",
+          "knitsearch_fuzzy",
+          binds
+        )
+        result.rows
+      rescue
+        []
+      end
+
+      def vocab_table_available?(connection, vocab_table)
+        connection.execute("SELECT 1 FROM #{connection.quote_table_name(vocab_table)} LIMIT 0")
+        true
+      rescue
+        false
+      end
+  end
+end

data/lib/knitsearch/has_many_dependent.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require "active_support/concern"
+
+module Knitsearch
+  module HasManyDependent
+    extend ActiveSupport::Concern
+
+    included do
+      # Proc form, not `after_save_commit :knitsearch_refresh_has_many_parents`.
+      # Symbol-form after_commit callbacks silently no-op when the target method
+      # is defined on a module included into the class — the callback registers
+      # but dispatch never reaches the method. Procs work.
+      after_save_commit { |record| record.knitsearch_refresh_has_many_parents }
+      after_destroy_commit { |record| record.knitsearch_refresh_has_many_parents }
+    end
+
+    def knitsearch_refresh_has_many_parents
+      dependents = Knitsearch.has_many_dependents[self.class]
+      return unless dependents
+
+      dependents.each do |dependent|
+        parent_class = dependent[:parent]
+        inverse_fk_sym = dependent[:inverse_fk]
+        shadow_map = dependent[:columns]
+        parent_assoc = dependent[:parent_assoc]
+
+        # Determine which parents to update
+        parents_to_update = []
+
+        # Current parent (if FK is set)
+        current_fk_value = read_attribute(inverse_fk_sym)
+        if current_fk_value.present?
+          parents_to_update << current_fk_value
+        end
+
+        # Previous parent (if FK was changed)
+        if saved_change_to_attribute?(inverse_fk_sym)
+          old_fk_value = saved_changes[inverse_fk_sym]&.first
+          if old_fk_value.present?
+            parents_to_update << old_fk_value
+          end
+        end
+
+        # Update each affected parent
+        parents_to_update.uniq.each do |parent_id|
+          parent = parent_class.find_by(id: parent_id)
+          next unless parent
+
+          # Recompute shadow columns for this parent
+          updates = {}
+          shadow_map.each do |shadow_col, source_col|
+            values = parent.send(parent_assoc).pluck(source_col).compact.map(&:to_s)
+            updates[shadow_col] = values.any? ? values.join(" ") : nil
+          end
+
+          parent.update_columns(updates)
+        end
+      end
+    end
+  end
+end

data/lib/knitsearch/has_many_through_join_dependent.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require "active_support/concern"
+
+module Knitsearch
+  module HasManyThroughJoinDependent
+    extend ActiveSupport::Concern
+
+    included do
+      # Proc form, not `after_save_commit :knitsearch_refresh_through_parent_from_join`.
+      # Symbol-form after_commit callbacks silently no-op when the target method
+      # is defined on a module included into the class — the callback registers
+      # but dispatch never reaches the method. Procs work.
+      after_create_commit { |record| record.knitsearch_refresh_through_parent_from_join }
+      after_destroy_commit { |record| record.knitsearch_refresh_through_parent_from_join }
+    end
+
+    def knitsearch_refresh_through_parent_from_join
+      dependents = Knitsearch.has_many_through_dependents[self.class]
+      return unless dependents
+
+      dependents.each do |dependent|
+        parent_class = dependent[:parent_class]
+        parent_fk = dependent[:parent_fk]
+        parent_assoc = dependent[:parent_assoc]
+        shadow_map = dependent[:columns]
+
+        # Read the parent FK from the join row
+        parent_id = read_attribute(parent_fk)
+        next unless parent_id.present?
+
+        # Find the parent and refresh its shadow columns
+        parent = parent_class.find_by(id: parent_id)
+        next unless parent
+
+        # Recompute shadow columns for this parent
+        updates = {}
+        shadow_map.each do |shadow_col, source_col|
+          values = parent.send(parent_assoc).pluck(source_col).compact.map(&:to_s)
+          updates[shadow_col] = values.any? ? values.join(" ") : nil
+        end
+
+        parent.update_columns(updates)
+      end
+    end
+  end
+end

data/lib/knitsearch/has_many_through_target_dependent.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+require "active_support/concern"
+
+module Knitsearch
+  module HasManyThroughTargetDependent
+    extend ActiveSupport::Concern
+
+    included do
+      # Proc form, not `after_update_commit :knitsearch_refresh_through_parents_from_target`.
+      # Symbol-form after_commit callbacks silently no-op when the target method
+      # is defined on a module included into the class — the callback registers
+      # but dispatch never reaches the method. Procs work.
+      after_update_commit { |record| record.knitsearch_refresh_through_parents_from_target }
+    end
+
+    def knitsearch_refresh_through_parents_from_target
+      dependents = Knitsearch.has_many_through_target_dependents[self.class]
+      return unless dependents
+
+      dependents.each do |dependent|
+        join_class = dependent[:join_class]
+        parent_class = dependent[:parent_class]
+        parent_fk = dependent[:parent_fk]
+        target_fk = dependent[:target_fk]
+        parent_assoc = dependent[:parent_assoc]
+        shadow_map = dependent[:columns]
+        source_columns = dependent[:source_columns]
+
+        # Guard: only refresh if any indexed source column actually changed
+        changed_indexed_columns = saved_changes.keys.map(&:to_sym) & source_columns.map(&:to_sym)
+        return if changed_indexed_columns.empty?
+
+        # Find all parent IDs that have this target through any join row
+        parent_ids = join_class.where(target_fk => id).pluck(parent_fk).uniq
+
+        # Refresh each affected parent
+        parent_ids.each do |parent_id|
+          parent = parent_class.find_by(id: parent_id)
+          next unless parent
+
+          # Recompute shadow columns for this parent
+          updates = {}
+          shadow_map.each do |shadow_col, source_col|
+            values = parent.send(parent_assoc).pluck(source_col).compact.map(&:to_s)
+            updates[shadow_col] = values.any? ? values.join(" ") : nil
+          end
+
+          parent.update_columns(updates)
+        end
+      end
+    end
+  end
+end

data/lib/knitsearch/highlighter.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require "cgi"
+
+module Knitsearch
+  # HTML highlighter for search results. Replaces placeholder marks inserted by
+  # FTS5's highlight() function with <mark> tags. The marks are control characters
+  # chosen to be unlikely in user content.
+  module Highlighter
+    extend self
+
+    def render(text)
+      return nil if text.nil?
+
+      # Escape user content FIRST, then convert sentinels to <mark>. Reordering
+      # this would render user-stored HTML verbatim and produce stored XSS.
+      CGI.escapeHTML(text.to_s)
+         .gsub(CGI.escapeHTML(opening_mark), "<mark>")
+         .gsub(CGI.escapeHTML(closing_mark), "</mark>")
+         .html_safe
+    end
+
+    def opening_mark
+      OPENING_MARK
+    end
+
+    def closing_mark
+      CLOSING_MARK
+    end
+
+    private
+
+    OPENING_MARK = "\x02knitsearch_open\x03"
+    CLOSING_MARK = "\x02knitsearch_close\x03"
+  end
+end

data/lib/knitsearch/levenshtein.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Knitsearch
+  module Levenshtein
+    extend self
+
+    def distance(a, b)
+      a = (a || "").to_s
+      b = (b || "").to_s
+      return b.length if a.empty?
+      return a.length if b.empty?
+
+      # Ensure b is the shorter — O(min(a,b)) space
+      a, b = b, a if a.length < b.length
+
+      prev = (0..b.length).to_a
+      curr = Array.new(b.length + 1)
+
+      a.each_char.with_index(1) do |ac, i|
+        curr[0] = i
+        b.each_char.with_index(1) do |bc, j|
+          cost = ac == bc ? 0 : 1
+          curr[j] = [
+            curr[j - 1] + 1,    # insert
+            prev[j] + 1,        # delete
+            prev[j - 1] + cost  # substitute
+          ].min
+        end
+        prev, curr = curr, prev
+      end
+
+      prev[b.length]
+    end
+  end
+end