fuzzily_reloaded 1.0.0

data/lib/fuzzily.rb

@@ -0,0 +1,7 @@
+require "fuzzily/version"
+require "fuzzily/searchable"
+require "fuzzily/migration"
+require "fuzzily/model"
+require "active_record"
+
+ActiveRecord::Base.send :include, Fuzzily::Searchable

data/lib/fuzzily/migration.rb

@@ -0,0 +1,43 @@
+require "active_record"
+
+module Fuzzily
+  module Migration
+    def self.extended(by)
+      by.singleton_class.class_eval do
+        def trigrams_table_name=(custom_name)
+          @trigrams_table_name = custom_name
+        end
+
+        def trigrams_table_name
+          @trigrams_table_name ||= :trigrams
+        end
+
+        def trigrams_owner_id_column_type=(custom_type)
+          @trigrams_owner_id_column_type = custom_type
+        end
+
+        def trigrams_owner_id_column_type
+          @trigrams_owner_id_column_type ||= :integer
+        end
+
+        def up
+          create_table trigrams_table_name do |t|
+            t.string  :trigram, limit: 3
+            t.integer :score,   limit: 2
+            t.send trigrams_owner_id_column_type, :owner_id
+            t.string  :owner_type
+            t.string  :fuzzy_field
+          end
+
+          # owner_id goes first as we'll GROUP BY that
+          add_index trigrams_table_name, [:owner_id, :owner_type, :fuzzy_field, :trigram, :score], name: :index_for_match
+          add_index trigrams_table_name, [:owner_id, :owner_type], name: :index_by_owner
+        end
+
+        def down
+          drop_table trigrams_table_name
+        end
+      end
+    end
+  end
+end

data/lib/fuzzily/model.rb

@@ -0,0 +1,49 @@
+module Fuzzily
+  module Model
+    # Needs fields: trigram, owner_type, owner_id, score
+    # Needs index on [owner_type, trigram] and [owner_type, owner_id]
+
+    def self.included(by)
+      by.ancestors.include?(ActiveRecord::Base) or raise "Not included in an ActiveRecord subclass"
+      by.extend(ClassMethods)
+
+      by.class_eval do
+        return if class_variable_defined?(:@@fuzzily_trigram_model)
+
+        belongs_to :owner, polymorphic: true
+        validates :owner, :score, :fuzzy_field, presence: true
+        validates :trigram, length: { is: 3 }, uniqueness: {
+          case_sensitive: true,
+          scope: [:owner_type, :owner_id, :fuzzy_field],
+        }
+
+        _add_fuzzy_scopes
+        class_variable_set(:@@fuzzily_trigram_model, true)
+      end
+    end
+
+    module ClassMethods
+      def matches_for(text)
+        _matches_for_trigrams Fuzzily::String.new(text).trigrams
+      end
+
+      private
+
+      def _matches_for_trigrams(trigrams)
+        self.
+          select("owner_id, owner_type, count(*) AS matches, MAX(score) AS score").
+          group("owner_id, owner_type").
+          order("matches DESC, score ASC").
+          with_trigram(trigrams)
+      end
+
+      def _add_fuzzy_scopes
+        scope :for_model, lambda { |model|
+          where(owner_type: model.kind_of?(Class) ? model.name : model)
+        }
+        scope :for_field,  lambda { |field_name| where(fuzzy_field: field_name) }
+        scope :with_trigram, lambda { |trigrams| where(trigram: trigrams) }
+      end
+    end
+  end
+end

data/lib/fuzzily/searchable.rb

@@ -0,0 +1,177 @@
+require "fuzzily/trigram"
+require "ostruct"
+
+module Fuzzily
+  module Searchable
+
+    def self.included(by)
+      case ActiveRecord::VERSION::MAJOR
+        when 5 then by.extend Rails5ClassMethods
+        when 6 then by.extend Rails6ClassMethods
+      end
+    end
+
+    private
+
+    def _update_fuzzy!(_o)
+      self.send(_o.trigram_association).delete_all
+      String.new(self.send(_o.field)).scored_trigrams.each do |trigram, score|
+        self.send(_o.trigram_association).build.tap do |record|
+          record.score       = score
+          record.trigram     = trigram
+          record.fuzzy_field = _o.field.to_s
+          record.save!
+        end
+      end
+    end
+
+
+    module ClassMethods
+      # fuzzily_searchable <field> [, <field>...] [, <options>]
+      def fuzzily_searchable(*fields)
+        options = fields.last.kind_of?(Hash) ? fields.pop : {}
+
+        fields.each do |field|
+          make_field_fuzzily_searchable(field, options)
+        end
+      end
+
+      private
+
+      def _find_by_fuzzy(_o, pattern, options={})
+        options[:limit] ||= 10 unless options.has_key? :limit
+        options[:offset] ||= 0
+
+        trigrams = _o.trigram_class_name.constantize.
+          limit(options[:limit]).
+          offset(options[:offset]).
+          for_model(self.name).
+          for_field(_o.field.to_s).
+          matches_for(pattern)
+        records = _load_for_ids(trigrams.map(&:owner_id))
+        # order records as per trigram query (no portable way to do this in SQL)
+        trigrams.map { |t| records[t.owner_id] }.compact
+      end
+
+      def _load_for_ids(ids)
+        {}.tap do |result|
+          results = if respond_to? :where
+            where(id: ids)
+          else
+            find(:all, ids)
+          end
+          results.each { |_r| result[_r.id] = _r }
+        end
+      end
+
+      def _bulk_update_fuzzy(_o)
+        trigram_class = _o.trigram_class_name.constantize
+        supports_bulk_inserts  =
+          connection.class.name !~ /sqlite/i ||
+          (
+            defined?(SQLite3::SQLITE_VERSION) &&
+            Gem::Version.new(SQLite3::SQLITE_VERSION) >= Gem::Version.new("3.7.11")
+          )
+
+        _with_included_trigrams(_o).find_in_batches(batch_size: 100) do |batch|
+          inserts = []
+          batch.each do |record|
+            data = Fuzzily::String.new(record.send(_o.field))
+            data.scored_trigrams.each do |trigram, score|
+              inserts << sanitize_sql_array(["(?,?,?,?,?)", self.name, record.id, _o.field.to_s, score, trigram])
+            end
+          end
+
+          # take care of quoting
+          c = trigram_class.connection
+          insert_sql = %Q{
+            INSERT INTO %s (%s, %s, %s, %s, %s)
+            VALUES
+          } % [
+            c.quote_table_name(trigram_class.table_name),
+            c.quote_column_name("owner_type"),
+            c.quote_column_name("owner_id"),
+            c.quote_column_name("fuzzy_field"),
+            c.quote_column_name("score"),
+            c.quote_column_name("trigram")
+          ]
+
+          trigram_class.transaction do
+            batch.each { |record| record.send(_o.trigram_association).delete_all }
+            break if inserts.empty?
+
+            if supports_bulk_inserts
+              trigram_class.connection.insert(insert_sql + inserts.join(", "))
+            else
+              inserts.each do |insert|
+                trigram_class.connection.insert(insert_sql + insert)
+              end
+            end
+          end
+        end
+      end
+
+      def make_field_fuzzily_searchable(field, options={})
+        class_variable_defined?(:"@@fuzzily_searchable_#{field}") and return
+
+        _o = OpenStruct.new(
+          field:                  field,
+          trigram_class_name:     options.fetch(:class_name, "Trigram"),
+          trigram_association:    "trigrams_for_#{field}".to_sym,
+          update_trigrams_method: "update_fuzzy_#{field}!".to_sym,
+          async:                  options.fetch(:async, false)
+        )
+
+        _add_trigram_association(_o)
+
+        singleton_class.send(:define_method, "find_by_fuzzy_#{field}".to_sym) do |*args|
+          _find_by_fuzzy(_o, *args)
+        end
+
+        singleton_class.send(:define_method, "bulk_update_fuzzy_#{field}".to_sym) do
+          _bulk_update_fuzzy(_o)
+        end
+
+        define_method _o.update_trigrams_method do
+          if _o.async && self.respond_to?(:delay)
+            self.delay._update_fuzzy!(_o)
+          else
+            _update_fuzzy!(_o)
+          end
+        end
+
+        after_save do |record|
+          next unless record.send("saved_change_to_#{field}?".to_sym)
+
+          record.send(_o.update_trigrams_method)
+        end
+
+        class_variable_set(:"@@fuzzily_searchable_#{field}", true)
+        self
+      end
+    end
+
+    module Rails5ClassMethods
+      include ClassMethods
+
+      private
+
+      def _add_trigram_association(_o)
+        has_many _o.trigram_association,
+          lambda { where(fuzzy_field: _o.field.to_s) },
+          class_name: _o.trigram_class_name,
+          as:         :owner,
+          dependent:  :delete_all,
+          autosave:   true
+      end
+
+      def _with_included_trigrams(_o)
+        self.includes(_o.trigram_association)
+      end
+    end
+
+    module Rails6ClassMethods
+      include Rails5ClassMethods
+    end
+  end
+end

data/lib/fuzzily/trigram.rb

@@ -0,0 +1,31 @@
+require "active_support/core_ext/string/multibyte"
+require "delegate"
+module Fuzzily
+  class String < SimpleDelegator
+
+    def trigrams
+      return [] if __getobj__.blank?
+      normalized = self.normalize
+      number_of_trigrams = normalized.length - 3
+      trigrams = (0..number_of_trigrams).map { |index| normalized[index, 3] }.uniq
+    end
+
+    def scored_trigrams
+      trigrams.map { |t| [t, self.length] }
+    end
+
+    protected
+
+    # Remove accents, downcase, replace spaces and word start with "*",
+    # return list of normalized words
+    def normalize
+      ActiveSupport::Multibyte::Chars.new(self.to_s)
+        .mb_chars.unicode_normalize(:nfkd).to_s.downcase
+        .gsub(/[^\x00-\x7F]/, "")
+        .gsub(/[^a-z\d]/, " ")
+        .gsub(/\s+/, "*")
+        .gsub(/^/, "**")
+        .gsub(/$/, "*")
+    end
+  end
+end

data/lib/fuzzily/version.rb

@@ -0,0 +1,3 @@
+module Fuzzily
+  VERSION = "1.0.0"
+end

data/lib/fuzzily_reloaded.rb

@@ -0,0 +1 @@
+require "fuzzily"

data/spec/fuzzily/migration_spec.rb

@@ -0,0 +1,41 @@
+require "spec_helper"
+
+describe Fuzzily::Migration do
+  subject { Class.new(ActiveRecord::Migration[5.1]).extend(described_class) }
+
+  it "is a proper migration" do
+    expect(subject.ancestors).to include(ActiveRecord::Migration[5.1])
+  end
+
+  it "applies cleanly" do
+    ActiveRecord::Migration.suppress_messages do
+      subject.up
+    end
+  end
+
+  it "rolls back cleanly" do
+    ActiveRecord::Migration.suppress_messages do
+      subject.up ; subject.down
+    end
+  end
+
+  it "has a customizable table name" do
+    ActiveRecord::Migration.suppress_messages do
+      subject.trigrams_table_name = :foobars
+      subject.up
+      expect {
+        ActiveRecord::Base.connection.execute("INSERT INTO foobars (score) VALUES (1)")
+      }.to_not raise_error
+    end
+  end
+
+  it "results in a functional model" do
+    ActiveRecord::Migration.suppress_messages do
+      subject.up
+      model_class = Class.new(ActiveRecord::Base)
+      model_class.table_name = "trigrams"
+      model_class.create(trigram: "abc")
+      expect(model_class.count).to eq 1
+    end
+  end
+end

data/spec/fuzzily/model_spec.rb

@@ -0,0 +1,77 @@
+require "spec_helper"
+
+describe Fuzzily::Model do
+  subject do
+    Class.new(ActiveRecord::Base).tap do |model|
+      model.table_name = :trigrams
+
+      def model.name ; "MyModel" ; end
+    end
+  end
+
+  before(:each) { prepare_trigrams_table }
+
+  it "can be included into an ActiveRecord model" do
+    subject.send(:include, described_class)
+  end
+
+  it "can be included twice" do
+    subject.send(:include, described_class)
+    subject.send(:include, described_class)
+  end
+
+  context "(derived model instance)" do
+    before { prepare_owners_table }
+    let(:model) { subject.send(:include, described_class) }
+
+    it "belongs to an owner" do
+      expect(model.new).to respond_to(:owner)
+    end
+
+    describe ".create" do
+      it "can create instances" do
+        model.create(owner: Stuff.create, score: 1, trigram: "abc", fuzzy_field: :name)
+      end
+    end
+
+    describe ".matches_for" do
+      before do
+        @paris = Stuff.create(name: "Paris")
+        %w(**p *pa par ari ris).each do |trigram|
+          model.create(owner: @paris, score: 1, fuzzy_field: :name, trigram: trigram)
+        end
+      end
+
+      it "finds matches" do
+        expect(model.matches_for("Paris").map(&:owner)).to eq [@paris]
+      end
+
+      it "finds close matches" do
+        expect(model.matches_for("Piriss").map(&:owner)).to eq [@paris]
+      end
+
+      it "does not confuse fields" do
+        expect(model.for_field(:name).matches_for("Paris").map(&:owner)).to eq [@paris]
+        expect(model.for_field(:data).matches_for("Paris").map(&:owner)).to be_empty
+      end
+
+      it "does not confuse owner types" do
+        expect(model.for_model(Stuff).matches_for("Paris").map(&:owner)).to eq [@paris]
+        expect(model.for_model(Object).matches_for("Paris").map(&:owner)).to be_empty
+      end
+
+      context "(with more than one entry)" do
+        before do
+          @palma = Stuff.create(name: "Palma")
+          %w(**p *pa pal alm lma).each do |trigram|
+            model.create(owner: @palma, score: 1, fuzzy_field: :name, trigram: trigram)
+          end
+        end
+
+        it "returns ordered results" do
+          expect(model.matches_for("Palmyre").map(&:owner)).to eq [@palma, @paris]
+        end
+      end
+    end
+  end
+end

data/spec/fuzzily/searchable_spec.rb

@@ -0,0 +1,201 @@
+require "spec_helper"
+
+describe Fuzzily::Searchable do
+  # Prepare ourselves a Trigram repository
+  before do
+    unless defined?(Trigram)
+      Trigram = Class.new(ActiveRecord::Base)
+      Trigram.class_eval { include Fuzzily::Model }
+    end
+  end
+
+  before(:each) { prepare_trigrams_table }
+  before(:each) { prepare_owners_table   }
+
+  subject do
+    Stuff ||= Class.new(ActiveRecord::Base)
+    def Stuff.name ; "Stuff" ; end
+    Stuff
+  end
+
+  describe ".fuzzily_searchable" do
+    it "is available to all of ActiveRecord" do
+      expect(subject).to respond_to(:fuzzily_searchable)
+    end
+
+    it "adds a find_by_fuzzy_<field> method" do
+      subject.fuzzily_searchable :name
+      expect(subject).to respond_to(:find_by_fuzzy_name)
+    end
+
+    it "is idempotent" do
+      subject.fuzzily_searchable :name
+      subject.fuzzily_searchable :name
+      expect(subject).to respond_to(:find_by_fuzzy_name)
+    end
+
+    it "creates the trigrams_for_<field> association" do
+      subject.fuzzily_searchable :name
+      expect(subject.new).to respond_to(:trigrams_for_name)
+    end
+  end
+
+  describe "(callbacks)" do
+    before { subject.fuzzily_searchable :name }
+
+    it "generates trigram records on creation" do
+      subject.create!(name: "Paris")
+      expect(subject.last.trigrams_for_name).to_not be_empty
+    end
+
+    it "generates the correct trigrams" do
+      record = subject.create!(name: "FOO")
+      expect(Trigram.first.trigram).to    eq "**f"
+      expect(Trigram.first.owner_id).to   eq record.id
+      expect(Trigram.first.owner_type).to eq "Stuff"
+    end
+
+    it "updates all trigram records on save" do
+      subject.create!(name: "Paris")
+      subject.first.update_attribute :name, "Rome"
+      expect(Trigram.all.order(:id).map(&:trigram)).to match %w(**r *ro rom ome me*)
+    end
+
+    it "deletes all trigrams on destroy" do
+      subject.create!(name: "Paris").destroy
+      expect(Trigram.all).to be_empty
+    end
+  end
+
+  describe "#update_fuzzy_<field>!" do
+    before do
+      subject.fuzzily_searchable :name
+    end
+
+    it "re-creates trigrams" do
+      subject.create!(name: "Paris")
+      old_ids = Trigram.all.map(&:id)
+      subject.last.update_fuzzy_name!
+      expect(old_ids & Trigram.all.map(&:id)).to be_empty
+    end
+
+    it "ignores nil values" do
+      subject.create!(name: nil)
+      subject.last.update_fuzzy_name!
+      expect(Trigram.all).to be_empty
+    end
+
+    it "ignores empty string values" do
+      subject.create!(name: "")
+      subject.last.update_fuzzy_name!
+      expect(Trigram.all).to be_empty
+    end
+
+    if ActiveRecord::VERSION::MAJOR <= 3
+      let(:fields) {[ :score, :fuzzy_field, :trigram ]}
+      before { Trigram.attr_protected  fields }
+
+      it "tolerates mass assignment security" do
+        subject.create!(name: "Paris")
+        subject.last.update_fuzzy_name!
+      end
+    end
+  end
+
+  describe ".bulk_update_fuzzy_<field>" do
+    before { subject.fuzzily_searchable :name }
+
+    it "creates all trigrams" do
+      subject.create!(name: "Paris")
+      Trigram.delete_all
+      subject.bulk_update_fuzzy_name
+      expect(Trigram.all).to_not be_empty
+    end
+
+    it "ignores nil values" do
+      subject.create!(name: nil)
+      Trigram.delete_all
+      subject.bulk_update_fuzzy_name
+      expect(Trigram.all).to be_empty
+    end
+
+    it "ignores empty string values" do
+      subject.create!(name: "")
+      Trigram.delete_all
+      subject.bulk_update_fuzzy_name
+      expect(Trigram.all).to be_empty
+    end
+  end
+
+  context "(integrationg test)" do
+    describe "#find_by_fuzzy_<field>" do
+      it "returns records" do
+        subject.fuzzily_searchable :name
+        @paris =   subject.create!(name: "Paris")
+        @palma =   subject.create!(name: "Palma de Majorca")
+        @palmyre = subject.create!(name: "La Palmyre")
+
+        expect(subject.find_by_fuzzy_name("Piris")).to_not be_empty
+        expect(subject.find_by_fuzzy_name("Piris")).to match [@paris, @palma]
+        expect(subject.find_by_fuzzy_name("Paradise")).to match [@paris, @palma, @palmyre]
+      end
+
+      it "favours exact matches" do
+        subject.fuzzily_searchable :name
+        @new_york   = subject.create!(name: "New York")
+        @yorkshire  = subject.create!(name: "Yorkshire")
+        @york       = subject.create!(name: "York")
+        @yorkisthan = subject.create!(name: "Yorkisthan")
+
+        expect(subject.find_by_fuzzy_name("York")).to      match [@york, @new_york, @yorkshire, @yorkisthan]
+        expect(subject.find_by_fuzzy_name("Yorkshire")).to match [@yorkshire, @york, @yorkisthan, @new_york]
+      end
+
+      it "does not favour short words" do
+        subject.fuzzily_searchable :name
+        @lo     = subject.create!(name: "Lo")      # **l *lo lo*
+        @london = subject.create!(name: "London")  # **l *lo lon ond ndo don on*
+                                                      # **l *lo lon
+        expect(subject.find_by_fuzzy_name("Lon")).to eq [@london, @lo]
+      end
+
+      it "honours limit option" do
+        subject.fuzzily_searchable :name
+        3.times { subject.create!(name: "Paris") }
+        expect(subject.find_by_fuzzy_name("Paris", limit: 2).length).to eq 2
+      end
+
+      it "limits results to 10 if limit option is not given" do
+        subject.fuzzily_searchable :name
+        30.times { subject.create!(name: "Paris") }
+        expect(subject.find_by_fuzzy_name("Paris").length).to eq 10
+      end
+
+      it "does not limit results it limit option is present and is nil" do
+        subject.fuzzily_searchable :name
+        30.times { subject.create!(name: "Paris") }
+        expect(subject.find_by_fuzzy_name("Paris", limit: nil).length).to eq 30
+      end
+
+      it "honours offset option" do
+        subject.fuzzily_searchable :name
+        3.times { subject.create!(name: "Paris") }
+        expect(subject.find_by_fuzzy_name("Paris", offset: 2).length).to eq 1
+      end
+
+      it "does not raise on missing objects" do
+        subject.fuzzily_searchable :name
+        belgium = subject.create(name: "Belgium")
+        belgium.delete
+        subject.find_by_fuzzy_name("Belgium")
+      end
+
+      it "finds others alongside missing" do
+        subject.fuzzily_searchable :name
+        belgium1, belgium2 = 2.times.map { subject.create(name: "Belgium") }
+        belgium1.delete
+        expect(subject.find_by_fuzzy_name("Belgium")).to eq [belgium2]
+      end
+    end
+  end
+end