fuzzily 0.0.1

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/.rspec

@@ -0,0 +1,3 @@
+--colour
+--format d
+--backtrace

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fuzzily.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2012 Julien Letessier
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,98 @@
+# Fuzzily
+
+A fast, [trigram](http://en.wikipedia.org/wiki/N-gram)-based, database-backed [fuzzy](http://en.wikipedia.org/wiki/Approximate_string_matching) string search/match engine for Rails.
+
+Loosely inspired from an [old blog post](http://unirec.blogspot.co.uk/2007/12/live-fuzzy-search-using-n-grams-in.html).
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'fuzzily'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install fuzzily
+
+## Usage
+
+You'll need to setup 2 things:
+
+- a trigram model (your search index) and its migration
+- the model you want to search for
+
+Create and ActiveRecord model in your app (this will be used to store a "fuzzy index" of all the models and fields you will be indexing):
+
+    class Trigram < ActiveRecord::Base
+      include Fuzzily::Model
+    end
+
+Create a migration for it:
+
+    class AddTrigramsModel < ActiveRecord::Migration
+      extend Fuzzily::Migration
+    end
+
+Instrument your model (your searchable fields do not have to be stored, they can be dynamic methods too):
+
+    class MyStuff < ActiveRecord::Base
+      # assuming my_stuffs has a 'name' attribute
+      fuzzily_searchable :name
+    end
+
+Index your model (will happen automatically for new/updated records):
+
+    MyStuff.find_each do |record|
+      record.update_fuzzy_name!
+    end
+
+Search!
+
+    MyStuff.find_by_fuzzy_name('Some Name', :limit => 10)
+    # => records
+
+
+
+## Indexing more than one field
+
+Just list all the field you want to index, or call `fuzzily_searchable` more than once: 
+
+    class MyStuff < ActiveRecord::Base
+      fuzzily_searchable :name_fr, :name_en
+      fuzzily_searchable :name_de
+    end
+
+
+## Custom name for the index model
+
+If you want or need to name your index model differently (e.g. because you already have a class called `Trigram`):
+
+    class CustomTrigram < ActiveRecord::Base
+      include Fuzzily::Model
+    end
+
+    class AddTrigramsModel < ActiveRecord::Migration
+      extend Fuzzily::Migration
+      trigrams_table_name = :custom_trigrams
+    end
+
+    class MyStuff < ActiveRecord::Base
+      fuzzily_searchable :name, :class_name => 'CustomTrigram'
+    end
+
+
+## License
+
+MIT licence. Quite permissive if you ask me.
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/fuzzily.gemspec

@@ -0,0 +1,27 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'fuzzily/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = "fuzzily"
+  gem.version       = Fuzzily::VERSION
+  gem.authors       = ["Julien Letessier"]
+  gem.email         = ["julien.letessier@gmail.com"]
+  gem.description   = %q{Fast fuzzy string matching for rails}
+  gem.summary       = %q{A fast, trigram-based, database-backed fuzzy string search/match engine for Rails.}
+  gem.homepage      = ""
+
+  gem.add_runtime_dependency 'activerecord', '~> 2.3'
+
+  gem.add_development_dependency 'rspec'
+  gem.add_development_dependency 'appraisal'
+  gem.add_development_dependency 'pry'
+  gem.add_development_dependency 'pry-nav'
+  gem.add_development_dependency 'sqlite3'
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+end

data/lib/fuzzily/migration.rb

@@ -0,0 +1,35 @@
+require 'active_record'
+
+module Fuzzily
+  module Migration
+
+    def trigrams_table_name=(custom_name)
+      @trigrams_table_name = custom_name
+    end
+
+    def trigrams_table_name
+      @trigrams_table_name ||= :trigrams
+    end
+
+    def up
+      create_table trigrams_table_name do |t|
+        t.string  :trigram, :limit => 3
+        t.integer :score
+        t.integer :owner_id
+        t.string  :owner_type
+        t.string  :fuzzy_field
+      end
+
+      add_index trigrams_table_name,
+        [:owner_type, :fuzzy_field, :trigram, :owner_id, :score],
+        :name => :index_for_match
+      add_index trigrams_table_name,
+        [:owner_type, :owner_id],
+        :name => :index_by_owner
+    end
+
+    def down
+      drop_table trigrams_table_name
+    end
+  end
+end

data/lib/fuzzily/model.rb

@@ -0,0 +1,51 @@
+module Fuzzily
+  module Model
+    # Needs fields: trigram, owner_type, owner_id, score
+    # Needs index on [owner_type, trigram] and [owner_type, owner_id]
+
+    def self.included(by)
+      by.ancestors.include?(ActiveRecord::Base) or raise 'Not included in an ActiveRecord subclass'
+      by.class_eval do
+        return if class_variable_defined?(:@@fuzzily_trigram_model)
+
+        belongs_to :owner, :polymorphic => true
+        validates_presence_of     :owner
+        validates_uniqueness_of   :trigram, :scope => [:owner_type, :owner_id]
+        validates_length_of       :trigram, :is => 3
+        validates_presence_of     :score
+        validates_presence_of     :fuzzy_field
+
+        named_scope :for_model,  lambda { |model| { 
+          :conditions => { :owner_type => model.kind_of?(Class) ? model.name : model  } 
+        }}
+        named_scope :for_field,  lambda { |field_name| {
+          :conditions => { :fuzzy_field => field_name }
+        }}
+        named_scope :with_trigram, lambda { |trigrams| {
+          :conditions => { :trigram => trigrams }
+        }}
+
+        class_variable_set(:@@fuzzily_trigram_model, true)
+      end
+
+      by.extend(ClassMethods)
+    end
+
+    module ClassMethods
+      # options:
+      # - model (mandatory)
+      # - field (mandatory)
+      # - limit (default 10)
+      def matches_for(text, options = {})
+        options[:limit] ||= 10
+        self.
+          scoped(:select => 'owner_id, owner_type, SUM(score) AS score').
+          scoped(:group => :owner_id).
+          scoped(:order => 'score DESC', :limit => options[:limit]).
+          with_trigram(text.extend(String).trigrams).
+          map(&:owner)
+      end
+    end
+  end
+end
+

data/lib/fuzzily/searchable.rb

@@ -0,0 +1,55 @@
+require 'fuzzily/trigram'
+
+module Fuzzily
+  module Searchable
+    # fuzzily_searchable <field> [, <field>...] [, <options>]
+    def fuzzily_searchable(*fields)
+      options = fields.last.kind_of?(Hash) ? fields.pop : {}
+
+      fields.each do |field|
+        make_field_fuzzily_searchable(field, options)
+      end
+    end
+
+    private
+
+    def make_field_fuzzily_searchable(field, options={})
+      class_variable_defined?(:"@@fuzzily_searchable_#{field}") and return
+
+      trigram_class_name = options.fetch(:class_name, 'Trigram')
+      trigram_association = "trigrams_for_#{field}".to_sym
+      update_trigrams_method = "update_fuzzy_#{field}!".to_sym
+
+      has_many trigram_association,
+        :class_name => trigram_class_name,
+        :as => :owner,
+        :conditions => { :fuzzy_field => field.to_s },
+        :dependent => :destroy
+
+      singleton_class.send(:define_method,"find_by_fuzzy_#{field}".to_sym) do |*args|
+        case args.size
+          when 1 then pattern = args.first ; options = {}
+          when 2 then pattern, options = args
+          else        raise 'Wrong # of arguments'
+        end
+        Trigram.scoped(options).for_model(self.name).for_field(field).matches(pattern)
+      end
+
+      define_method update_trigrams_method do
+        self.send(trigram_association).destroy_all
+        self.send(field).extend(String).trigrams.each do |trigram|
+          self.send(trigram_association).create!(:score => 1, :trigram => trigram)
+        end
+      end
+
+      after_save do |record|
+        next unless record.send("#{field}_changed?".to_sym)
+        record.send(update_trigrams_method)
+      end
+
+      class_variable_set(:"@@fuzzily_searchable_#{field}", true)
+      self
+    end
+
+  end
+end

data/lib/fuzzily/trigram.rb

@@ -0,0 +1,25 @@
+require 'iconv'
+
+module Fuzzily
+  module String
+    def trigrams
+      normalized_words.map do |word|
+        (0..(word.length - 3)).map { |index| word[index,3] }
+      end.flatten.uniq
+    end
+
+    private
+
+    # Remove accents, downcase, replace spaces and word start with '*',
+    # return list of normalized words
+    def normalized_words
+      self.split(/\s+/).map { |word|
+        Iconv.iconv('ascii//translit//ignore', 'utf-8', word).first.downcase.gsub(/\W/,'')
+      }.
+      delete_if(&:empty?).
+      map { |word|
+        "**#{word}"
+      }
+    end
+  end
+end

data/lib/fuzzily/version.rb

@@ -0,0 +1,3 @@
+module Fuzzily
+  VERSION = "0.0.1"
+end

data/lib/fuzzily.rb

@@ -0,0 +1,7 @@
+require "fuzzily/version"
+require "fuzzily/searchable"
+require "fuzzily/migration"
+require "fuzzily/model"
+require "active_record"
+
+ActiveRecord::Base.extend(Fuzzily::Searchable)

data/spec/fuzzily/migration_spec.rb

@@ -0,0 +1,33 @@
+require 'spec_helper'
+
+describe Fuzzily::Migration do
+  subject { Class.new(ActiveRecord::Migration).extend(described_class) }
+
+  it 'is a proper migration' do
+    subject.ancestors.should include(ActiveRecord::Migration)
+  end
+
+  it 'applies cleanly' do
+    silence_stream(STDOUT) { subject.up }
+  end
+
+  it 'rolls back cleanly' do
+    silence_stream(STDOUT) { subject.up ; subject.down }
+  end
+
+  it 'has a customizable table name' do
+    subject.trigrams_table_name = :foobars
+    silence_stream(STDOUT) { subject.up }
+    expect {
+      ActiveRecord::Base.connection.execute('INSERT INTO `foobars` (score) VALUES (1)')
+    }.to_not raise_error
+  end
+
+  it 'results in a functional model' do
+    silence_stream(STDOUT) { subject.up }
+    model_class = Class.new(ActiveRecord::Base)
+    model_class.table_name = 'trigrams'
+    model_class.create(:trigram => 'abc')
+    model_class.count.should == 1
+  end
+end

data/spec/fuzzily/model_spec.rb

@@ -0,0 +1,79 @@
+require 'spec_helper'
+
+describe Fuzzily::Model do
+  subject do
+    Class.new(ActiveRecord::Base).tap do |model|
+      model.table_name = :trigrams
+    end
+  end
+
+  before(:each) { prepare_trigrams_table }
+
+  it 'can be included into an ActiveRecord model' do
+    subject.send(:include, described_class)
+  end
+
+  it 'can be included twice' do
+    subject.send(:include, described_class)
+    subject.send(:include, described_class)
+  end
+
+  context '(derived model instance)' do
+    before { prepare_owners_table }
+    let(:model) { subject.send(:include, described_class) }
+
+    it 'belongs to an owner' do
+      model.new.should respond_to(:owner)
+    end
+
+    describe '.create' do
+      it 'can create instances' do
+        model.create(:owner => Stuff.create, :score => 1, :trigram => 'abc', :fuzzy_field => :name)
+      end
+    end
+
+    describe '.matches_for' do
+      before do
+        @paris = Stuff.create(:name => 'Paris')
+        %w(**p *pa par ari ris).each do |trigram|
+          model.create(:owner => @paris, :score => 1, :fuzzy_field => :name, :trigram => trigram)
+        end
+      end
+
+      it 'finds matches' do
+        model.matches_for('Paris').should == [@paris]
+      end
+
+      it 'finds close matches' do
+        model.matches_for('Piriss').should == [@paris]
+      end
+
+      it 'does not confuse fields' do
+        model.for_field(:name).matches_for('Paris').should == [@paris]
+        model.for_field(:data).matches_for('Paris').should be_empty
+      end
+
+      it 'does not confuse owner types' do
+        model.for_model(Stuff).matches_for('Paris').should == [@paris]
+        model.for_model(Object).matches_for('Paris').should be_empty
+      end
+
+      context '(with more than one entry)' do
+        before do
+          @palma = Stuff.create(:name => 'Palma')
+          %w(**p *pa pal alm lma).each do |trigram|
+            model.create(:owner => @palma, :score => 1, :fuzzy_field => :name, :trigram => trigram)
+          end
+        end
+
+        it 'honors the limit option' do
+          model.matches_for('Palmyre', :limit => 1).should == [@palma]
+        end
+
+        it 'returns ordered results' do
+          model.matches_for('Palmyre').should == [@palma, @paris]
+        end
+      end
+    end
+  end
+end

data/spec/fuzzily/searchable_spec.rb

@@ -0,0 +1,72 @@
+require 'spec_helper'
+
+describe Fuzzily::Searchable do
+  # Prepare ourselves a Trigram repository
+  class Trigram < ActiveRecord::Base
+    include Fuzzily::Model
+  end
+
+  before(:each) { prepare_trigrams_table }
+  before(:each) { prepare_owners_table   }
+
+  subject do 
+    Stuff.clone.class_eval do
+      def self.name ; 'Stuff' ; end
+      self
+    end
+  end
+
+  describe '.fuzzily_searchable' do
+    it 'is available to all of ActiveRecord' do
+      subject.should respond_to(:fuzzily_searchable)
+    end
+
+    it 'adds a find_by_fuzzy_<field> method' do
+      subject.fuzzily_searchable :name
+      subject.should respond_to(:find_by_fuzzy_name)
+    end
+
+    it 'is idempotent' do
+      subject.fuzzily_searchable :name
+      subject.fuzzily_searchable :name
+      subject.should respond_to(:find_by_fuzzy_name)
+    end
+
+    it 'creates the trigrams_for_<field> association' do
+      subject.fuzzily_searchable :name
+      subject.new.should respond_to(:trigrams_for_name)
+    end
+  end
+
+  describe '(callbacks)' do
+    it 'generates trigram records on creation' do
+      subject.fuzzily_searchable :name
+      subject.create(:name => 'Paris')
+      subject.last.trigrams_for_name.should_not be_empty
+    end
+
+    it 'generates the correct trigrams' do
+      subject.fuzzily_searchable :name
+      record = subject.create(:name => 'FOO')
+      Trigram.first.trigram.should    == '**f'
+      Trigram.first.owner_id.should   == record.id
+      Trigram.first.owner_type.should == 'Stuff'
+    end
+
+    it 'updates all trigram records on save' do
+      subject.fuzzily_searchable :name
+      subject.create(:name => 'Paris')
+      subject.first.update_attribute :name, 'Rome'
+      Trigram.all.map(&:trigram).should =~ %w(**r *ro rom ome)
+    end
+  end
+
+  describe '#find_by_fuzzy_<field>' do
+    it 'works'
+  end
+
+  describe '#update_fuzzy_<field>!' do
+    it 'works'
+  end
+
+end

data/spec/fuzzily/trigram_spec.rb

@@ -0,0 +1,8 @@
+require 'spec_helper'
+
+describe Fuzzily::String do
+  it 'splits strings into trigrams'
+  it 'removes accents'
+  it 'removes symbols'
+  it 'handles multi word strings'
+end

data/spec/meta_spec.rb

@@ -0,0 +1,8 @@
+require 'spec_helper'
+# This tests our RSpec setup works
+
+describe 'Test suite' do
+  it 'has a working ActiveRecord connection' do
+    ActiveRecord::Base.connection.execute('SELECT * FROM `sqlite_master`')
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,48 @@
+require 'fuzzily'
+require 'pathname'
+require 'yaml'
+
+Database = Pathname.new 'test.sqlite3'
+
+# A test model we'll need as a source of trigrams
+class Stuff < ActiveRecord::Base ; end
+class StuffMigration < ActiveRecord::Migration
+  def self.up
+    create_table :stuffs do |t|
+      t.string :name
+      t.string :data
+      t.timestamps
+    end
+  end
+
+  def self.down
+    drop_table :stuffs
+  end
+end
+
+RSpec.configure do |config|
+  config.before(:each) do
+    # Setup test database
+    ActiveRecord::Base.establish_connection(
+      :adapter  => 'sqlite3',
+      :database => Database.to_s
+    )
+
+    def prepare_trigrams_table
+      silence_stream(STDOUT) do
+        Class.new(ActiveRecord::Migration).extend(Fuzzily::Migration).up
+      end
+    end
+
+    def prepare_owners_table
+      silence_stream(STDOUT) do
+        StuffMigration.up
+      end
+    end
+
+  end
+
+  config.after(:each) do
+    Database.delete if Database.exist?
+  end
+end

metadata

@@ -0,0 +1,175 @@
+--- !ruby/object:Gem::Specification 
+name: fuzzily
+version: !ruby/object:Gem::Version 
+  hash: 29
+  prerelease: 
+  segments: 
+  - 0
+  - 0
+  - 1
+  version: 0.0.1
+platform: ruby
+authors: 
+- Julien Letessier
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2012-10-25 00:00:00 +01:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 5
+        segments: 
+        - 2
+        - 3
+        version: "2.3"
+  prerelease: false
+  name: activerecord
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  prerelease: false
+  name: rspec
+  type: :development
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  prerelease: false
+  name: appraisal
+  type: :development
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  prerelease: false
+  name: pry
+  type: :development
+  version_requirements: *id004
+- !ruby/object:Gem::Dependency 
+  requirement: &id005 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  prerelease: false
+  name: pry-nav
+  type: :development
+  version_requirements: *id005
+- !ruby/object:Gem::Dependency 
+  requirement: &id006 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  prerelease: false
+  name: sqlite3
+  type: :development
+  version_requirements: *id006
+description: Fast fuzzy string matching for rails
+email: 
+- julien.letessier@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- .gitignore
+- .rspec
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- fuzzily.gemspec
+- lib/fuzzily.rb
+- lib/fuzzily/migration.rb
+- lib/fuzzily/model.rb
+- lib/fuzzily/searchable.rb
+- lib/fuzzily/trigram.rb
+- lib/fuzzily/version.rb
+- spec/fuzzily/migration_spec.rb
+- spec/fuzzily/model_spec.rb
+- spec/fuzzily/searchable_spec.rb
+- spec/fuzzily/trigram_spec.rb
+- spec/meta_spec.rb
+- spec/spec_helper.rb
+has_rdoc: true
+homepage: ""
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.9.5
+signing_key: 
+specification_version: 3
+summary: A fast, trigram-based, database-backed fuzzy string search/match engine for Rails.
+test_files: 
+- spec/fuzzily/migration_spec.rb
+- spec/fuzzily/model_spec.rb
+- spec/fuzzily/searchable_spec.rb
+- spec/fuzzily/trigram_spec.rb
+- spec/meta_spec.rb
+- spec/spec_helper.rb