acts_as_tokenizable 0.7.0 → 0.9.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 17eaf4f8283785561ba5703894d935e0c54a527d
+  data.tar.gz: a19c66a85adf99293addc2bb3c3214cfc6544c1c
+SHA512:
+  metadata.gz: db4f0327800efe794ff813630cb24df22c7852fcdb674851034cca68bb73790de70bad6fc84f5ddbcb57091679879628ce3a66a5b3e19ff727ac4bf634def1f4
+  data.tar.gz: 4f2443a43d80f329a32ad65f4b50a0aaea794fe9671a556654c416176cdc1182eb132bc8767e8a9abbe15a9c1e41e324df7af6b976908650b457dea2f874689f

data/.coveralls.yml

@@ -0,0 +1 @@
+service_name: travis-ci

data/.gitignore

@@ -0,0 +1,4 @@
+*.gem
+Gemfile.lock
+.byebug_history
+

data/.hound.yml

@@ -0,0 +1,3 @@
+ruby:
+  config_file: .ruby-style.yml
+

data/.rspec

@@ -0,0 +1,2 @@
+--colour
+--format progress

data/.rubocop.yml

@@ -0,0 +1,3 @@
+inherit_from: .rubocop_todo.yml
+Documentation:
+  Enabled: false

data/.rubocop_todo.yml

@@ -0,0 +1,16 @@
+# This configuration was generated by
+# `rubocop --auto-gen-config`
+# on 2016-05-16 07:44:12 +0200 using RuboCop version 0.39.0.
+# The point is for the user to remove these configuration records
+# one by one as the offenses are removed from the code base.
+# Note that changes in the inspected code, or installation of new
+# versions of RuboCop, may require this file to be generated again.
+
+# Offense count: 2
+Metrics/AbcSize:
+  Max: 22
+
+# Offense count: 2
+# Configuration parameters: CountComments.
+Metrics/MethodLength:
+  Max: 17

data/.ruby-style.yml

@@ -0,0 +1,7 @@
+Style/StringLiterals:
+  Enabled: true
+  EnforcedStyle: single_quotes
+Style/DotPosition:
+  Enabled: true
+  EnforcedStyle: leading
+

data/.travis.yml

@@ -0,0 +1,9 @@
+language: ruby
+rvm:
+- 2.2.3
+cache: bundler
+script: bundle exec rake
+addons:
+  code_climate:
+    repo_token:
+      secure: EEQ24se1YhmTSlpHI47yrlAQi1s36WOj9Sk9WDecpKjZFYJ8gSyS0nVhUAPi4rG84KyJARUd0uGNHBOhJ/ZH2YN3Ge4kxzJZEbSKQ9F65MW0VZelcL5DtIsdEoiuZ6BZ1fGM9RYnPnk4jwHn8JeZ/XyVPmSvhmShmwwcWNjY5hqGw2KWHqOvgiw9fWNn/9r1iq+pUQAjqrsjjeih5fC5nrh570mt1sY51ogFbnfI6vO5JpqLcK6v1PO4gejYharfE4zqMiBwWf4HWduyQNUVC/4pKa8ohwh8yQoInieLcweLosOTRPulBuE9Hu3EXSv4JRbNhBtREjRva8YfG7epSlQddlyc5LITwZgpw2nLDLxkGjXH0JZ9g02y5uorCiwcU3amVtrxusqpYmq6zl7o6LxNnPhCbWrMdRveXmTPpS/rL5G8DAKS7/VRylXvy4nrPbbpvwrNicyRJeFrgXREA0KXaYAOTGL4C4yrhY0LPxjbwGzL6goD2U6dxja0SDovyWPbiXIlDxPX4tn09LxdzPMdHBqfFb/oqJiLrn0vp7kn736VrvhMfU3BweLEjlCXbSmLy+h5l8d118EpZOs/BnOdqjqtAb5eztiyBHZaaJG1H6cAxCST40yCx7nhAeEti5Zt/EVyBbUyZFcJiNbvQv5jbbaX7LA4nI4Y4pmduQA=

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in activerecord_reset_pk_sequence.gemspec
+gemspec

data/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2009 Splendeo
+Copyright (c) 2012 Splendeo
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the

data/README.rdoc

@@ -1,9 +1,14 @@
 = acts_as_tokenizable
 
-Description goes here.
+Add this gem to enable a simple analizer on fields suitable for searching.
+
+{<img src="https://travis-ci.org/fjuan/acts_as_tokenizable.svg?branch=master" alt="Build Status" />}[https://travis-ci.org/fjuan/acts_as_tokenizable]
+{<img src="https://codeclimate.com/github/fjuan/acts_as_tokenizable/badges/gpa.svg" />}[https://codeclimate.com/github/fjuan/acts_as_tokenizable]
+{<img src="https://gemnasium.com/badges/github.com/fjuan/acts_as_tokenizable.svg" alt="Dependency Status" />}[https://gemnasium.com/github.com/fjuan/acts_as_tokenizable]
+{<img src="https://coveralls.io/repos/github/fjuan/acts_as_tokenizable/badge.svg?branch=master" alt="Coverage Status" />}[https://coveralls.io/github/fjuan/acts_as_tokenizable?branch=master]
 
 == Note on Patches/Pull Requests
- 
+
 * Fork the project.
 * Make your feature addition or bug fix.
 * Add tests for it. This is important so I don't break it in a

data/Rakefile

@@ -1,52 +1,6 @@
-require 'rubygems'
-require 'rake'
+require 'rspec/core/rake_task'
+require 'bundler/gem_tasks'
 
-begin
-  require 'jeweler'
-  Jeweler::Tasks.new do |gem|
-    gem.name = "acts_as_tokenizable"
-    gem.summary = "Acts as tokenizable"
-    gem.description = "Make ActiveRecord models easily searchable via tokens."
-    gem.email = "github@splendeo.es"
-    gem.homepage = "http://github.com/splendeo/acts_as_tokenizable"
-    gem.authors = ["Enrique Garcia Cota", "Francisco de Juan"]
-    gem.add_dependency "babosa", "~> 0.3.7"
-  end
-  Jeweler::GemcutterTasks.new
-rescue LoadError
-  puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler"
-end
+RSpec::Core::RakeTask.new(:spec)
 
-require 'rake/testtask'
-Rake::TestTask.new(:test) do |test|
-  test.libs << 'lib' << 'test'
-  test.pattern = 'test/**/test_*.rb'
-  test.verbose = true
-end
-
-begin
-  require 'rcov/rcovtask'
-  Rcov::RcovTask.new do |test|
-    test.libs << 'test'
-    test.pattern = 'test/**/test_*.rb'
-    test.verbose = true
-  end
-rescue LoadError
-  task :rcov do
-    abort "RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov"
-  end
-end
-
-task :test => :check_dependencies
-
-task :default => :test
-
-require 'rake/rdoctask'
-Rake::RDocTask.new do |rdoc|
-  version = File.exist?('VERSION') ? File.read('VERSION') : ""
-
-  rdoc.rdoc_dir = 'rdoc'
-  rdoc.title = "acts_as_tokenizable #{version}"
-  rdoc.rdoc_files.include('README*')
-  rdoc.rdoc_files.include('lib/**/*.rb')
-end
+task default: :spec

data/acts_as_tokenizable.gemspec

@@ -1,56 +1,31 @@
-# Generated by jeweler
-# DO NOT EDIT THIS FILE DIRECTLY
-# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'acts_as_tokenizable/version'
 
-Gem::Specification.new do |s|
-  s.name = %q{acts_as_tokenizable}
-  s.version = "0.7.0"
+Gem::Specification.new do |gem|
+  gem.authors = ['Enrique Garcia Cota', 'Francisco Juan']
+  gem.email = 'francisco.juan@gmail.com'
+  gem.description = 'Make ActiveRecord models easily searchable via tokens.'
+  gem.summary = 'Acts as tokenizable'
+  gem.homepage = 'https://github.com/fjuan/acts_as_tokenizable'
+  gem.license = 'MIT'
+  gem.extra_rdoc_files = ['README.rdoc']
 
-  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
-  s.authors = [%q{Enrique Garcia Cota}, %q{Francisco de Juan}]
-  s.date = %q{2012-03-21}
-  s.description = %q{Make ActiveRecord models easily searchable via tokens.}
-  s.email = %q{github@splendeo.es}
-  s.extra_rdoc_files = [
-    "LICENSE",
-    "README",
-    "README.rdoc"
-  ]
-  s.files = [
-    "LICENSE",
-    "README",
-    "README.rdoc",
-    "Rakefile",
-    "VERSION",
-    "acts_as_tokenizable.gemspec",
-    "init.rb",
-    "lib/acts_as_tokenizable.rb",
-    "lib/acts_as_tokenizable/acts_as_tokenizable.rb",
-    "lib/acts_as_tokenizable/string_utils.rb",
-    "lib/tasks/acts_as_tokenizable.rake",
-    "test/helper.rb",
-    "test/test_acts_as_tokenizable.rb"
-  ]
-  s.homepage = %q{http://github.com/splendeo/acts_as_tokenizable}
-  s.require_paths = [%q{lib}]
-  s.rubygems_version = %q{1.8.6}
-  s.summary = %q{Acts as tokenizable}
-  s.test_files = [
-    "test/helper.rb",
-    "test/test_acts_as_tokenizable.rb"
-  ]
+  gem.files = `git ls-files`.split($OUTPUT_RECORD_SEPARATOR)
+  gem.executables = gem.files.grep(%r{^bin/}).map { |f| File.basename(f) }
+  gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name = 'acts_as_tokenizable'
+  gem.require_paths = ['lib']
+  gem.version = ActsAsTokenizable::VERSION
 
-  if s.respond_to? :specification_version then
-    s.specification_version = 3
+  gem.add_runtime_dependency 'activerecord'
+  gem.add_runtime_dependency 'babosa'
 
-    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
-      s.add_runtime_dependency(%q<babosa>, ["~> 0.3.7"])
-    else
-      s.add_dependency(%q<babosa>, ["~> 0.3.7"])
-    end
-  else
-    s.add_dependency(%q<babosa>, ["~> 0.3.7"])
-  end
+  gem.add_development_dependency 'rake'
+  gem.add_development_dependency 'rspec'
+  gem.add_development_dependency 'sqlite3'
+  gem.add_development_dependency 'coveralls'
+  gem.add_development_dependency 'byebug'
+  gem.add_development_dependency 'pry-byebug'
 end
-

data/lib/acts_as_tokenizable.rb

@@ -1,9 +1,19 @@
+$LOAD_PATH << File.dirname(__FILE__)
+
+require 'active_record'
 require 'acts_as_tokenizable/acts_as_tokenizable'
+require 'acts_as_tokenizable/string_utils'
 
-class ActiveRecord::Base
-  def self.acts_as_tokenizable(field_name=:token)
-    include ActsAsTokenizable
-    self.token_field_name = field_name
-    self.before_save :tokenize
+module ActsAsTokenizable
+  if defined?(Rails::Railtie)
+    class Railtie < Rails::Railtie
+      initializer 'acts_as_tokenizable.insert_into_active_record' do
+        ActiveSupport.on_load :active_record do
+          ActiveRecord::Base.send(:include, ActsAsTokenizable)
+        end
+      end
+    end
+  elsif defined?(ActiveRecord)
+    ActiveRecord::Base.send(:include, ActsAsTokenizable)
   end
 end

data/lib/acts_as_tokenizable/acts_as_tokenizable.rb

@@ -1,45 +1,64 @@
-require 'acts_as_tokenizable/string_utils'
-
 module ActsAsTokenizable
-
-  # default to_token method. needs to have a "name" property on the object.
-  # override for more complex token generation
-  def to_token
-    raise NoMethodError.new("You must redefine to_token in your model. Example: self.name.to_token()")
-  end
-
-  #makes self.<token_field_name>=self.to_token
-  def tokenize
-    self.send("#{self.class.token_field_name}=", self.to_token)
+  def self.included(base)
+    base.class_eval { extend ClassMethods }
   end
 
   module ClassMethods
     attr_accessor :token_field_name
 
-    # search_token parameter is used by tokenized_by. This function allows for preparation
-    # before tokenized_by function is invoked. Usually this means removing
-    # stop words, replacing words.
+    def acts_as_tokenizable(field_name = :token)
+      include InstanceMethods
+      include TokenizedBy
+
+      before_save :tokenize
+
+      self.token_field_name = field_name
+    end
+
+    # search_token parameter is used by tokenized_by. This function allows for
+    # preparation before tokenized_by function is invoked. Usually this means
+    # removing stop words, replacing words.
     # By default it tokenizes each word and removes duplicates.
     def prepare_search_token(search_token)
       StringUtils.words_to_token(search_token)
     end
   end
 
-  def self.included(base)
-    base.class_eval do
-      extend ClassMethods
+  module InstanceMethods
+    # default to_token method. needs to have a "name" property on the object.
+    # override for more complex token generation
+    def to_token
+      raise(
+        NoMethodError,
+        'You must define to_token in your model. Example: self.name.to_token()'
+      )
+    end
+
+    # makes self.<token_field_name>=self.to_token
+    def tokenize
+      send("#{self.class.token_field_name}=", to_token)
+    end
+
+    def tokenize!
+      update_column(self.class.token_field_name, to_token)
+    end
+  end
+
+  module TokenizedBy
+    extend ActiveSupport::Concern
 
-      scope :tokenized_by, lambda {|search_token|
+    included do
+      scope :tokenized_by, lambda { |search_token|
         search_strings = []
         search_values = []
-        StringUtils.words(prepare_search_token(search_token)).each do |w|
-          if w[0,1] == '-'
+        StringUtils.words(search_token).each do |w|
+          if w[0, 1] == '-'
             search_strings.push("#{table_name}.#{token_field_name} NOT LIKE ?")
-            search_values.push("%#{w[1,w.length]}%")
           else
             search_strings.push("#{table_name}.#{token_field_name} LIKE ?")
-            search_values.push("%#{w}%")
           end
+          tokenized_word = StringUtils.to_token(w)
+          search_values.push("%#{tokenized_word}%")
         end
         where([search_strings.join(' AND '), *search_values])
       }

data/lib/acts_as_tokenizable/string_utils.rb

@@ -1,33 +1,35 @@
 require 'babosa' # make sure that babosa is loaded
 
 module ActsAsTokenizable
-
   module StringUtils
-
-    #returns true if numeric, false, otherwise
+    # returns true if numeric, false, otherwise
     def self.numeric?(str)
-      true if Float(str) rescue
+      true if Float(str)
+    rescue
       false
     end
 
-    #returns an array of strings containing the words on this string. removes spaces, strange chars, etc
+    # returns an array of strings containing the words on this string. Removes
+    # spaces, strange chars, etc
     def self.words(str)
-      str.gsub(/[^\w|-]/, ' ').split
+      str.split(/[\s|\.|,]+/)
     end
 
-    #removes certain words from a string.
+    # removes certain words from a string.
     # As a side-effect, all word-separators are converted to the separator char
     def self.remove_words(str, words_array, separator = ' ')
       (words(str) - words_array).join separator
     end
 
-    # replaces certain words on a string. 
+    # replaces certain words on a string.
     # As a side-effect, all word-separators are converted to the separator char
     def self.replace_words(str, replacements, separator = ' ')
       replaced_words = words(str)
-      replacements.each do |candidates,replacement|
+      replacements.each do |candidates, replacement|
         candidates.each do |candidate|
-          replaced_words=replaced_words.collect {|w| w==candidate ? replacement : w}
+          replaced_words = replaced_words.collect do |w|
+            w == candidate ? replacement : w
+          end
         end
       end
       replaced_words.join separator
@@ -44,17 +46,23 @@ module ActsAsTokenizable
       str.split(/(\d+)/).map { |v| v =~ /\d/ ? v.to_i : v }
     end
 
-    #convert into something that can be used as an indexation key
-    def self.to_token(str, max_length=255)
+    # convert into something that can be used as an indexation key
+    def self.to_token(str, max_length = 255)
       # to_slug and normalize are provided by the 'babosa' gem
-      str = str.to_slug.normalize.strip.downcase.gsub(/[^\w|-]/, '') #remove all non-alphanumeric but hyphen (-)
-      str = str.squeeze unless numeric?(str) #remove duplicates, except on pure numbers
-      return str[0..(max_length-1)]
+      # remove all non-alphanumeric but hyphen (-)
+      str = str.to_slug.normalize.strip.downcase.gsub(/[\s|\.|,]+/, '')
+      # remove duplicates, except on pure numbers
+      str = str.squeeze unless numeric?(str)
+      str[0..(max_length - 1)]
     end
 
-    #tokenizes each word individually, and joins the word with the separator char.
-    def self.words_to_token(str, max_length=255, separator = ' ')
-      words(str).collect{|w| to_token(w)}.uniq.join(separator)[0..(max_length-1)]
+    # tokenizes each word individually and joins the word with the separator
+    def self.words_to_token(str, max_length = 255, separator = ' ')
+      words(str)
+        .collect { |w| to_token(w) }
+        .uniq
+        .join(separator)
+        .slice(0, max_length)
     end
   end
 end

data/lib/acts_as_tokenizable/version.rb

@@ -0,0 +1,3 @@
+module ActsAsTokenizable
+  VERSION = '0.9.0'.freeze
+end

data/lib/tasks/acts_as_tokenizable.rake

@@ -1,54 +1,52 @@
 namespace :tokens do
-  desc "Generates the token for objects without tokens."
-  task :generate => :environment do
+  desc 'Generates the token for objects without tokens.'
+  task generate: :environment do
     tokenize_models
   end
 
-  desc "Re-builds the token for all objects."
-  task :regenerate => :environment do
+  desc 'Re-builds the token for all objects.'
+  task regenerate: :environment do
     tokenize_models(true)
   end
 end
 
 def array_of_active_record_models
-  Dir.glob(RAILS_ROOT + '/app/models/*.rb').each { |file| require file }
- 
-  models_with_token = ActiveRecord::Base.send(:subclasses).select{|m| m.respond_to?(:tokenized_by)}
+  Rails.application.eager_load!
+  ActiveRecord::Base.descendants.select { |m| m.respond_to? :tokenized_by }
 end
- 
+
 def tokenize_records(records)
   total_count = records.size
-  
+
   count = 0
-  
+
   records.each do |record|
-    record.tokenize     #this generates tokens
-    record.save false   #this saves without checking validations
+    record.tokenize!
     count += 1
     print "\r#{count}/#{total_count}"
-    GC.start if count % 1000 == 0 #launch garbage collection each 1000 registers
+    # launch garbage collection each 1000 registers
+    GC.start if count % 1000 == 0
   end
-  puts ""
+  puts ''
 end
- 
+
 def tokenize_models(regenerate = false)
-  start = Time.now
-  puts "Start token generation"
-  puts "++++++++++++++++++++++++++++++++"
-  
+  start = Time.current
+  puts 'Start token generation'
+  puts '++++++++++++++++++++++++++++++++'
+
   array_of_active_record_models.each do |model|
-  puts "Generating new tokens for #{model.name.pluralize}"
-  
-  conditions = "#{model.token_field_name} IS NULL OR #{model.token_field_name} = ''" unless regenerate
- 
-  records_without_token = model.all(:conditions => conditions)
-    if records_without_token.size > 0 
+    puts "Generating new tokens for #{model.name.pluralize}"
+    field_name = model.token_field_name
+    sql_conds = "#{field_name} IS NULL OR #{field_name} = ''" unless regenerate
+
+    records_without_token = model.all(conditions: sql_conds)
+    if !records_without_token.empty?
       tokenize_records(records_without_token)
     else
-      puts "There are no records without token"
-      puts "++++++++++++++++++++++++++++++++"
+      puts 'There are no records without token'
+      puts '++++++++++++++++++++++++++++++++'
     end
   end
-  puts "Elapsed time " + (Time.now - start).seconds.to_s + " seconds"
-  
+  puts "Elapsed time #{(Time.current - start).seconds} seconds"
 end

data/spec/acts_as_tokenizable_spec.rb

@@ -0,0 +1,72 @@
+require 'spec_helper'
+require 'support_helper'
+
+describe ActiveRecord do
+  describe 'classes with acts_as_tokenizable' do
+    it 'respond to `tokenized_by`' do
+      expect(Friend.methods).to include(:tokenized_by)
+    end
+
+    it 'raises an error if `to_token` method is not defined' do
+      expect { Enemy.to_token }.to raise_error(NoMethodError)
+    end
+
+    describe 'instances' do
+      it 'include to_token and tokenize methods' do
+        expect(Friend.instance_methods)
+          .to include(:tokenize, :tokenize!, :to_token)
+      end
+
+      it 'update the token field after creating an object' do
+        friend = Friend.new name: 'John', email: 'john@example.com', age: 30
+        expect(friend.token).to be_nil
+        friend.save
+        expect(friend.token).to eq('john 30')
+      end
+
+      it 'allow different token field name' do
+        malmo = City.create name: 'Malmö'
+        expect(malmo.tokenized_name).to eq('malmo')
+      end
+    end
+
+    describe 'scope tokenized_by' do
+      before do
+        Friend.delete_all
+
+        Friend.create name: 'Tomás', age: 31
+        Friend.create name: 'Rafa', age: 25
+        Friend.create name: 'Matthias', age: 35
+        Friend.create name: 'Mamá', age: 30
+      end
+
+      it 'finds records with tokenized string' do
+        expect(Friend.tokenized_by('as').count).to eq(2)
+      end
+
+      it 'finds records with tokenized with number' do
+        expect(Friend.tokenized_by('3').count).to eq(3)
+      end
+
+      it 'finds records with duplicated characters' do
+        expect(Friend.tokenized_by('mathias').count).to eq(1)
+        expect(Friend.tokenized_by('matthias').count).to eq(1)
+      end
+
+      it 'excludes records with negative token' do
+        expect(Friend.tokenized_by('-To').count).to eq(3)
+      end
+
+      it 'combines tokens' do
+        expect(Friend.tokenized_by('-To 5').count).to eq(2)
+      end
+    end
+  end
+
+  describe 'classes without acts_as_tokenizable' do
+    it 'does not include to_token and tokenize methods' do
+      expect(Pet.instance_methods)
+        .to_not include(:tokenize, :tokenize!, :to_token)
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,7 @@
+require 'coveralls'
+Coveralls.wear!
+
+$LOAD_PATH << File.dirname(__FILE__) + '/../lib'
+
+require 'acts_as_tokenizable'
+require 'byebug'

data/spec/string_utils_spec.rb

@@ -0,0 +1,82 @@
+require 'spec_helper'
+
+module ActsAsTokenizable
+  describe StringUtils do
+    describe '.numeric?' do
+      it 'returns true with a float' do
+        expect(described_class.numeric?('1.2')).to be_truthy
+      end
+
+      it 'returns true with an integer' do
+        expect(described_class.numeric?('1')).to be_truthy
+      end
+
+      it 'returns true with a negative number' do
+        expect(described_class.numeric?('-11')).to be_truthy
+      end
+
+      it 'returns false with a string' do
+        expect(described_class.numeric?('1a')).to be_falsy
+      end
+    end
+
+    describe '.words' do
+      it 'splits a string into by word separators' do
+        example = 'una mamá española aterrizó en Götemborg-int, Suecia.'
+
+        expect(described_class.words(example))
+          .to match_array %w(una mamá española aterrizó en Götemborg-int Suecia)
+      end
+    end
+
+    describe '.remove_words' do
+      it 'returns a string without the specified list of words' do
+        sentence = 'a b c d e f g'
+        words_to_remove = %w(c e f)
+        expect(described_class.remove_words(sentence, words_to_remove, ' '))
+          .to eq('a b d g')
+      end
+
+      it 'remove multiples occurences of a word' do
+        sentence = 'a b a d a f g'
+        words_to_remove = %w(a f)
+        expect(described_class.remove_words(sentence, words_to_remove, ' '))
+          .to eq('b d g')
+      end
+    end
+
+    describe '.to_token' do
+      it 'transforms tildes and letter modifications' do
+        examples = {
+          'mamá' => 'mama',
+          'éxtasis' => 'extasis',
+          'maría' => 'maria',
+          'camión' => 'camion',
+          'Úrsula' => 'ursula',
+          'Umeå' => 'umea',
+          'Gävle' => 'gavle',
+          'Malmö' => 'malmo',
+          'terraza' => 'teraza',
+          'España' => 'espana'
+        }
+
+        examples.each do |string, expected_token|
+          expect(described_class.to_token(string)).to eq(expected_token)
+        end
+      end
+
+      it 'removes duplicate characters' do
+        expect(described_class.to_token('terraza')).to eq('teraza')
+      end
+    end
+
+    describe '.words_to_token' do
+      it 'converts a string into something that can be used as an index key' do
+        example = 'una mamá española aterrizó en Götemborg-int, Suecia.'
+
+        expect(described_class.words_to_token(example))
+          .to eq('una mama espanola aterizo en gotemborg-int suecia')
+      end
+    end
+  end
+end

data/spec/support_helper.rb

@@ -0,0 +1,56 @@
+class Pet < ActiveRecord::Base
+end
+
+class Enemy < ActiveRecord::Base
+  acts_as_tokenizable :token
+end
+
+class Friend < ActiveRecord::Base
+  acts_as_tokenizable :token
+
+  def to_token
+    string_to_token = [name, age].join(' ')
+    ActsAsTokenizable::StringUtils.words_to_token(string_to_token)
+  end
+end
+
+class City < ActiveRecord::Base
+  acts_as_tokenizable :tokenized_name
+
+  def to_token
+    ActsAsTokenizable::StringUtils.words_to_token(name)
+  end
+end
+
+ActiveRecord::Base.establish_connection(
+  adapter: 'sqlite3',
+  database: ':memory:'
+)
+
+ActiveRecord::Base.logger
+ActiveRecord::Schema.define do
+  self.verbose = false
+end
+
+ActiveRecord::Schema.define(version: 1) do
+  create_table :pets do |t|
+    t.column :name, :string
+    t.column :owner, :string
+  end
+end
+
+ActiveRecord::Schema.define(version: 2) do
+  create_table :friends do |t|
+    t.column :name, :string
+    t.column :email, :string
+    t.column :age, :integer
+    t.column :token, :text
+  end
+end
+
+ActiveRecord::Schema.define(version: 3) do
+  create_table :cities do |t|
+    t.column :name, :string
+    t.column :tokenized_name, :text
+  end
+end

metadata

@@ -1,74 +1,183 @@
 --- !ruby/object:Gem::Specification
 name: acts_as_tokenizable
 version: !ruby/object:Gem::Version
-  version: 0.7.0
-  prerelease: 
+  version: 0.9.0
 platform: ruby
 authors:
 - Enrique Garcia Cota
-- Francisco de Juan
+- Francisco Juan
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-03-21 00:00:00.000000000Z
+date: 2016-06-26 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: babosa
-  requirement: &17284020 !ruby/object:Gem::Requirement
-    none: false
+  requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.3.7
+        version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *17284020
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: coveralls
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry-byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Make ActiveRecord models easily searchable via tokens.
-email: github@splendeo.es
+email: francisco.juan@gmail.com
 executables: []
 extensions: []
 extra_rdoc_files:
-- LICENSE
-- README
 - README.rdoc
 files:
+- ".coveralls.yml"
+- ".gitignore"
+- ".hound.yml"
+- ".rspec"
+- ".rubocop.yml"
+- ".rubocop_todo.yml"
+- ".ruby-style.yml"
+- ".travis.yml"
+- Gemfile
 - LICENSE
-- README
 - README.rdoc
 - Rakefile
-- VERSION
 - acts_as_tokenizable.gemspec
-- init.rb
 - lib/acts_as_tokenizable.rb
 - lib/acts_as_tokenizable/acts_as_tokenizable.rb
 - lib/acts_as_tokenizable/string_utils.rb
+- lib/acts_as_tokenizable/version.rb
 - lib/tasks/acts_as_tokenizable.rake
-- test/helper.rb
-- test/test_acts_as_tokenizable.rb
-homepage: http://github.com/splendeo/acts_as_tokenizable
-licenses: []
+- spec/acts_as_tokenizable_spec.rb
+- spec/spec_helper.rb
+- spec/string_utils_spec.rb
+- spec/support_helper.rb
+homepage: https://github.com/fjuan/acts_as_tokenizable
+licenses:
+- MIT
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.6
+rubygems_version: 2.6.4
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Acts as tokenizable
 test_files:
-- test/helper.rb
-- test/test_acts_as_tokenizable.rb
+- spec/acts_as_tokenizable_spec.rb
+- spec/spec_helper.rb
+- spec/string_utils_spec.rb
+- spec/support_helper.rb

data/VERSION

@@ -1 +0,0 @@
-0.7.0

data/init.rb

@@ -1 +0,0 @@
-require 'acts_as_tokenizable'

data/test/helper.rb

@@ -1,10 +0,0 @@
-require 'rubygems'
-require 'test/unit'
-require 'shoulda'
-
-$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
-$LOAD_PATH.unshift(File.dirname(__FILE__))
-require 'acts_as_tokenizable'
-
-class Test::Unit::TestCase
-end

data/test/test_acts_as_tokenizable.rb

@@ -1,7 +0,0 @@
-require 'helper'
-
-class TestActsAsTokenizable < Test::Unit::TestCase
-  should "probably rename this file and start testing for real" do
-    flunk "hey buddy, you should probably rename this file and start testing for real"
-  end
-end