smart_search 0.0.61 → 0.0.65

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d1dba8372dab82dd2cce0ebc37101e2ebcf2fba1
-  data.tar.gz: a71c65f0d8adac611ce88e75a43ea30d36a5d63a
+  metadata.gz: 09683d827340bc1dc832274e31723252da084649
+  data.tar.gz: 580b05753d0355b99502205b99276bf71eb6bf58
 SHA512:
-  metadata.gz: 6a932397b5b2eafae190335567419028ef0ad2547c2156c1aab1ca5b1aed4204caaa646c82bd1f0b062eecc88662f06ba06422b0c59336ff1cfc910c670c1e27
-  data.tar.gz: 6b544662bb2360c1bba224a580d02fc8d5b46153200d77734f2297471b8c67d7362deadc7c93d6fbbf7cd961617e9cb48fa8bb373e14584b88688cbe21edbbde
+  metadata.gz: 54a44b78402531aa5098329681e7929f3f97671e42cd5bbe5b517e5143bfa4b9e671179a3d99b537fe8c43de770929ee3c7041be4d2f28197be242bedb774333
+  data.tar.gz: 89df4b626f3f3979a509f637aa5112c6452cf1394108223ba80f603d36534d24331e5864d0ca1d6f85539cf6430cf291a4fdaf385b92bfef6ec17c9358b21fa2

data/config/database.yml

@@ -0,0 +1,6 @@
+test:
+  adapter: mysql2
+  database: smart_search_test
+  username: travis
+  password: 
+  encoding: utf8

data/lib/smart_search/smart_search_engine.rb

@@ -1,5 +1,8 @@
 module SmartSearch
+  require "rails"
   class SmartSearchEngine < Rails::Engine
     isolate_namespace SmartSearch
+    require "friendly_extensions"
+    require "amatch"
   end  
 end  

data/lib/smart_search.rb

@@ -1,10 +1,13 @@
 # -*- encoding : utf-8 -*-
+require "rails"
+
 require "smart_search"
+require "smart_search/smart_search_engine"
+
 require "smart_similarity"
 require "smart_search_history"
-require "add_search_tags"
+require "smart_search_tag"
 
-require "smart_search/smart_search_engine"
 
 module SmartSearch
   
@@ -27,12 +30,9 @@ module SmartSearch
           
           cattr_accessor :condition_default, :group_default, :tags, :order_default, :enable_similarity
           send :include, InstanceMethods
-          if self.column_names.index("search_tags").nil?
-            ::AddSearchTags.add_to_table(self.table_name)
-          end
-            self.send(:before_save, :create_search_tags)
-            
-            self.enable_similarity = true
+            self.send(:after_save, :create_search_tags)
+            self.send(:before_destroy, :create_search_tags)
+            self.enable_similarity ||= true
             
             # options zuweisen
             if options[:conditions].is_a?(String) && !options[:conditions].blank?
@@ -89,8 +89,13 @@ module SmartSearch
           tags.map! {|t| "search_tags LIKE '%#{t}%'"}
         end  
         
+        # Load ranking from Search tags
+        result_ids = SmartSearchTag.connection.select_all("select entry_id, sum(boost), group_concat(search_tags) as grouped_tags 
+        from smart_search_tags where `table_name`= '#{self.table_name}' and 
+        
+        (#{tags.join(' OR ')}) group by entry_id having (#{tags.join(' AND ').gsub('search_tags', 'grouped_tags')}) order by sum(boost) DESC").map {|r| r["entry_id"]}
         
-        results =  self.where("(#{tags.join(' AND ')})")
+        results =  self.where(:id => result_ids)
         
         if options[:conditions]
           results = results.where(options[:conditions])
@@ -127,10 +132,10 @@ module SmartSearch
       end  
     end  
     
-    # Create all search tags for this table into similarity index
+    # Load all search tags for this table into similarity index
     def set_similarity_index
       
-      search_tags_list = self.connection.select_all("SELECT search_tags from #{self.table_name}").map {|r| r["search_tags"]}
+      search_tags_list = self.connection.select_all("SELECT search_tags from #{SmartSearchTag.table_name} where `table_name` = #{self.table_name}").map {|r| r["search_tags"]}
       
       SmartSimilarity.create_from_text(search_tags_list.join(" "))
     end  
@@ -145,29 +150,51 @@ module SmartSearch
     
     # create search tags for this very record based on the attributes defined in ':on' option passed to the 'Class.smart_search' method
     def create_search_tags
-      tags = []
+      tags      = []
+      
       self.class.tags.each do |tag|
-        if tag.is_a?(Symbol)
-          tags << self.send(tag)
-        elsif tag.is_a?(String)
-          tag_methods = tag.split(".")  
+        
+        if !tag.is_a?(Hash)
+          tag = {:field_name => tag, :boost => 1, :search_tags => ""} 
+        else
+          tag[:search_tags] = ""
+          tag[:boost] ||= 1
+        end    
+        
+        if tag[:field_name].is_a?(Symbol)
+          tag[:search_tags] << self.send(tag[:field_name]).to_s
+        elsif tag[:field_name].is_a?(String)
+          tag_methods = tag[:field_name].split(".")  
           tagx = self.send(tag_methods[0])
           tag_methods[1..-1].each do |x|
             tagx = tagx.send(x) rescue ""
           end
-          tags << tagx  
-        end  
+          tag[:search_tags] << tagx.to_s  
+        end
+        
+        tag[:search_tags] = tag[:search_tags].split(" ").uniq.join(" ").downcase   
+        tags << tag
       end
-      searchtags = tags.join(" ").split(" ")  
-      searchtags = searchtags.uniq.join(" ")
-      search_tags_min = searchtags.gsub(" ", "").downcase
       
-      self.search_tags = "#{searchtags.downcase}"
-    end  
+      
+      self.clear_search_tags
+        
+      tags.each do |t|
+        SmartSearchTag.create(t.merge!(:table_name => self.class.table_name, :entry_id => self.id))
+      end
+   
+      
+    end
+    
+    def clear_search_tags
+      if !self.id.nil?
+        SmartSearchTag.connection.execute("DELETE from #{SmartSearchTag.table_name} where `table_name` = '#{self.class.table_name}' and entry_id = #{self.id}")
+      end  
+    end    
     
   end    
         
-  
+
   class Config
     
     cattr_accessor  :search_models

data/lib/smart_search_ignore_word.rb

@@ -0,0 +1,28 @@
+class SmartSearchIgnoreWord < ActiveRecord::Base
+
+  #= Configuration
+  self.table_name = "smart_search_ignore_words" 
+      #== Associations
+          # => Stuff in Here
+  
+      #== Plugins and modules
+        #=== PlugIns
+          # => Stuff in Here        
+  
+        #=== include Modules
+          # => Stuff in Here
+  
+      #== Konstanten
+          # => Stuff in Here
+  
+      #== Validation and Callbacks
+        #=== Validation
+          validates_uniqueness_of :word
+          
+        #=== Callbacks
+          # => Stuff in Here
+          
+          
+    # => END
+  
+end  

data/lib/smart_search_tag.rb

@@ -0,0 +1,2 @@
+class SmartSearchTag < ActiveRecord::Base
+end  

data/lib/smart_similarity.rb

@@ -3,7 +3,6 @@ class SmartSimilarity < ActiveRecord::Base
   #= Configuration
   serialize :similarities, Array
   self.table_name = "smart_search_similarities"
-  require "amatch"
   
       #== Associations
           # => Stuff in Here
@@ -28,6 +27,8 @@ class SmartSimilarity < ActiveRecord::Base
           # Limit Number of similar words
           SIMILARITY_LIMIT  = 8
           
+          SPLITTING_REGEXP = /\b/
+          
       #== Validation and Callbacks
         #=== Validation
         
@@ -39,7 +40,7 @@ class SmartSimilarity < ActiveRecord::Base
     # This method is used to generate date from every source, e.g. file, url, single words etc..
     def self.create_from_text(text)
       # prepare text
-      prepared_text = text.downcase.split(/\b/).uniq
+      prepared_text = text.downcase.split(SPLITTING_REGEXP).uniq
       prepared_text = prepared_text.select {|w| w.size >= 3 && !w.match(/[0-9\-_<>\.\/(){}&\?"'@*+$!=,:'#;]/)}
       list = {}
       prepared_text.each do |word|
@@ -78,18 +79,19 @@ class SmartSimilarity < ActiveRecord::Base
     # Best used for loading big dictionary files.
     # Uses 'spawnling' to split the data into 8 stacks and load them simultaniously
     def self.load_file(path)
-      count = %x{wc -l #{path}}.split[0].to_i
+      count = %x{wc -l #{path}}.split[0].to_i.max(1)
       puts "loading file: #{path}"
       puts "=> #{count} rows"
-      File.open(path, "r").to_a.seperate([8,count].min).each_with_index do |stack, si| 
-        Spawnling.new(:argv => "sim-file-#{si}") do
-          QueryLog.info "sim-file-#{si}"
-          stack.each_with_index do |l,i|
-            QueryLog.info "#{si}: #{i.fdiv(count).round(4)} %"
-            self.add_word(l)
-          end
-        end  
-      end  
+      
+      if count == 1
+        File.open(path, "r").read.split(SPLITTING_REGEXP).each {|w| self.add_word(w)}
+      else  
+        File.open(path, "r").read.split(SPLITTING_REGEXP).seperate([8,count].min).each_with_index do |stack, si| 
+            stack.each_with_index do |l,i|
+              self.add_word(l)
+            end
+        end
+      end    
     end
     
     # Load words from website and save them to index
@@ -99,7 +101,7 @@ class SmartSimilarity < ActiveRecord::Base
     
     # Loads your created query history and saves them to the index
     def self.load_from_query_history
-      queries = self.connection.select_all("SELECT query from `#{::SmartSearchHistory.table_name}`").map {|r| r["query"]}.join(" ")
+      queries = ActiveRecord::Base.connection.select_all("SELECT query from `#{::SmartSearchHistory.table_name}`").map {|r| r["query"]}.join(" ")
       self.create_from_text(queries)
       self.connection.execute("TRUNCATE `#{::SmartSearchHistory.table_name}`")
     end
@@ -110,7 +112,7 @@ class SmartSimilarity < ActiveRecord::Base
       if list.nil?
         return [word]
       else
-        self.increment_counter(:count, list.id)
+        self.connection.execute("UPDATE `smart_search_similarities` SET `count` = #{list.count+1} where `smart_search_similarities`.`phrase` = '#{list.phrase}'")
         return [word, list.similarities].flatten
       end    
     end

data/lib/tasks/smart_search.rake

@@ -26,5 +26,33 @@ namespace :smart_search do
       SmartSimilarity.load_url(ENV['URL'])
     end  
   end
+  
+  
+  
+  desc "load ignore words list"
+  task :load_ignore_words => :environment do
+    require File.expand_path("../../smart_search_ignore_word", __FILE__)
+    
+    dic_path = File.expand_path("../../../dictionaries/*", __FILE__)
+    
+    raise dic_path.inspect
+    
+    dic_folders = Dir.glob(dic_path).select {|d| File.directory?(d)}
+    
+    dic_folders.each do |folder|
+      locale = folder.split("/").last
+      word_file = File.join(folder, "#{locale}.ignore_words.dic")
+      if File.exists?(word_file)
+        File.open(word_file, "r").each_line do |word|
+          SmartSearchIgnoreWord.create(:word => word.strip.downcase, :locale => locale)
+        end  
+      end  
+    end  
+  end   
+  
+  
+
     
-end  
+end  
+
+

data/lib/tasks/testing.rake

@@ -0,0 +1,27 @@
+require "active_record"
+namespace :db do
+  desc "Create test database. Overwrite dasebase config with USERNAME=, PASSWORD=, DATABASE="
+  task :create_test_db do
+    config = YAML::load(File.open(File.expand_path("config/database.yml")))["test"]
+    
+    # Overwrite config
+    config.merge!('database' => ENV['DATABASE']) if ENV['DATABASE']
+    config.merge!('username' => ENV['USERNAME']) if ENV['USERNAME']
+    config.merge!('password' => ENV['PASSWORD']) if ENV['PASSWORD']
+    
+    ActiveRecord::Base.establish_connection(config.merge('database' => nil))
+    ActiveRecord::Base.connection.drop_database(config['database']) rescue nil
+    ActiveRecord::Base.connection.create_database(config['database'])
+    ActiveRecord::Base.establish_connection(config)
+  end
+  
+  task :migrate do
+    ActiveRecord::Migrator.migrate(ActiveRecord::Migrator.migrations_paths)
+  end    
+end  
+
+task :test_smart_search do
+  Rake::Task["db:create_test_db"].execute
+  Rake::Task["db:migrate"].execute
+  Rake::Task["test"].execute
+end  

data/test/test_document_multi_line.txt

@@ -0,0 +1,6 @@
+The quick brown fox jumps over the lazy dog.
+The five boxing wizards jump quickly.
+Sphinx of all black quartz judge my vow.
+A quick movement of the enemy will jeopardize six gunboats.
+Five quacking Zephyrs jolt my wax bed.
+Heavy boxes perform waltzes and jigs.

data/test/test_document_one_line.txt

@@ -0,0 +1 @@
+Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

data/test/test_helper.rb

@@ -1,14 +1,16 @@
 # -*- encoding : utf-8 -*-
 require 'rubygems'
+require 'bundler'
 require 'test/unit'
 require 'active_support'
 require 'active_record'
 require 'active_model'
 
 require "smart_search"
-require "add_search_tags"
+require "smart_search/smart_search_engine"
 
-ActiveRecord::Base.establish_connection(:adapter => "sqlite3", :database => ":memory:")
+
+ActiveRecord::Base.establish_connection(:adapter => "mysql2", :database => "smart_search_test")
 
   ActiveRecord::Schema.define(:version => 1) do
     create_table :users do |t|
@@ -16,7 +18,6 @@ ActiveRecord::Base.establish_connection(:adapter => "sqlite3", :database => ":me
       t.string  :last_name
       t.integer :office_id
       t.date    :birthday
-      t.text    :search_tags
       t.timestamps
     end
     
@@ -25,15 +26,15 @@ ActiveRecord::Base.establish_connection(:adapter => "sqlite3", :database => ":me
       t.string  :last_name
       t.integer :user_id
       t.date    :birthday
-      t.text    :search_tags
       t.timestamps
     end
     
     create_table  :offices do |t|  
       t.string  :name
-      t.text    :search_tags
       t.timestamps
-    end  
+    end
+    
+      
   end  
 # 
 # def drop_db

data/test/unit/{smart_search_test.rb → 01_smart_search_test.rb}

@@ -6,6 +6,8 @@ class SmartSearchTest < Test::Unit::TestCase
     office_name = "Office1"
     office = Office.create(:name => office_name)
     
+    Office.enable_similarity = false
+    
     assert_equal office, Office.find_by_tags(office_name).first
   end
   
@@ -36,6 +38,7 @@ class SmartSearchTest < Test::Unit::TestCase
     office_id_nok = 5
     
     User.smart_search :on => [:full_name], :conditions => "office_id <> #{office_id_nok}", :force => true
+    User.enable_similarity = false
     
     user    = User.create(:first_name => "Unknown", :last_name => "User", :office_id => office_id_nok)
     user    = User.create(:first_name => "Public", :last_name => "User", :office_id => office_id_ok)
@@ -46,6 +49,7 @@ class SmartSearchTest < Test::Unit::TestCase
   
   def test_should_use_default_order_and_order_should_be_overwriteable
     User.smart_search :on => [:full_name], :order => :first_name, :force => true
+    User.enable_similarity = false
     
     user_c    = User.create(:first_name => "C", :last_name => "Test1")
     user_a    = User.create(:first_name => "A", :last_name => "Test3")
@@ -65,9 +69,7 @@ class SmartSearchTest < Test::Unit::TestCase
     user_b    = User.create(:first_name => "B", :last_name => "Next2")
     
     assert_equal [],  User.find_by_tags("A").where("last_name <> 'Bah' ")
-  end  
-  
-  
+  end 
   
   
 end  

data/test/unit/02_smart_search_similarity_test.rb

@@ -0,0 +1,32 @@
+# -*- encoding : utf-8 -*-
+require "test_helper"
+class SmartSearchSimilartyTest < Test::Unit::TestCase
+  
+  def test_similarity_should_load_from_file
+    testfile_1 = File.expand_path("../../test_document_one_line.txt", __FILE__)
+    testfile_2 = File.expand_path("../../test_document_multi_line.txt", __FILE__)
+    SmartSimilarity.connection.execute "Truncate table #{SmartSimilarity.table_name}"
+    
+    assert_equal 0, SmartSimilarity.count
+    SmartSimilarity.load_file(testfile_2)
+    new_count = SmartSimilarity.count
+    assert_not_equal 0, new_count
+    
+    SmartSimilarity.load_file(testfile_1)
+    assert_not_equal new_count, SmartSimilarity.count
+  end
+  
+  def test_similarity_should_load_from_url
+    count = SmartSimilarity.count
+    SmartSimilarity.load_url("https://github.com/florianeck/smart_search")
+    assert_not_equal count, SmartSimilarity.count
+  end
+  
+  def test_similarity_should_load_from_history
+    count = SmartSimilarity.count
+    User.find_by_tags("this is history now")
+    SmartSimilarity.load_from_query_history
+    assert_not_equal count, SmartSimilarity.count
+  end      
+  
+end  

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: smart_search
 version: !ruby/object:Gem::Version
-  version: 0.0.61
+  version: 0.0.65
 platform: ruby
 authors:
 - Florian Eck
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-02-09 00:00:00.000000000 Z
+date: 2014-02-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -39,49 +39,55 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: spawnling
+  name: friendly_extensions
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.0.61
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.0.61
 - !ruby/object:Gem::Dependency
-  name: friendly_extensions
+  name: mysql2
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.0.61
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.0.61
-description: Adds easy to use full-text search to ActiveRecord models, based the attributes
-  you want to search.
+        version: '0'
+description: SmartSearch adds full-text search functions to ActiveRecord, including
+  search for similiar words. Its fast, simple, and works with almost zero-config!
 email: it-support@friends-systems.de
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- config/database.yml
 - config/routes.rb
-- lib/add_search_tags.rb
 - lib/smart_search.rb
 - lib/smart_search/smart_search_engine.rb
 - lib/smart_search_history.rb
+- lib/smart_search_ignore_word.rb
+- lib/smart_search_tag.rb
 - lib/smart_similarity.rb
 - lib/tasks/smart_search.rake
+- lib/tasks/testing.rake
+- test/test_document_multi_line.txt
+- test/test_document_one_line.txt
 - test/test_helper.rb
-- test/unit/smart_search_test.rb
+- test/unit/01_smart_search_test.rb
+- test/unit/02_smart_search_similarity_test.rb
 homepage: https://rubygems.org/gems/smart_search
 licenses: []
 metadata: {}
@@ -104,7 +110,10 @@ rubyforge_project:
 rubygems_version: 2.2.1
 signing_key: 
 specification_version: 4
-summary: Simple, easy to use search.
+summary: Simple, easy to use search MySQL based search for ActiveRecord
 test_files:
+- test/test_document_multi_line.txt
+- test/test_document_one_line.txt
 - test/test_helper.rb
-- test/unit/smart_search_test.rb
+- test/unit/01_smart_search_test.rb
+- test/unit/02_smart_search_similarity_test.rb

data/lib/add_search_tags.rb

@@ -1,8 +0,0 @@
-# -*- encoding : utf-8 -*-
-class AddSearchTags < ActiveRecord::Migration
- 
-  def self.add_to_table(table_name)
-    add_column table_name, :search_tags, :text
-  end  
-  
-end