smart_search 0.0.55 → 0.0.61

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ebbed44ee232de1e1b002de0114c895f22669cbe
-  data.tar.gz: 6fe910045758f11f1c1fccc40229ca3b14919361
+  metadata.gz: d1dba8372dab82dd2cce0ebc37101e2ebcf2fba1
+  data.tar.gz: a71c65f0d8adac611ce88e75a43ea30d36a5d63a
 SHA512:
-  metadata.gz: 92f42ecd8d1f22b42221f93453ff31df341d546771525183c8fef42cd900aa05db1148c3251832e75e28e4f38170916b9aee4bef9862bc8d72b06959f89a17eb
-  data.tar.gz: b1d4e94ec2556002bce8a453d6830eae3eb6808f1af30580e988bf5ccb785ff4eee7014d6ad7951a3eae5b9d0a22e5b64eb72ad9763f6185c7c73fa00ed46495
+  metadata.gz: 6a932397b5b2eafae190335567419028ef0ad2547c2156c1aab1ca5b1aed4204caaa646c82bd1f0b062eecc88662f06ba06422b0c59336ff1cfc910c670c1e27
+  data.tar.gz: 6b544662bb2360c1bba224a580d02fc8d5b46153200d77734f2297471b8c67d7362deadc7c93d6fbbf7cd961617e9cb48fa8bb373e14584b88688cbe21edbbde

data/lib/smart_search/smart_search_engine.rb

@@ -0,0 +1,5 @@
+module SmartSearch
+  class SmartSearchEngine < Rails::Engine
+    isolate_namespace SmartSearch
+  end  
+end  

data/lib/smart_search.rb

@@ -1,7 +1,11 @@
 # -*- encoding : utf-8 -*-
 require "smart_search"
+require "smart_similarity"
+require "smart_search_history"
 require "add_search_tags"
 
+require "smart_search/smart_search_engine"
+
 module SmartSearch
   
   def self.included(base)
@@ -9,18 +13,27 @@ module SmartSearch
   end  
   
   module ClassMethods
+    # Enable SmartSearch for the current ActiveRecord model.
+    # accepts options:
+    # - :on, define which attributes to add to the search index
+    # - :conditions, define default scope for all queries made
+    # - :group, group by column
+    # - :order, order by column
+    # see readme for details
     def smart_search(options = {:on => [], :conditions => nil, :group => nil, :order => "created_at", :force => false})
       if table_exists?
         # Check if search_tags exists
         if !is_smart_search? || options[:force] == true
-          puts "\nAdding SmartSearch to #{self.name}: #{options.inspect}"
-          cattr_accessor :condition_default, :group_default, :tags, :order_default
+          
+          cattr_accessor :condition_default, :group_default, :tags, :order_default, :enable_similarity
           send :include, InstanceMethods
           if self.column_names.index("search_tags").nil?
             ::AddSearchTags.add_to_table(self.table_name)
           end
             self.send(:before_save, :create_search_tags)
-
+            
+            self.enable_similarity = true
+            
             # options zuweisen
             if options[:conditions].is_a?(String) && !options[:conditions].blank?
               self.condition_default = options[:conditions]
@@ -41,22 +54,40 @@ module SmartSearch
       end  
     end
     
+    # Verify if SmartSearch already loaded for this model
     def is_smart_search?
       self.included_modules.include?(InstanceMethods)
     end
     
+    # defines where to look for a partial to load when displaying results for this model
     def result_template_path
       "/search/results/#{self.name.split("::").last.underscore}"
     end  
     
+    # Serach database for given search tags
     def find_by_tags(tags = "", options = {})
       if self.is_smart_search?
+        
+        # Save Data for similarity analysis
+        if tags.size > 3
+          self.connection.execute("INSERT INTO `#{::SmartSearchHistory.table_name}` (`query`) VALUES ('#{tags.gsub(/[^a-zA-ZäöüÖÄÜß\ ]/, '')}');")
+        end  
+        
         tags = tags.split(" ")
         
         # Fallback for Empty String
         tags << "#" if tags.empty?
         
-        tags.map! {|t| "search_tags LIKE '%#{t.downcase}%'"}
+        # Similarity
+        if self.enable_similarity == true
+          tags.map! do |t|   
+            similars = SmartSimilarity.similars(t, :increment_counter => true).join("|")
+            "search_tags REGEXP '#{similars}'"
+          end  
+          
+        else
+          tags.map! {|t| "search_tags LIKE '%#{t}%'"}
+        end  
         
         
         results =  self.where("(#{tags.join(' AND ')})")
@@ -85,15 +116,24 @@ module SmartSearch
       end  
     end
     
+    # reload search_tags for entire table based on the attributes defined in ':on' option passed to the 'smart_search' method
     def set_search_index
       s = self.all.size.to_f
       self.all.each_with_index do |a, i|
-        a.create_search_tags
+        a.create_search_tags rescue nil
         a.send(:update_without_callbacks)
         done = ((i+1).to_f/s)*100
         printf "Set search index for #{self.name}: #{done}%%                  \r"
       end  
     end  
+    
+    # Create all search tags for this table into similarity index
+    def set_similarity_index
+      
+      search_tags_list = self.connection.select_all("SELECT search_tags from #{self.table_name}").map {|r| r["search_tags"]}
+      
+      SmartSimilarity.create_from_text(search_tags_list.join(" "))
+    end  
          
   end  
   
@@ -103,6 +143,7 @@ module SmartSearch
       self.class.result_template_path
     end  
     
+    # create search tags for this very record based on the attributes defined in ':on' option passed to the 'Class.smart_search' method
     def create_search_tags
       tags = []
       self.class.tags.each do |tag|

data/lib/smart_search_history.rb

@@ -0,0 +1,7 @@
+# Saves all queries made so the data can be used to build a similarity index based on the queries made
+# by actual users of the website
+class SmartSearchHistory < ActiveRecord::Base
+  
+  
+  
+end  

data/lib/smart_similarity.rb

@@ -0,0 +1,125 @@
+# This class is used to build similiarity index
+class SmartSimilarity < ActiveRecord::Base
+  #= Configuration
+  serialize :similarities, Array
+  self.table_name = "smart_search_similarities"
+  require "amatch"
+  
+      #== Associations
+          # => Stuff in Here
+
+      #== Plugins and modules
+        #=== PlugIns
+          # => Stuff in Here        
+
+        #=== include Modules
+          # => Stuff in Here
+
+      #== Konstanten
+          # Defines the min. result of word simililarity check
+          SIMILARITY_FACTOR = 0.8
+          # Defines first simililarity check method 
+          SIMILARITY_METHOD_1 = :jarowinkler
+          # Defines first simililarity check method 
+          SIMILARITY_METHOD_2 = :levenshtein
+          
+          # An average of both results will generated and compered with 'SIMILARITY_FACTOR'
+          
+          # Limit Number of similar words
+          SIMILARITY_LIMIT  = 8
+          
+      #== Validation and Callbacks
+        #=== Validation
+        
+        #=== Callbacks
+          # => Stuff in Here
+    # => END
+  
+    # Create similarity data based on the given text
+    # This method is used to generate date from every source, e.g. file, url, single words etc..
+    def self.create_from_text(text)
+      # prepare text
+      prepared_text = text.downcase.split(/\b/).uniq
+      prepared_text = prepared_text.select {|w| w.size >= 3 && !w.match(/[0-9\-_<>\.\/(){}&\?"'@*+$!=,:'#;]/)}
+      list = {}
+      prepared_text.each do |word|
+        # Load index from database
+        words_in_db = self.find_by_phrase(word)
+        if words_in_db.nil?
+          self.connection.execute "INSERT INTO `#{self.table_name}` (`phrase`, `ind`) VALUES ('#{word}', '#{word[0..1]}');"
+          current = []
+        else  
+          current = words_in_db.similarities
+        end  
+        
+        current += prepared_text.select {|w| w != word && self.match_words(w,word) >= SIMILARITY_FACTOR}
+        
+        list[word] = current.uniq
+      end  
+      
+      # Write to Database
+      list.each do |word, sims|
+        sims = sims.sort_by {|s| self.match_words(s,word) }.reverse.first(SIMILARITY_LIMIT)
+        
+        self.connection.execute 'UPDATE %s set similarities = "%s" where phrase = "%s"' % [self.table_name, sims.to_yaml, word] rescue nil
+      end  
+    end
+    
+    # Add one simgle word to database and check if there are already similars
+    def self.add_word(word)
+      words = [word]
+      phrases = self.connection.select_all("SELECT phrase from smart_search_similarities").map {|r| r["phrase"] }  
+      words +=  phrases.select {|p| self.match_words(p,word) >= SIMILARITY_FACTOR }
+      
+      self.create_from_text(words.join(" "))
+    end
+    
+    # Load an entire file to the index.
+    # Best used for loading big dictionary files.
+    # Uses 'spawnling' to split the data into 8 stacks and load them simultaniously
+    def self.load_file(path)
+      count = %x{wc -l #{path}}.split[0].to_i
+      puts "loading file: #{path}"
+      puts "=> #{count} rows"
+      File.open(path, "r").to_a.seperate([8,count].min).each_with_index do |stack, si| 
+        Spawnling.new(:argv => "sim-file-#{si}") do
+          QueryLog.info "sim-file-#{si}"
+          stack.each_with_index do |l,i|
+            QueryLog.info "#{si}: #{i.fdiv(count).round(4)} %"
+            self.add_word(l)
+          end
+        end  
+      end  
+    end
+    
+    # Load words from website and save them to index
+    def self.load_url(url)
+      self.create_from_text(%x(curl #{url}))
+    end  
+    
+    # Loads your created query history and saves them to the index
+    def self.load_from_query_history
+      queries = self.connection.select_all("SELECT query from `#{::SmartSearchHistory.table_name}`").map {|r| r["query"]}.join(" ")
+      self.create_from_text(queries)
+      self.connection.execute("TRUNCATE `#{::SmartSearchHistory.table_name}`")
+    end
+    
+    # Get array of similar words including orig word
+    def self.similars(word, options = {})      
+      list = self.where(:phrase => word).first
+      if list.nil?
+        return [word]
+      else
+        self.increment_counter(:count, list.id)
+        return [word, list.similarities].flatten
+      end    
+    end
+    
+    # Return match score for two words bases und the two defined similarity methods
+    def self.match_words(word1, word2)
+      x1 = word1.send("#{SIMILARITY_METHOD_1}_similar", word2)
+      x2 = word1.send("#{SIMILARITY_METHOD_2}_similar", word2)
+      return (x1+x2)/2.0
+    end    
+  
+end 

data/lib/tasks/smart_search.rake

@@ -0,0 +1,30 @@
+namespace :smart_search do
+  desc "Load similarity data from query history"
+  task :similarity_from_query_history => :environment do
+    require File.expand_path("../../smart_similarity", __FILE__)
+    SmartSimilarity.load_from_query_history
+  end
+  
+  desc "Load similarity data from file - Use FILE=path/to/file to specify file"
+  task :similarity_from_file => :environment do
+    require File.expand_path("../../smart_similarity", __FILE__)
+    if ENV['FILE_PATH'].nil?
+      raise ArgumentError, "No file specified. "
+    elsif !File.exist?(ENV['FILE_PATH'])
+      raise ArgumentError, "File not found "
+    else    
+      SmartSimilarity.load_file(ENV['FILE_PATH'])
+    end  
+  end
+  
+  desc "Load similarity data from url - Use URL=http://.../ to specify url - Requires 'curl'"
+  task :similarity_from_url => :environment do
+    require File.expand_path("../../smart_similarity", __FILE__)
+    if ENV['URL'].nil?
+      raise ArgumentError, "No URL specified. "
+    else    
+      SmartSimilarity.load_url(ENV['URL'])
+    end  
+  end
+    
+end  

metadata

@@ -1,15 +1,71 @@
 --- !ruby/object:Gem::Specification
 name: smart_search
 version: !ruby/object:Gem::Version
-  version: 0.0.55
+  version: 0.0.61
 platform: ruby
 authors:
 - Florian Eck
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-03-11 00:00:00.000000000 Z
-dependencies: []
+date: 2014-02-09 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.2.9
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.2.9
+- !ruby/object:Gem::Dependency
+  name: amatch
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: spawnling
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: friendly_extensions
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.61
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.61
 description: Adds easy to use full-text search to ActiveRecord models, based the attributes
   you want to search.
 email: it-support@friends-systems.de
@@ -20,6 +76,10 @@ files:
 - config/routes.rb
 - lib/add_search_tags.rb
 - lib/smart_search.rb
+- lib/smart_search/smart_search_engine.rb
+- lib/smart_search_history.rb
+- lib/smart_similarity.rb
+- lib/tasks/smart_search.rake
 - test/test_helper.rb
 - test/unit/smart_search_test.rb
 homepage: https://rubygems.org/gems/smart_search