redis-search 0.5.2 → 0.6.1

data/README.markdown

@@ -9,6 +9,7 @@ High performance real-time search (Support Chinese), index in Redis for Rails ap
 * Segment words search and prefix match search
 * Support ActiveRecord and Mongoid
 * Sort results by one field
+* Homophone search, pinyin search
 
 ## Requirements
 
@@ -19,8 +20,9 @@ High performance real-time search (Support Chinese), index in Redis for Rails ap
 in Rails application Gemfile
 
   gem 'redis','>= 2.1.1'
-  gem "rmmseg-cpp-huacnlee", "0.2.8"
-  gem 'redis-search', '0.5'
+  gem 'chinese_pinyin', '0.3.0'
+  gem 'rmmseg-cpp-huacnlee', '0.2.8'
+  gem 'redis-search', '0.6'
 
 install bundlers
 
@@ -37,6 +39,7 @@ create file in: config/initializers/redis_search.rb
     Redis::Search.configure do |config|
       config.redis = redis
       config.complete_max_length = 100
+      config.pinyin_match = true
     end
 
 ## Usage
@@ -106,4 +109,4 @@ see [Rdoc.info redis-search](http://rubydoc.info/gems/redis-search)
     
 ## Demo
 
-You can try the search feature in [`zheye.org`](http://zheye.org)
+You can try the search feature in [`zheye.org`](http://zheye.org) | [`shu.im`](http://shu.im)

data/lib/redis/search/config.rb

@@ -12,18 +12,24 @@ class Redis
     class Config
       # Redis 
       attr_accessor :redis
-      # Debug toggle
+      # Debug toggle, default false
       attr_accessor :debug
       # config for max length of content with Redis::Search.complete method，default 100
       # Please change this with your real data length, short is fast
       # For example: You use complete search for your User model name field, and the "name" as max length in 15 chars, then you can set here to 15
       # warring! The long content will can't be found, if the config length less than real content.
       attr_accessor :complete_max_length
+      # Pinyin search/index (true|false) - default = false
+      # If set this is true, the indexer will convert Chinese to Pinyin, and index them
+      # When you search "de" will -> 得|的|德...
+      # When you search "得" will -> "de" -> 得|的|德...
+      attr_accessor :pinyin_match
     
       def initialize
         self.debug = false
         self.redis = nil
         self.complete_max_length = 100
+        self.pinyin_match = false
       end
     end
   end

data/lib/redis/search/finder.rb

@@ -1,34 +1,13 @@
 # coding: utf-8
 require "rmmseg"
+require 'chinese_pinyin'
+
 class Redis
   module Search
+    # use rmmseg to split words
     def self.split(text)
-      algor = RMMSeg::Algorithm.new(text)
-      words = []
-      loop do
-        tok = algor.next_token
-        break if tok.nil?
-        words << tok.text
-      end
-      words
-    end
-    
-    def self.warn(msg)
-      puts "[Redis::Search][warn]: #{msg}"
-    end
-  
-    # 生成 uuid，用于作为 hashes 的 field, sets 关键词的值
-    def self.mk_sets_key(type, key)
-      "#{type}:#{key.downcase}"
-    end
-    
-    def self.mk_score_key(type, id)
-      "#{type}:_score_:#{id}"
-    end
-  
-    def self.mk_complete_key(type)
-      "Compl#{type}"
-    end
+      _split(text)
+    end    
 
     # Use for short title search, this method is search by chars, for example Tag, User, Category ...
     # 
@@ -104,39 +83,98 @@ class Redis
     # h3. usage:
     # * Redis::Search.query("Tag","Ruby vs Python")
     def self.query(type, text,options = {})
+      tm = Time.now
       result = []
       return result if text.strip.blank?
 
-      words = Search.split(text)
       limit = options[:limit] || 10
       sort_field = options[:sort_field] || "id"
+
+      words = Search.split(text)
       words = words.collect { |w| Search.mk_sets_key(type,w) }
-      word_score = words.collect { |w| "#{w}:*" }
+      
       return result if words.blank?
+      
       temp_store_key = "tmpinterstore:#{words.join("+")}"
+      
       if words.length > 1
         if !Redis::Search.config.redis.exists(temp_store_key)
           # 将多个词语组合对比，得到交集，并存入临时区域
           Redis::Search.config.redis.sinterstore(temp_store_key,*words)
           # 将临时搜索设为1天后自动清除
           Redis::Search.config.redis.expire(temp_store_key,86400)
+          
+          # 拼音搜索
+          if Search.config.pinyin_match
+            pinyin_words = Search.split_pinyin(text)
+            pinyin_words = pinyin_words.collect { |w| Search.mk_sets_key(type,w) }
+            temp_sunion_key = "tmpsunionstore:#{words.join("+")}"
+            if Search.config.pinyin_match
+              temp_pinyin_store_key = "tmpinterstore:#{pinyin_words.join("+")}"
+            end
+            # 找出拼音的
+            Redis::Search.config.redis.sinterstore(temp_pinyin_store_key,*pinyin_words)
+            # 合并中文和拼音的搜索结果
+            Redis::Search.config.redis.sunionstore(temp_sunion_key,*[temp_store_key,temp_pinyin_store_key])
+            # 将临时搜索设为1天后自动清除
+            Redis::Search.config.redis.expire(temp_pinyin_store_key,86400)
+            Redis::Search.config.redis.expire(temp_sunion_key,86400)
+            temp_store_key = temp_sunion_key
+          end
         end
-        # 根据需要的数量取出 ids
-        ids = Redis::Search.config.redis.sort(temp_store_key,
-                                              :limit => [0,limit], 
-                                              :by => Search.mk_score_key(type,"*"),
-                                              :order => "desc")
       else
-        # 根据需要的数量取出 ids
-        ids = Redis::Search.config.redis.sort(words.first,
-                                              :limit => [0,limit], 
-                                              :by => Search.mk_score_key(type,"*"),
-                                              :order => "desc")
+        temp_store_key = words.first
       end
-      hmget(type,ids, :sort_field => sort_field)
+      
+      # 根据需要的数量取出 ids
+      ids = Redis::Search.config.redis.sort(temp_store_key,
+                                            :limit => [0,limit], 
+                                            :by => Search.mk_score_key(type,"*"),
+                                            :order => "desc")
+      result = hmget(type,ids, :sort_field => sort_field)
+      Search.info("{#{type} : \"#{text}\"} | Time spend: #{Time.now - tm}s")
+      result 
     end
   
+    protected
+      def self.split_pinyin(text)
+        # Pinyin search split as pinyin again
+        _split(Pinyin.t(text))
+      end
+  
     private
+      def self._split(text)
+        algor = RMMSeg::Algorithm.new(text)
+        words = []
+        loop do
+          tok = algor.next_token
+          break if tok.nil?
+          words << tok.text
+        end
+        words
+      end
+    
+      def self.warn(msg)
+        ::Rails.logger.warn("\e[33m[Redis::Search] #{msg}\e[0m")
+      end
+      
+      def self.info(msg)
+        ::Rails.logger.debug("\e[32m[Redis::Search] #{msg}\e[0m")
+      end
+  
+      # 生成 uuid，用于作为 hashes 的 field, sets 关键词的值
+      def self.mk_sets_key(type, key)
+        "#{type}:#{key.downcase}"
+      end
+    
+      def self.mk_score_key(type, id)
+        "#{type}:_score_:#{id}"
+      end
+  
+      def self.mk_complete_key(type)
+        "Compl#{type}"
+      end
+      
       def self.hmget(type, ids, options = {})
         result = []
         sort_field = options[:sort_field] || "id"

data/lib/redis/search/index.rb

@@ -19,11 +19,13 @@ class Redis
         # 将原始数据存入 hashes
         res = Redis::Search.config.redis.hset(self.type, self.id, data.to_json)
         # 保存 sets 索引，以分词的单词为key，用于后面搜索，里面存储 ids
-        words = Search.split(self.title)
+        words = Search::Index.split_words_for_index(self.title)
         return if words.blank?
         words.each do |word|
           key = Search.mk_sets_key(self.type,word)
+          # word index for item id
           Redis::Search.config.redis.sadd(key, self.id)
+          # score for search sort
           Redis::Search.config.redis.set(Search.mk_score_key(self.type,self.id),self.score)
         end
 
@@ -32,28 +34,48 @@ class Redis
           save_prefix_index
         end
       end
-
-      def save_prefix_index
-        word = self.title.downcase
-        Redis::Search.config.redis.sadd(Search.mk_sets_key(self.type,self.title), self.id)
-        key = Search.mk_complete_key(self.type)
-        (1..(word.length)).each do |l|
-          prefix = word[0...l]
-          Redis::Search.config.redis.zadd(key, 0, prefix)
-        end
-        Redis::Search.config.redis.zadd(key, 0, word + "*")
-      end
       
       def self.remove(options = {})
         type = options[:type]
         Redis::Search.config.redis.hdel(type,options[:id])
-        words = Search.split(options[:title])
+        words = Search::Index.split_words_for_index(options[:title])
         words.each do |word|
           key = Search.mk_sets_key(type,word)
           Redis::Search.config.redis.srem(key, options[:id])
           Redis::Search.config.redis.del(Search.mk_score_key(type,options[:id]))
         end
       end
+      
+      private
+        def self.split_words_for_index(title)
+          words = Search.split(title)
+          if Search.config.pinyin_match
+            # covert Chinese to pinyin to as an index
+            words += Search.split_pinyin(title)
+          end
+          words.uniq
+        end
+        
+        def save_prefix_index
+          words = []
+          words << self.title.downcase
+          Redis::Search.config.redis.sadd(Search.mk_sets_key(self.type,self.title), self.id)
+          if Search.config.pinyin_match
+            pinyin = Pinyin.t(self.title.downcase,'')
+            words << pinyin
+            Redis::Search.config.redis.sadd(Search.mk_sets_key(self.type,pinyin), self.id)
+          end
+          
+          words.each do |word|
+            
+            key = Search.mk_complete_key(self.type)
+            (1..(word.length)).each do |l|
+              prefix = word[0...l]
+              Redis::Search.config.redis.zadd(key, 0, prefix)
+            end
+            Redis::Search.config.redis.zadd(key, 0, word + "*")
+          end
+        end
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: redis-search
 version: !ruby/object:Gem::Version
-  version: 0.5.2
+  version: 0.6.1
   prerelease: 
 platform: ruby
 authors:
@@ -9,22 +9,33 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-09-25 00:00:00.000000000Z
+date: 2011-09-30 00:00:00.000000000Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: chinese_pinyin
+  requirement: &2157459160 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.3.0
+  type: :runtime
+  prerelease: false
+  version_requirements: *2157459160
 - !ruby/object:Gem::Dependency
   name: rmmseg-cpp-huacnlee
-  requirement: &2151875920 !ruby/object:Gem::Requirement
+  requirement: &2157458680 !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ~>
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: 0.2.8
   type: :runtime
   prerelease: false
-  version_requirements: *2151875920
+  version_requirements: *2157458680
 - !ruby/object:Gem::Dependency
   name: redis
-  requirement: &2151873980 !ruby/object:Gem::Requirement
+  requirement: &2157458200 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,7 +43,7 @@ dependencies:
         version: 2.1.1
   type: :runtime
   prerelease: false
-  version_requirements: *2151873980
+  version_requirements: *2157458200
 description: High performance real-time search (Support Chinese), index in Redis for
   Rails application.
 email: