redis-search 0.3.2 → 0.4

data/README.markdown

@@ -1,4 +1,4 @@
-# RedisSearch
+# Redis-Search
 
 High performance real-time search (Support Chinese), index in Redis for Rails application
 
@@ -11,16 +11,15 @@ High performance real-time search (Support Chinese), index in Redis for Rails ap
 
 ## Requirements
 
-* Ruby 1.9.2
 * Redis 2.2+
 
 ## Install
 
 in Rails application Gemfile
 
-	gem 'redis','2.1.1'
+	gem 'redis','>= 2.1.1'
 	gem "rmmseg-cpp-huacnlee", "0.2.8"
-	gem 'redis-search', '0.3'
+	gem 'redis-search', '0.4'
 
 install bundlers
 
@@ -30,22 +29,22 @@ install bundlers
 
 create file in: config/initializers/redis_search.rb
 
-    require "redis_search"
+    require "redis-search"
     redis = Redis.new(:host => "127.0.0.1",:port => "6379")
     # change redis database to 3, you need use a special database for search feature.
     redis.select(3)
-    RedisSearch.configure do |config|
+    Redis::Search.configure do |config|
       config.redis = redis
       config.complete_max_length = 100
     end
 
 ## Usage
 
-bind RedisSearch callback event, it will to rebuild search indexes when data create or update.
+bind Redis::Search callback event, it will to rebuild search indexes when data create or update.
 
     class Post
       include Mongoid::Document
-      include RedisSearch
+      include Redis::Search
   
       field :title
       field :body
@@ -63,7 +62,7 @@ bind RedisSearch callback event, it will to rebuild search indexes when data cre
     
     class User
       include Mongoid::Document
-      include RedisSearch
+      include Redis::Search
       
       field :name
       field :tagline
@@ -77,15 +76,21 @@ bind RedisSearch callback event, it will to rebuild search indexes when data cre
     class SearchController < ApplicationController
       # GET /searchs?q=title
       def index
-        RedisSearch::Search.query(params[:q], :type => "Post")
+        Redis::Search.query("Post", params[:q])
       end
       
       # GET /search_users?q=j
       def search_users
-        RedisSearch::Search.complete(params[:q], :type => "Post")
+        Redis::Search.complete("Post", params[:q])
       end
     end
 
+## Index data to Redis
+
+If you are first install it in you old project, or your Redis cache lose, you can use this command to rebuild indexes.
+
+    $ rake redis_search:index
+
 ## Benchmark test
 
 * [https://gist.github.com/1150933](https://gist.github.com/1150933)

data/lib/redis/search/base.rb

@@ -0,0 +1,66 @@
+# coding: utf-8
+class Redis
+  module Search
+    extend ActiveSupport::Concern
+
+    module ClassMethods
+      def redis_search_index(options = {})
+        title_field = options[:title_field] || :title
+        prefix_index_enable = options[:prefix_index_enable] || false
+        ext_fields = options[:ext_fields] || []
+      
+        # store Model name to indexed_models for Rake tasks
+        Search.indexed_models = [] if Search.indexed_models == nil
+        Search.indexed_models << self
+        # bind instance methods and callback events
+        class_eval %(
+          def redis_search_ext_fields(ext_fields)
+            exts = {}
+            ext_fields.each do |f|
+              exts[f] = instance_eval(f.to_s)
+            end
+            exts
+          end
+
+          after_create :redis_search_index_create
+          def redis_search_index_create
+            s = Search::Index.new(:title => self.#{title_field}, 
+                           :id => self.id, 
+                           :exts => self.redis_search_ext_fields(#{ext_fields.inspect}), 
+                           :type => self.class.to_s,
+                           :prefix_index_enable => #{prefix_index_enable})
+            s.save
+            # release s
+            s = nil
+          end
+
+          before_destroy :redis_search_index_remove
+          def redis_search_index_remove
+            Search::Index.remove(:id => self.id, :title => self.#{title_field}, :type => self.class.to_s)
+          end
+
+          before_update :redis_search_index_update
+          def redis_search_index_update
+            index_fields_changed = false
+            #{ext_fields.inspect}.each do |f|
+              next if f.to_s == "id"
+              if instance_eval(f.to_s + "_changed?")
+                index_fields_changed = true
+              end
+            end
+            begin
+              if(self.#{title_field}_changed?)
+                index_fields_changed = true
+              end
+            rescue
+            end
+            if index_fields_changed
+              Search::Index.remove(:id => self.id, :title => self.#{title_field}_was, :type => self.class.to_s)
+              self.redis_search_index_create
+            end
+          end
+        )
+      end
+    end
+  end
+end

data/lib/redis/search/config.rb

@@ -0,0 +1,30 @@
+# coding: utf-8
+class Redis
+  module Search
+    class << self
+      attr_accessor :config, :indexed_models
+    
+      def configure
+        yield self.config ||= Config.new
+      end
+    end
+  
+    class Config
+      # Redis 
+      attr_accessor :redis
+      # Debug toggle
+      attr_accessor :debug
+      # config for max length of content with Redis::Search.complete method，default 100
+      # Please change this with your real data length, short is fast
+      # For example: You use complete search for your User model name field, and the "name" as max length in 15 chars, then you can set here to 15
+      # warring! The long content will can't be found, if the config length less than real content.
+      attr_accessor :complete_max_length
+    
+      def initialize
+        self.debug = false
+        self.redis = nil
+        self.complete_max_length = 100
+      end
+    end
+  end
+end

data/lib/{redis_search/search.rb → redis/search/finder.rb}

@@ -1,15 +1,7 @@
 # coding: utf-8
 require "rmmseg"
-module RedisSearch  
-  class Search
-    attr_accessor :type, :title, :id, :exts, :prefix_index_enable
-    def initialize(options = {})
-      self.exts = []
-      options.keys.each do |k|
-        eval("self.#{k} = options[k]")
-      end
-    end
-  
+class Redis
+  module Search
     def self.split(text)
       algor = RMMSeg::Algorithm.new(text)
       words = []
@@ -22,7 +14,7 @@ module RedisSearch
     end
     
     def self.warn(msg)
-      puts "[RedisSearch][warn]: #{msg}"
+      puts "[Redis::Search][warn]: #{msg}"
     end
   
     # 生成 uuid，用于作为 hashes 的 field, sets 关键词的值
@@ -33,56 +25,6 @@ module RedisSearch
     def self.mk_complete_key(type)
       "Compl#{type}"
     end
-  
-    def self.word?(word)
-      return !/^[\w\u4e00-\u9fa5]+$/i.match(word.force_encoding("UTF-8")).blank?
-    end
-
-    def save
-      return if self.title.blank?
-      data = {:title => self.title, :id => self.id, :type => self.type}
-      self.exts.each do |f|
-        data[f[0]] = f[1]
-      end
-    
-      # 将原始数据存入 hashes
-      res = RedisSearch.config.redis.hset(self.type, self.id, data.to_json)
-      # 保存 sets 索引，以分词的单词为key，用于后面搜索，里面存储 ids
-      words = Search.split(self.title)
-      return if words.blank?
-      words.each do |word|
-        next if not Search.word?(word)
-        key = Search.mk_sets_key(self.type,word)
-        RedisSearch.config.redis.sadd(key, self.id)
-      end
-      
-      # 建立前最索引
-      if prefix_index_enable
-        save_prefix_index
-      end
-    end
-  
-    def save_prefix_index
-      word = self.title.downcase
-      RedisSearch.config.redis.sadd(Search.mk_sets_key(self.type,self.title), self.id)
-      key = Search.mk_complete_key(self.type)
-      (1..(word.length)).each do |l|
-        prefix = word[0...l]
-        RedisSearch.config.redis.zadd(key, 0, prefix)
-      end
-      RedisSearch.config.redis.zadd(key, 0, word + "*")
-    end
-
-    def self.remove(options = {})
-      type = options[:type]
-      RedisSearch.config.redis.hdel(type,options[:id])
-      words = Search.split(options[:title])
-      words.each do |word|
-        next if not Search.word?(word)
-        key = Search.mk_sets_key(type,word)
-        RedisSearch.config.redis.srem(key, options[:id])
-      end
-    end
 
     # Use for short title search, this method is search by chars, for example Tag, User, Category ...
     # 
@@ -91,24 +33,24 @@ module RedisSearch
     #   w         search char
     #   :limit    result limit
     # h3. usage:
-    # * RedisSearch::Search.complete("Tag","r") => ["Ruby","Rails", "REST", "Redis", "Redmine"]
-    # * RedisSearch::Search.complete("Tag","re") => ["Redis", "Redmine"]
-    # * RedisSearch::Search.complete("Tag","red") => ["Redis", "Redmine"]
-    # * RedisSearch::Search.complete("Tag","redi") => ["Redis"]
+    # * Redis::Search.complete("Tag","r") => ["Ruby","Rails", "REST", "Redis", "Redmine"]
+    # * Redis::Search.complete("Tag","re") => ["Redis", "Redmine"]
+    # * Redis::Search.complete("Tag","red") => ["Redis", "Redmine"]
+    # * Redis::Search.complete("Tag","redi") => ["Redis"]
     def self.complete(type, w, options = {})
       limit = options[:limit] || 10 
 
       prefix_matchs = []
       # This is not random, try to get replies < MTU size
-      rangelen = RedisSearch.config.complete_max_length
+      rangelen = Redis::Search.config.complete_max_length
       prefix = w.downcase
       key = Search.mk_complete_key(type)
       
-      start = RedisSearch.config.redis.zrank(key,prefix)
+      start = Redis::Search.config.redis.zrank(key,prefix)
       return [] if !start
       count = limit
       max_range = start+(rangelen*limit)-1
-      range = RedisSearch.config.redis.zrange(key,start,max_range)
+      range = Redis::Search.config.redis.zrange(key,start,max_range)
       while prefix_matchs.length <= count
         start += rangelen
         break if !range or range.length == 0
@@ -128,16 +70,16 @@ module RedisSearch
       words = prefix_matchs.uniq.collect { |w| Search.mk_sets_key(type,w) }
       if words.length > 1
         temp_store_key = "tmpsunionstore:#{words.join("+")}"   
-        if !RedisSearch.config.redis.exists(temp_store_key)
+        if !Redis::Search.config.redis.exists(temp_store_key)
           # 将多个词语组合对比，得到并集，并存入临时区域   
-          RedisSearch.config.redis.sunionstore(temp_store_key,*words)
+          Redis::Search.config.redis.sunionstore(temp_store_key,*words)
           # 将临时搜索设为1天后自动清除
-          RedisSearch.config.redis.expire(temp_store_key,86400)
+          Redis::Search.config.redis.expire(temp_store_key,86400)
         end
         # 根据需要的数量取出 ids
-        ids = RedisSearch.config.redis.sort(temp_store_key,:limit => [0,limit])
+        ids = Redis::Search.config.redis.sort(temp_store_key,:limit => [0,limit])
       else
-        ids = RedisSearch.config.redis.sort(words.first,:limit => [0,limit])
+        ids = Redis::Search.config.redis.sort(words.first,:limit => [0,limit])
       end
       return [] if ids.blank?
       hmget(type,ids)
@@ -150,7 +92,7 @@ module RedisSearch
     #   text         search text
     #   :limit    result limit
     # h3. usage:
-    # * RedisSearch::Search.query("Tag","Ruby vs Python")
+    # * Redis::Search.query("Tag","Ruby vs Python")
     def self.query(type, text,options = {})
       result = []
       return result if text.strip.blank?
@@ -162,17 +104,17 @@ module RedisSearch
       return result if words.blank?
       temp_store_key = "tmpinterstore:#{words.join("+")}"
       if words.length > 1
-        if !RedisSearch.config.redis.exists(temp_store_key)
+        if !Redis::Search.config.redis.exists(temp_store_key)
           # 将多个词语组合对比，得到交集，并存入临时区域
-          RedisSearch.config.redis.sinterstore(temp_store_key,*words)
+          Redis::Search.config.redis.sinterstore(temp_store_key,*words)
           # 将临时搜索设为1天后自动清除
-          RedisSearch.config.redis.expire(temp_store_key,86400)
+          Redis::Search.config.redis.expire(temp_store_key,86400)
         end
         # 根据需要的数量取出 ids
-        ids = RedisSearch.config.redis.sort(temp_store_key,:limit => [0,limit])
+        ids = Redis::Search.config.redis.sort(temp_store_key,:limit => [0,limit])
       else
         # 根据需要的数量取出 ids
-        ids = RedisSearch.config.redis.sort(words.first,:limit => [0,limit])
+        ids = Redis::Search.config.redis.sort(words.first,:limit => [0,limit])
       end
       hmget(type,ids, :sort_field => sort_field)
     end
@@ -182,7 +124,7 @@ module RedisSearch
         result = []
         sort_field = options[:sort_field] || "id"
         return result if ids.blank?
-        RedisSearch.config.redis.hmget(type,*ids).each do |r|
+        Redis::Search.config.redis.hmget(type,*ids).each do |r|
           begin
             result << JSON.parse(r)
           rescue => e

data/lib/redis/search/index.rb

@@ -0,0 +1,57 @@
+class Redis
+  module Search
+    class Index
+      attr_accessor :type, :title, :id, :exts, :prefix_index_enable
+      def initialize(options = {})
+        self.exts = []
+        options.keys.each do |k|
+          eval("self.#{k} = options[k]")
+        end
+      end
+      
+      def save
+        return if self.title.blank?
+        data = {:title => self.title, :id => self.id, :type => self.type}
+        self.exts.each do |f|
+          data[f[0]] = f[1]
+        end
+
+        # 将原始数据存入 hashes
+        res = Redis::Search.config.redis.hset(self.type, self.id, data.to_json)
+        # 保存 sets 索引，以分词的单词为key，用于后面搜索，里面存储 ids
+        words = Search.split(self.title)
+        return if words.blank?
+        words.each do |word|
+          key = Search.mk_sets_key(self.type,word)
+          Redis::Search.config.redis.sadd(key, self.id)
+        end
+
+        # 建立前最索引
+        if prefix_index_enable
+          save_prefix_index
+        end
+      end
+
+      def save_prefix_index
+        word = self.title.downcase
+        Redis::Search.config.redis.sadd(Search.mk_sets_key(self.type,self.title), self.id)
+        key = Search.mk_complete_key(self.type)
+        (1..(word.length)).each do |l|
+          prefix = word[0...l]
+          Redis::Search.config.redis.zadd(key, 0, prefix)
+        end
+        Redis::Search.config.redis.zadd(key, 0, word + "*")
+      end
+      
+      def self.remove(options = {})
+        type = options[:type]
+        Redis::Search.config.redis.hdel(type,options[:id])
+        words = Search.split(options[:title])
+        words.each do |word|
+          key = Search.mk_sets_key(type,word)
+          Redis::Search.config.redis.srem(key, options[:id])
+        end
+      end
+    end
+  end
+end

data/lib/redis/search/railtie.rb

@@ -0,0 +1,9 @@
+class Redis
+  module Search
+    class Railtie < Rails::Railtie
+      rake_tasks do
+        load File.expand_path('../tasks.rb', __FILE__)
+      end
+    end
+  end
+end

data/lib/redis/search/tasks.rb

@@ -0,0 +1,40 @@
+# coding: utf-8
+require "redis-search"
+namespace :redis_search do
+  desc "Redis-Search index data to Redis"
+  task :index => :environment do
+    tm = Time.now
+    count = 0
+    puts "redis-search index".upcase.rjust(120)
+    puts "-"*120
+    puts "Now indexing search to Redis...".rjust(120)
+    puts ""
+    Redis::Search.indexed_models.each do |klass|
+      print "[#{klass.to_s}]"
+      if klass.superclass.to_s == "ActiveRecord::Base"
+        klass.find_in_batches(:batch_size => 1000) do |items|
+          items.each do |item|
+            item.redis_search_index_create
+      			item = nil
+      			count += 1
+            print "."
+          end
+        end
+      elsif klass.included_modules.collect { |m| m.to_s }.include?("Mongoid::Document")
+        klass.all.each do |item|
+          item.redis_search_index_create
+    			item = nil
+    			count += 1
+          print "."
+        end
+      else
+        puts "skiped, not support this ORM in current."
+      end
+      puts ""
+    end
+    puts ""
+    puts "-"*120
+    puts "Indexed #{count} rows  |  Time spend: #{(Time.now - tm)}s".rjust(120)
+    puts "Rebuild Index done.".rjust(120)
+  end
+end

data/lib/redis-search.rb

@@ -0,0 +1,6 @@
+# coding: utf-8
+require "redis/search/base"
+require "redis/search/finder"
+require "redis/search/index"
+require "redis/search/config"
+require 'redis/search/railtie' if defined?(Rails)

metadata

@@ -1,103 +1,77 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: redis-search
-version: !ruby/object:Gem::Version 
-  hash: 23
+version: !ruby/object:Gem::Version
+  version: '0.4'
   prerelease: 
-  segments: 
-  - 0
-  - 3
-  - 2
-  version: 0.3.2
 platform: ruby
-authors: 
+authors:
 - Jason Lee
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2011-09-21 00:00:00 Z
-dependencies: 
-- !ruby/object:Gem::Dependency 
+date: 2011-09-22 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
   name: rmmseg-cpp-huacnlee
-  prerelease: false
-  requirement: &id001 !ruby/object:Gem::Requirement 
+  requirement: &2153202040 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 7
-        segments: 
-        - 0
-        - 2
-        - 8
+      - !ruby/object:Gem::Version
         version: 0.2.8
   type: :runtime
-  version_requirements: *id001
-- !ruby/object:Gem::Dependency 
-  name: redis
   prerelease: false
-  requirement: &id002 !ruby/object:Gem::Requirement 
+  version_requirements: *2153202040
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: &2153198300 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 9
-        segments: 
-        - 2
-        - 1
-        - 1
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
         version: 2.1.1
   type: :runtime
-  version_requirements: *id002
-description: High performance real-time search (Support Chinese), index in Redis for Rails application.
-email: 
+  prerelease: false
+  version_requirements: *2153198300
+description: High performance real-time search (Support Chinese), index in Redis for
+  Rails application.
+email:
 - huacnlee@gmail.com
 executables: []
-
 extensions: []
-
 extra_rdoc_files: []
-
-files: 
-- lib/redis_search/base.rb
-- lib/redis_search/config.rb
-- lib/redis_search/search.rb
-- lib/redis_search.rb
+files:
+- lib/redis/search/base.rb
+- lib/redis/search/config.rb
+- lib/redis/search/finder.rb
+- lib/redis/search/index.rb
+- lib/redis/search/railtie.rb
+- lib/redis/search/tasks.rb
+- lib/redis-search.rb
 - README.markdown
 homepage: http://github.com/huacnlee/redis-search
 licenses: []
-
 post_install_message: 
 rdoc_options: []
-
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
+required_ruby_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 23
-      segments: 
-      - 1
-      - 3
-      - 6
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
       version: 1.3.6
 requirements: []
-
 rubyforge_project: 
-rubygems_version: 1.8.8
+rubygems_version: 1.8.6
 signing_key: 
 specification_version: 3
-summary: High performance real-time search (Support Chinese), index in Redis for Rails application.
+summary: High performance real-time search (Support Chinese), index in Redis for Rails
+  application.
 test_files: []
-

data/lib/redis_search/base.rb

@@ -1,57 +0,0 @@
-module RedisSearch
-  extend ActiveSupport::Concern
-
-  module ClassMethods
-    def redis_search_index(options = {})
-      title_field = options[:title_field] || :title
-      prefix_index_enable = options[:prefix_index_enable] || false
-      ext_fields = options[:ext_fields] || []
-      class_eval %(
-        def redis_search_ext_fields(ext_fields)
-          exts = {}
-          ext_fields.each do |f|
-            exts[f] = instance_eval(f.to_s)
-          end
-          exts
-        end
-
-        after_create :redis_search_index_create
-        def redis_search_index_create
-          s = Search.new(:title => self.#{title_field}, :id => self.id, 
-                          :exts => self.redis_search_ext_fields(#{ext_fields}), 
-                          :type => self.class.to_s,
-                          :prefix_index_enable => #{prefix_index_enable})
-          s.save
-          # release s
-          s = nil
-        end
-
-        before_destroy :redis_search_index_remove
-        def redis_search_index_remove
-          Search.remove(:id => self.id, :title => self.#{title_field}, :type => self.class.to_s)
-        end
-
-        before_update :redis_search_index_update
-        def redis_search_index_update
-          index_fields_changed = false
-          #{ext_fields}.each do |f|
-            next if f.to_s == "id"
-            if instance_eval(f.to_s + "_changed?")
-              index_fields_changed = true
-            end
-          end
-          begin
-            if(self.#{title_field}_changed?)
-              index_fields_changed = true
-            end
-          rescue
-          end
-          if index_fields_changed
-            Search.remove(:id => self.id, :title => self.#{title_field}_was, :type => self.class.to_s)
-            self.redis_search_index_create
-          end
-        end
-      )
-    end
-  end
-end

data/lib/redis_search/config.rb

@@ -1,23 +0,0 @@
-module RedisSearch
-  class << self
-    attr_accessor :config
-  end
-  
-  class Config
-    # Redis 
-    attr_accessor :redis
-    # Debug toggle
-    attr_accessor :debug
-    # config for max length of content with RedisSearch:Search.complete method，default 100
-    # Please change this with your real data length, short is fast
-    # For example: You use complete search for your User model name field, and the "name" as max length in 15 chars, then you can set here to 15
-    # warring! The long content will can't be found, if the config length less than real content.
-    attr_accessor :complete_max_length
-    
-    def initialize
-      self.debug = false
-      self.redis = nil
-      self.complete_max_length = 100
-    end
-  end
-end

data/lib/redis_search.rb

@@ -1,11 +0,0 @@
-require "redis_search/base"
-require "redis_search/search"
-require "redis_search/config"
-
-module RedisSearch
-  class << self
-    def configure
-      yield self.config ||= Config.new
-    end
-  end
-end