redis-search 0.1

data/README.textile

@@ -0,0 +1,55 @@
+h1. RedisSearch
+
+High performance real-time search (Support Chinese), index in Redis for Rails application
+
+h2. Features
+
+* Real-time search
+* High performance
+
+h2. Requirements
+
+* Redis 2.2
+* Libmmseg
+
+h2. Install
+
+* gem install redis-search
+
+h2. Configure
+
+  # config/initializers/redis_search.rb
+  require "redis_search"
+  redis = Redis.new(:host => redis_config[:host],:port => redis_config[:port])
+  redis.select("app_name.search")
+  RedisSearch.configure do |config|
+    config.redis = redis
+  end
+
+h2. Usage
+
+	class Post
+	  include Mongoid::Document
+	  include RedisSearch
+	  
+	  field :title
+	  field :body
+	  
+	  belongs_to :user
+	  belongs_to :category
+	  
+	  # bind RedisSearch callback event, it will to rebuild search indexes when data create or update.
+	  redis_search_index(:title_field => :title,
+	                     :ext_fields => [:category_name])
+	  
+	  def category_name
+	    self.category.name
+	  end
+	end
+	
+	# GET /searchs?q=title
+  class SearchController < ApplicationController
+    def index
+      RedisSearch::Search.query(params[:q], :type => "Post")
+    end
+  end

data/lib/redis_search.rb

@@ -0,0 +1,11 @@
+require "redis_search/base"
+require "redis_search/search"
+require "redis_search/config"
+
+module RedisSearch
+  class << self
+    def configure
+      yield self.config ||= Config.new
+    end
+  end
+end

data/lib/redis_search/base.rb

@@ -0,0 +1,53 @@
+module RedisSearch
+  extend ActiveSupport::Concern
+
+  module ClassMethods
+    def redis_search_index(options = {})
+      title_field = options[:title_field] || :title
+      ext_fields = options[:ext_fields] || []
+      class_eval %(
+        def redis_search_ext_fields(ext_fields)
+          exts = {}
+          ext_fields.each do |f|
+            exts[f] = instance_eval(f.to_s)
+          end
+          exts
+        end
+
+        after_create :redis_search_index_create
+        def redis_search_index_create
+          s = Search.new(:title => self.#{title_field}, :id => self.id, 
+                          :exts => self.redis_search_ext_fields(#{ext_fields}), 
+                          :type => self.class.to_s)
+          s.save
+        end
+
+        before_destroy :redis_search_index_remove
+        def redis_search_index_remove
+          Search.remove(:id => self.id, :title => self.#{title_field}, :type => self.class.to_s)
+        end
+
+        before_update :redis_search_index_update
+        def redis_search_index_update
+          index_fields_changed = false
+          #{ext_fields}.each do |f|
+            next if f.to_s == "id"
+            if instance_eval(f.to_s + "_changed?")
+              index_fields_changed = true
+            end
+          end
+          begin
+            if(self.#{title_field}_changed?)
+              index_fields_changed = true
+            end
+          rescue
+          end
+          if index_fields_changed
+            Search.remove(:id => self.id, :title => self.#{title_field}_was, :type => self.class.to_s)
+            self.redis_search_index_create
+          end
+        end
+      )
+    end
+  end
+end

data/lib/redis_search/config.rb

@@ -0,0 +1,14 @@
+module RedisSearch
+  class << self
+    attr_accessor :config
+  end
+  
+  class Config
+    attr_accessor :redis, :debug
+    
+    def initialize
+      self.debug = false
+      self.redis = nil
+    end
+  end
+end

data/lib/redis_search/search.rb

@@ -0,0 +1,173 @@
+# coding: utf-8
+require "mmseg"
+module RedisSearch  
+  class Search
+    attr_accessor :type, :title, :id, :exts
+    def initialize(options = {})
+      self.exts = []
+      options.keys.each do |k|
+        eval("self.#{k} = options[k]")
+      end
+    end
+  
+    def self.split(text)
+      algor = RMMSeg::Algorithm.new(text)
+      words = []
+      loop do
+        tok = algor.next_token
+        break if tok.nil?
+        words << tok.text
+      end
+      words
+    end
+    
+    def self.warn(msg)
+      puts "[RedisSearch][warn]: #{msg}"
+    end
+  
+    # 生成 uuid，用于作为 hashes 的 field, sets 关键词的值
+    def self.mk_sets_key(type, key)
+      "#{type}:#{key.downcase}"
+    end
+  
+    def self.mk_complete_key(type)
+      "Compl#{type}"
+    end
+  
+    def self.word?(word)
+      return !/^[\w\u4e00-\u9fa5]+$/i.match(word.force_encoding("UTF-8")).blank?
+    end
+
+    def save
+      return if self.title.blank?
+      data = {:title => self.title, :id => self.id, :type => self.type}
+      self.exts.each do |f|
+        data[f[0]] = f[1]
+      end
+    
+      # 将原始数据存入 hashes
+      res = RedisSearch.config.redis.hset(self.type, self.id, data.to_json)
+      # 保存 sets 索引，以分词的单词为key，用于后面搜索，里面存储 ids
+      words = Search.split(self.title)
+      return if words.blank?
+      words.each do |word|
+        next if not Search.word?(word)
+        save_zindex(word)
+        key = Search.mk_sets_key(self.type,word)
+        RedisSearch.config.redis.sadd(key, self.id)
+      end
+    end
+  
+    def save_zindex(word)
+      return if not Search.word?(word)
+      word = word.downcase
+      key = Search.mk_complete_key(self.type)
+      (1..(word.length)).each do |l|
+        prefix = word[0...l]
+        RedisSearch.config.redis.zadd(key, 0, prefix)
+      end
+      RedisSearch.config.redis.zadd(key, 0, word + "*")
+    end
+
+    def self.remove(options = {})
+      puts options.inspect
+      type = options[:type]
+      RedisSearch.config.redis.hdel(type,options[:id])
+      words = Search.split(options[:title])
+      words.each do |word|
+        next if not Search.word?(word)
+        key = Search.mk_sets_key(type,word)
+        RedisSearch.config.redis.srem(key, options[:id])
+      end
+    end
+
+    # Use for short title search, this method is search by chars, for example Tag, User, Category ...
+    # 
+    # h3. params:
+    #   type      model name
+    #   w         search char
+    #   :limit    result limit
+    # h3. usage:
+    # * RedisSearch::Search.complete("Tag","r") => ["Ruby","Rails", "REST", "Redis", "Redmine"]
+    # * RedisSearch::Search.complete("Tag","re") => ["Redis", "Redmine"]
+    # * RedisSearch::Search.complete("Tag","red") => ["Redis", "Redmine"]
+    # * RedisSearch::Search.complete("Tag","redi") => ["Redis"]
+    def self.complete(type, w, options = {})
+      limit = options[:limit] || 10 
+
+      prefix_matchs = []
+      rangelen = 100 # This is not random, try to get replies < MTU size
+      prefix = w.downcase
+      key = Search.mk_complete_key(type)
+      start = RedisSearch.config.redis.zrank(key,prefix)
+
+      return [] if !start
+      count = limit
+      while prefix_matchs.length <= count
+        range = RedisSearch.config.redis.zrange(key,start,start+rangelen-1)
+        start += rangelen
+        break if !range or range.length == 0
+        range.each {|entry|
+          minlen = [entry.length,prefix.length].min
+          if entry[0...minlen] != prefix[0...minlen]
+            count = prefix_matchs.count
+            break
+          end
+          if entry[-1..-1] == "*" and prefix_matchs.length != count
+            prefix_matchs << entry[0...-1]
+          end
+        }
+      end
+      words = []
+      words = prefix_matchs.uniq.collect { |w| Search.mk_sets_key(type,w) }
+      ids = RedisSearch.config.redis.sunion(*words)
+      return [] if ids.blank?
+      hmget(type,ids, :limit => limit)
+    end
+
+    # Search items, this will split words by Libmmseg
+    # 
+    # h3. params:
+    #   type      model name
+    #   text         search text
+    #   :limit    result limit
+    # h3. usage:
+    # * RedisSearch::Search.query("Tag","Ruby vs Python")
+    def self.query(type, text,options = {})
+      result = []
+      return result if text.strip.blank?
+
+      words = Search.split(text)
+      limit = options[:limit] || 10
+      sort_field = options[:sort_field] || "id"
+      words = words.collect { |w| Search.mk_sets_key(type,w) }
+      return result if words.blank?
+      ids = RedisSearch.config.redis.sinter(*words)
+      hmget(type,ids, :limit => limit, :sort_field => sort_field)
+    end
+  
+    private
+      def self.hmget(type, ids, options = {})
+        result = []
+        limit = options[:limit] || 10
+        sort_field = options[:sort_field] || "id"
+        return result if ids.blank?
+        RedisSearch.config.redis.hmget(type,*ids).each do |r|
+          begin
+            result << JSON.parse(r)
+          rescue => e
+            Search.warn("Search.query failed: #{e}")
+          end
+        end
+        items = sort_result(result, type, sort_field)
+        items = items[0..limit-1] if items.length > limit
+        items
+      end
+  
+      def self.sort_result(items, type, sort_field)
+        return items if items.blank?
+        items = items.sort { |x,y| y[sort_field] <=> x[sort_field] }
+        items
+      end
+  end
+end

metadata

@@ -0,0 +1,74 @@
+--- !ruby/object:Gem::Specification
+name: redis-search
+version: !ruby/object:Gem::Version
+  version: '0.1'
+  prerelease: 
+platform: ruby
+authors:
+- Jason Lee
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2011-08-16 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rmmseg-cpp-huacnlee
+  requirement: &2159834180 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.2.8
+  type: :runtime
+  prerelease: false
+  version_requirements: *2159834180
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: &2159833700 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.1.1
+  type: :runtime
+  prerelease: false
+  version_requirements: *2159833700
+description: High performance real-time search (Support Chinese), index in Redis for
+  Rails application.
+email:
+- huacnlee@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/redis_search/base.rb
+- lib/redis_search/config.rb
+- lib/redis_search/search.rb
+- lib/redis_search.rb
+- README.textile
+homepage: http://github.com/huacnlee/redis-search
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: 1.3.6
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.6
+signing_key: 
+specification_version: 3
+summary: High performance real-time search (Support Chinese), index in Redis for Rails
+  application.
+test_files: []