redis-search 0.9.7 → 1.0.0.beta1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +87 -97
- data/lib/redis-search.rb +6 -6
- data/lib/{redis/search → redis-search}/base.rb +41 -42
- data/lib/{redis/search → redis-search}/config.rb +2 -3
- data/lib/{redis/search → redis-search}/finder.rb +63 -58
- data/lib/{redis/search → redis-search}/index.rb +27 -26
- data/lib/{redis/search → redis-search}/railtie.rb +0 -0
- data/lib/{redis/search → redis-search}/tasks.rb +11 -11
- data/lib/redis-search/version.rb +5 -0
- metadata +10 -10
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c3db9e3cf11d6d82edb9366ec887db853a684468
|
4
|
+
data.tar.gz: 349c2ad7333803b7f868cabd5ee4053f216b9e14
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: e1d2dcf8826e6c4e0e3a9414fdc0a8f1b11fc9d0cf024f5c6d5ab27e56e7da3c436f5ab20d1c487f6e2feab09d24462de24cee03209ecccf5c4ce7c51a56d423
|
7
|
+
data.tar.gz: 9389c3f640c4ff76719507d681fb0d8ba4bc24cb7d36544617c76c4d209709121ec87e052e2ae4e6d52ecbb9cba3e09b212aafc1fa07e025c1dbc0a840dffbf1
|
data/README.md
CHANGED
@@ -4,17 +4,10 @@ High performance real-time search (Support Chinese), index in Redis for Rails ap
|
|
4
4
|
|
5
5
|
[中文介绍和使用说明](https://github.com/huacnlee/redis-search/wiki/Usage-in-Chinese)
|
6
6
|
|
7
|
-
## Demo
|
8
|
-
|
9
|
-
![](http://l.ruby-china.org/photo/34368688ee1c1928c2841eb2f41306ec.png)
|
10
|
-
|
11
|
-
You can try the search feature in [`IMAX.im`](http://imax.im) | [`shu.im`](http://shu.im)
|
12
|
-
|
13
|
-
And there is an [Example App](https://github.com/huacnlee/redis-search-example) to show you how to use redis-search.
|
14
|
-
|
15
7
|
## Master Status
|
16
8
|
|
17
|
-
[![CI Status](https://secure.travis-ci.org/huacnlee/redis-search.
|
9
|
+
[![Gem Version](https://badge.fury.io/rb/redis-search.svg)](https://badge.fury.io/rb/redis-search) [![CI Status](https://secure.travis-ci.org/huacnlee/redis-search.svg)](http://travis-ci.org/huacnlee/redis-search) [![CodeCov](https://codecov.io/gh/huacnlee/redis-search/branch/master/graph/badge.svg)](https://codecov.io/gh/huacnlee/redis-search)
|
10
|
+
|
18
11
|
|
19
12
|
## Features
|
20
13
|
|
@@ -34,99 +27,87 @@ And there is an [Example App](https://github.com/huacnlee/redis-search-example)
|
|
34
27
|
|
35
28
|
## Install
|
36
29
|
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
gem 'ruby-pinyin', '0.3.0'
|
41
|
-
gem 'redis-search', '0.9.6'
|
42
|
-
```
|
30
|
+
```ruby
|
31
|
+
gem 'redis-search'
|
32
|
+
```
|
43
33
|
|
44
|
-
|
45
|
-
|
46
|
-
|
34
|
+
```bash
|
35
|
+
$ bundle install
|
36
|
+
```
|
47
37
|
|
48
38
|
## Configure
|
49
39
|
|
50
|
-
* Create file in: config/initializers/
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
40
|
+
* Create file in: config/initializers/redis-search.rb
|
41
|
+
|
42
|
+
```ruby
|
43
|
+
require "redis"
|
44
|
+
require "redis-namespace"
|
45
|
+
require "redis-search"
|
46
|
+
|
47
|
+
# don't forget change namespace
|
48
|
+
redis = Redis.new(host: '127.0.0.1', port: '6379')
|
49
|
+
# We suggest you use a special db in Redis, when you need to clear all data, you can use `flushdb` command to cleanup.
|
50
|
+
redis.select(3)
|
51
|
+
# Give a special namespace as prefix for Redis key, when your have more than one project used redis-search, this config will make them work fine.
|
52
|
+
redis = Redis::Namespace.new("your_app_name:redis_search", redis: redis)
|
53
|
+
Redis::Search.configure do |config|
|
54
|
+
config.redis = redis
|
55
|
+
config.complete_max_length = 100
|
56
|
+
config.pinyin_match = true
|
57
|
+
# use rmmseg, true to disable it, it can save memroy
|
58
|
+
config.disable_rmmseg = false
|
59
|
+
end
|
60
|
+
```
|
70
61
|
|
71
62
|
## Usage
|
72
63
|
|
73
|
-
* Bind Redis::Search callback event, it will to rebuild search indices when data create or update.
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
def index
|
121
|
-
Redis::Search.query("Post", params[:q], :conditions => {:user_id => 12})
|
122
|
-
end
|
123
|
-
|
124
|
-
# GET /search_users?q=j
|
125
|
-
def search_users
|
126
|
-
Redis::Search.complete("Post", params[:q], :conditions => {:user_id => 12, :category_id => 4})
|
127
|
-
end
|
128
|
-
end
|
129
|
-
```
|
64
|
+
* Bind `Redis::Search` callback event, it will to rebuild search indices when data create or update.
|
65
|
+
|
66
|
+
```ruby
|
67
|
+
class Post < ActiveRecord::Base
|
68
|
+
include Redis::Search
|
69
|
+
|
70
|
+
belongs_to :user
|
71
|
+
belongs_to :category
|
72
|
+
|
73
|
+
redis_search_index(title_field: :title,
|
74
|
+
score_field: :hits,
|
75
|
+
condition_fields: [:user_id, :category_id],
|
76
|
+
ext_fields: [:category_name])
|
77
|
+
|
78
|
+
def category_name
|
79
|
+
self.category.name
|
80
|
+
end
|
81
|
+
end
|
82
|
+
```
|
83
|
+
|
84
|
+
```ruby
|
85
|
+
class User < ActiveRecord::Base
|
86
|
+
include Redis::Search
|
87
|
+
|
88
|
+
serialize :alias_names, Array
|
89
|
+
|
90
|
+
redis_search_index(title_field: :name,
|
91
|
+
alias_field: :alias_names,
|
92
|
+
prefix_index_enable: true,
|
93
|
+
score_field: :followers_count,
|
94
|
+
ext_fields: [:email, :tagline])
|
95
|
+
end
|
96
|
+
```
|
97
|
+
|
98
|
+
```ruby
|
99
|
+
class SearchController < ApplicationController
|
100
|
+
# GET /searchs?q=title
|
101
|
+
def index
|
102
|
+
Redis::Search.query("Post", params[:q], conditions: { user_id: 12 })
|
103
|
+
end
|
104
|
+
|
105
|
+
# GET /search_users?q=j
|
106
|
+
def search_users
|
107
|
+
Redis::Search.complete("Post", params[:q], conditions: { user_id: 12, category_id: 4 })
|
108
|
+
end
|
109
|
+
end
|
110
|
+
```
|
130
111
|
|
131
112
|
## Index data to Redis
|
132
113
|
|
@@ -135,13 +116,13 @@ And there is an [Example App](https://github.com/huacnlee/redis-search-example)
|
|
135
116
|
Redis-Search index data to Redis from your model (pass name as CLASS environment variable).
|
136
117
|
|
137
118
|
```bash
|
138
|
-
$ rake redis_search:index:model CLASS='
|
119
|
+
$ rake redis_search:index:model CLASS='User'
|
139
120
|
```
|
140
121
|
|
141
122
|
Customize the batch size:
|
142
123
|
|
143
124
|
```bash
|
144
|
-
$ rake redis_search:index:model CLASS='
|
125
|
+
$ rake redis_search:index:model CLASS='User' BATCH=100
|
145
126
|
```
|
146
127
|
|
147
128
|
### All Models
|
@@ -170,6 +151,15 @@ There is my performance test result.
|
|
170
151
|
|
171
152
|
* [https://gist.github.com/1150933](https://gist.github.com/1150933)
|
172
153
|
|
154
|
+
## Demo
|
155
|
+
|
156
|
+
![](http://l.ruby-china.org/photo/34368688ee1c1928c2841eb2f41306ec.png)
|
157
|
+
|
158
|
+
Projects used redis-search:
|
159
|
+
|
160
|
+
- [redis-search-example](https://github.com/huacnlee/redis-search-example) - An example for show you how to use redis-search.
|
161
|
+
- [IMAX.im](https://github.com/huacnlee/imax.im)
|
162
|
+
|
173
163
|
## License
|
174
164
|
|
175
165
|
* MIT
|
data/lib/redis-search.rb
CHANGED
@@ -1,6 +1,6 @@
|
|
1
|
-
|
2
|
-
require
|
3
|
-
require
|
4
|
-
require
|
5
|
-
require
|
6
|
-
require 'redis
|
1
|
+
require 'redis-search/base'
|
2
|
+
require 'redis-search/finder'
|
3
|
+
require 'redis-search/index'
|
4
|
+
require 'redis-search/config'
|
5
|
+
require 'redis-search/version'
|
6
|
+
require 'redis-search/railtie' if defined?(Rails)
|
@@ -1,9 +1,9 @@
|
|
1
|
-
# coding: utf-8
|
2
1
|
class Redis
|
2
|
+
# nodoc
|
3
3
|
module Search
|
4
4
|
autoload :PinYin, 'ruby-pinyin'
|
5
5
|
|
6
|
-
extend
|
6
|
+
extend ActiveSupport::Concern
|
7
7
|
|
8
8
|
included do
|
9
9
|
cattr_reader :redis_search_options
|
@@ -22,28 +22,28 @@ class Redis
|
|
22
22
|
end
|
23
23
|
|
24
24
|
def redis_search_alias_value(field)
|
25
|
-
return [] if field.blank? || field ==
|
26
|
-
val = (instance_eval("self.#{field}") ||
|
27
|
-
return []
|
28
|
-
if val.is_a?(String)
|
29
|
-
val = val.to_s.split(",")
|
30
|
-
end
|
25
|
+
return [] if field.blank? || field == '_was'.freeze
|
26
|
+
val = (instance_eval("self.#{field}") || ''.freeze).clone
|
27
|
+
return [] unless val.class.in?([String, Array])
|
28
|
+
val = val.to_s.split(',') if val.is_a?(String)
|
31
29
|
val
|
32
30
|
end
|
33
31
|
|
34
32
|
# Rebuild search index with create
|
35
33
|
def redis_search_index_create
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
34
|
+
opts = {
|
35
|
+
title: send(redis_search_options[:title_field]),
|
36
|
+
aliases: redis_search_alias_value(redis_search_options[:alias_field]),
|
37
|
+
id: id,
|
38
|
+
exts: redis_search_fields_to_hash(redis_search_options[:ext_fields]),
|
39
|
+
type: redis_search_options[:class_name] || self.class.name,
|
40
|
+
condition_fields: redis_search_options[:condition_fields],
|
41
|
+
score: send(redis_search_options[:score_field]).to_i,
|
42
|
+
prefix_index_enable: redis_search_options[:prefix_index_enable]
|
43
|
+
}
|
44
|
+
|
45
|
+
s = Search::Index.new(opts)
|
44
46
|
s.save
|
45
|
-
# release s
|
46
|
-
s = nil
|
47
47
|
true
|
48
48
|
end
|
49
49
|
|
@@ -51,15 +51,14 @@ class Redis
|
|
51
51
|
titles.uniq!
|
52
52
|
titles.each do |title|
|
53
53
|
next if title.blank?
|
54
|
-
Search::Index.remove(id:
|
54
|
+
Search::Index.remove(id: id, title: title, type: self.class.name)
|
55
55
|
end
|
56
56
|
true
|
57
57
|
end
|
58
58
|
|
59
59
|
def redis_search_index_before_destroy
|
60
|
-
titles = []
|
61
|
-
titles
|
62
|
-
titles << self.send(self.redis_search_options[:title_field])
|
60
|
+
titles = redis_search_alias_value(redis_search_options[:alias_field])
|
61
|
+
titles << send(redis_search_options[:title_field])
|
63
62
|
|
64
63
|
redis_search_index_delete(titles)
|
65
64
|
true
|
@@ -67,10 +66,10 @@ class Redis
|
|
67
66
|
|
68
67
|
def redis_search_index_need_reindex
|
69
68
|
index_fields_changed = false
|
70
|
-
|
71
|
-
next if f.to_s ==
|
69
|
+
redis_search_options[:ext_fields].each do |f|
|
70
|
+
next if f.to_s == 'id'.freeze
|
72
71
|
field_method = "#{f}_changed?"
|
73
|
-
if
|
72
|
+
if methods.index(field_method.to_sym).nil?
|
74
73
|
Redis::Search.warn("#{self.class.name} model reindex on update need #{field_method} method.")
|
75
74
|
next
|
76
75
|
end
|
@@ -79,24 +78,24 @@ class Redis
|
|
79
78
|
end
|
80
79
|
|
81
80
|
begin
|
82
|
-
if
|
81
|
+
if send("#{redis_search_options[:title_field]}_changed?")
|
83
82
|
index_fields_changed = true
|
84
83
|
end
|
85
84
|
|
86
|
-
if
|
85
|
+
if send(redis_search_options[:alias_field]) ||
|
86
|
+
send("#{redis_search_options[:title_field]}_changed?")
|
87
87
|
index_fields_changed = true
|
88
88
|
end
|
89
89
|
rescue
|
90
90
|
end
|
91
91
|
|
92
|
-
|
92
|
+
index_fields_changed
|
93
93
|
end
|
94
94
|
|
95
95
|
def redis_search_index_after_update
|
96
|
-
if
|
97
|
-
titles = []
|
98
|
-
titles
|
99
|
-
titles << self.send("#{self.redis_search_options[:title_field]}_was")
|
96
|
+
if redis_search_index_need_reindex
|
97
|
+
titles = redis_search_alias_value("#{redis_search_options[:alias_field]}_was")
|
98
|
+
titles << send("#{redis_search_options[:title_field]}_was")
|
100
99
|
redis_search_index_delete(titles)
|
101
100
|
end
|
102
101
|
|
@@ -104,8 +103,8 @@ class Redis
|
|
104
103
|
end
|
105
104
|
|
106
105
|
def redis_search_index_after_save
|
107
|
-
if
|
108
|
-
|
106
|
+
if redis_search_index_need_reindex || new_record?
|
107
|
+
redis_search_index_create
|
109
108
|
end
|
110
109
|
true
|
111
110
|
end
|
@@ -134,32 +133,32 @@ class Redis
|
|
134
133
|
opts[:ext_fields] += opts[:condition_fields] if opts[:condition_fields].is_a?(Array)
|
135
134
|
|
136
135
|
# store Model name to indexed_models for Rake tasks
|
137
|
-
Search.indexed_models = [] if Search.indexed_models
|
136
|
+
Search.indexed_models = [] if Search.indexed_models.nil?
|
138
137
|
Search.indexed_models << self
|
139
138
|
|
140
|
-
class_variable_set(
|
139
|
+
class_variable_set('@@redis_search_options'.freeze, opts)
|
141
140
|
end
|
142
141
|
|
143
142
|
def redis_search_index_batch_create(batch_size = 1000, progressbar = false)
|
144
143
|
count = 0
|
145
|
-
if
|
146
|
-
find_in_batches(:
|
144
|
+
if ancestors.collect(&:to_s).include?('ActiveRecord::Base'.freeze)
|
145
|
+
find_in_batches(batch_size: batch_size) do |items|
|
147
146
|
items.each do |item|
|
148
147
|
item.redis_search_index_create
|
149
148
|
count += 1
|
150
|
-
print
|
149
|
+
print '.' if progressbar
|
151
150
|
end
|
152
151
|
end
|
153
|
-
elsif
|
152
|
+
elsif included_modules.collect(&:to_s).include?('Mongoid::Document'.freeze)
|
154
153
|
all.each_slice(batch_size) do |items|
|
155
154
|
items.each do |item|
|
156
155
|
item.redis_search_index_create
|
157
156
|
count += 1
|
158
|
-
print
|
157
|
+
print '.' if progressbar
|
159
158
|
end
|
160
159
|
end
|
161
160
|
else
|
162
|
-
puts
|
161
|
+
puts 'skiped, not support this ORM in current.'
|
163
162
|
end
|
164
163
|
|
165
164
|
count
|
@@ -1,4 +1,3 @@
|
|
1
|
-
# coding: utf-8
|
2
1
|
class Redis
|
3
2
|
module Search
|
4
3
|
class << self
|
@@ -7,8 +6,8 @@ class Redis
|
|
7
6
|
def configure
|
8
7
|
yield @config ||= Config.new
|
9
8
|
|
10
|
-
|
11
|
-
require
|
9
|
+
unless @config.disable_rmmseg
|
10
|
+
require 'rmmseg'
|
12
11
|
# loading RMMSeg chinese word dicts.
|
13
12
|
RMMSeg::Dictionary.load_dictionaries
|
14
13
|
end
|
@@ -13,7 +13,9 @@ class Redis
|
|
13
13
|
# type model name
|
14
14
|
# w search char
|
15
15
|
# :limit result limit
|
16
|
+
#
|
16
17
|
# h3. usage:
|
18
|
+
#
|
17
19
|
# * Redis::Search.complete("Tag","r") => ["Ruby","Rails", "REST", "Redis", "Redmine"]
|
18
20
|
# * Redis::Search.complete("Tag","re") => ["Redis", "Redmine"]
|
19
21
|
# * Redis::Search.complete("Tag","red") => ["Redis", "Redmine"]
|
@@ -25,24 +27,24 @@ class Redis
|
|
25
27
|
|
26
28
|
prefix_matchs = []
|
27
29
|
# This is not random, try to get replies < MTU size
|
28
|
-
rangelen =
|
30
|
+
rangelen = config.complete_max_length
|
29
31
|
prefix = w.downcase
|
30
|
-
key =
|
32
|
+
key = mk_complete_key(type)
|
31
33
|
|
32
|
-
if start =
|
34
|
+
if start = config.redis.zrank(key, prefix)
|
33
35
|
count = limit
|
34
36
|
max_range = start + (rangelen * limit) - 1
|
35
|
-
range =
|
37
|
+
range = config.redis.zrange(key, start, max_range)
|
36
38
|
while prefix_matchs.length <= count
|
37
39
|
start += rangelen
|
38
|
-
break if !range || range.
|
40
|
+
break if !range || range.empty?
|
39
41
|
range.each do |entry|
|
40
|
-
minlen = [entry.length,prefix.length].min
|
42
|
+
minlen = [entry.length, prefix.length].min
|
41
43
|
if entry[0...minlen] != prefix[0...minlen]
|
42
44
|
count = prefix_matchs.count
|
43
45
|
break
|
44
46
|
end
|
45
|
-
if entry[-1..-1] ==
|
47
|
+
if entry[-1..-1] == '*' && prefix_matchs.length != count
|
46
48
|
prefix_matchs << entry[0...-1]
|
47
49
|
end
|
48
50
|
end
|
@@ -53,25 +55,25 @@ class Redis
|
|
53
55
|
prefix_matchs.uniq!
|
54
56
|
|
55
57
|
# 组合 words 的特别 key 名
|
56
|
-
words = prefix_matchs.collect { |w|
|
58
|
+
words = prefix_matchs.collect { |w| mk_sets_key(type, w) }
|
57
59
|
|
58
60
|
# 组合特别 key ,但这里不会像 query 那样放入 words, 因为在 complete 里面 words 是用 union 取的,condition_keys 和 words 应该取交集
|
59
61
|
condition_keys = []
|
60
|
-
|
62
|
+
unless conditions.blank?
|
61
63
|
conditions = conditions[0] if conditions.is_a?(Array)
|
62
64
|
conditions.keys.each do |c|
|
63
|
-
condition_keys <<
|
65
|
+
condition_keys << mk_condition_key(type, c, conditions[c])
|
64
66
|
end
|
65
67
|
end
|
66
68
|
|
67
69
|
# 按词语搜索
|
68
|
-
temp_store_key = "tmpsunionstore:#{words.join(
|
70
|
+
temp_store_key = "tmpsunionstore:#{words.join('+')}"
|
69
71
|
if words.length > 1
|
70
|
-
|
72
|
+
unless config.redis.exists(temp_store_key)
|
71
73
|
# 将多个词语组合对比,得到并集,并存入临时区域
|
72
|
-
|
74
|
+
config.redis.sunionstore(temp_store_key, *words)
|
73
75
|
# 将临时搜索设为1天后自动清除
|
74
|
-
|
76
|
+
config.redis.expire(temp_store_key, 86_400)
|
75
77
|
end
|
76
78
|
# 根据需要的数量取出 ids
|
77
79
|
else
|
@@ -79,21 +81,21 @@ class Redis
|
|
79
81
|
end
|
80
82
|
|
81
83
|
# 如果有条件,这里再次组合一下
|
82
|
-
|
83
|
-
condition_keys << temp_store_key
|
84
|
+
unless condition_keys.blank?
|
85
|
+
condition_keys << temp_store_key unless words.blank?
|
84
86
|
temp_store_key = "tmpsinterstore:#{condition_keys.join('+')}"
|
85
|
-
|
86
|
-
|
87
|
-
|
87
|
+
unless config.redis.exists(temp_store_key)
|
88
|
+
config.redis.sinterstore(temp_store_key, *condition_keys)
|
89
|
+
config.redis.expire(temp_store_key, 86_400)
|
88
90
|
end
|
89
91
|
end
|
90
92
|
|
91
|
-
ids =
|
92
|
-
|
93
|
-
|
94
|
-
|
93
|
+
ids = config.redis.sort(temp_store_key,
|
94
|
+
limit: [0, limit],
|
95
|
+
by: mk_score_key(type, '*'),
|
96
|
+
order: 'desc')
|
95
97
|
return [] if ids.blank?
|
96
|
-
|
98
|
+
hmget(type, ids)
|
97
99
|
end
|
98
100
|
|
99
101
|
# Search items, this will split words by Libmmseg
|
@@ -102,26 +104,28 @@ class Redis
|
|
102
104
|
# type model name
|
103
105
|
# text search text
|
104
106
|
# :limit result limit
|
107
|
+
#
|
105
108
|
# h3. usage:
|
106
109
|
# * Redis::Search.query("Tag","Ruby vs Python")
|
110
|
+
#
|
107
111
|
def query(type, text, options = {})
|
108
112
|
tm = Time.now
|
109
113
|
result = []
|
110
114
|
limit = options[:limit] || 10
|
111
|
-
sort_field = options[:sort_field] ||
|
115
|
+
sort_field = options[:sort_field] || 'id'
|
112
116
|
conditions = options[:conditions] || []
|
113
117
|
|
114
118
|
# 如果搜索文本和查询条件均没有,那就直接返回 []
|
115
119
|
return result if text.strip.blank? && conditions.blank?
|
116
120
|
|
117
|
-
words =
|
118
|
-
words = words.collect { |w|
|
121
|
+
words = split(text)
|
122
|
+
words = words.collect { |w| mk_sets_key(type, w) }
|
119
123
|
|
120
124
|
condition_keys = []
|
121
|
-
|
125
|
+
unless conditions.blank?
|
122
126
|
conditions = conditions[0] if conditions.is_a?(Array)
|
123
127
|
conditions.keys.each do |c|
|
124
|
-
condition_keys <<
|
128
|
+
condition_keys << mk_condition_key(type, c, conditions[c])
|
125
129
|
end
|
126
130
|
# 将条件的 key 放入关键词搜索集合内,用于 sinterstore 搜索
|
127
131
|
words += condition_keys
|
@@ -129,30 +133,30 @@ class Redis
|
|
129
133
|
|
130
134
|
return result if words.blank?
|
131
135
|
|
132
|
-
temp_store_key = "tmpinterstore:#{words.join(
|
136
|
+
temp_store_key = "tmpinterstore:#{words.join('+')}"
|
133
137
|
|
134
138
|
if words.length > 1
|
135
|
-
|
136
|
-
|
139
|
+
unless config.redis.exists(temp_store_key)
|
140
|
+
config.redis.pipelined do
|
137
141
|
# 将多个词语组合对比,得到交集,并存入临时区域
|
138
|
-
|
142
|
+
config.redis.sinterstore(temp_store_key, *words)
|
139
143
|
# 将临时搜索设为1天后自动清除
|
140
|
-
|
144
|
+
config.redis.expire(temp_store_key, 86_400)
|
141
145
|
|
142
146
|
# 拼音搜索
|
143
|
-
if
|
144
|
-
pinyin_words =
|
145
|
-
pinyin_words = pinyin_words.collect { |w|
|
147
|
+
if config.pinyin_match
|
148
|
+
pinyin_words = split_pinyin(text)
|
149
|
+
pinyin_words = pinyin_words.collect { |w| mk_sets_key(type, w) }
|
146
150
|
pinyin_words += condition_keys
|
147
|
-
temp_sunion_key = "tmpsunionstore:#{words.join(
|
148
|
-
temp_pinyin_store_key = "tmpinterstore:#{pinyin_words.join(
|
151
|
+
temp_sunion_key = "tmpsunionstore:#{words.join('+')}"
|
152
|
+
temp_pinyin_store_key = "tmpinterstore:#{pinyin_words.join('+')}"
|
149
153
|
# 找出拼音的
|
150
|
-
|
154
|
+
config.redis.sinterstore(temp_pinyin_store_key, *pinyin_words)
|
151
155
|
# 合并中文和拼音的搜索结果
|
152
|
-
|
156
|
+
config.redis.sunionstore(temp_sunion_key, *[temp_store_key, temp_pinyin_store_key])
|
153
157
|
# 将临时搜索设为1天后自动清除
|
154
|
-
|
155
|
-
|
158
|
+
config.redis.expire(temp_pinyin_store_key, 86_400)
|
159
|
+
config.redis.expire(temp_sunion_key, 86_400)
|
156
160
|
end
|
157
161
|
temp_store_key = temp_sunion_key
|
158
162
|
end
|
@@ -162,27 +166,29 @@ class Redis
|
|
162
166
|
end
|
163
167
|
|
164
168
|
# 根据需要的数量取出 ids
|
165
|
-
ids =
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
result =
|
170
|
-
|
169
|
+
ids = config.redis.sort(temp_store_key,
|
170
|
+
limit: [0, limit],
|
171
|
+
by: mk_score_key(type, '*'),
|
172
|
+
order: 'desc')
|
173
|
+
result = hmget(type, ids, sort_field: sort_field)
|
174
|
+
info("{#{type} : \"#{text}\"} | Time spend: #{Time.now - tm}s")
|
171
175
|
result
|
172
176
|
end
|
173
177
|
end # end class << self
|
174
178
|
|
175
179
|
protected
|
180
|
+
|
176
181
|
def self.split_pinyin(text)
|
177
182
|
# Pinyin search split as pinyin again
|
178
183
|
_split(PinYin.sentence(text))
|
179
184
|
end
|
180
185
|
|
181
186
|
private
|
187
|
+
|
182
188
|
def self._split(text)
|
183
189
|
return [] if text.blank?
|
184
190
|
# return chars if disabled rmmseg
|
185
|
-
return text.split(
|
191
|
+
return text.split('') if Search.config.disable_rmmseg
|
186
192
|
|
187
193
|
algor = RMMSeg::Algorithm.new(text)
|
188
194
|
words = []
|
@@ -195,8 +201,8 @@ class Redis
|
|
195
201
|
end
|
196
202
|
|
197
203
|
def self.warn(msg)
|
198
|
-
return
|
199
|
-
msg = "\e[33m[
|
204
|
+
return unless Redis::Search.config.debug
|
205
|
+
msg = "\e[33m[redis-search] #{msg}\e[0m"
|
200
206
|
if defined?(Rails) == 'constant' && Rails.class == Class
|
201
207
|
::Rails.logger.warn(msg)
|
202
208
|
else
|
@@ -205,8 +211,8 @@ class Redis
|
|
205
211
|
end
|
206
212
|
|
207
213
|
def self.info(msg)
|
208
|
-
return
|
209
|
-
msg = "\e[32m[
|
214
|
+
return unless Redis::Search.config.debug
|
215
|
+
msg = "\e[32m[redis-search] #{msg}\e[0m"
|
210
216
|
if defined?(Rails) == 'constant' && Rails.class == Class
|
211
217
|
::Rails.logger.debug(msg)
|
212
218
|
else
|
@@ -233,13 +239,12 @@ class Redis
|
|
233
239
|
|
234
240
|
def self.hmget(type, ids, options = {})
|
235
241
|
result = []
|
236
|
-
sort_field = options[:sort_field] || "id"
|
237
242
|
return result if ids.blank?
|
238
|
-
|
243
|
+
config.redis.hmget(type, *ids).each do |r|
|
239
244
|
begin
|
240
|
-
result << JSON.parse(r)
|
245
|
+
result << JSON.parse(r) unless r.blank?
|
241
246
|
rescue => e
|
242
|
-
|
247
|
+
warn("Search.query failed: #{e}")
|
243
248
|
end
|
244
249
|
end
|
245
250
|
result
|
@@ -2,7 +2,8 @@
|
|
2
2
|
class Redis
|
3
3
|
module Search
|
4
4
|
class Index
|
5
|
-
attr_accessor :type, :title, :id, :score, :aliases, :exts,
|
5
|
+
attr_accessor :type, :title, :id, :score, :aliases, :exts,
|
6
|
+
:condition_fields, :prefix_index_enable
|
6
7
|
|
7
8
|
class << self
|
8
9
|
def redis
|
@@ -12,17 +13,17 @@ class Redis
|
|
12
13
|
def remove(options = {})
|
13
14
|
type = options[:type]
|
14
15
|
|
15
|
-
|
16
|
-
|
17
|
-
|
16
|
+
redis.pipelined do
|
17
|
+
redis.hdel(type, options[:id])
|
18
|
+
redis.del(Search.mk_score_key(type, options[:id]))
|
18
19
|
|
19
|
-
words =
|
20
|
+
words = split_words_for_index(options[:title])
|
20
21
|
words.each do |word|
|
21
|
-
|
22
|
+
redis.srem(Search.mk_sets_key(type, word), options[:id])
|
22
23
|
end
|
23
24
|
|
24
25
|
# remove set for prefix index key
|
25
|
-
|
26
|
+
redis.srem(Search.mk_sets_key(type, options[:title]), options[:id])
|
26
27
|
end
|
27
28
|
end
|
28
29
|
|
@@ -31,7 +32,7 @@ class Redis
|
|
31
32
|
if Search.config.pinyin_match
|
32
33
|
# covert Chinese to pinyin to as an index
|
33
34
|
pinyin_full = Search.split_pinyin(title)
|
34
|
-
pinyin_first = pinyin_full.collect { |p| p[0] }.join(
|
35
|
+
pinyin_first = pinyin_full.collect { |p| p[0] }.join('')
|
35
36
|
words += pinyin_full
|
36
37
|
words << pinyin_first
|
37
38
|
pinyin_full = nil
|
@@ -61,38 +62,38 @@ class Redis
|
|
61
62
|
|
62
63
|
# set attributes value from params
|
63
64
|
options.keys.each do |k|
|
64
|
-
|
65
|
+
send("#{k}=", options[k])
|
65
66
|
end
|
66
|
-
@aliases <<
|
67
|
+
@aliases << title
|
67
68
|
@aliases.uniq!
|
68
69
|
end
|
69
70
|
|
70
71
|
def save
|
71
72
|
return if @title.blank?
|
72
73
|
|
73
|
-
|
74
|
-
data = {title: @title, id: @id, type: @type}
|
75
|
-
|
74
|
+
redis.pipelined do
|
75
|
+
data = { title: @title, id: @id, type: @type }
|
76
|
+
exts.each do |f|
|
76
77
|
data[f[0]] = f[1]
|
77
78
|
end
|
78
79
|
|
79
80
|
# 将原始数据存入 hashes
|
80
|
-
res =
|
81
|
+
res = redis.hset(@type, @id, data.to_json)
|
81
82
|
|
82
83
|
# 将目前的编号保存到条件(conditions)字段所创立的索引上面
|
83
|
-
|
84
|
-
|
84
|
+
condition_fields.each do |field|
|
85
|
+
redis.sadd(Search.mk_condition_key(@type, field, data[field.to_sym]), @id)
|
85
86
|
end
|
86
87
|
|
87
88
|
# score for search sort
|
88
|
-
|
89
|
+
redis.set(Search.mk_score_key(@type, @id), @score)
|
89
90
|
|
90
91
|
# 保存 sets 索引,以分词的单词为key,用于后面搜索,里面存储 ids
|
91
|
-
|
92
|
+
aliases.each do |val|
|
92
93
|
words = Search::Index.split_words_for_index(val)
|
93
94
|
next if words.blank?
|
94
95
|
words.each do |word|
|
95
|
-
|
96
|
+
redis.sadd(Search.mk_sets_key(@type, word), @id)
|
96
97
|
end
|
97
98
|
end
|
98
99
|
|
@@ -102,25 +103,26 @@ class Redis
|
|
102
103
|
end
|
103
104
|
|
104
105
|
private
|
106
|
+
|
105
107
|
def save_prefix_index
|
106
108
|
sorted_set_key = Search.mk_complete_key(@type)
|
107
109
|
sorted_vals = []
|
108
110
|
|
109
|
-
|
111
|
+
aliases.each do |val|
|
110
112
|
words = []
|
111
113
|
words << val.downcase
|
112
114
|
|
113
|
-
|
115
|
+
redis.sadd(Search.mk_sets_key(@type, val), @id)
|
114
116
|
|
115
117
|
if Search.config.pinyin_match
|
116
118
|
pinyin_full = Search.split_pinyin(val.downcase)
|
117
|
-
pinyin_first = pinyin_full.collect { |p| p[0] }.join(
|
118
|
-
pinyin = pinyin_full.join(
|
119
|
+
pinyin_first = pinyin_full.collect { |p| p[0] }.join('')
|
120
|
+
pinyin = pinyin_full.join('')
|
119
121
|
|
120
122
|
words << pinyin
|
121
123
|
words << pinyin_first
|
122
124
|
|
123
|
-
|
125
|
+
redis.sadd(Search.mk_sets_key(@type, pinyin), @id)
|
124
126
|
|
125
127
|
pinyin_full = nil
|
126
128
|
pinyin_first = nil
|
@@ -136,9 +138,8 @@ class Redis
|
|
136
138
|
end
|
137
139
|
end
|
138
140
|
|
139
|
-
|
141
|
+
redis.zadd(sorted_set_key, sorted_vals)
|
140
142
|
end
|
141
|
-
|
142
143
|
end # end Index
|
143
144
|
end
|
144
145
|
end
|
File without changes
|
@@ -1,5 +1,5 @@
|
|
1
|
-
|
2
|
-
|
1
|
+
require 'redis-search'
|
2
|
+
|
3
3
|
namespace :redis_search do
|
4
4
|
task index: 'index:all'
|
5
5
|
|
@@ -27,18 +27,18 @@ namespace :redis_search do
|
|
27
27
|
desc index_model_desc
|
28
28
|
task model: :environment do
|
29
29
|
if ENV['CLASS'].to_s == ''
|
30
|
-
puts '='*90, 'USAGE', '='*90, index_model_desc,
|
30
|
+
puts '=' * 90, 'USAGE', '=' * 90, index_model_desc, ''
|
31
31
|
exit(1)
|
32
32
|
end
|
33
33
|
|
34
34
|
klass = eval(ENV['CLASS'].to_s)
|
35
35
|
batch = ENV['BATCH'].to_i > 0 ? ENV['BATCH'].to_i : 1000
|
36
36
|
tm = Time.now
|
37
|
-
puts "Redis-Search index data to Redis from [#{klass
|
37
|
+
puts "Redis-Search index data to Redis from [#{klass}]"
|
38
38
|
count = klass.redis_search_index_batch_create(batch, true)
|
39
|
-
puts
|
39
|
+
puts ''
|
40
40
|
puts "Indexed #{count} rows | Time spend: #{(Time.now - tm)}s"
|
41
|
-
puts
|
41
|
+
puts 'Rebuild Index done.'
|
42
42
|
end
|
43
43
|
|
44
44
|
desc index_all_desc
|
@@ -51,23 +51,23 @@ namespace :redis_search do
|
|
51
51
|
Dir.glob(File.join("#{dir}/**/*.rb")).each do |path|
|
52
52
|
model_filename = path[/#{Regexp.escape(dir.to_s)}\/([^\.]+).rb/, 1]
|
53
53
|
|
54
|
-
next if model_filename
|
54
|
+
next if model_filename =~ /^concerns\//i # Skip concerns/ folder
|
55
55
|
|
56
56
|
begin
|
57
57
|
klass = model_filename.camelize.constantize
|
58
58
|
rescue NameError
|
59
|
-
require(path) ? retry : raise(
|
59
|
+
require(path) ? retry : raise("Cannot load class '#{klass}'")
|
60
60
|
end
|
61
61
|
end
|
62
62
|
|
63
63
|
puts "Redis-Search index data to Redis from [#{dir}]"
|
64
64
|
Redis::Search.indexed_models.each do |klass|
|
65
|
-
puts "[#{klass
|
65
|
+
puts "[#{klass}]"
|
66
66
|
count += klass.redis_search_index_batch_create(batch, true)
|
67
|
-
puts
|
67
|
+
puts ''
|
68
68
|
end
|
69
69
|
puts "Indexed #{count} rows | Time spend: #{(Time.now - tm)}s"
|
70
|
-
puts
|
70
|
+
puts 'Rebuild Index done.'
|
71
71
|
end
|
72
72
|
end
|
73
73
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: redis-search
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 1.0.0.beta1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Jason Lee
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2016-05-09 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: ruby-pinyin
|
@@ -69,12 +69,13 @@ files:
|
|
69
69
|
- LICENSE
|
70
70
|
- README.md
|
71
71
|
- lib/redis-search.rb
|
72
|
-
- lib/redis
|
73
|
-
- lib/redis
|
74
|
-
- lib/redis
|
75
|
-
- lib/redis
|
76
|
-
- lib/redis
|
77
|
-
- lib/redis
|
72
|
+
- lib/redis-search/base.rb
|
73
|
+
- lib/redis-search/config.rb
|
74
|
+
- lib/redis-search/finder.rb
|
75
|
+
- lib/redis-search/index.rb
|
76
|
+
- lib/redis-search/railtie.rb
|
77
|
+
- lib/redis-search/tasks.rb
|
78
|
+
- lib/redis-search/version.rb
|
78
79
|
homepage: http://github.com/huacnlee/redis-search
|
79
80
|
licenses:
|
80
81
|
- MIT
|
@@ -95,10 +96,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
95
96
|
version: 1.3.6
|
96
97
|
requirements: []
|
97
98
|
rubyforge_project:
|
98
|
-
rubygems_version: 2.
|
99
|
+
rubygems_version: 2.5.1
|
99
100
|
signing_key:
|
100
101
|
specification_version: 4
|
101
102
|
summary: High performance real-time search (Support Chinese), indexes store in Redis
|
102
103
|
for Rails applications.
|
103
104
|
test_files: []
|
104
|
-
has_rdoc:
|