redis-search 0.9.7 → 1.0.0.beta1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +87 -97
- data/lib/redis-search.rb +6 -6
- data/lib/{redis/search → redis-search}/base.rb +41 -42
- data/lib/{redis/search → redis-search}/config.rb +2 -3
- data/lib/{redis/search → redis-search}/finder.rb +63 -58
- data/lib/{redis/search → redis-search}/index.rb +27 -26
- data/lib/{redis/search → redis-search}/railtie.rb +0 -0
- data/lib/{redis/search → redis-search}/tasks.rb +11 -11
- data/lib/redis-search/version.rb +5 -0
- metadata +10 -10
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c3db9e3cf11d6d82edb9366ec887db853a684468
|
4
|
+
data.tar.gz: 349c2ad7333803b7f868cabd5ee4053f216b9e14
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: e1d2dcf8826e6c4e0e3a9414fdc0a8f1b11fc9d0cf024f5c6d5ab27e56e7da3c436f5ab20d1c487f6e2feab09d24462de24cee03209ecccf5c4ce7c51a56d423
|
7
|
+
data.tar.gz: 9389c3f640c4ff76719507d681fb0d8ba4bc24cb7d36544617c76c4d209709121ec87e052e2ae4e6d52ecbb9cba3e09b212aafc1fa07e025c1dbc0a840dffbf1
|
data/README.md
CHANGED
@@ -4,17 +4,10 @@ High performance real-time search (Support Chinese), index in Redis for Rails ap
|
|
4
4
|
|
5
5
|
[中文介绍和使用说明](https://github.com/huacnlee/redis-search/wiki/Usage-in-Chinese)
|
6
6
|
|
7
|
-
## Demo
|
8
|
-
|
9
|
-

|
10
|
-
|
11
|
-
You can try the search feature in [`IMAX.im`](http://imax.im) | [`shu.im`](http://shu.im)
|
12
|
-
|
13
|
-
And there is an [Example App](https://github.com/huacnlee/redis-search-example) to show you how to use redis-search.
|
14
|
-
|
15
7
|
## Master Status
|
16
8
|
|
17
|
-
[](https://badge.fury.io/rb/redis-search) [](http://travis-ci.org/huacnlee/redis-search) [](https://codecov.io/gh/huacnlee/redis-search)
|
10
|
+
|
18
11
|
|
19
12
|
## Features
|
20
13
|
|
@@ -34,99 +27,87 @@ And there is an [Example App](https://github.com/huacnlee/redis-search-example)
|
|
34
27
|
|
35
28
|
## Install
|
36
29
|
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
gem 'ruby-pinyin', '0.3.0'
|
41
|
-
gem 'redis-search', '0.9.6'
|
42
|
-
```
|
30
|
+
```ruby
|
31
|
+
gem 'redis-search'
|
32
|
+
```
|
43
33
|
|
44
|
-
|
45
|
-
|
46
|
-
|
34
|
+
```bash
|
35
|
+
$ bundle install
|
36
|
+
```
|
47
37
|
|
48
38
|
## Configure
|
49
39
|
|
50
|
-
* Create file in: config/initializers/
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
40
|
+
* Create file in: config/initializers/redis-search.rb
|
41
|
+
|
42
|
+
```ruby
|
43
|
+
require "redis"
|
44
|
+
require "redis-namespace"
|
45
|
+
require "redis-search"
|
46
|
+
|
47
|
+
# don't forget change namespace
|
48
|
+
redis = Redis.new(host: '127.0.0.1', port: '6379')
|
49
|
+
# We suggest you use a special db in Redis, when you need to clear all data, you can use `flushdb` command to cleanup.
|
50
|
+
redis.select(3)
|
51
|
+
# Give a special namespace as prefix for Redis key, when your have more than one project used redis-search, this config will make them work fine.
|
52
|
+
redis = Redis::Namespace.new("your_app_name:redis_search", redis: redis)
|
53
|
+
Redis::Search.configure do |config|
|
54
|
+
config.redis = redis
|
55
|
+
config.complete_max_length = 100
|
56
|
+
config.pinyin_match = true
|
57
|
+
# use rmmseg, true to disable it, it can save memroy
|
58
|
+
config.disable_rmmseg = false
|
59
|
+
end
|
60
|
+
```
|
70
61
|
|
71
62
|
## Usage
|
72
63
|
|
73
|
-
* Bind Redis::Search callback event, it will to rebuild search indices when data create or update.
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
def index
|
121
|
-
Redis::Search.query("Post", params[:q], :conditions => {:user_id => 12})
|
122
|
-
end
|
123
|
-
|
124
|
-
# GET /search_users?q=j
|
125
|
-
def search_users
|
126
|
-
Redis::Search.complete("Post", params[:q], :conditions => {:user_id => 12, :category_id => 4})
|
127
|
-
end
|
128
|
-
end
|
129
|
-
```
|
64
|
+
* Bind `Redis::Search` callback event, it will to rebuild search indices when data create or update.
|
65
|
+
|
66
|
+
```ruby
|
67
|
+
class Post < ActiveRecord::Base
|
68
|
+
include Redis::Search
|
69
|
+
|
70
|
+
belongs_to :user
|
71
|
+
belongs_to :category
|
72
|
+
|
73
|
+
redis_search_index(title_field: :title,
|
74
|
+
score_field: :hits,
|
75
|
+
condition_fields: [:user_id, :category_id],
|
76
|
+
ext_fields: [:category_name])
|
77
|
+
|
78
|
+
def category_name
|
79
|
+
self.category.name
|
80
|
+
end
|
81
|
+
end
|
82
|
+
```
|
83
|
+
|
84
|
+
```ruby
|
85
|
+
class User < ActiveRecord::Base
|
86
|
+
include Redis::Search
|
87
|
+
|
88
|
+
serialize :alias_names, Array
|
89
|
+
|
90
|
+
redis_search_index(title_field: :name,
|
91
|
+
alias_field: :alias_names,
|
92
|
+
prefix_index_enable: true,
|
93
|
+
score_field: :followers_count,
|
94
|
+
ext_fields: [:email, :tagline])
|
95
|
+
end
|
96
|
+
```
|
97
|
+
|
98
|
+
```ruby
|
99
|
+
class SearchController < ApplicationController
|
100
|
+
# GET /searchs?q=title
|
101
|
+
def index
|
102
|
+
Redis::Search.query("Post", params[:q], conditions: { user_id: 12 })
|
103
|
+
end
|
104
|
+
|
105
|
+
# GET /search_users?q=j
|
106
|
+
def search_users
|
107
|
+
Redis::Search.complete("Post", params[:q], conditions: { user_id: 12, category_id: 4 })
|
108
|
+
end
|
109
|
+
end
|
110
|
+
```
|
130
111
|
|
131
112
|
## Index data to Redis
|
132
113
|
|
@@ -135,13 +116,13 @@ And there is an [Example App](https://github.com/huacnlee/redis-search-example)
|
|
135
116
|
Redis-Search index data to Redis from your model (pass name as CLASS environment variable).
|
136
117
|
|
137
118
|
```bash
|
138
|
-
$ rake redis_search:index:model CLASS='
|
119
|
+
$ rake redis_search:index:model CLASS='User'
|
139
120
|
```
|
140
121
|
|
141
122
|
Customize the batch size:
|
142
123
|
|
143
124
|
```bash
|
144
|
-
$ rake redis_search:index:model CLASS='
|
125
|
+
$ rake redis_search:index:model CLASS='User' BATCH=100
|
145
126
|
```
|
146
127
|
|
147
128
|
### All Models
|
@@ -170,6 +151,15 @@ There is my performance test result.
|
|
170
151
|
|
171
152
|
* [https://gist.github.com/1150933](https://gist.github.com/1150933)
|
172
153
|
|
154
|
+
## Demo
|
155
|
+
|
156
|
+

|
157
|
+
|
158
|
+
Projects used redis-search:
|
159
|
+
|
160
|
+
- [redis-search-example](https://github.com/huacnlee/redis-search-example) - An example for show you how to use redis-search.
|
161
|
+
- [IMAX.im](https://github.com/huacnlee/imax.im)
|
162
|
+
|
173
163
|
## License
|
174
164
|
|
175
165
|
* MIT
|
data/lib/redis-search.rb
CHANGED
@@ -1,6 +1,6 @@
|
|
1
|
-
|
2
|
-
require
|
3
|
-
require
|
4
|
-
require
|
5
|
-
require
|
6
|
-
require 'redis
|
1
|
+
require 'redis-search/base'
|
2
|
+
require 'redis-search/finder'
|
3
|
+
require 'redis-search/index'
|
4
|
+
require 'redis-search/config'
|
5
|
+
require 'redis-search/version'
|
6
|
+
require 'redis-search/railtie' if defined?(Rails)
|
@@ -1,9 +1,9 @@
|
|
1
|
-
# coding: utf-8
|
2
1
|
class Redis
|
2
|
+
# nodoc
|
3
3
|
module Search
|
4
4
|
autoload :PinYin, 'ruby-pinyin'
|
5
5
|
|
6
|
-
extend
|
6
|
+
extend ActiveSupport::Concern
|
7
7
|
|
8
8
|
included do
|
9
9
|
cattr_reader :redis_search_options
|
@@ -22,28 +22,28 @@ class Redis
|
|
22
22
|
end
|
23
23
|
|
24
24
|
def redis_search_alias_value(field)
|
25
|
-
return [] if field.blank? || field ==
|
26
|
-
val = (instance_eval("self.#{field}") ||
|
27
|
-
return []
|
28
|
-
if val.is_a?(String)
|
29
|
-
val = val.to_s.split(",")
|
30
|
-
end
|
25
|
+
return [] if field.blank? || field == '_was'.freeze
|
26
|
+
val = (instance_eval("self.#{field}") || ''.freeze).clone
|
27
|
+
return [] unless val.class.in?([String, Array])
|
28
|
+
val = val.to_s.split(',') if val.is_a?(String)
|
31
29
|
val
|
32
30
|
end
|
33
31
|
|
34
32
|
# Rebuild search index with create
|
35
33
|
def redis_search_index_create
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
34
|
+
opts = {
|
35
|
+
title: send(redis_search_options[:title_field]),
|
36
|
+
aliases: redis_search_alias_value(redis_search_options[:alias_field]),
|
37
|
+
id: id,
|
38
|
+
exts: redis_search_fields_to_hash(redis_search_options[:ext_fields]),
|
39
|
+
type: redis_search_options[:class_name] || self.class.name,
|
40
|
+
condition_fields: redis_search_options[:condition_fields],
|
41
|
+
score: send(redis_search_options[:score_field]).to_i,
|
42
|
+
prefix_index_enable: redis_search_options[:prefix_index_enable]
|
43
|
+
}
|
44
|
+
|
45
|
+
s = Search::Index.new(opts)
|
44
46
|
s.save
|
45
|
-
# release s
|
46
|
-
s = nil
|
47
47
|
true
|
48
48
|
end
|
49
49
|
|
@@ -51,15 +51,14 @@ class Redis
|
|
51
51
|
titles.uniq!
|
52
52
|
titles.each do |title|
|
53
53
|
next if title.blank?
|
54
|
-
Search::Index.remove(id:
|
54
|
+
Search::Index.remove(id: id, title: title, type: self.class.name)
|
55
55
|
end
|
56
56
|
true
|
57
57
|
end
|
58
58
|
|
59
59
|
def redis_search_index_before_destroy
|
60
|
-
titles = []
|
61
|
-
titles
|
62
|
-
titles << self.send(self.redis_search_options[:title_field])
|
60
|
+
titles = redis_search_alias_value(redis_search_options[:alias_field])
|
61
|
+
titles << send(redis_search_options[:title_field])
|
63
62
|
|
64
63
|
redis_search_index_delete(titles)
|
65
64
|
true
|
@@ -67,10 +66,10 @@ class Redis
|
|
67
66
|
|
68
67
|
def redis_search_index_need_reindex
|
69
68
|
index_fields_changed = false
|
70
|
-
|
71
|
-
next if f.to_s ==
|
69
|
+
redis_search_options[:ext_fields].each do |f|
|
70
|
+
next if f.to_s == 'id'.freeze
|
72
71
|
field_method = "#{f}_changed?"
|
73
|
-
if
|
72
|
+
if methods.index(field_method.to_sym).nil?
|
74
73
|
Redis::Search.warn("#{self.class.name} model reindex on update need #{field_method} method.")
|
75
74
|
next
|
76
75
|
end
|
@@ -79,24 +78,24 @@ class Redis
|
|
79
78
|
end
|
80
79
|
|
81
80
|
begin
|
82
|
-
if
|
81
|
+
if send("#{redis_search_options[:title_field]}_changed?")
|
83
82
|
index_fields_changed = true
|
84
83
|
end
|
85
84
|
|
86
|
-
if
|
85
|
+
if send(redis_search_options[:alias_field]) ||
|
86
|
+
send("#{redis_search_options[:title_field]}_changed?")
|
87
87
|
index_fields_changed = true
|
88
88
|
end
|
89
89
|
rescue
|
90
90
|
end
|
91
91
|
|
92
|
-
|
92
|
+
index_fields_changed
|
93
93
|
end
|
94
94
|
|
95
95
|
def redis_search_index_after_update
|
96
|
-
if
|
97
|
-
titles = []
|
98
|
-
titles
|
99
|
-
titles << self.send("#{self.redis_search_options[:title_field]}_was")
|
96
|
+
if redis_search_index_need_reindex
|
97
|
+
titles = redis_search_alias_value("#{redis_search_options[:alias_field]}_was")
|
98
|
+
titles << send("#{redis_search_options[:title_field]}_was")
|
100
99
|
redis_search_index_delete(titles)
|
101
100
|
end
|
102
101
|
|
@@ -104,8 +103,8 @@ class Redis
|
|
104
103
|
end
|
105
104
|
|
106
105
|
def redis_search_index_after_save
|
107
|
-
if
|
108
|
-
|
106
|
+
if redis_search_index_need_reindex || new_record?
|
107
|
+
redis_search_index_create
|
109
108
|
end
|
110
109
|
true
|
111
110
|
end
|
@@ -134,32 +133,32 @@ class Redis
|
|
134
133
|
opts[:ext_fields] += opts[:condition_fields] if opts[:condition_fields].is_a?(Array)
|
135
134
|
|
136
135
|
# store Model name to indexed_models for Rake tasks
|
137
|
-
Search.indexed_models = [] if Search.indexed_models
|
136
|
+
Search.indexed_models = [] if Search.indexed_models.nil?
|
138
137
|
Search.indexed_models << self
|
139
138
|
|
140
|
-
class_variable_set(
|
139
|
+
class_variable_set('@@redis_search_options'.freeze, opts)
|
141
140
|
end
|
142
141
|
|
143
142
|
def redis_search_index_batch_create(batch_size = 1000, progressbar = false)
|
144
143
|
count = 0
|
145
|
-
if
|
146
|
-
find_in_batches(:
|
144
|
+
if ancestors.collect(&:to_s).include?('ActiveRecord::Base'.freeze)
|
145
|
+
find_in_batches(batch_size: batch_size) do |items|
|
147
146
|
items.each do |item|
|
148
147
|
item.redis_search_index_create
|
149
148
|
count += 1
|
150
|
-
print
|
149
|
+
print '.' if progressbar
|
151
150
|
end
|
152
151
|
end
|
153
|
-
elsif
|
152
|
+
elsif included_modules.collect(&:to_s).include?('Mongoid::Document'.freeze)
|
154
153
|
all.each_slice(batch_size) do |items|
|
155
154
|
items.each do |item|
|
156
155
|
item.redis_search_index_create
|
157
156
|
count += 1
|
158
|
-
print
|
157
|
+
print '.' if progressbar
|
159
158
|
end
|
160
159
|
end
|
161
160
|
else
|
162
|
-
puts
|
161
|
+
puts 'skiped, not support this ORM in current.'
|
163
162
|
end
|
164
163
|
|
165
164
|
count
|
@@ -1,4 +1,3 @@
|
|
1
|
-
# coding: utf-8
|
2
1
|
class Redis
|
3
2
|
module Search
|
4
3
|
class << self
|
@@ -7,8 +6,8 @@ class Redis
|
|
7
6
|
def configure
|
8
7
|
yield @config ||= Config.new
|
9
8
|
|
10
|
-
|
11
|
-
require
|
9
|
+
unless @config.disable_rmmseg
|
10
|
+
require 'rmmseg'
|
12
11
|
# loading RMMSeg chinese word dicts.
|
13
12
|
RMMSeg::Dictionary.load_dictionaries
|
14
13
|
end
|
@@ -13,7 +13,9 @@ class Redis
|
|
13
13
|
# type model name
|
14
14
|
# w search char
|
15
15
|
# :limit result limit
|
16
|
+
#
|
16
17
|
# h3. usage:
|
18
|
+
#
|
17
19
|
# * Redis::Search.complete("Tag","r") => ["Ruby","Rails", "REST", "Redis", "Redmine"]
|
18
20
|
# * Redis::Search.complete("Tag","re") => ["Redis", "Redmine"]
|
19
21
|
# * Redis::Search.complete("Tag","red") => ["Redis", "Redmine"]
|
@@ -25,24 +27,24 @@ class Redis
|
|
25
27
|
|
26
28
|
prefix_matchs = []
|
27
29
|
# This is not random, try to get replies < MTU size
|
28
|
-
rangelen =
|
30
|
+
rangelen = config.complete_max_length
|
29
31
|
prefix = w.downcase
|
30
|
-
key =
|
32
|
+
key = mk_complete_key(type)
|
31
33
|
|
32
|
-
if start =
|
34
|
+
if start = config.redis.zrank(key, prefix)
|
33
35
|
count = limit
|
34
36
|
max_range = start + (rangelen * limit) - 1
|
35
|
-
range =
|
37
|
+
range = config.redis.zrange(key, start, max_range)
|
36
38
|
while prefix_matchs.length <= count
|
37
39
|
start += rangelen
|
38
|
-
break if !range || range.
|
40
|
+
break if !range || range.empty?
|
39
41
|
range.each do |entry|
|
40
|
-
minlen = [entry.length,prefix.length].min
|
42
|
+
minlen = [entry.length, prefix.length].min
|
41
43
|
if entry[0...minlen] != prefix[0...minlen]
|
42
44
|
count = prefix_matchs.count
|
43
45
|
break
|
44
46
|
end
|
45
|
-
if entry[-1..-1] ==
|
47
|
+
if entry[-1..-1] == '*' && prefix_matchs.length != count
|
46
48
|
prefix_matchs << entry[0...-1]
|
47
49
|
end
|
48
50
|
end
|
@@ -53,25 +55,25 @@ class Redis
|
|
53
55
|
prefix_matchs.uniq!
|
54
56
|
|
55
57
|
# 组合 words 的特别 key 名
|
56
|
-
words = prefix_matchs.collect { |w|
|
58
|
+
words = prefix_matchs.collect { |w| mk_sets_key(type, w) }
|
57
59
|
|
58
60
|
# 组合特别 key ,但这里不会像 query 那样放入 words, 因为在 complete 里面 words 是用 union 取的,condition_keys 和 words 应该取交集
|
59
61
|
condition_keys = []
|
60
|
-
|
62
|
+
unless conditions.blank?
|
61
63
|
conditions = conditions[0] if conditions.is_a?(Array)
|
62
64
|
conditions.keys.each do |c|
|
63
|
-
condition_keys <<
|
65
|
+
condition_keys << mk_condition_key(type, c, conditions[c])
|
64
66
|
end
|
65
67
|
end
|
66
68
|
|
67
69
|
# 按词语搜索
|
68
|
-
temp_store_key = "tmpsunionstore:#{words.join(
|
70
|
+
temp_store_key = "tmpsunionstore:#{words.join('+')}"
|
69
71
|
if words.length > 1
|
70
|
-
|
72
|
+
unless config.redis.exists(temp_store_key)
|
71
73
|
# 将多个词语组合对比,得到并集,并存入临时区域
|
72
|
-
|
74
|
+
config.redis.sunionstore(temp_store_key, *words)
|
73
75
|
# 将临时搜索设为1天后自动清除
|
74
|
-
|
76
|
+
config.redis.expire(temp_store_key, 86_400)
|
75
77
|
end
|
76
78
|
# 根据需要的数量取出 ids
|
77
79
|
else
|
@@ -79,21 +81,21 @@ class Redis
|
|
79
81
|
end
|
80
82
|
|
81
83
|
# 如果有条件,这里再次组合一下
|
82
|
-
|
83
|
-
condition_keys << temp_store_key
|
84
|
+
unless condition_keys.blank?
|
85
|
+
condition_keys << temp_store_key unless words.blank?
|
84
86
|
temp_store_key = "tmpsinterstore:#{condition_keys.join('+')}"
|
85
|
-
|
86
|
-
|
87
|
-
|
87
|
+
unless config.redis.exists(temp_store_key)
|
88
|
+
config.redis.sinterstore(temp_store_key, *condition_keys)
|
89
|
+
config.redis.expire(temp_store_key, 86_400)
|
88
90
|
end
|
89
91
|
end
|
90
92
|
|
91
|
-
ids =
|
92
|
-
|
93
|
-
|
94
|
-
|
93
|
+
ids = config.redis.sort(temp_store_key,
|
94
|
+
limit: [0, limit],
|
95
|
+
by: mk_score_key(type, '*'),
|
96
|
+
order: 'desc')
|
95
97
|
return [] if ids.blank?
|
96
|
-
|
98
|
+
hmget(type, ids)
|
97
99
|
end
|
98
100
|
|
99
101
|
# Search items, this will split words by Libmmseg
|
@@ -102,26 +104,28 @@ class Redis
|
|
102
104
|
# type model name
|
103
105
|
# text search text
|
104
106
|
# :limit result limit
|
107
|
+
#
|
105
108
|
# h3. usage:
|
106
109
|
# * Redis::Search.query("Tag","Ruby vs Python")
|
110
|
+
#
|
107
111
|
def query(type, text, options = {})
|
108
112
|
tm = Time.now
|
109
113
|
result = []
|
110
114
|
limit = options[:limit] || 10
|
111
|
-
sort_field = options[:sort_field] ||
|
115
|
+
sort_field = options[:sort_field] || 'id'
|
112
116
|
conditions = options[:conditions] || []
|
113
117
|
|
114
118
|
# 如果搜索文本和查询条件均没有,那就直接返回 []
|
115
119
|
return result if text.strip.blank? && conditions.blank?
|
116
120
|
|
117
|
-
words =
|
118
|
-
words = words.collect { |w|
|
121
|
+
words = split(text)
|
122
|
+
words = words.collect { |w| mk_sets_key(type, w) }
|
119
123
|
|
120
124
|
condition_keys = []
|
121
|
-
|
125
|
+
unless conditions.blank?
|
122
126
|
conditions = conditions[0] if conditions.is_a?(Array)
|
123
127
|
conditions.keys.each do |c|
|
124
|
-
condition_keys <<
|
128
|
+
condition_keys << mk_condition_key(type, c, conditions[c])
|
125
129
|
end
|
126
130
|
# 将条件的 key 放入关键词搜索集合内,用于 sinterstore 搜索
|
127
131
|
words += condition_keys
|
@@ -129,30 +133,30 @@ class Redis
|
|
129
133
|
|
130
134
|
return result if words.blank?
|
131
135
|
|
132
|
-
temp_store_key = "tmpinterstore:#{words.join(
|
136
|
+
temp_store_key = "tmpinterstore:#{words.join('+')}"
|
133
137
|
|
134
138
|
if words.length > 1
|
135
|
-
|
136
|
-
|
139
|
+
unless config.redis.exists(temp_store_key)
|
140
|
+
config.redis.pipelined do
|
137
141
|
# 将多个词语组合对比,得到交集,并存入临时区域
|
138
|
-
|
142
|
+
config.redis.sinterstore(temp_store_key, *words)
|
139
143
|
# 将临时搜索设为1天后自动清除
|
140
|
-
|
144
|
+
config.redis.expire(temp_store_key, 86_400)
|
141
145
|
|
142
146
|
# 拼音搜索
|
143
|
-
if
|
144
|
-
pinyin_words =
|
145
|
-
pinyin_words = pinyin_words.collect { |w|
|
147
|
+
if config.pinyin_match
|
148
|
+
pinyin_words = split_pinyin(text)
|
149
|
+
pinyin_words = pinyin_words.collect { |w| mk_sets_key(type, w) }
|
146
150
|
pinyin_words += condition_keys
|
147
|
-
temp_sunion_key = "tmpsunionstore:#{words.join(
|
148
|
-
temp_pinyin_store_key = "tmpinterstore:#{pinyin_words.join(
|
151
|
+
temp_sunion_key = "tmpsunionstore:#{words.join('+')}"
|
152
|
+
temp_pinyin_store_key = "tmpinterstore:#{pinyin_words.join('+')}"
|
149
153
|
# 找出拼音的
|
150
|
-
|
154
|
+
config.redis.sinterstore(temp_pinyin_store_key, *pinyin_words)
|
151
155
|
# 合并中文和拼音的搜索结果
|
152
|
-
|
156
|
+
config.redis.sunionstore(temp_sunion_key, *[temp_store_key, temp_pinyin_store_key])
|
153
157
|
# 将临时搜索设为1天后自动清除
|
154
|
-
|
155
|
-
|
158
|
+
config.redis.expire(temp_pinyin_store_key, 86_400)
|
159
|
+
config.redis.expire(temp_sunion_key, 86_400)
|
156
160
|
end
|
157
161
|
temp_store_key = temp_sunion_key
|
158
162
|
end
|
@@ -162,27 +166,29 @@ class Redis
|
|
162
166
|
end
|
163
167
|
|
164
168
|
# 根据需要的数量取出 ids
|
165
|
-
ids =
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
result =
|
170
|
-
|
169
|
+
ids = config.redis.sort(temp_store_key,
|
170
|
+
limit: [0, limit],
|
171
|
+
by: mk_score_key(type, '*'),
|
172
|
+
order: 'desc')
|
173
|
+
result = hmget(type, ids, sort_field: sort_field)
|
174
|
+
info("{#{type} : \"#{text}\"} | Time spend: #{Time.now - tm}s")
|
171
175
|
result
|
172
176
|
end
|
173
177
|
end # end class << self
|
174
178
|
|
175
179
|
protected
|
180
|
+
|
176
181
|
def self.split_pinyin(text)
|
177
182
|
# Pinyin search split as pinyin again
|
178
183
|
_split(PinYin.sentence(text))
|
179
184
|
end
|
180
185
|
|
181
186
|
private
|
187
|
+
|
182
188
|
def self._split(text)
|
183
189
|
return [] if text.blank?
|
184
190
|
# return chars if disabled rmmseg
|
185
|
-
return text.split(
|
191
|
+
return text.split('') if Search.config.disable_rmmseg
|
186
192
|
|
187
193
|
algor = RMMSeg::Algorithm.new(text)
|
188
194
|
words = []
|
@@ -195,8 +201,8 @@ class Redis
|
|
195
201
|
end
|
196
202
|
|
197
203
|
def self.warn(msg)
|
198
|
-
return
|
199
|
-
msg = "\e[33m[
|
204
|
+
return unless Redis::Search.config.debug
|
205
|
+
msg = "\e[33m[redis-search] #{msg}\e[0m"
|
200
206
|
if defined?(Rails) == 'constant' && Rails.class == Class
|
201
207
|
::Rails.logger.warn(msg)
|
202
208
|
else
|
@@ -205,8 +211,8 @@ class Redis
|
|
205
211
|
end
|
206
212
|
|
207
213
|
def self.info(msg)
|
208
|
-
return
|
209
|
-
msg = "\e[32m[
|
214
|
+
return unless Redis::Search.config.debug
|
215
|
+
msg = "\e[32m[redis-search] #{msg}\e[0m"
|
210
216
|
if defined?(Rails) == 'constant' && Rails.class == Class
|
211
217
|
::Rails.logger.debug(msg)
|
212
218
|
else
|
@@ -233,13 +239,12 @@ class Redis
|
|
233
239
|
|
234
240
|
def self.hmget(type, ids, options = {})
|
235
241
|
result = []
|
236
|
-
sort_field = options[:sort_field] || "id"
|
237
242
|
return result if ids.blank?
|
238
|
-
|
243
|
+
config.redis.hmget(type, *ids).each do |r|
|
239
244
|
begin
|
240
|
-
result << JSON.parse(r)
|
245
|
+
result << JSON.parse(r) unless r.blank?
|
241
246
|
rescue => e
|
242
|
-
|
247
|
+
warn("Search.query failed: #{e}")
|
243
248
|
end
|
244
249
|
end
|
245
250
|
result
|
@@ -2,7 +2,8 @@
|
|
2
2
|
class Redis
|
3
3
|
module Search
|
4
4
|
class Index
|
5
|
-
attr_accessor :type, :title, :id, :score, :aliases, :exts,
|
5
|
+
attr_accessor :type, :title, :id, :score, :aliases, :exts,
|
6
|
+
:condition_fields, :prefix_index_enable
|
6
7
|
|
7
8
|
class << self
|
8
9
|
def redis
|
@@ -12,17 +13,17 @@ class Redis
|
|
12
13
|
def remove(options = {})
|
13
14
|
type = options[:type]
|
14
15
|
|
15
|
-
|
16
|
-
|
17
|
-
|
16
|
+
redis.pipelined do
|
17
|
+
redis.hdel(type, options[:id])
|
18
|
+
redis.del(Search.mk_score_key(type, options[:id]))
|
18
19
|
|
19
|
-
words =
|
20
|
+
words = split_words_for_index(options[:title])
|
20
21
|
words.each do |word|
|
21
|
-
|
22
|
+
redis.srem(Search.mk_sets_key(type, word), options[:id])
|
22
23
|
end
|
23
24
|
|
24
25
|
# remove set for prefix index key
|
25
|
-
|
26
|
+
redis.srem(Search.mk_sets_key(type, options[:title]), options[:id])
|
26
27
|
end
|
27
28
|
end
|
28
29
|
|
@@ -31,7 +32,7 @@ class Redis
|
|
31
32
|
if Search.config.pinyin_match
|
32
33
|
# covert Chinese to pinyin to as an index
|
33
34
|
pinyin_full = Search.split_pinyin(title)
|
34
|
-
pinyin_first = pinyin_full.collect { |p| p[0] }.join(
|
35
|
+
pinyin_first = pinyin_full.collect { |p| p[0] }.join('')
|
35
36
|
words += pinyin_full
|
36
37
|
words << pinyin_first
|
37
38
|
pinyin_full = nil
|
@@ -61,38 +62,38 @@ class Redis
|
|
61
62
|
|
62
63
|
# set attributes value from params
|
63
64
|
options.keys.each do |k|
|
64
|
-
|
65
|
+
send("#{k}=", options[k])
|
65
66
|
end
|
66
|
-
@aliases <<
|
67
|
+
@aliases << title
|
67
68
|
@aliases.uniq!
|
68
69
|
end
|
69
70
|
|
70
71
|
def save
|
71
72
|
return if @title.blank?
|
72
73
|
|
73
|
-
|
74
|
-
data = {title: @title, id: @id, type: @type}
|
75
|
-
|
74
|
+
redis.pipelined do
|
75
|
+
data = { title: @title, id: @id, type: @type }
|
76
|
+
exts.each do |f|
|
76
77
|
data[f[0]] = f[1]
|
77
78
|
end
|
78
79
|
|
79
80
|
# 将原始数据存入 hashes
|
80
|
-
res =
|
81
|
+
res = redis.hset(@type, @id, data.to_json)
|
81
82
|
|
82
83
|
# 将目前的编号保存到条件(conditions)字段所创立的索引上面
|
83
|
-
|
84
|
-
|
84
|
+
condition_fields.each do |field|
|
85
|
+
redis.sadd(Search.mk_condition_key(@type, field, data[field.to_sym]), @id)
|
85
86
|
end
|
86
87
|
|
87
88
|
# score for search sort
|
88
|
-
|
89
|
+
redis.set(Search.mk_score_key(@type, @id), @score)
|
89
90
|
|
90
91
|
# 保存 sets 索引,以分词的单词为key,用于后面搜索,里面存储 ids
|
91
|
-
|
92
|
+
aliases.each do |val|
|
92
93
|
words = Search::Index.split_words_for_index(val)
|
93
94
|
next if words.blank?
|
94
95
|
words.each do |word|
|
95
|
-
|
96
|
+
redis.sadd(Search.mk_sets_key(@type, word), @id)
|
96
97
|
end
|
97
98
|
end
|
98
99
|
|
@@ -102,25 +103,26 @@ class Redis
|
|
102
103
|
end
|
103
104
|
|
104
105
|
private
|
106
|
+
|
105
107
|
def save_prefix_index
|
106
108
|
sorted_set_key = Search.mk_complete_key(@type)
|
107
109
|
sorted_vals = []
|
108
110
|
|
109
|
-
|
111
|
+
aliases.each do |val|
|
110
112
|
words = []
|
111
113
|
words << val.downcase
|
112
114
|
|
113
|
-
|
115
|
+
redis.sadd(Search.mk_sets_key(@type, val), @id)
|
114
116
|
|
115
117
|
if Search.config.pinyin_match
|
116
118
|
pinyin_full = Search.split_pinyin(val.downcase)
|
117
|
-
pinyin_first = pinyin_full.collect { |p| p[0] }.join(
|
118
|
-
pinyin = pinyin_full.join(
|
119
|
+
pinyin_first = pinyin_full.collect { |p| p[0] }.join('')
|
120
|
+
pinyin = pinyin_full.join('')
|
119
121
|
|
120
122
|
words << pinyin
|
121
123
|
words << pinyin_first
|
122
124
|
|
123
|
-
|
125
|
+
redis.sadd(Search.mk_sets_key(@type, pinyin), @id)
|
124
126
|
|
125
127
|
pinyin_full = nil
|
126
128
|
pinyin_first = nil
|
@@ -136,9 +138,8 @@ class Redis
|
|
136
138
|
end
|
137
139
|
end
|
138
140
|
|
139
|
-
|
141
|
+
redis.zadd(sorted_set_key, sorted_vals)
|
140
142
|
end
|
141
|
-
|
142
143
|
end # end Index
|
143
144
|
end
|
144
145
|
end
|
File without changes
|
@@ -1,5 +1,5 @@
|
|
1
|
-
|
2
|
-
|
1
|
+
require 'redis-search'
|
2
|
+
|
3
3
|
namespace :redis_search do
|
4
4
|
task index: 'index:all'
|
5
5
|
|
@@ -27,18 +27,18 @@ namespace :redis_search do
|
|
27
27
|
desc index_model_desc
|
28
28
|
task model: :environment do
|
29
29
|
if ENV['CLASS'].to_s == ''
|
30
|
-
puts '='*90, 'USAGE', '='*90, index_model_desc,
|
30
|
+
puts '=' * 90, 'USAGE', '=' * 90, index_model_desc, ''
|
31
31
|
exit(1)
|
32
32
|
end
|
33
33
|
|
34
34
|
klass = eval(ENV['CLASS'].to_s)
|
35
35
|
batch = ENV['BATCH'].to_i > 0 ? ENV['BATCH'].to_i : 1000
|
36
36
|
tm = Time.now
|
37
|
-
puts "Redis-Search index data to Redis from [#{klass
|
37
|
+
puts "Redis-Search index data to Redis from [#{klass}]"
|
38
38
|
count = klass.redis_search_index_batch_create(batch, true)
|
39
|
-
puts
|
39
|
+
puts ''
|
40
40
|
puts "Indexed #{count} rows | Time spend: #{(Time.now - tm)}s"
|
41
|
-
puts
|
41
|
+
puts 'Rebuild Index done.'
|
42
42
|
end
|
43
43
|
|
44
44
|
desc index_all_desc
|
@@ -51,23 +51,23 @@ namespace :redis_search do
|
|
51
51
|
Dir.glob(File.join("#{dir}/**/*.rb")).each do |path|
|
52
52
|
model_filename = path[/#{Regexp.escape(dir.to_s)}\/([^\.]+).rb/, 1]
|
53
53
|
|
54
|
-
next if model_filename
|
54
|
+
next if model_filename =~ /^concerns\//i # Skip concerns/ folder
|
55
55
|
|
56
56
|
begin
|
57
57
|
klass = model_filename.camelize.constantize
|
58
58
|
rescue NameError
|
59
|
-
require(path) ? retry : raise(
|
59
|
+
require(path) ? retry : raise("Cannot load class '#{klass}'")
|
60
60
|
end
|
61
61
|
end
|
62
62
|
|
63
63
|
puts "Redis-Search index data to Redis from [#{dir}]"
|
64
64
|
Redis::Search.indexed_models.each do |klass|
|
65
|
-
puts "[#{klass
|
65
|
+
puts "[#{klass}]"
|
66
66
|
count += klass.redis_search_index_batch_create(batch, true)
|
67
|
-
puts
|
67
|
+
puts ''
|
68
68
|
end
|
69
69
|
puts "Indexed #{count} rows | Time spend: #{(Time.now - tm)}s"
|
70
|
-
puts
|
70
|
+
puts 'Rebuild Index done.'
|
71
71
|
end
|
72
72
|
end
|
73
73
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: redis-search
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 1.0.0.beta1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Jason Lee
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2016-05-09 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: ruby-pinyin
|
@@ -69,12 +69,13 @@ files:
|
|
69
69
|
- LICENSE
|
70
70
|
- README.md
|
71
71
|
- lib/redis-search.rb
|
72
|
-
- lib/redis
|
73
|
-
- lib/redis
|
74
|
-
- lib/redis
|
75
|
-
- lib/redis
|
76
|
-
- lib/redis
|
77
|
-
- lib/redis
|
72
|
+
- lib/redis-search/base.rb
|
73
|
+
- lib/redis-search/config.rb
|
74
|
+
- lib/redis-search/finder.rb
|
75
|
+
- lib/redis-search/index.rb
|
76
|
+
- lib/redis-search/railtie.rb
|
77
|
+
- lib/redis-search/tasks.rb
|
78
|
+
- lib/redis-search/version.rb
|
78
79
|
homepage: http://github.com/huacnlee/redis-search
|
79
80
|
licenses:
|
80
81
|
- MIT
|
@@ -95,10 +96,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
95
96
|
version: 1.3.6
|
96
97
|
requirements: []
|
97
98
|
rubyforge_project:
|
98
|
-
rubygems_version: 2.
|
99
|
+
rubygems_version: 2.5.1
|
99
100
|
signing_key:
|
100
101
|
specification_version: 4
|
101
102
|
summary: High performance real-time search (Support Chinese), indexes store in Redis
|
102
103
|
for Rails applications.
|
103
104
|
test_files: []
|
104
|
-
has_rdoc:
|