picky 3.0.1 → 3.1.0
Sign up to get free protection for your applications and to get access to all the features.
- data/lib/picky/application.rb +12 -12
- data/lib/picky/backends/backend.rb +17 -0
- data/lib/picky/{backend → backends}/file/basic.rb +1 -1
- data/lib/picky/{backend → backends}/file/json.rb +1 -1
- data/lib/picky/{backend → backends}/file/marshal.rb +1 -1
- data/lib/picky/{backend → backends}/file/text.rb +1 -1
- data/lib/picky/backends/memory.rb +53 -0
- data/lib/picky/{backend → backends}/redis/basic.rb +9 -14
- data/lib/picky/backends/redis/float_hash.rb +26 -0
- data/lib/picky/{backend → backends}/redis/list_hash.rb +7 -11
- data/lib/picky/{backend → backends}/redis/string_hash.rb +7 -11
- data/lib/picky/backends/redis.rb +87 -0
- data/lib/picky/bundle.rb +107 -11
- data/lib/picky/category.rb +5 -5
- data/lib/picky/index.rb +329 -0
- data/lib/picky/index_indexed.rb +31 -0
- data/lib/picky/index_indexing.rb +161 -0
- data/lib/picky/indexed/bundle.rb +112 -0
- data/lib/picky/indexed/wrappers/exact_first.rb +1 -1
- data/lib/picky/indexers/parallel.rb +2 -1
- data/lib/picky/indexers/serial.rb +2 -1
- data/lib/picky/indexes_indexing.rb +1 -1
- data/lib/picky/indexing/bundle.rb +188 -0
- data/lib/picky/indexing/wrappers/category/location.rb +1 -1
- data/lib/picky/interfaces/live_parameters.rb +8 -8
- data/lib/picky/loader.rb +24 -38
- data/lib/picky/migrations/from_30_to_31.rb +61 -0
- data/lib/picky/query/allocation.rb +10 -5
- data/lib/picky/query/combinations.rb +70 -0
- data/lib/picky/query/indexes.rb +8 -7
- data/lib/picky/query/indexes_check.rb +47 -0
- data/lib/picky/query/token.rb +16 -29
- data/lib/picky/query/tokens.rb +4 -20
- data/lib/picky/search.rb +51 -58
- data/lib/picky/tokenizer.rb +231 -0
- data/lib/picky/tokenizers/location.rb +1 -1
- data/lib/tasks/try.rake +4 -12
- data/lib/tasks/try.rb +37 -0
- data/spec/lib/application_spec.rb +5 -5
- data/spec/lib/{backend → backends}/file/basic_spec.rb +2 -2
- data/spec/lib/{backend → backends}/file/json_spec.rb +2 -2
- data/spec/lib/{backend → backends}/file/marshal_spec.rb +2 -2
- data/spec/lib/{backend → backends}/file/text_spec.rb +1 -1
- data/spec/lib/backends/memory_spec.rb +77 -0
- data/spec/lib/{backend → backends}/redis/basic_spec.rb +19 -21
- data/spec/lib/backends/redis/float_hash_spec.rb +38 -0
- data/spec/lib/backends/redis/list_hash_spec.rb +27 -0
- data/spec/lib/backends/redis/string_hash_spec.rb +38 -0
- data/spec/lib/backends/redis_spec.rb +79 -0
- data/spec/lib/categories_indexed_spec.rb +3 -3
- data/spec/lib/category_indexed_spec.rb +6 -6
- data/spec/lib/category_indexing_spec.rb +1 -1
- data/spec/lib/category_spec.rb +1 -1
- data/spec/lib/frontend_adapters/rack_spec.rb +2 -2
- data/spec/lib/{indexes/index_indexed_spec.rb → index_indexed_spec.rb} +1 -1
- data/spec/lib/{indexes/index_indexing_spec.rb → index_indexing_spec.rb} +1 -1
- data/spec/lib/{indexes/index_spec.rb → index_spec.rb} +1 -1
- data/spec/lib/indexed/{bundle/memory_spec.rb → memory_spec.rb} +18 -18
- data/spec/lib/indexed/wrappers/exact_first_spec.rb +2 -2
- data/spec/lib/indexing/{bundle/memory_partial_generation_speed_spec.rb → bundle_partial_generation_speed_spec.rb} +3 -3
- data/spec/lib/indexing/bundle_spec.rb +302 -0
- data/spec/lib/query/allocation_spec.rb +21 -11
- data/spec/lib/query/combination_spec.rb +2 -2
- data/spec/lib/query/{combinations/base_spec.rb → combinations_spec.rb} +1 -1
- data/spec/lib/query/indexes_check_spec.rb +25 -0
- data/spec/lib/query/indexes_spec.rb +5 -1
- data/spec/lib/query/token_spec.rb +18 -20
- data/spec/lib/query/tokens_spec.rb +14 -65
- data/spec/lib/search_spec.rb +36 -37
- data/spec/lib/tasks/try_spec.rb +51 -0
- data/spec/lib/{tokenizers/base_spec.rb → tokenizer_spec.rb} +15 -44
- metadata +64 -81
- data/lib/picky/backend/base.rb +0 -121
- data/lib/picky/backend/files.rb +0 -28
- data/lib/picky/backend/redis.rb +0 -44
- data/lib/picky/indexed/bundle/base.rb +0 -47
- data/lib/picky/indexed/bundle/memory.rb +0 -88
- data/lib/picky/indexed/bundle/redis.rb +0 -91
- data/lib/picky/indexes/index.rb +0 -328
- data/lib/picky/indexes/index_indexed.rb +0 -35
- data/lib/picky/indexes/index_indexing.rb +0 -165
- data/lib/picky/indexes/memory.rb +0 -20
- data/lib/picky/indexes/redis.rb +0 -20
- data/lib/picky/indexing/bundle/base.rb +0 -242
- data/lib/picky/indexing/bundle/memory.rb +0 -26
- data/lib/picky/indexing/bundle/redis.rb +0 -26
- data/lib/picky/query/combinations/base.rb +0 -74
- data/lib/picky/query/combinations/memory.rb +0 -52
- data/lib/picky/query/combinations/redis.rb +0 -90
- data/lib/picky/query.rb +0 -6
- data/lib/picky/tokenizers/base.rb +0 -231
- data/lib/picky/tokenizers/index.rb +0 -34
- data/lib/picky/tokenizers/query.rb +0 -61
- data/spec/lib/backend/files_spec.rb +0 -189
- data/spec/lib/backend/redis/list_hash_spec.rb +0 -40
- data/spec/lib/backend/redis/string_hash_spec.rb +0 -47
- data/spec/lib/backend/redis_spec.rb +0 -170
- data/spec/lib/indexed/bundle/redis_spec.rb +0 -41
- data/spec/lib/indexes/redis_spec.rb +0 -15
- data/spec/lib/indexing/bundle/base_spec.rb +0 -38
- data/spec/lib/indexing/bundle/memory_spec.rb +0 -287
- data/spec/lib/indexing/bundle/redis_spec.rb +0 -283
- data/spec/lib/query/combinations/memory_spec.rb +0 -158
- data/spec/lib/query/combinations/redis_spec.rb +0 -172
- data/spec/lib/tokenizers/index_spec.rb +0 -69
- data/spec/lib/tokenizers/query_spec.rb +0 -121
data/lib/picky/application.rb
CHANGED
@@ -15,14 +15,14 @@ module Picky
|
|
15
15
|
# will generate an example <tt>project_name/app/application.rb</tt> file for you
|
16
16
|
# with some example code inside.
|
17
17
|
#
|
18
|
-
# ==
|
18
|
+
# == Index.new(name)
|
19
19
|
#
|
20
|
-
# Next, define where your data comes from, creating an <tt>Index</tt>. You use the <tt>
|
21
|
-
# my_index =
|
20
|
+
# Next, define where your data comes from, creating an <tt>Index</tt>. You use the <tt>Index.new</tt> method for that:
|
21
|
+
# my_index = Index.new :some_index_name
|
22
22
|
# You give the index a name (or identifier), and a source (see Sources), where its data comes from. Let's do that:
|
23
23
|
# class MyGreatSearch < Application
|
24
24
|
#
|
25
|
-
# books =
|
25
|
+
# books = Index.new :books do
|
26
26
|
# source Sources::CSV.new(:title, :author, :isbn, file:'app/library.csv')
|
27
27
|
# end
|
28
28
|
#
|
@@ -31,7 +31,7 @@ module Picky
|
|
31
31
|
#
|
32
32
|
# That on itself won't do much good.
|
33
33
|
#
|
34
|
-
# Note that a Redis index is also available:
|
34
|
+
# Note that a Redis index is also available: Index.new.
|
35
35
|
#
|
36
36
|
# == category(identifier, options = {})
|
37
37
|
#
|
@@ -43,7 +43,7 @@ module Picky
|
|
43
43
|
# Let's go ahead and define a category:
|
44
44
|
# class MyGreatSearch < Application
|
45
45
|
#
|
46
|
-
# books =
|
46
|
+
# books = Index.new :books do
|
47
47
|
# source Sources::CSV.new(:title, :author, :isbn, file:'app/library.csv')
|
48
48
|
# category :title
|
49
49
|
# end
|
@@ -70,7 +70,7 @@ module Picky
|
|
70
70
|
# In full glory:
|
71
71
|
# class MyGreatSearch < Application
|
72
72
|
#
|
73
|
-
# books =
|
73
|
+
# books = Index.new :books do
|
74
74
|
# source Sources::CSV.new(:title, :author, :isbn, file:'app/library.csv')
|
75
75
|
# category :title
|
76
76
|
# end
|
@@ -134,7 +134,7 @@ module Picky
|
|
134
134
|
# substitutes_characters_with: CharacterSubstituters::WestEuropean.new,
|
135
135
|
# maximum_tokens: 4
|
136
136
|
#
|
137
|
-
# books =
|
137
|
+
# books = Index.new :books do
|
138
138
|
# source Sources::CSV.new(:title, :author, :isbn, file:'app/library.csv')
|
139
139
|
# category :title,
|
140
140
|
# qualifiers: [:t, :title, :titre],
|
@@ -163,14 +163,14 @@ module Picky
|
|
163
163
|
# is used for indexing by default.
|
164
164
|
#
|
165
165
|
def indexing options = {}
|
166
|
-
|
166
|
+
Tokenizer.index_default = Tokenizer.new(options)
|
167
167
|
end
|
168
168
|
|
169
169
|
# Returns a configured tokenizer that
|
170
170
|
# is used for querying by default.
|
171
171
|
#
|
172
172
|
def searching options = {}
|
173
|
-
|
173
|
+
Tokenizer.query_default = Tokenizer.new(options)
|
174
174
|
end
|
175
175
|
|
176
176
|
# Routes.
|
@@ -240,10 +240,10 @@ module Picky
|
|
240
240
|
def to_stats
|
241
241
|
<<-APP
|
242
242
|
\033[1mIndexing (default)\033[m:
|
243
|
-
#{
|
243
|
+
#{Tokenizer.index_default.indented_to_s}
|
244
244
|
|
245
245
|
\033[1mQuerying (default)\033[m:
|
246
|
-
#{
|
246
|
+
#{Tokenizer.query_default.indented_to_s}
|
247
247
|
|
248
248
|
\033[1mIndexes\033[m:
|
249
249
|
#{Indexes.to_s.indented_to_s}
|
@@ -0,0 +1,53 @@
|
|
1
|
+
module Picky
|
2
|
+
|
3
|
+
module Backends
|
4
|
+
|
5
|
+
class Memory < Backend
|
6
|
+
|
7
|
+
def create_inverted bundle
|
8
|
+
File::JSON.new bundle.index_path(:inverted)
|
9
|
+
end
|
10
|
+
def create_weights bundle
|
11
|
+
File::JSON.new bundle.index_path(:weights)
|
12
|
+
end
|
13
|
+
def create_similarity bundle
|
14
|
+
File::Marshal.new bundle.index_path(:similarity)
|
15
|
+
end
|
16
|
+
def create_configuration bundle
|
17
|
+
File::JSON.new bundle.index_path(:configuration)
|
18
|
+
end
|
19
|
+
|
20
|
+
# Returns the result ids for the allocation.
|
21
|
+
#
|
22
|
+
# Sorts the ids by size and & through them in the following order (sizes):
|
23
|
+
# 0. [100_000, 400, 30, 2]
|
24
|
+
# 1. [2, 30, 400, 100_000]
|
25
|
+
# 2. (100_000 & (400 & (30 & 2))) # => result
|
26
|
+
#
|
27
|
+
# Note: Uses a C-optimized intersection routine (in performant.c)
|
28
|
+
# for speed and memory efficiency.
|
29
|
+
#
|
30
|
+
# Note: In the memory based version we ignore the amount and offset hints.
|
31
|
+
# We cannot use the information to speed up the algorithm, unfortunately.
|
32
|
+
#
|
33
|
+
def ids combinations, _, _
|
34
|
+
return [] if combinations.empty?
|
35
|
+
|
36
|
+
# Get the ids for each combination.
|
37
|
+
#
|
38
|
+
id_arrays = combinations.inject([]) do |total, combination|
|
39
|
+
total << combination.ids
|
40
|
+
end
|
41
|
+
|
42
|
+
# Call the optimized C algorithm.
|
43
|
+
#
|
44
|
+
# Note: It orders the passed arrays by size.
|
45
|
+
#
|
46
|
+
Performant::Array.memory_efficient_intersect id_arrays
|
47
|
+
end
|
48
|
+
|
49
|
+
end
|
50
|
+
|
51
|
+
end
|
52
|
+
|
53
|
+
end
|
@@ -1,6 +1,6 @@
|
|
1
1
|
module Picky
|
2
2
|
|
3
|
-
module
|
3
|
+
module Backends
|
4
4
|
|
5
5
|
class Redis
|
6
6
|
|
@@ -13,34 +13,29 @@ module Picky
|
|
13
13
|
#
|
14
14
|
class Basic
|
15
15
|
|
16
|
-
attr_reader :
|
16
|
+
attr_reader :client, :namespace
|
17
17
|
|
18
18
|
# An index cache takes a path, without file extension,
|
19
19
|
# which will be provided by the subclasses.
|
20
20
|
#
|
21
|
-
def initialize namespace
|
21
|
+
def initialize client, namespace
|
22
|
+
@client = client
|
22
23
|
@namespace = namespace
|
23
|
-
|
24
|
-
# TODO Turn this inside out such that people can pass in
|
25
|
-
# their own Redis instance.
|
26
|
-
#
|
27
|
-
# TODO Make the :db a real option.
|
28
|
-
#
|
29
|
-
@backend = ::Redis.new :db => 15
|
30
24
|
end
|
31
25
|
|
32
|
-
#
|
26
|
+
# Returns itself.
|
33
27
|
#
|
34
28
|
def load
|
35
|
-
|
29
|
+
self
|
36
30
|
end
|
31
|
+
|
37
32
|
# We do not use Redis to retrieve data.
|
38
33
|
#
|
39
34
|
def retrieve
|
40
35
|
# Nothing.
|
41
36
|
end
|
42
37
|
|
43
|
-
# Redis
|
38
|
+
# Redis doesn't do backup.
|
44
39
|
#
|
45
40
|
def backup
|
46
41
|
# Nothing.
|
@@ -79,7 +74,7 @@ module Picky
|
|
79
74
|
# DBSIZE KEYPATTERN, we are stuck with this.
|
80
75
|
#
|
81
76
|
def size
|
82
|
-
|
77
|
+
client.dbsize
|
83
78
|
end
|
84
79
|
|
85
80
|
#
|
@@ -0,0 +1,26 @@
|
|
1
|
+
module Picky
|
2
|
+
|
3
|
+
module Backends
|
4
|
+
|
5
|
+
class Redis
|
6
|
+
|
7
|
+
class FloatHash < StringHash
|
8
|
+
|
9
|
+
# Get a single value.
|
10
|
+
#
|
11
|
+
# Internal API method for the index.
|
12
|
+
#
|
13
|
+
# Note: Works like the StringHash method, but
|
14
|
+
# returns a float corresponding to that string.
|
15
|
+
#
|
16
|
+
def [] key
|
17
|
+
super.to_f
|
18
|
+
end
|
19
|
+
|
20
|
+
end
|
21
|
+
|
22
|
+
end
|
23
|
+
|
24
|
+
end
|
25
|
+
|
26
|
+
end
|
@@ -1,6 +1,6 @@
|
|
1
1
|
module Picky
|
2
2
|
|
3
|
-
module
|
3
|
+
module Backends
|
4
4
|
|
5
5
|
class Redis
|
6
6
|
|
@@ -15,7 +15,7 @@ module Picky
|
|
15
15
|
i = 0
|
16
16
|
values.each do |value|
|
17
17
|
i += 1
|
18
|
-
|
18
|
+
client.zadd redis_key, i, value
|
19
19
|
end
|
20
20
|
end
|
21
21
|
end
|
@@ -27,21 +27,17 @@ module Picky
|
|
27
27
|
#
|
28
28
|
def clear
|
29
29
|
redis_key = "#{namespace}:*"
|
30
|
-
|
31
|
-
|
30
|
+
client.keys(redis_key).each do |key|
|
31
|
+
client.del key
|
32
32
|
end
|
33
33
|
end
|
34
34
|
|
35
35
|
# Get a collection.
|
36
36
|
#
|
37
|
-
|
38
|
-
backend.zrange "#{namespace}:#{key}", 0, -1
|
39
|
-
end
|
40
|
-
|
41
|
-
# Get a single value.
|
37
|
+
# Internal API method for the index.
|
42
38
|
#
|
43
|
-
def
|
44
|
-
|
39
|
+
def [] key
|
40
|
+
client.zrange "#{namespace}:#{key}", 0, -1
|
45
41
|
end
|
46
42
|
|
47
43
|
end
|
@@ -1,6 +1,6 @@
|
|
1
1
|
module Picky
|
2
2
|
|
3
|
-
module
|
3
|
+
module Backends
|
4
4
|
|
5
5
|
class Redis
|
6
6
|
|
@@ -13,26 +13,22 @@ module Picky
|
|
13
13
|
def dump hash
|
14
14
|
clear
|
15
15
|
hash.each_pair do |key, value|
|
16
|
-
|
16
|
+
client.hset namespace, key, value
|
17
17
|
end
|
18
18
|
end
|
19
19
|
|
20
20
|
# Clears the hash.
|
21
21
|
#
|
22
22
|
def clear
|
23
|
-
|
24
|
-
end
|
25
|
-
|
26
|
-
# Get a collection.
|
27
|
-
#
|
28
|
-
def collection key
|
29
|
-
raise "Can't retrieve collection for :#{key} from a StringHash. Use Indexes::Redis::ListHash."
|
23
|
+
client.del namespace
|
30
24
|
end
|
31
25
|
|
32
26
|
# Get a single value.
|
33
27
|
#
|
34
|
-
|
35
|
-
|
28
|
+
# Internal API method for the index.
|
29
|
+
#
|
30
|
+
def [] key
|
31
|
+
client.hget namespace, key
|
36
32
|
end
|
37
33
|
|
38
34
|
end
|
@@ -0,0 +1,87 @@
|
|
1
|
+
module Picky
|
2
|
+
|
3
|
+
module Backends
|
4
|
+
|
5
|
+
#
|
6
|
+
#
|
7
|
+
class Redis < Backend
|
8
|
+
|
9
|
+
attr_reader :client
|
10
|
+
|
11
|
+
def initialize options = {}
|
12
|
+
@client = options[:client] || ::Redis.new(:db => (options[:db] || 15))
|
13
|
+
end
|
14
|
+
|
15
|
+
def create_inverted bundle
|
16
|
+
Redis::ListHash.new client, "#{bundle.identifier}:inverted"
|
17
|
+
end
|
18
|
+
def create_weights bundle
|
19
|
+
Redis::FloatHash.new client, "#{bundle.identifier}:weights"
|
20
|
+
end
|
21
|
+
def create_similarity bundle
|
22
|
+
Redis::ListHash.new client, "#{bundle.identifier}:similarity"
|
23
|
+
end
|
24
|
+
def create_configuration bundle
|
25
|
+
Redis::StringHash.new client, "#{bundle.identifier}:configuration"
|
26
|
+
end
|
27
|
+
|
28
|
+
# Returns the result ids for the allocation.
|
29
|
+
#
|
30
|
+
# Developers wanting to program fast intersection
|
31
|
+
# routines, can do so analogue to this in their own
|
32
|
+
# backend implementations.
|
33
|
+
#
|
34
|
+
# Note: We use the amount and offset hints to speed Redis up.
|
35
|
+
#
|
36
|
+
def ids combinations, amount, offset
|
37
|
+
return [] if combinations.empty?
|
38
|
+
|
39
|
+
identifiers = combinations.inject([]) do |identifiers, combination|
|
40
|
+
identifiers << "#{combination.identifier}"
|
41
|
+
end
|
42
|
+
|
43
|
+
result_id = generate_intermediate_result_id
|
44
|
+
|
45
|
+
# Intersect and store.
|
46
|
+
#
|
47
|
+
client.zinterstore result_id, identifiers
|
48
|
+
|
49
|
+
# Get the stored result.
|
50
|
+
#
|
51
|
+
results = client.zrange result_id, offset, (offset + amount)
|
52
|
+
|
53
|
+
# Delete the stored result as it was only for temporary purposes.
|
54
|
+
#
|
55
|
+
# Note: I could also not delete it, but that would not be clean at all.
|
56
|
+
#
|
57
|
+
client.del result_id
|
58
|
+
|
59
|
+
results
|
60
|
+
end
|
61
|
+
|
62
|
+
# Generate a multiple host/process safe result id.
|
63
|
+
#
|
64
|
+
# Note: Generated when this class loads.
|
65
|
+
#
|
66
|
+
require 'socket'
|
67
|
+
def self.extract_host
|
68
|
+
@host ||= Socket.gethostname
|
69
|
+
end
|
70
|
+
def host
|
71
|
+
self.class.extract_host
|
72
|
+
end
|
73
|
+
extract_host
|
74
|
+
def pid
|
75
|
+
@pid ||= Process.pid
|
76
|
+
end
|
77
|
+
# Use the host and pid (generated lazily in child processes) for the result.
|
78
|
+
#
|
79
|
+
def generate_intermediate_result_id
|
80
|
+
:"#{host}:#{pid}:picky:result"
|
81
|
+
end
|
82
|
+
|
83
|
+
end
|
84
|
+
|
85
|
+
end
|
86
|
+
|
87
|
+
end
|
data/lib/picky/bundle.rb
CHANGED
@@ -23,28 +23,37 @@ module Picky
|
|
23
23
|
class Bundle
|
24
24
|
|
25
25
|
attr_reader :name,
|
26
|
-
:category
|
26
|
+
:category,
|
27
|
+
:backend
|
27
28
|
|
28
29
|
attr_accessor :inverted,
|
29
30
|
:weights,
|
30
31
|
:similarity,
|
31
32
|
:configuration,
|
33
|
+
|
34
|
+
:backend_inverted,
|
35
|
+
:backend_weights,
|
36
|
+
:backend_similarity,
|
37
|
+
:backend_configuration,
|
38
|
+
|
32
39
|
:similarity_strategy
|
33
40
|
|
34
|
-
delegate :clear, :to => :inverted
|
35
41
|
delegate :[], :[]=, :to => :configuration
|
36
42
|
delegate :index_directory, :to => :category
|
37
43
|
|
38
|
-
def initialize name, category, similarity_strategy, options = {}
|
39
|
-
@name
|
40
|
-
@category
|
41
|
-
|
42
|
-
@inverted = {}
|
43
|
-
@weights = {}
|
44
|
-
@similarity = {}
|
45
|
-
@configuration = {} # A hash with config options.
|
46
|
-
|
44
|
+
def initialize name, category, backend, similarity_strategy, options = {}
|
45
|
+
@name = name
|
46
|
+
@category = category
|
47
47
|
@similarity_strategy = similarity_strategy
|
48
|
+
|
49
|
+
# Extract specific indexes from backend.
|
50
|
+
#
|
51
|
+
# TODO Clean up all related.
|
52
|
+
#
|
53
|
+
@backend_inverted = backend.create_inverted self
|
54
|
+
@backend_weights = backend.create_weights self
|
55
|
+
@backend_similarity = backend.create_similarity self
|
56
|
+
@backend_configuration = backend.create_configuration self
|
48
57
|
end
|
49
58
|
def identifier
|
50
59
|
"#{category.identifier}:#{name}"
|
@@ -79,6 +88,93 @@ module Picky
|
|
79
88
|
::File.join index_directory, "#{category.name}_#{name}_#{type}"
|
80
89
|
end
|
81
90
|
|
91
|
+
# Copies the indexes to the "backup" directory.
|
92
|
+
#
|
93
|
+
def backup
|
94
|
+
@backend_inverted.backup if @backend_inverted.respond_to? :backup
|
95
|
+
@backend_weights.backup if @backend_weights.respond_to? :backup
|
96
|
+
@backend_similarity.backup if @backend_similarity.respond_to? :backup
|
97
|
+
@backend_configuration.backup if @backend_configuration.respond_to? :backup
|
98
|
+
end
|
99
|
+
|
100
|
+
# Restores the indexes from the "backup" directory.
|
101
|
+
#
|
102
|
+
def restore
|
103
|
+
@backend_inverted.restore if @backend_inverted.respond_to? :restore
|
104
|
+
@backend_weights.restore if @backend_weights.respond_to? :restore
|
105
|
+
@backend_similarity.restore if @backend_similarity.respond_to? :restore
|
106
|
+
@backend_configuration.restore if @backend_configuration.respond_to? :restore
|
107
|
+
end
|
108
|
+
|
109
|
+
# Delete all index files.
|
110
|
+
#
|
111
|
+
def delete
|
112
|
+
@backend_inverted.delete if @backend_inverted.respond_to? :delete
|
113
|
+
@backend_weights.delete if @backend_weights.respond_to? :delete
|
114
|
+
@backend_similarity.delete if @backend_similarity.respond_to? :delete
|
115
|
+
@backend_configuration.delete if @backend_configuration.respond_to? :delete
|
116
|
+
end
|
117
|
+
|
118
|
+
# Alerts the user if an index is missing.
|
119
|
+
#
|
120
|
+
def raise_unless_cache_exists
|
121
|
+
raise_unless_index_exists
|
122
|
+
raise_unless_similarity_exists
|
123
|
+
end
|
124
|
+
# Alerts the user if one of the necessary indexes
|
125
|
+
# (core, weights) is missing.
|
126
|
+
#
|
127
|
+
def raise_unless_index_exists
|
128
|
+
if partial_strategy.saved?
|
129
|
+
warn_if_index_small
|
130
|
+
raise_unless_index_ok
|
131
|
+
end
|
132
|
+
end
|
133
|
+
# Alerts the user if the similarity
|
134
|
+
# index is missing (given that it's used).
|
135
|
+
#
|
136
|
+
def raise_unless_similarity_exists
|
137
|
+
if similarity_strategy.saved?
|
138
|
+
warn_if_similarity_small
|
139
|
+
raise_unless_similarity_ok
|
140
|
+
end
|
141
|
+
end
|
142
|
+
|
143
|
+
# Outputs a warning for the given cache.
|
144
|
+
#
|
145
|
+
def warn_cache_small what
|
146
|
+
warn "Warning: #{what} cache for #{identifier} smaller than 16 bytes."
|
147
|
+
end
|
148
|
+
# Raises an appropriate error message for the given cache.
|
149
|
+
#
|
150
|
+
def raise_cache_missing what
|
151
|
+
raise "Error: The #{what} cache for #{identifier} is missing."
|
152
|
+
end
|
153
|
+
|
154
|
+
# Warns the user if the similarity index is small.
|
155
|
+
#
|
156
|
+
def warn_if_similarity_small
|
157
|
+
warn_cache_small :similarity if backend_similarity.respond_to?(:cache_small?) && backend_similarity.cache_small?
|
158
|
+
end
|
159
|
+
# Alerts the user if the similarity index is not there.
|
160
|
+
#
|
161
|
+
def raise_unless_similarity_ok
|
162
|
+
raise_cache_missing :similarity if backend_similarity.respond_to?(:cache_ok?) && !backend_similarity.cache_ok?
|
163
|
+
end
|
164
|
+
|
165
|
+
# Warns the user if the core or weights indexes are small.
|
166
|
+
#
|
167
|
+
def warn_if_index_small
|
168
|
+
warn_cache_small :inverted if backend_inverted.respond_to?(:cache_small?) && backend_inverted.cache_small?
|
169
|
+
warn_cache_small :weights if backend_weights.respond_to?(:cache_small?) && backend_weights.cache_small?
|
170
|
+
end
|
171
|
+
# Alerts the user if the core or weights indexes are not there.
|
172
|
+
#
|
173
|
+
def raise_unless_index_ok
|
174
|
+
raise_cache_missing :inverted if backend_inverted.respond_to?(:cache_ok?) && !backend_inverted.cache_ok?
|
175
|
+
raise_cache_missing :weights if backend_weights.respond_to?(:cache_ok?) && !backend_weights.cache_ok?
|
176
|
+
end
|
177
|
+
|
82
178
|
def to_s
|
83
179
|
"#{self.class}(#{identifier})"
|
84
180
|
end
|
data/lib/picky/category.rb
CHANGED
@@ -35,16 +35,16 @@ module Picky
|
|
35
35
|
partial = options[:partial] || Generators::Partial::Default
|
36
36
|
similarity = options[:similarity] || Generators::Similarity::Default
|
37
37
|
|
38
|
-
@indexing_exact =
|
39
|
-
@indexing_partial =
|
38
|
+
@indexing_exact = Indexing::Bundle.new :exact, self, index.backend, weights, Generators::Partial::None.new, similarity, options
|
39
|
+
@indexing_partial = Indexing::Bundle.new :partial, self, index.backend, weights, partial, Generators::Similarity::None.new, options
|
40
40
|
|
41
41
|
# Indexed.
|
42
42
|
#
|
43
|
-
@indexed_exact =
|
43
|
+
@indexed_exact = Indexed::Bundle.new :exact, self, index.backend, similarity
|
44
44
|
if partial.use_exact_for_partial?
|
45
45
|
@indexed_partial = @indexed_exact
|
46
46
|
else
|
47
|
-
@indexed_partial =
|
47
|
+
@indexed_partial = Indexed::Bundle.new :partial, self, index.backend, similarity
|
48
48
|
end
|
49
49
|
|
50
50
|
# @exact = exact_lambda.call(@exact, @partial) if exact_lambda = options[:exact_lambda]
|
@@ -92,7 +92,7 @@ module Picky
|
|
92
92
|
# Note: If you don't use it with the block, do not forget to close it.
|
93
93
|
#
|
94
94
|
def prepared_index_file &block
|
95
|
-
@prepared_index_file ||=
|
95
|
+
@prepared_index_file ||= Backends::File::Text.new prepared_index_path
|
96
96
|
@prepared_index_file.open &block
|
97
97
|
end
|
98
98
|
# Creates the index directory including all necessary paths above it.
|