picky 3.0.1 → 3.1.0
Sign up to get free protection for your applications and to get access to all the features.
- data/lib/picky/application.rb +12 -12
- data/lib/picky/backends/backend.rb +17 -0
- data/lib/picky/{backend → backends}/file/basic.rb +1 -1
- data/lib/picky/{backend → backends}/file/json.rb +1 -1
- data/lib/picky/{backend → backends}/file/marshal.rb +1 -1
- data/lib/picky/{backend → backends}/file/text.rb +1 -1
- data/lib/picky/backends/memory.rb +53 -0
- data/lib/picky/{backend → backends}/redis/basic.rb +9 -14
- data/lib/picky/backends/redis/float_hash.rb +26 -0
- data/lib/picky/{backend → backends}/redis/list_hash.rb +7 -11
- data/lib/picky/{backend → backends}/redis/string_hash.rb +7 -11
- data/lib/picky/backends/redis.rb +87 -0
- data/lib/picky/bundle.rb +107 -11
- data/lib/picky/category.rb +5 -5
- data/lib/picky/index.rb +329 -0
- data/lib/picky/index_indexed.rb +31 -0
- data/lib/picky/index_indexing.rb +161 -0
- data/lib/picky/indexed/bundle.rb +112 -0
- data/lib/picky/indexed/wrappers/exact_first.rb +1 -1
- data/lib/picky/indexers/parallel.rb +2 -1
- data/lib/picky/indexers/serial.rb +2 -1
- data/lib/picky/indexes_indexing.rb +1 -1
- data/lib/picky/indexing/bundle.rb +188 -0
- data/lib/picky/indexing/wrappers/category/location.rb +1 -1
- data/lib/picky/interfaces/live_parameters.rb +8 -8
- data/lib/picky/loader.rb +24 -38
- data/lib/picky/migrations/from_30_to_31.rb +61 -0
- data/lib/picky/query/allocation.rb +10 -5
- data/lib/picky/query/combinations.rb +70 -0
- data/lib/picky/query/indexes.rb +8 -7
- data/lib/picky/query/indexes_check.rb +47 -0
- data/lib/picky/query/token.rb +16 -29
- data/lib/picky/query/tokens.rb +4 -20
- data/lib/picky/search.rb +51 -58
- data/lib/picky/tokenizer.rb +231 -0
- data/lib/picky/tokenizers/location.rb +1 -1
- data/lib/tasks/try.rake +4 -12
- data/lib/tasks/try.rb +37 -0
- data/spec/lib/application_spec.rb +5 -5
- data/spec/lib/{backend → backends}/file/basic_spec.rb +2 -2
- data/spec/lib/{backend → backends}/file/json_spec.rb +2 -2
- data/spec/lib/{backend → backends}/file/marshal_spec.rb +2 -2
- data/spec/lib/{backend → backends}/file/text_spec.rb +1 -1
- data/spec/lib/backends/memory_spec.rb +77 -0
- data/spec/lib/{backend → backends}/redis/basic_spec.rb +19 -21
- data/spec/lib/backends/redis/float_hash_spec.rb +38 -0
- data/spec/lib/backends/redis/list_hash_spec.rb +27 -0
- data/spec/lib/backends/redis/string_hash_spec.rb +38 -0
- data/spec/lib/backends/redis_spec.rb +79 -0
- data/spec/lib/categories_indexed_spec.rb +3 -3
- data/spec/lib/category_indexed_spec.rb +6 -6
- data/spec/lib/category_indexing_spec.rb +1 -1
- data/spec/lib/category_spec.rb +1 -1
- data/spec/lib/frontend_adapters/rack_spec.rb +2 -2
- data/spec/lib/{indexes/index_indexed_spec.rb → index_indexed_spec.rb} +1 -1
- data/spec/lib/{indexes/index_indexing_spec.rb → index_indexing_spec.rb} +1 -1
- data/spec/lib/{indexes/index_spec.rb → index_spec.rb} +1 -1
- data/spec/lib/indexed/{bundle/memory_spec.rb → memory_spec.rb} +18 -18
- data/spec/lib/indexed/wrappers/exact_first_spec.rb +2 -2
- data/spec/lib/indexing/{bundle/memory_partial_generation_speed_spec.rb → bundle_partial_generation_speed_spec.rb} +3 -3
- data/spec/lib/indexing/bundle_spec.rb +302 -0
- data/spec/lib/query/allocation_spec.rb +21 -11
- data/spec/lib/query/combination_spec.rb +2 -2
- data/spec/lib/query/{combinations/base_spec.rb → combinations_spec.rb} +1 -1
- data/spec/lib/query/indexes_check_spec.rb +25 -0
- data/spec/lib/query/indexes_spec.rb +5 -1
- data/spec/lib/query/token_spec.rb +18 -20
- data/spec/lib/query/tokens_spec.rb +14 -65
- data/spec/lib/search_spec.rb +36 -37
- data/spec/lib/tasks/try_spec.rb +51 -0
- data/spec/lib/{tokenizers/base_spec.rb → tokenizer_spec.rb} +15 -44
- metadata +64 -81
- data/lib/picky/backend/base.rb +0 -121
- data/lib/picky/backend/files.rb +0 -28
- data/lib/picky/backend/redis.rb +0 -44
- data/lib/picky/indexed/bundle/base.rb +0 -47
- data/lib/picky/indexed/bundle/memory.rb +0 -88
- data/lib/picky/indexed/bundle/redis.rb +0 -91
- data/lib/picky/indexes/index.rb +0 -328
- data/lib/picky/indexes/index_indexed.rb +0 -35
- data/lib/picky/indexes/index_indexing.rb +0 -165
- data/lib/picky/indexes/memory.rb +0 -20
- data/lib/picky/indexes/redis.rb +0 -20
- data/lib/picky/indexing/bundle/base.rb +0 -242
- data/lib/picky/indexing/bundle/memory.rb +0 -26
- data/lib/picky/indexing/bundle/redis.rb +0 -26
- data/lib/picky/query/combinations/base.rb +0 -74
- data/lib/picky/query/combinations/memory.rb +0 -52
- data/lib/picky/query/combinations/redis.rb +0 -90
- data/lib/picky/query.rb +0 -6
- data/lib/picky/tokenizers/base.rb +0 -231
- data/lib/picky/tokenizers/index.rb +0 -34
- data/lib/picky/tokenizers/query.rb +0 -61
- data/spec/lib/backend/files_spec.rb +0 -189
- data/spec/lib/backend/redis/list_hash_spec.rb +0 -40
- data/spec/lib/backend/redis/string_hash_spec.rb +0 -47
- data/spec/lib/backend/redis_spec.rb +0 -170
- data/spec/lib/indexed/bundle/redis_spec.rb +0 -41
- data/spec/lib/indexes/redis_spec.rb +0 -15
- data/spec/lib/indexing/bundle/base_spec.rb +0 -38
- data/spec/lib/indexing/bundle/memory_spec.rb +0 -287
- data/spec/lib/indexing/bundle/redis_spec.rb +0 -283
- data/spec/lib/query/combinations/memory_spec.rb +0 -158
- data/spec/lib/query/combinations/redis_spec.rb +0 -172
- data/spec/lib/tokenizers/index_spec.rb +0 -69
- data/spec/lib/tokenizers/query_spec.rb +0 -121
data/lib/picky/backend/base.rb
DELETED
@@ -1,121 +0,0 @@
|
|
1
|
-
module Picky
|
2
|
-
|
3
|
-
module Backend
|
4
|
-
|
5
|
-
class Base
|
6
|
-
|
7
|
-
attr_reader :bundle,
|
8
|
-
:inverted,
|
9
|
-
:weights,
|
10
|
-
:similarity,
|
11
|
-
:configuration
|
12
|
-
|
13
|
-
delegate :identifier,
|
14
|
-
:to => :bundle
|
15
|
-
|
16
|
-
def initialize bundle
|
17
|
-
@bundle = bundle
|
18
|
-
end
|
19
|
-
def identifier
|
20
|
-
bundle.identifier
|
21
|
-
end
|
22
|
-
|
23
|
-
# Delegators.
|
24
|
-
#
|
25
|
-
|
26
|
-
# Dumping.
|
27
|
-
#
|
28
|
-
def dump_inverted inverted_hash
|
29
|
-
timed_exclaim %Q{"#{identifier}": => #{inverted}.}
|
30
|
-
inverted.dump inverted_hash
|
31
|
-
end
|
32
|
-
def dump_weights weights_hash
|
33
|
-
timed_exclaim %Q{"#{identifier}": => #{weights}.}
|
34
|
-
weights.dump weights_hash
|
35
|
-
end
|
36
|
-
def dump_similarity similarity_hash
|
37
|
-
timed_exclaim %Q{"#{identifier}": => #{similarity}.}
|
38
|
-
similarity.dump similarity_hash
|
39
|
-
end
|
40
|
-
def dump_configuration configuration_hash
|
41
|
-
timed_exclaim %Q{"#{identifier}": => #{configuration}.}
|
42
|
-
configuration.dump configuration_hash
|
43
|
-
end
|
44
|
-
|
45
|
-
# Loading.
|
46
|
-
#
|
47
|
-
def load_inverted
|
48
|
-
inverted.load
|
49
|
-
end
|
50
|
-
def load_similarity
|
51
|
-
similarity.load
|
52
|
-
end
|
53
|
-
def load_weights
|
54
|
-
weights.load
|
55
|
-
end
|
56
|
-
def load_configuration
|
57
|
-
configuration.load
|
58
|
-
end
|
59
|
-
|
60
|
-
# Cache ok?
|
61
|
-
#
|
62
|
-
def inverted_cache_ok?
|
63
|
-
inverted.cache_ok?
|
64
|
-
end
|
65
|
-
def similarity_cache_ok?
|
66
|
-
similarity.cache_ok?
|
67
|
-
end
|
68
|
-
def weights_cache_ok?
|
69
|
-
weights.cache_ok?
|
70
|
-
end
|
71
|
-
|
72
|
-
# Cache small?
|
73
|
-
#
|
74
|
-
def inverted_cache_small?
|
75
|
-
inverted.cache_small?
|
76
|
-
end
|
77
|
-
def similarity_cache_small?
|
78
|
-
similarity.cache_small?
|
79
|
-
end
|
80
|
-
def weights_cache_small?
|
81
|
-
weights.cache_small?
|
82
|
-
end
|
83
|
-
|
84
|
-
# Copies the indexes to the "backup" directory.
|
85
|
-
#
|
86
|
-
def backup
|
87
|
-
inverted.backup
|
88
|
-
weights.backup
|
89
|
-
similarity.backup
|
90
|
-
configuration.backup
|
91
|
-
end
|
92
|
-
|
93
|
-
# Restores the indexes from the "backup" directory.
|
94
|
-
#
|
95
|
-
def restore
|
96
|
-
inverted.restore
|
97
|
-
weights.restore
|
98
|
-
similarity.restore
|
99
|
-
configuration.restore
|
100
|
-
end
|
101
|
-
|
102
|
-
# Delete all index files.
|
103
|
-
#
|
104
|
-
def delete
|
105
|
-
inverted.delete
|
106
|
-
weights.delete
|
107
|
-
similarity.delete
|
108
|
-
configuration.delete
|
109
|
-
end
|
110
|
-
|
111
|
-
#
|
112
|
-
#
|
113
|
-
def to_s
|
114
|
-
"#{self.class}(#{bundle.identifier})"
|
115
|
-
end
|
116
|
-
|
117
|
-
end
|
118
|
-
|
119
|
-
end
|
120
|
-
|
121
|
-
end
|
data/lib/picky/backend/files.rb
DELETED
@@ -1,28 +0,0 @@
|
|
1
|
-
module Picky
|
2
|
-
|
3
|
-
module Backend
|
4
|
-
|
5
|
-
class Files < Base
|
6
|
-
|
7
|
-
def initialize bundle
|
8
|
-
super bundle
|
9
|
-
|
10
|
-
# Note: We marshal the similarity, as the
|
11
|
-
# Yajl json lib cannot load symbolized
|
12
|
-
# values, just keys.
|
13
|
-
#
|
14
|
-
@inverted = File::JSON.new bundle.index_path(:inverted)
|
15
|
-
@weights = File::JSON.new bundle.index_path(:weights)
|
16
|
-
@similarity = File::Marshal.new bundle.index_path(:similarity)
|
17
|
-
@configuration = File::JSON.new bundle.index_path(:configuration)
|
18
|
-
end
|
19
|
-
|
20
|
-
def to_s
|
21
|
-
"#{self.class}(#{[@inverted, @weights, @similarity, @configuration].join(', ')})"
|
22
|
-
end
|
23
|
-
|
24
|
-
end
|
25
|
-
|
26
|
-
end
|
27
|
-
|
28
|
-
end
|
data/lib/picky/backend/redis.rb
DELETED
@@ -1,44 +0,0 @@
|
|
1
|
-
module Picky
|
2
|
-
|
3
|
-
module Backend
|
4
|
-
|
5
|
-
#
|
6
|
-
#
|
7
|
-
class Redis < Base
|
8
|
-
|
9
|
-
def initialize bundle
|
10
|
-
super bundle
|
11
|
-
|
12
|
-
# Refine a few Redis "types".
|
13
|
-
#
|
14
|
-
@inverted = Redis::ListHash.new "#{bundle.identifier}:inverted"
|
15
|
-
@weights = Redis::StringHash.new "#{bundle.identifier}:weights"
|
16
|
-
@similarity = Redis::ListHash.new "#{bundle.identifier}:similarity"
|
17
|
-
@configuration = Redis::StringHash.new "#{bundle.identifier}:configuration"
|
18
|
-
end
|
19
|
-
|
20
|
-
# Delegate to the right collection.
|
21
|
-
#
|
22
|
-
def ids sym
|
23
|
-
inverted.collection sym
|
24
|
-
end
|
25
|
-
|
26
|
-
# Delegate to the right member value.
|
27
|
-
#
|
28
|
-
# Note: Converts to float.
|
29
|
-
#
|
30
|
-
def weight sym
|
31
|
-
weights.member(sym).to_f
|
32
|
-
end
|
33
|
-
|
34
|
-
# Delegate to a member value.
|
35
|
-
#
|
36
|
-
def setting sym
|
37
|
-
configuration.member sym
|
38
|
-
end
|
39
|
-
|
40
|
-
end
|
41
|
-
|
42
|
-
end
|
43
|
-
|
44
|
-
end
|
@@ -1,47 +0,0 @@
|
|
1
|
-
module Picky
|
2
|
-
|
3
|
-
module Indexed # :nodoc:all
|
4
|
-
|
5
|
-
# An indexed bundle is a number of memory/redis
|
6
|
-
# indexes that compose the indexes for a single category:
|
7
|
-
# * core (inverted) index
|
8
|
-
# * weights index
|
9
|
-
# * similarity index
|
10
|
-
# * index configuration
|
11
|
-
#
|
12
|
-
# Indexed refers to them being indexed.
|
13
|
-
# This class notably offers the methods:
|
14
|
-
# * load
|
15
|
-
# * clear
|
16
|
-
#
|
17
|
-
# To (re)load or clear the current indexes.
|
18
|
-
#
|
19
|
-
module Bundle
|
20
|
-
|
21
|
-
class Base < Picky::Bundle
|
22
|
-
|
23
|
-
# Loads all indexes.
|
24
|
-
#
|
25
|
-
def load
|
26
|
-
load_inverted
|
27
|
-
load_weights
|
28
|
-
load_similarity
|
29
|
-
load_configuration
|
30
|
-
end
|
31
|
-
|
32
|
-
# Clears all indexes.
|
33
|
-
#
|
34
|
-
def clear
|
35
|
-
clear_inverted
|
36
|
-
clear_weights
|
37
|
-
clear_similarity
|
38
|
-
clear_configuration
|
39
|
-
end
|
40
|
-
|
41
|
-
end
|
42
|
-
|
43
|
-
end
|
44
|
-
|
45
|
-
end
|
46
|
-
|
47
|
-
end
|
@@ -1,88 +0,0 @@
|
|
1
|
-
# encoding: utf-8
|
2
|
-
#
|
3
|
-
module Picky
|
4
|
-
|
5
|
-
module Indexed # :nodoc:all
|
6
|
-
|
7
|
-
#
|
8
|
-
#
|
9
|
-
module Bundle
|
10
|
-
|
11
|
-
# This is the _actual_ index (based on memory).
|
12
|
-
#
|
13
|
-
# Handles exact/partial index, weights index, and similarity index.
|
14
|
-
#
|
15
|
-
# Delegates file handling and checking to an *Indexed*::*Files* object.
|
16
|
-
#
|
17
|
-
class Memory < Base
|
18
|
-
|
19
|
-
delegate :[], :to => :configuration
|
20
|
-
|
21
|
-
def initialize name, category, *args
|
22
|
-
super name, category, *args
|
23
|
-
|
24
|
-
@configuration = {} # A hash with config options.
|
25
|
-
|
26
|
-
@backend = Backend::Files.new self
|
27
|
-
end
|
28
|
-
|
29
|
-
# Get the ids for the given symbol.
|
30
|
-
#
|
31
|
-
def ids sym
|
32
|
-
@inverted[sym] || []
|
33
|
-
end
|
34
|
-
# Get a weight for the given symbol.
|
35
|
-
#
|
36
|
-
def weight sym
|
37
|
-
@weights[sym]
|
38
|
-
end
|
39
|
-
|
40
|
-
# Loads the core index.
|
41
|
-
#
|
42
|
-
def load_inverted
|
43
|
-
self.inverted = @backend.load_inverted
|
44
|
-
end
|
45
|
-
# Loads the weights index.
|
46
|
-
#
|
47
|
-
def load_weights
|
48
|
-
self.weights = @backend.load_weights
|
49
|
-
end
|
50
|
-
# Loads the similarity index.
|
51
|
-
#
|
52
|
-
def load_similarity
|
53
|
-
self.similarity = @backend.load_similarity
|
54
|
-
end
|
55
|
-
# Loads the configuration.
|
56
|
-
#
|
57
|
-
def load_configuration
|
58
|
-
self.configuration = @backend.load_configuration
|
59
|
-
end
|
60
|
-
|
61
|
-
# Loads the core index.
|
62
|
-
#
|
63
|
-
def clear_inverted
|
64
|
-
self.inverted = {}
|
65
|
-
end
|
66
|
-
# Loads the weights index.
|
67
|
-
#
|
68
|
-
def clear_weights
|
69
|
-
self.weights = {}
|
70
|
-
end
|
71
|
-
# Loads the similarity index.
|
72
|
-
#
|
73
|
-
def clear_similarity
|
74
|
-
self.similarity = {}
|
75
|
-
end
|
76
|
-
# Loads the configuration.
|
77
|
-
#
|
78
|
-
def clear_configuration
|
79
|
-
self.configuration = {}
|
80
|
-
end
|
81
|
-
|
82
|
-
end
|
83
|
-
|
84
|
-
end
|
85
|
-
|
86
|
-
end
|
87
|
-
|
88
|
-
end
|
@@ -1,91 +0,0 @@
|
|
1
|
-
# encoding: utf-8
|
2
|
-
#
|
3
|
-
module Picky
|
4
|
-
|
5
|
-
module Indexed # :nodoc:all
|
6
|
-
|
7
|
-
#
|
8
|
-
#
|
9
|
-
module Bundle
|
10
|
-
|
11
|
-
# This is the _actual_ index (based on Redis).
|
12
|
-
#
|
13
|
-
# Handles exact/partial index, weights index, and similarity index.
|
14
|
-
#
|
15
|
-
class Redis < Base
|
16
|
-
|
17
|
-
def initialize name, category, *args
|
18
|
-
super name, category, *args
|
19
|
-
|
20
|
-
@backend = Backend::Redis.new self
|
21
|
-
end
|
22
|
-
|
23
|
-
# Get the ids for the given symbol.
|
24
|
-
#
|
25
|
-
# Ids are an array of string values in Redis.
|
26
|
-
#
|
27
|
-
def ids sym
|
28
|
-
@backend.ids sym
|
29
|
-
end
|
30
|
-
# Get a weight for the given symbol.
|
31
|
-
#
|
32
|
-
# A weight is a string value in Redis.
|
33
|
-
#
|
34
|
-
def weight sym
|
35
|
-
@backend.weight sym
|
36
|
-
end
|
37
|
-
# Settings of this bundle can be accessed via [].
|
38
|
-
#
|
39
|
-
def [] sym
|
40
|
-
@backend.setting sym
|
41
|
-
end
|
42
|
-
|
43
|
-
# Loads the inverted index.
|
44
|
-
#
|
45
|
-
def load_inverted
|
46
|
-
# No loading needed.
|
47
|
-
end
|
48
|
-
# Loads the weights index.
|
49
|
-
#
|
50
|
-
def load_weights
|
51
|
-
# No loading needed.
|
52
|
-
end
|
53
|
-
# Loads the similarity index.
|
54
|
-
#
|
55
|
-
def load_similarity
|
56
|
-
# No loading needed.
|
57
|
-
end
|
58
|
-
# Loads the configuration.
|
59
|
-
#
|
60
|
-
def load_configuration
|
61
|
-
# No loading needed.
|
62
|
-
end
|
63
|
-
|
64
|
-
# Loads the inverted index.
|
65
|
-
#
|
66
|
-
def clear_inverted
|
67
|
-
# No clearing possible, currently.
|
68
|
-
end
|
69
|
-
# Loads the weights index.
|
70
|
-
#
|
71
|
-
def clear_weights
|
72
|
-
# No clearing possible, currently.
|
73
|
-
end
|
74
|
-
# Loads the similarity index.
|
75
|
-
#
|
76
|
-
def clear_similarity
|
77
|
-
# No clearing possible, currently.
|
78
|
-
end
|
79
|
-
# Loads the configuration.
|
80
|
-
#
|
81
|
-
def clear_configuration
|
82
|
-
# No clearing possible, currently.
|
83
|
-
end
|
84
|
-
|
85
|
-
end
|
86
|
-
|
87
|
-
end
|
88
|
-
|
89
|
-
end
|
90
|
-
|
91
|
-
end
|