picky 3.0.1 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/lib/picky/application.rb +12 -12
- data/lib/picky/backends/backend.rb +17 -0
- data/lib/picky/{backend → backends}/file/basic.rb +1 -1
- data/lib/picky/{backend → backends}/file/json.rb +1 -1
- data/lib/picky/{backend → backends}/file/marshal.rb +1 -1
- data/lib/picky/{backend → backends}/file/text.rb +1 -1
- data/lib/picky/backends/memory.rb +53 -0
- data/lib/picky/{backend → backends}/redis/basic.rb +9 -14
- data/lib/picky/backends/redis/float_hash.rb +26 -0
- data/lib/picky/{backend → backends}/redis/list_hash.rb +7 -11
- data/lib/picky/{backend → backends}/redis/string_hash.rb +7 -11
- data/lib/picky/backends/redis.rb +87 -0
- data/lib/picky/bundle.rb +107 -11
- data/lib/picky/category.rb +5 -5
- data/lib/picky/index.rb +329 -0
- data/lib/picky/index_indexed.rb +31 -0
- data/lib/picky/index_indexing.rb +161 -0
- data/lib/picky/indexed/bundle.rb +112 -0
- data/lib/picky/indexed/wrappers/exact_first.rb +1 -1
- data/lib/picky/indexers/parallel.rb +2 -1
- data/lib/picky/indexers/serial.rb +2 -1
- data/lib/picky/indexes_indexing.rb +1 -1
- data/lib/picky/indexing/bundle.rb +188 -0
- data/lib/picky/indexing/wrappers/category/location.rb +1 -1
- data/lib/picky/interfaces/live_parameters.rb +8 -8
- data/lib/picky/loader.rb +24 -38
- data/lib/picky/migrations/from_30_to_31.rb +61 -0
- data/lib/picky/query/allocation.rb +10 -5
- data/lib/picky/query/combinations.rb +70 -0
- data/lib/picky/query/indexes.rb +8 -7
- data/lib/picky/query/indexes_check.rb +47 -0
- data/lib/picky/query/token.rb +16 -29
- data/lib/picky/query/tokens.rb +4 -20
- data/lib/picky/search.rb +51 -58
- data/lib/picky/tokenizer.rb +231 -0
- data/lib/picky/tokenizers/location.rb +1 -1
- data/lib/tasks/try.rake +4 -12
- data/lib/tasks/try.rb +37 -0
- data/spec/lib/application_spec.rb +5 -5
- data/spec/lib/{backend → backends}/file/basic_spec.rb +2 -2
- data/spec/lib/{backend → backends}/file/json_spec.rb +2 -2
- data/spec/lib/{backend → backends}/file/marshal_spec.rb +2 -2
- data/spec/lib/{backend → backends}/file/text_spec.rb +1 -1
- data/spec/lib/backends/memory_spec.rb +77 -0
- data/spec/lib/{backend → backends}/redis/basic_spec.rb +19 -21
- data/spec/lib/backends/redis/float_hash_spec.rb +38 -0
- data/spec/lib/backends/redis/list_hash_spec.rb +27 -0
- data/spec/lib/backends/redis/string_hash_spec.rb +38 -0
- data/spec/lib/backends/redis_spec.rb +79 -0
- data/spec/lib/categories_indexed_spec.rb +3 -3
- data/spec/lib/category_indexed_spec.rb +6 -6
- data/spec/lib/category_indexing_spec.rb +1 -1
- data/spec/lib/category_spec.rb +1 -1
- data/spec/lib/frontend_adapters/rack_spec.rb +2 -2
- data/spec/lib/{indexes/index_indexed_spec.rb → index_indexed_spec.rb} +1 -1
- data/spec/lib/{indexes/index_indexing_spec.rb → index_indexing_spec.rb} +1 -1
- data/spec/lib/{indexes/index_spec.rb → index_spec.rb} +1 -1
- data/spec/lib/indexed/{bundle/memory_spec.rb → memory_spec.rb} +18 -18
- data/spec/lib/indexed/wrappers/exact_first_spec.rb +2 -2
- data/spec/lib/indexing/{bundle/memory_partial_generation_speed_spec.rb → bundle_partial_generation_speed_spec.rb} +3 -3
- data/spec/lib/indexing/bundle_spec.rb +302 -0
- data/spec/lib/query/allocation_spec.rb +21 -11
- data/spec/lib/query/combination_spec.rb +2 -2
- data/spec/lib/query/{combinations/base_spec.rb → combinations_spec.rb} +1 -1
- data/spec/lib/query/indexes_check_spec.rb +25 -0
- data/spec/lib/query/indexes_spec.rb +5 -1
- data/spec/lib/query/token_spec.rb +18 -20
- data/spec/lib/query/tokens_spec.rb +14 -65
- data/spec/lib/search_spec.rb +36 -37
- data/spec/lib/tasks/try_spec.rb +51 -0
- data/spec/lib/{tokenizers/base_spec.rb → tokenizer_spec.rb} +15 -44
- metadata +64 -81
- data/lib/picky/backend/base.rb +0 -121
- data/lib/picky/backend/files.rb +0 -28
- data/lib/picky/backend/redis.rb +0 -44
- data/lib/picky/indexed/bundle/base.rb +0 -47
- data/lib/picky/indexed/bundle/memory.rb +0 -88
- data/lib/picky/indexed/bundle/redis.rb +0 -91
- data/lib/picky/indexes/index.rb +0 -328
- data/lib/picky/indexes/index_indexed.rb +0 -35
- data/lib/picky/indexes/index_indexing.rb +0 -165
- data/lib/picky/indexes/memory.rb +0 -20
- data/lib/picky/indexes/redis.rb +0 -20
- data/lib/picky/indexing/bundle/base.rb +0 -242
- data/lib/picky/indexing/bundle/memory.rb +0 -26
- data/lib/picky/indexing/bundle/redis.rb +0 -26
- data/lib/picky/query/combinations/base.rb +0 -74
- data/lib/picky/query/combinations/memory.rb +0 -52
- data/lib/picky/query/combinations/redis.rb +0 -90
- data/lib/picky/query.rb +0 -6
- data/lib/picky/tokenizers/base.rb +0 -231
- data/lib/picky/tokenizers/index.rb +0 -34
- data/lib/picky/tokenizers/query.rb +0 -61
- data/spec/lib/backend/files_spec.rb +0 -189
- data/spec/lib/backend/redis/list_hash_spec.rb +0 -40
- data/spec/lib/backend/redis/string_hash_spec.rb +0 -47
- data/spec/lib/backend/redis_spec.rb +0 -170
- data/spec/lib/indexed/bundle/redis_spec.rb +0 -41
- data/spec/lib/indexes/redis_spec.rb +0 -15
- data/spec/lib/indexing/bundle/base_spec.rb +0 -38
- data/spec/lib/indexing/bundle/memory_spec.rb +0 -287
- data/spec/lib/indexing/bundle/redis_spec.rb +0 -283
- data/spec/lib/query/combinations/memory_spec.rb +0 -158
- data/spec/lib/query/combinations/redis_spec.rb +0 -172
- data/spec/lib/tokenizers/index_spec.rb +0 -69
- data/spec/lib/tokenizers/query_spec.rb +0 -121
data/lib/picky/backend/base.rb
DELETED
@@ -1,121 +0,0 @@
|
|
1
|
-
module Picky
|
2
|
-
|
3
|
-
module Backend
|
4
|
-
|
5
|
-
class Base
|
6
|
-
|
7
|
-
attr_reader :bundle,
|
8
|
-
:inverted,
|
9
|
-
:weights,
|
10
|
-
:similarity,
|
11
|
-
:configuration
|
12
|
-
|
13
|
-
delegate :identifier,
|
14
|
-
:to => :bundle
|
15
|
-
|
16
|
-
def initialize bundle
|
17
|
-
@bundle = bundle
|
18
|
-
end
|
19
|
-
def identifier
|
20
|
-
bundle.identifier
|
21
|
-
end
|
22
|
-
|
23
|
-
# Delegators.
|
24
|
-
#
|
25
|
-
|
26
|
-
# Dumping.
|
27
|
-
#
|
28
|
-
def dump_inverted inverted_hash
|
29
|
-
timed_exclaim %Q{"#{identifier}": => #{inverted}.}
|
30
|
-
inverted.dump inverted_hash
|
31
|
-
end
|
32
|
-
def dump_weights weights_hash
|
33
|
-
timed_exclaim %Q{"#{identifier}": => #{weights}.}
|
34
|
-
weights.dump weights_hash
|
35
|
-
end
|
36
|
-
def dump_similarity similarity_hash
|
37
|
-
timed_exclaim %Q{"#{identifier}": => #{similarity}.}
|
38
|
-
similarity.dump similarity_hash
|
39
|
-
end
|
40
|
-
def dump_configuration configuration_hash
|
41
|
-
timed_exclaim %Q{"#{identifier}": => #{configuration}.}
|
42
|
-
configuration.dump configuration_hash
|
43
|
-
end
|
44
|
-
|
45
|
-
# Loading.
|
46
|
-
#
|
47
|
-
def load_inverted
|
48
|
-
inverted.load
|
49
|
-
end
|
50
|
-
def load_similarity
|
51
|
-
similarity.load
|
52
|
-
end
|
53
|
-
def load_weights
|
54
|
-
weights.load
|
55
|
-
end
|
56
|
-
def load_configuration
|
57
|
-
configuration.load
|
58
|
-
end
|
59
|
-
|
60
|
-
# Cache ok?
|
61
|
-
#
|
62
|
-
def inverted_cache_ok?
|
63
|
-
inverted.cache_ok?
|
64
|
-
end
|
65
|
-
def similarity_cache_ok?
|
66
|
-
similarity.cache_ok?
|
67
|
-
end
|
68
|
-
def weights_cache_ok?
|
69
|
-
weights.cache_ok?
|
70
|
-
end
|
71
|
-
|
72
|
-
# Cache small?
|
73
|
-
#
|
74
|
-
def inverted_cache_small?
|
75
|
-
inverted.cache_small?
|
76
|
-
end
|
77
|
-
def similarity_cache_small?
|
78
|
-
similarity.cache_small?
|
79
|
-
end
|
80
|
-
def weights_cache_small?
|
81
|
-
weights.cache_small?
|
82
|
-
end
|
83
|
-
|
84
|
-
# Copies the indexes to the "backup" directory.
|
85
|
-
#
|
86
|
-
def backup
|
87
|
-
inverted.backup
|
88
|
-
weights.backup
|
89
|
-
similarity.backup
|
90
|
-
configuration.backup
|
91
|
-
end
|
92
|
-
|
93
|
-
# Restores the indexes from the "backup" directory.
|
94
|
-
#
|
95
|
-
def restore
|
96
|
-
inverted.restore
|
97
|
-
weights.restore
|
98
|
-
similarity.restore
|
99
|
-
configuration.restore
|
100
|
-
end
|
101
|
-
|
102
|
-
# Delete all index files.
|
103
|
-
#
|
104
|
-
def delete
|
105
|
-
inverted.delete
|
106
|
-
weights.delete
|
107
|
-
similarity.delete
|
108
|
-
configuration.delete
|
109
|
-
end
|
110
|
-
|
111
|
-
#
|
112
|
-
#
|
113
|
-
def to_s
|
114
|
-
"#{self.class}(#{bundle.identifier})"
|
115
|
-
end
|
116
|
-
|
117
|
-
end
|
118
|
-
|
119
|
-
end
|
120
|
-
|
121
|
-
end
|
data/lib/picky/backend/files.rb
DELETED
@@ -1,28 +0,0 @@
|
|
1
|
-
module Picky
|
2
|
-
|
3
|
-
module Backend
|
4
|
-
|
5
|
-
class Files < Base
|
6
|
-
|
7
|
-
def initialize bundle
|
8
|
-
super bundle
|
9
|
-
|
10
|
-
# Note: We marshal the similarity, as the
|
11
|
-
# Yajl json lib cannot load symbolized
|
12
|
-
# values, just keys.
|
13
|
-
#
|
14
|
-
@inverted = File::JSON.new bundle.index_path(:inverted)
|
15
|
-
@weights = File::JSON.new bundle.index_path(:weights)
|
16
|
-
@similarity = File::Marshal.new bundle.index_path(:similarity)
|
17
|
-
@configuration = File::JSON.new bundle.index_path(:configuration)
|
18
|
-
end
|
19
|
-
|
20
|
-
def to_s
|
21
|
-
"#{self.class}(#{[@inverted, @weights, @similarity, @configuration].join(', ')})"
|
22
|
-
end
|
23
|
-
|
24
|
-
end
|
25
|
-
|
26
|
-
end
|
27
|
-
|
28
|
-
end
|
data/lib/picky/backend/redis.rb
DELETED
@@ -1,44 +0,0 @@
|
|
1
|
-
module Picky
|
2
|
-
|
3
|
-
module Backend
|
4
|
-
|
5
|
-
#
|
6
|
-
#
|
7
|
-
class Redis < Base
|
8
|
-
|
9
|
-
def initialize bundle
|
10
|
-
super bundle
|
11
|
-
|
12
|
-
# Refine a few Redis "types".
|
13
|
-
#
|
14
|
-
@inverted = Redis::ListHash.new "#{bundle.identifier}:inverted"
|
15
|
-
@weights = Redis::StringHash.new "#{bundle.identifier}:weights"
|
16
|
-
@similarity = Redis::ListHash.new "#{bundle.identifier}:similarity"
|
17
|
-
@configuration = Redis::StringHash.new "#{bundle.identifier}:configuration"
|
18
|
-
end
|
19
|
-
|
20
|
-
# Delegate to the right collection.
|
21
|
-
#
|
22
|
-
def ids sym
|
23
|
-
inverted.collection sym
|
24
|
-
end
|
25
|
-
|
26
|
-
# Delegate to the right member value.
|
27
|
-
#
|
28
|
-
# Note: Converts to float.
|
29
|
-
#
|
30
|
-
def weight sym
|
31
|
-
weights.member(sym).to_f
|
32
|
-
end
|
33
|
-
|
34
|
-
# Delegate to a member value.
|
35
|
-
#
|
36
|
-
def setting sym
|
37
|
-
configuration.member sym
|
38
|
-
end
|
39
|
-
|
40
|
-
end
|
41
|
-
|
42
|
-
end
|
43
|
-
|
44
|
-
end
|
@@ -1,47 +0,0 @@
|
|
1
|
-
module Picky
|
2
|
-
|
3
|
-
module Indexed # :nodoc:all
|
4
|
-
|
5
|
-
# An indexed bundle is a number of memory/redis
|
6
|
-
# indexes that compose the indexes for a single category:
|
7
|
-
# * core (inverted) index
|
8
|
-
# * weights index
|
9
|
-
# * similarity index
|
10
|
-
# * index configuration
|
11
|
-
#
|
12
|
-
# Indexed refers to them being indexed.
|
13
|
-
# This class notably offers the methods:
|
14
|
-
# * load
|
15
|
-
# * clear
|
16
|
-
#
|
17
|
-
# To (re)load or clear the current indexes.
|
18
|
-
#
|
19
|
-
module Bundle
|
20
|
-
|
21
|
-
class Base < Picky::Bundle
|
22
|
-
|
23
|
-
# Loads all indexes.
|
24
|
-
#
|
25
|
-
def load
|
26
|
-
load_inverted
|
27
|
-
load_weights
|
28
|
-
load_similarity
|
29
|
-
load_configuration
|
30
|
-
end
|
31
|
-
|
32
|
-
# Clears all indexes.
|
33
|
-
#
|
34
|
-
def clear
|
35
|
-
clear_inverted
|
36
|
-
clear_weights
|
37
|
-
clear_similarity
|
38
|
-
clear_configuration
|
39
|
-
end
|
40
|
-
|
41
|
-
end
|
42
|
-
|
43
|
-
end
|
44
|
-
|
45
|
-
end
|
46
|
-
|
47
|
-
end
|
@@ -1,88 +0,0 @@
|
|
1
|
-
# encoding: utf-8
|
2
|
-
#
|
3
|
-
module Picky
|
4
|
-
|
5
|
-
module Indexed # :nodoc:all
|
6
|
-
|
7
|
-
#
|
8
|
-
#
|
9
|
-
module Bundle
|
10
|
-
|
11
|
-
# This is the _actual_ index (based on memory).
|
12
|
-
#
|
13
|
-
# Handles exact/partial index, weights index, and similarity index.
|
14
|
-
#
|
15
|
-
# Delegates file handling and checking to an *Indexed*::*Files* object.
|
16
|
-
#
|
17
|
-
class Memory < Base
|
18
|
-
|
19
|
-
delegate :[], :to => :configuration
|
20
|
-
|
21
|
-
def initialize name, category, *args
|
22
|
-
super name, category, *args
|
23
|
-
|
24
|
-
@configuration = {} # A hash with config options.
|
25
|
-
|
26
|
-
@backend = Backend::Files.new self
|
27
|
-
end
|
28
|
-
|
29
|
-
# Get the ids for the given symbol.
|
30
|
-
#
|
31
|
-
def ids sym
|
32
|
-
@inverted[sym] || []
|
33
|
-
end
|
34
|
-
# Get a weight for the given symbol.
|
35
|
-
#
|
36
|
-
def weight sym
|
37
|
-
@weights[sym]
|
38
|
-
end
|
39
|
-
|
40
|
-
# Loads the core index.
|
41
|
-
#
|
42
|
-
def load_inverted
|
43
|
-
self.inverted = @backend.load_inverted
|
44
|
-
end
|
45
|
-
# Loads the weights index.
|
46
|
-
#
|
47
|
-
def load_weights
|
48
|
-
self.weights = @backend.load_weights
|
49
|
-
end
|
50
|
-
# Loads the similarity index.
|
51
|
-
#
|
52
|
-
def load_similarity
|
53
|
-
self.similarity = @backend.load_similarity
|
54
|
-
end
|
55
|
-
# Loads the configuration.
|
56
|
-
#
|
57
|
-
def load_configuration
|
58
|
-
self.configuration = @backend.load_configuration
|
59
|
-
end
|
60
|
-
|
61
|
-
# Loads the core index.
|
62
|
-
#
|
63
|
-
def clear_inverted
|
64
|
-
self.inverted = {}
|
65
|
-
end
|
66
|
-
# Loads the weights index.
|
67
|
-
#
|
68
|
-
def clear_weights
|
69
|
-
self.weights = {}
|
70
|
-
end
|
71
|
-
# Loads the similarity index.
|
72
|
-
#
|
73
|
-
def clear_similarity
|
74
|
-
self.similarity = {}
|
75
|
-
end
|
76
|
-
# Loads the configuration.
|
77
|
-
#
|
78
|
-
def clear_configuration
|
79
|
-
self.configuration = {}
|
80
|
-
end
|
81
|
-
|
82
|
-
end
|
83
|
-
|
84
|
-
end
|
85
|
-
|
86
|
-
end
|
87
|
-
|
88
|
-
end
|
@@ -1,91 +0,0 @@
|
|
1
|
-
# encoding: utf-8
|
2
|
-
#
|
3
|
-
module Picky
|
4
|
-
|
5
|
-
module Indexed # :nodoc:all
|
6
|
-
|
7
|
-
#
|
8
|
-
#
|
9
|
-
module Bundle
|
10
|
-
|
11
|
-
# This is the _actual_ index (based on Redis).
|
12
|
-
#
|
13
|
-
# Handles exact/partial index, weights index, and similarity index.
|
14
|
-
#
|
15
|
-
class Redis < Base
|
16
|
-
|
17
|
-
def initialize name, category, *args
|
18
|
-
super name, category, *args
|
19
|
-
|
20
|
-
@backend = Backend::Redis.new self
|
21
|
-
end
|
22
|
-
|
23
|
-
# Get the ids for the given symbol.
|
24
|
-
#
|
25
|
-
# Ids are an array of string values in Redis.
|
26
|
-
#
|
27
|
-
def ids sym
|
28
|
-
@backend.ids sym
|
29
|
-
end
|
30
|
-
# Get a weight for the given symbol.
|
31
|
-
#
|
32
|
-
# A weight is a string value in Redis.
|
33
|
-
#
|
34
|
-
def weight sym
|
35
|
-
@backend.weight sym
|
36
|
-
end
|
37
|
-
# Settings of this bundle can be accessed via [].
|
38
|
-
#
|
39
|
-
def [] sym
|
40
|
-
@backend.setting sym
|
41
|
-
end
|
42
|
-
|
43
|
-
# Loads the inverted index.
|
44
|
-
#
|
45
|
-
def load_inverted
|
46
|
-
# No loading needed.
|
47
|
-
end
|
48
|
-
# Loads the weights index.
|
49
|
-
#
|
50
|
-
def load_weights
|
51
|
-
# No loading needed.
|
52
|
-
end
|
53
|
-
# Loads the similarity index.
|
54
|
-
#
|
55
|
-
def load_similarity
|
56
|
-
# No loading needed.
|
57
|
-
end
|
58
|
-
# Loads the configuration.
|
59
|
-
#
|
60
|
-
def load_configuration
|
61
|
-
# No loading needed.
|
62
|
-
end
|
63
|
-
|
64
|
-
# Loads the inverted index.
|
65
|
-
#
|
66
|
-
def clear_inverted
|
67
|
-
# No clearing possible, currently.
|
68
|
-
end
|
69
|
-
# Loads the weights index.
|
70
|
-
#
|
71
|
-
def clear_weights
|
72
|
-
# No clearing possible, currently.
|
73
|
-
end
|
74
|
-
# Loads the similarity index.
|
75
|
-
#
|
76
|
-
def clear_similarity
|
77
|
-
# No clearing possible, currently.
|
78
|
-
end
|
79
|
-
# Loads the configuration.
|
80
|
-
#
|
81
|
-
def clear_configuration
|
82
|
-
# No clearing possible, currently.
|
83
|
-
end
|
84
|
-
|
85
|
-
end
|
86
|
-
|
87
|
-
end
|
88
|
-
|
89
|
-
end
|
90
|
-
|
91
|
-
end
|