picky 3.0.1 → 3.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (106) hide show
  1. data/lib/picky/application.rb +12 -12
  2. data/lib/picky/backends/backend.rb +17 -0
  3. data/lib/picky/{backend → backends}/file/basic.rb +1 -1
  4. data/lib/picky/{backend → backends}/file/json.rb +1 -1
  5. data/lib/picky/{backend → backends}/file/marshal.rb +1 -1
  6. data/lib/picky/{backend → backends}/file/text.rb +1 -1
  7. data/lib/picky/backends/memory.rb +53 -0
  8. data/lib/picky/{backend → backends}/redis/basic.rb +9 -14
  9. data/lib/picky/backends/redis/float_hash.rb +26 -0
  10. data/lib/picky/{backend → backends}/redis/list_hash.rb +7 -11
  11. data/lib/picky/{backend → backends}/redis/string_hash.rb +7 -11
  12. data/lib/picky/backends/redis.rb +87 -0
  13. data/lib/picky/bundle.rb +107 -11
  14. data/lib/picky/category.rb +5 -5
  15. data/lib/picky/index.rb +329 -0
  16. data/lib/picky/index_indexed.rb +31 -0
  17. data/lib/picky/index_indexing.rb +161 -0
  18. data/lib/picky/indexed/bundle.rb +112 -0
  19. data/lib/picky/indexed/wrappers/exact_first.rb +1 -1
  20. data/lib/picky/indexers/parallel.rb +2 -1
  21. data/lib/picky/indexers/serial.rb +2 -1
  22. data/lib/picky/indexes_indexing.rb +1 -1
  23. data/lib/picky/indexing/bundle.rb +188 -0
  24. data/lib/picky/indexing/wrappers/category/location.rb +1 -1
  25. data/lib/picky/interfaces/live_parameters.rb +8 -8
  26. data/lib/picky/loader.rb +24 -38
  27. data/lib/picky/migrations/from_30_to_31.rb +61 -0
  28. data/lib/picky/query/allocation.rb +10 -5
  29. data/lib/picky/query/combinations.rb +70 -0
  30. data/lib/picky/query/indexes.rb +8 -7
  31. data/lib/picky/query/indexes_check.rb +47 -0
  32. data/lib/picky/query/token.rb +16 -29
  33. data/lib/picky/query/tokens.rb +4 -20
  34. data/lib/picky/search.rb +51 -58
  35. data/lib/picky/tokenizer.rb +231 -0
  36. data/lib/picky/tokenizers/location.rb +1 -1
  37. data/lib/tasks/try.rake +4 -12
  38. data/lib/tasks/try.rb +37 -0
  39. data/spec/lib/application_spec.rb +5 -5
  40. data/spec/lib/{backend → backends}/file/basic_spec.rb +2 -2
  41. data/spec/lib/{backend → backends}/file/json_spec.rb +2 -2
  42. data/spec/lib/{backend → backends}/file/marshal_spec.rb +2 -2
  43. data/spec/lib/{backend → backends}/file/text_spec.rb +1 -1
  44. data/spec/lib/backends/memory_spec.rb +77 -0
  45. data/spec/lib/{backend → backends}/redis/basic_spec.rb +19 -21
  46. data/spec/lib/backends/redis/float_hash_spec.rb +38 -0
  47. data/spec/lib/backends/redis/list_hash_spec.rb +27 -0
  48. data/spec/lib/backends/redis/string_hash_spec.rb +38 -0
  49. data/spec/lib/backends/redis_spec.rb +79 -0
  50. data/spec/lib/categories_indexed_spec.rb +3 -3
  51. data/spec/lib/category_indexed_spec.rb +6 -6
  52. data/spec/lib/category_indexing_spec.rb +1 -1
  53. data/spec/lib/category_spec.rb +1 -1
  54. data/spec/lib/frontend_adapters/rack_spec.rb +2 -2
  55. data/spec/lib/{indexes/index_indexed_spec.rb → index_indexed_spec.rb} +1 -1
  56. data/spec/lib/{indexes/index_indexing_spec.rb → index_indexing_spec.rb} +1 -1
  57. data/spec/lib/{indexes/index_spec.rb → index_spec.rb} +1 -1
  58. data/spec/lib/indexed/{bundle/memory_spec.rb → memory_spec.rb} +18 -18
  59. data/spec/lib/indexed/wrappers/exact_first_spec.rb +2 -2
  60. data/spec/lib/indexing/{bundle/memory_partial_generation_speed_spec.rb → bundle_partial_generation_speed_spec.rb} +3 -3
  61. data/spec/lib/indexing/bundle_spec.rb +302 -0
  62. data/spec/lib/query/allocation_spec.rb +21 -11
  63. data/spec/lib/query/combination_spec.rb +2 -2
  64. data/spec/lib/query/{combinations/base_spec.rb → combinations_spec.rb} +1 -1
  65. data/spec/lib/query/indexes_check_spec.rb +25 -0
  66. data/spec/lib/query/indexes_spec.rb +5 -1
  67. data/spec/lib/query/token_spec.rb +18 -20
  68. data/spec/lib/query/tokens_spec.rb +14 -65
  69. data/spec/lib/search_spec.rb +36 -37
  70. data/spec/lib/tasks/try_spec.rb +51 -0
  71. data/spec/lib/{tokenizers/base_spec.rb → tokenizer_spec.rb} +15 -44
  72. metadata +64 -81
  73. data/lib/picky/backend/base.rb +0 -121
  74. data/lib/picky/backend/files.rb +0 -28
  75. data/lib/picky/backend/redis.rb +0 -44
  76. data/lib/picky/indexed/bundle/base.rb +0 -47
  77. data/lib/picky/indexed/bundle/memory.rb +0 -88
  78. data/lib/picky/indexed/bundle/redis.rb +0 -91
  79. data/lib/picky/indexes/index.rb +0 -328
  80. data/lib/picky/indexes/index_indexed.rb +0 -35
  81. data/lib/picky/indexes/index_indexing.rb +0 -165
  82. data/lib/picky/indexes/memory.rb +0 -20
  83. data/lib/picky/indexes/redis.rb +0 -20
  84. data/lib/picky/indexing/bundle/base.rb +0 -242
  85. data/lib/picky/indexing/bundle/memory.rb +0 -26
  86. data/lib/picky/indexing/bundle/redis.rb +0 -26
  87. data/lib/picky/query/combinations/base.rb +0 -74
  88. data/lib/picky/query/combinations/memory.rb +0 -52
  89. data/lib/picky/query/combinations/redis.rb +0 -90
  90. data/lib/picky/query.rb +0 -6
  91. data/lib/picky/tokenizers/base.rb +0 -231
  92. data/lib/picky/tokenizers/index.rb +0 -34
  93. data/lib/picky/tokenizers/query.rb +0 -61
  94. data/spec/lib/backend/files_spec.rb +0 -189
  95. data/spec/lib/backend/redis/list_hash_spec.rb +0 -40
  96. data/spec/lib/backend/redis/string_hash_spec.rb +0 -47
  97. data/spec/lib/backend/redis_spec.rb +0 -170
  98. data/spec/lib/indexed/bundle/redis_spec.rb +0 -41
  99. data/spec/lib/indexes/redis_spec.rb +0 -15
  100. data/spec/lib/indexing/bundle/base_spec.rb +0 -38
  101. data/spec/lib/indexing/bundle/memory_spec.rb +0 -287
  102. data/spec/lib/indexing/bundle/redis_spec.rb +0 -283
  103. data/spec/lib/query/combinations/memory_spec.rb +0 -158
  104. data/spec/lib/query/combinations/redis_spec.rb +0 -172
  105. data/spec/lib/tokenizers/index_spec.rb +0 -69
  106. data/spec/lib/tokenizers/query_spec.rb +0 -121
@@ -1,121 +0,0 @@
1
- module Picky
2
-
3
- module Backend
4
-
5
- class Base
6
-
7
- attr_reader :bundle,
8
- :inverted,
9
- :weights,
10
- :similarity,
11
- :configuration
12
-
13
- delegate :identifier,
14
- :to => :bundle
15
-
16
- def initialize bundle
17
- @bundle = bundle
18
- end
19
- def identifier
20
- bundle.identifier
21
- end
22
-
23
- # Delegators.
24
- #
25
-
26
- # Dumping.
27
- #
28
- def dump_inverted inverted_hash
29
- timed_exclaim %Q{"#{identifier}": => #{inverted}.}
30
- inverted.dump inverted_hash
31
- end
32
- def dump_weights weights_hash
33
- timed_exclaim %Q{"#{identifier}": => #{weights}.}
34
- weights.dump weights_hash
35
- end
36
- def dump_similarity similarity_hash
37
- timed_exclaim %Q{"#{identifier}": => #{similarity}.}
38
- similarity.dump similarity_hash
39
- end
40
- def dump_configuration configuration_hash
41
- timed_exclaim %Q{"#{identifier}": => #{configuration}.}
42
- configuration.dump configuration_hash
43
- end
44
-
45
- # Loading.
46
- #
47
- def load_inverted
48
- inverted.load
49
- end
50
- def load_similarity
51
- similarity.load
52
- end
53
- def load_weights
54
- weights.load
55
- end
56
- def load_configuration
57
- configuration.load
58
- end
59
-
60
- # Cache ok?
61
- #
62
- def inverted_cache_ok?
63
- inverted.cache_ok?
64
- end
65
- def similarity_cache_ok?
66
- similarity.cache_ok?
67
- end
68
- def weights_cache_ok?
69
- weights.cache_ok?
70
- end
71
-
72
- # Cache small?
73
- #
74
- def inverted_cache_small?
75
- inverted.cache_small?
76
- end
77
- def similarity_cache_small?
78
- similarity.cache_small?
79
- end
80
- def weights_cache_small?
81
- weights.cache_small?
82
- end
83
-
84
- # Copies the indexes to the "backup" directory.
85
- #
86
- def backup
87
- inverted.backup
88
- weights.backup
89
- similarity.backup
90
- configuration.backup
91
- end
92
-
93
- # Restores the indexes from the "backup" directory.
94
- #
95
- def restore
96
- inverted.restore
97
- weights.restore
98
- similarity.restore
99
- configuration.restore
100
- end
101
-
102
- # Delete all index files.
103
- #
104
- def delete
105
- inverted.delete
106
- weights.delete
107
- similarity.delete
108
- configuration.delete
109
- end
110
-
111
- #
112
- #
113
- def to_s
114
- "#{self.class}(#{bundle.identifier})"
115
- end
116
-
117
- end
118
-
119
- end
120
-
121
- end
@@ -1,28 +0,0 @@
1
- module Picky
2
-
3
- module Backend
4
-
5
- class Files < Base
6
-
7
- def initialize bundle
8
- super bundle
9
-
10
- # Note: We marshal the similarity, as the
11
- # Yajl json lib cannot load symbolized
12
- # values, just keys.
13
- #
14
- @inverted = File::JSON.new bundle.index_path(:inverted)
15
- @weights = File::JSON.new bundle.index_path(:weights)
16
- @similarity = File::Marshal.new bundle.index_path(:similarity)
17
- @configuration = File::JSON.new bundle.index_path(:configuration)
18
- end
19
-
20
- def to_s
21
- "#{self.class}(#{[@inverted, @weights, @similarity, @configuration].join(', ')})"
22
- end
23
-
24
- end
25
-
26
- end
27
-
28
- end
@@ -1,44 +0,0 @@
1
- module Picky
2
-
3
- module Backend
4
-
5
- #
6
- #
7
- class Redis < Base
8
-
9
- def initialize bundle
10
- super bundle
11
-
12
- # Refine a few Redis "types".
13
- #
14
- @inverted = Redis::ListHash.new "#{bundle.identifier}:inverted"
15
- @weights = Redis::StringHash.new "#{bundle.identifier}:weights"
16
- @similarity = Redis::ListHash.new "#{bundle.identifier}:similarity"
17
- @configuration = Redis::StringHash.new "#{bundle.identifier}:configuration"
18
- end
19
-
20
- # Delegate to the right collection.
21
- #
22
- def ids sym
23
- inverted.collection sym
24
- end
25
-
26
- # Delegate to the right member value.
27
- #
28
- # Note: Converts to float.
29
- #
30
- def weight sym
31
- weights.member(sym).to_f
32
- end
33
-
34
- # Delegate to a member value.
35
- #
36
- def setting sym
37
- configuration.member sym
38
- end
39
-
40
- end
41
-
42
- end
43
-
44
- end
@@ -1,47 +0,0 @@
1
- module Picky
2
-
3
- module Indexed # :nodoc:all
4
-
5
- # An indexed bundle is a number of memory/redis
6
- # indexes that compose the indexes for a single category:
7
- # * core (inverted) index
8
- # * weights index
9
- # * similarity index
10
- # * index configuration
11
- #
12
- # Indexed refers to them being indexed.
13
- # This class notably offers the methods:
14
- # * load
15
- # * clear
16
- #
17
- # To (re)load or clear the current indexes.
18
- #
19
- module Bundle
20
-
21
- class Base < Picky::Bundle
22
-
23
- # Loads all indexes.
24
- #
25
- def load
26
- load_inverted
27
- load_weights
28
- load_similarity
29
- load_configuration
30
- end
31
-
32
- # Clears all indexes.
33
- #
34
- def clear
35
- clear_inverted
36
- clear_weights
37
- clear_similarity
38
- clear_configuration
39
- end
40
-
41
- end
42
-
43
- end
44
-
45
- end
46
-
47
- end
@@ -1,88 +0,0 @@
1
- # encoding: utf-8
2
- #
3
- module Picky
4
-
5
- module Indexed # :nodoc:all
6
-
7
- #
8
- #
9
- module Bundle
10
-
11
- # This is the _actual_ index (based on memory).
12
- #
13
- # Handles exact/partial index, weights index, and similarity index.
14
- #
15
- # Delegates file handling and checking to an *Indexed*::*Files* object.
16
- #
17
- class Memory < Base
18
-
19
- delegate :[], :to => :configuration
20
-
21
- def initialize name, category, *args
22
- super name, category, *args
23
-
24
- @configuration = {} # A hash with config options.
25
-
26
- @backend = Backend::Files.new self
27
- end
28
-
29
- # Get the ids for the given symbol.
30
- #
31
- def ids sym
32
- @inverted[sym] || []
33
- end
34
- # Get a weight for the given symbol.
35
- #
36
- def weight sym
37
- @weights[sym]
38
- end
39
-
40
- # Loads the core index.
41
- #
42
- def load_inverted
43
- self.inverted = @backend.load_inverted
44
- end
45
- # Loads the weights index.
46
- #
47
- def load_weights
48
- self.weights = @backend.load_weights
49
- end
50
- # Loads the similarity index.
51
- #
52
- def load_similarity
53
- self.similarity = @backend.load_similarity
54
- end
55
- # Loads the configuration.
56
- #
57
- def load_configuration
58
- self.configuration = @backend.load_configuration
59
- end
60
-
61
- # Loads the core index.
62
- #
63
- def clear_inverted
64
- self.inverted = {}
65
- end
66
- # Loads the weights index.
67
- #
68
- def clear_weights
69
- self.weights = {}
70
- end
71
- # Loads the similarity index.
72
- #
73
- def clear_similarity
74
- self.similarity = {}
75
- end
76
- # Loads the configuration.
77
- #
78
- def clear_configuration
79
- self.configuration = {}
80
- end
81
-
82
- end
83
-
84
- end
85
-
86
- end
87
-
88
- end
@@ -1,91 +0,0 @@
1
- # encoding: utf-8
2
- #
3
- module Picky
4
-
5
- module Indexed # :nodoc:all
6
-
7
- #
8
- #
9
- module Bundle
10
-
11
- # This is the _actual_ index (based on Redis).
12
- #
13
- # Handles exact/partial index, weights index, and similarity index.
14
- #
15
- class Redis < Base
16
-
17
- def initialize name, category, *args
18
- super name, category, *args
19
-
20
- @backend = Backend::Redis.new self
21
- end
22
-
23
- # Get the ids for the given symbol.
24
- #
25
- # Ids are an array of string values in Redis.
26
- #
27
- def ids sym
28
- @backend.ids sym
29
- end
30
- # Get a weight for the given symbol.
31
- #
32
- # A weight is a string value in Redis.
33
- #
34
- def weight sym
35
- @backend.weight sym
36
- end
37
- # Settings of this bundle can be accessed via [].
38
- #
39
- def [] sym
40
- @backend.setting sym
41
- end
42
-
43
- # Loads the inverted index.
44
- #
45
- def load_inverted
46
- # No loading needed.
47
- end
48
- # Loads the weights index.
49
- #
50
- def load_weights
51
- # No loading needed.
52
- end
53
- # Loads the similarity index.
54
- #
55
- def load_similarity
56
- # No loading needed.
57
- end
58
- # Loads the configuration.
59
- #
60
- def load_configuration
61
- # No loading needed.
62
- end
63
-
64
- # Loads the inverted index.
65
- #
66
- def clear_inverted
67
- # No clearing possible, currently.
68
- end
69
- # Loads the weights index.
70
- #
71
- def clear_weights
72
- # No clearing possible, currently.
73
- end
74
- # Loads the similarity index.
75
- #
76
- def clear_similarity
77
- # No clearing possible, currently.
78
- end
79
- # Loads the configuration.
80
- #
81
- def clear_configuration
82
- # No clearing possible, currently.
83
- end
84
-
85
- end
86
-
87
- end
88
-
89
- end
90
-
91
- end