picky 3.0.1 → 3.1.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (106) hide show
  1. data/lib/picky/application.rb +12 -12
  2. data/lib/picky/backends/backend.rb +17 -0
  3. data/lib/picky/{backend → backends}/file/basic.rb +1 -1
  4. data/lib/picky/{backend → backends}/file/json.rb +1 -1
  5. data/lib/picky/{backend → backends}/file/marshal.rb +1 -1
  6. data/lib/picky/{backend → backends}/file/text.rb +1 -1
  7. data/lib/picky/backends/memory.rb +53 -0
  8. data/lib/picky/{backend → backends}/redis/basic.rb +9 -14
  9. data/lib/picky/backends/redis/float_hash.rb +26 -0
  10. data/lib/picky/{backend → backends}/redis/list_hash.rb +7 -11
  11. data/lib/picky/{backend → backends}/redis/string_hash.rb +7 -11
  12. data/lib/picky/backends/redis.rb +87 -0
  13. data/lib/picky/bundle.rb +107 -11
  14. data/lib/picky/category.rb +5 -5
  15. data/lib/picky/index.rb +329 -0
  16. data/lib/picky/index_indexed.rb +31 -0
  17. data/lib/picky/index_indexing.rb +161 -0
  18. data/lib/picky/indexed/bundle.rb +112 -0
  19. data/lib/picky/indexed/wrappers/exact_first.rb +1 -1
  20. data/lib/picky/indexers/parallel.rb +2 -1
  21. data/lib/picky/indexers/serial.rb +2 -1
  22. data/lib/picky/indexes_indexing.rb +1 -1
  23. data/lib/picky/indexing/bundle.rb +188 -0
  24. data/lib/picky/indexing/wrappers/category/location.rb +1 -1
  25. data/lib/picky/interfaces/live_parameters.rb +8 -8
  26. data/lib/picky/loader.rb +24 -38
  27. data/lib/picky/migrations/from_30_to_31.rb +61 -0
  28. data/lib/picky/query/allocation.rb +10 -5
  29. data/lib/picky/query/combinations.rb +70 -0
  30. data/lib/picky/query/indexes.rb +8 -7
  31. data/lib/picky/query/indexes_check.rb +47 -0
  32. data/lib/picky/query/token.rb +16 -29
  33. data/lib/picky/query/tokens.rb +4 -20
  34. data/lib/picky/search.rb +51 -58
  35. data/lib/picky/tokenizer.rb +231 -0
  36. data/lib/picky/tokenizers/location.rb +1 -1
  37. data/lib/tasks/try.rake +4 -12
  38. data/lib/tasks/try.rb +37 -0
  39. data/spec/lib/application_spec.rb +5 -5
  40. data/spec/lib/{backend → backends}/file/basic_spec.rb +2 -2
  41. data/spec/lib/{backend → backends}/file/json_spec.rb +2 -2
  42. data/spec/lib/{backend → backends}/file/marshal_spec.rb +2 -2
  43. data/spec/lib/{backend → backends}/file/text_spec.rb +1 -1
  44. data/spec/lib/backends/memory_spec.rb +77 -0
  45. data/spec/lib/{backend → backends}/redis/basic_spec.rb +19 -21
  46. data/spec/lib/backends/redis/float_hash_spec.rb +38 -0
  47. data/spec/lib/backends/redis/list_hash_spec.rb +27 -0
  48. data/spec/lib/backends/redis/string_hash_spec.rb +38 -0
  49. data/spec/lib/backends/redis_spec.rb +79 -0
  50. data/spec/lib/categories_indexed_spec.rb +3 -3
  51. data/spec/lib/category_indexed_spec.rb +6 -6
  52. data/spec/lib/category_indexing_spec.rb +1 -1
  53. data/spec/lib/category_spec.rb +1 -1
  54. data/spec/lib/frontend_adapters/rack_spec.rb +2 -2
  55. data/spec/lib/{indexes/index_indexed_spec.rb → index_indexed_spec.rb} +1 -1
  56. data/spec/lib/{indexes/index_indexing_spec.rb → index_indexing_spec.rb} +1 -1
  57. data/spec/lib/{indexes/index_spec.rb → index_spec.rb} +1 -1
  58. data/spec/lib/indexed/{bundle/memory_spec.rb → memory_spec.rb} +18 -18
  59. data/spec/lib/indexed/wrappers/exact_first_spec.rb +2 -2
  60. data/spec/lib/indexing/{bundle/memory_partial_generation_speed_spec.rb → bundle_partial_generation_speed_spec.rb} +3 -3
  61. data/spec/lib/indexing/bundle_spec.rb +302 -0
  62. data/spec/lib/query/allocation_spec.rb +21 -11
  63. data/spec/lib/query/combination_spec.rb +2 -2
  64. data/spec/lib/query/{combinations/base_spec.rb → combinations_spec.rb} +1 -1
  65. data/spec/lib/query/indexes_check_spec.rb +25 -0
  66. data/spec/lib/query/indexes_spec.rb +5 -1
  67. data/spec/lib/query/token_spec.rb +18 -20
  68. data/spec/lib/query/tokens_spec.rb +14 -65
  69. data/spec/lib/search_spec.rb +36 -37
  70. data/spec/lib/tasks/try_spec.rb +51 -0
  71. data/spec/lib/{tokenizers/base_spec.rb → tokenizer_spec.rb} +15 -44
  72. metadata +64 -81
  73. data/lib/picky/backend/base.rb +0 -121
  74. data/lib/picky/backend/files.rb +0 -28
  75. data/lib/picky/backend/redis.rb +0 -44
  76. data/lib/picky/indexed/bundle/base.rb +0 -47
  77. data/lib/picky/indexed/bundle/memory.rb +0 -88
  78. data/lib/picky/indexed/bundle/redis.rb +0 -91
  79. data/lib/picky/indexes/index.rb +0 -328
  80. data/lib/picky/indexes/index_indexed.rb +0 -35
  81. data/lib/picky/indexes/index_indexing.rb +0 -165
  82. data/lib/picky/indexes/memory.rb +0 -20
  83. data/lib/picky/indexes/redis.rb +0 -20
  84. data/lib/picky/indexing/bundle/base.rb +0 -242
  85. data/lib/picky/indexing/bundle/memory.rb +0 -26
  86. data/lib/picky/indexing/bundle/redis.rb +0 -26
  87. data/lib/picky/query/combinations/base.rb +0 -74
  88. data/lib/picky/query/combinations/memory.rb +0 -52
  89. data/lib/picky/query/combinations/redis.rb +0 -90
  90. data/lib/picky/query.rb +0 -6
  91. data/lib/picky/tokenizers/base.rb +0 -231
  92. data/lib/picky/tokenizers/index.rb +0 -34
  93. data/lib/picky/tokenizers/query.rb +0 -61
  94. data/spec/lib/backend/files_spec.rb +0 -189
  95. data/spec/lib/backend/redis/list_hash_spec.rb +0 -40
  96. data/spec/lib/backend/redis/string_hash_spec.rb +0 -47
  97. data/spec/lib/backend/redis_spec.rb +0 -170
  98. data/spec/lib/indexed/bundle/redis_spec.rb +0 -41
  99. data/spec/lib/indexes/redis_spec.rb +0 -15
  100. data/spec/lib/indexing/bundle/base_spec.rb +0 -38
  101. data/spec/lib/indexing/bundle/memory_spec.rb +0 -287
  102. data/spec/lib/indexing/bundle/redis_spec.rb +0 -283
  103. data/spec/lib/query/combinations/memory_spec.rb +0 -158
  104. data/spec/lib/query/combinations/redis_spec.rb +0 -172
  105. data/spec/lib/tokenizers/index_spec.rb +0 -69
  106. data/spec/lib/tokenizers/query_spec.rb +0 -121
@@ -1,121 +0,0 @@
1
- module Picky
2
-
3
- module Backend
4
-
5
- class Base
6
-
7
- attr_reader :bundle,
8
- :inverted,
9
- :weights,
10
- :similarity,
11
- :configuration
12
-
13
- delegate :identifier,
14
- :to => :bundle
15
-
16
- def initialize bundle
17
- @bundle = bundle
18
- end
19
- def identifier
20
- bundle.identifier
21
- end
22
-
23
- # Delegators.
24
- #
25
-
26
- # Dumping.
27
- #
28
- def dump_inverted inverted_hash
29
- timed_exclaim %Q{"#{identifier}": => #{inverted}.}
30
- inverted.dump inverted_hash
31
- end
32
- def dump_weights weights_hash
33
- timed_exclaim %Q{"#{identifier}": => #{weights}.}
34
- weights.dump weights_hash
35
- end
36
- def dump_similarity similarity_hash
37
- timed_exclaim %Q{"#{identifier}": => #{similarity}.}
38
- similarity.dump similarity_hash
39
- end
40
- def dump_configuration configuration_hash
41
- timed_exclaim %Q{"#{identifier}": => #{configuration}.}
42
- configuration.dump configuration_hash
43
- end
44
-
45
- # Loading.
46
- #
47
- def load_inverted
48
- inverted.load
49
- end
50
- def load_similarity
51
- similarity.load
52
- end
53
- def load_weights
54
- weights.load
55
- end
56
- def load_configuration
57
- configuration.load
58
- end
59
-
60
- # Cache ok?
61
- #
62
- def inverted_cache_ok?
63
- inverted.cache_ok?
64
- end
65
- def similarity_cache_ok?
66
- similarity.cache_ok?
67
- end
68
- def weights_cache_ok?
69
- weights.cache_ok?
70
- end
71
-
72
- # Cache small?
73
- #
74
- def inverted_cache_small?
75
- inverted.cache_small?
76
- end
77
- def similarity_cache_small?
78
- similarity.cache_small?
79
- end
80
- def weights_cache_small?
81
- weights.cache_small?
82
- end
83
-
84
- # Copies the indexes to the "backup" directory.
85
- #
86
- def backup
87
- inverted.backup
88
- weights.backup
89
- similarity.backup
90
- configuration.backup
91
- end
92
-
93
- # Restores the indexes from the "backup" directory.
94
- #
95
- def restore
96
- inverted.restore
97
- weights.restore
98
- similarity.restore
99
- configuration.restore
100
- end
101
-
102
- # Delete all index files.
103
- #
104
- def delete
105
- inverted.delete
106
- weights.delete
107
- similarity.delete
108
- configuration.delete
109
- end
110
-
111
- #
112
- #
113
- def to_s
114
- "#{self.class}(#{bundle.identifier})"
115
- end
116
-
117
- end
118
-
119
- end
120
-
121
- end
@@ -1,28 +0,0 @@
1
- module Picky
2
-
3
- module Backend
4
-
5
- class Files < Base
6
-
7
- def initialize bundle
8
- super bundle
9
-
10
- # Note: We marshal the similarity, as the
11
- # Yajl json lib cannot load symbolized
12
- # values, just keys.
13
- #
14
- @inverted = File::JSON.new bundle.index_path(:inverted)
15
- @weights = File::JSON.new bundle.index_path(:weights)
16
- @similarity = File::Marshal.new bundle.index_path(:similarity)
17
- @configuration = File::JSON.new bundle.index_path(:configuration)
18
- end
19
-
20
- def to_s
21
- "#{self.class}(#{[@inverted, @weights, @similarity, @configuration].join(', ')})"
22
- end
23
-
24
- end
25
-
26
- end
27
-
28
- end
@@ -1,44 +0,0 @@
1
- module Picky
2
-
3
- module Backend
4
-
5
- #
6
- #
7
- class Redis < Base
8
-
9
- def initialize bundle
10
- super bundle
11
-
12
- # Refine a few Redis "types".
13
- #
14
- @inverted = Redis::ListHash.new "#{bundle.identifier}:inverted"
15
- @weights = Redis::StringHash.new "#{bundle.identifier}:weights"
16
- @similarity = Redis::ListHash.new "#{bundle.identifier}:similarity"
17
- @configuration = Redis::StringHash.new "#{bundle.identifier}:configuration"
18
- end
19
-
20
- # Delegate to the right collection.
21
- #
22
- def ids sym
23
- inverted.collection sym
24
- end
25
-
26
- # Delegate to the right member value.
27
- #
28
- # Note: Converts to float.
29
- #
30
- def weight sym
31
- weights.member(sym).to_f
32
- end
33
-
34
- # Delegate to a member value.
35
- #
36
- def setting sym
37
- configuration.member sym
38
- end
39
-
40
- end
41
-
42
- end
43
-
44
- end
@@ -1,47 +0,0 @@
1
- module Picky
2
-
3
- module Indexed # :nodoc:all
4
-
5
- # An indexed bundle is a number of memory/redis
6
- # indexes that compose the indexes for a single category:
7
- # * core (inverted) index
8
- # * weights index
9
- # * similarity index
10
- # * index configuration
11
- #
12
- # Indexed refers to them being indexed.
13
- # This class notably offers the methods:
14
- # * load
15
- # * clear
16
- #
17
- # To (re)load or clear the current indexes.
18
- #
19
- module Bundle
20
-
21
- class Base < Picky::Bundle
22
-
23
- # Loads all indexes.
24
- #
25
- def load
26
- load_inverted
27
- load_weights
28
- load_similarity
29
- load_configuration
30
- end
31
-
32
- # Clears all indexes.
33
- #
34
- def clear
35
- clear_inverted
36
- clear_weights
37
- clear_similarity
38
- clear_configuration
39
- end
40
-
41
- end
42
-
43
- end
44
-
45
- end
46
-
47
- end
@@ -1,88 +0,0 @@
1
- # encoding: utf-8
2
- #
3
- module Picky
4
-
5
- module Indexed # :nodoc:all
6
-
7
- #
8
- #
9
- module Bundle
10
-
11
- # This is the _actual_ index (based on memory).
12
- #
13
- # Handles exact/partial index, weights index, and similarity index.
14
- #
15
- # Delegates file handling and checking to an *Indexed*::*Files* object.
16
- #
17
- class Memory < Base
18
-
19
- delegate :[], :to => :configuration
20
-
21
- def initialize name, category, *args
22
- super name, category, *args
23
-
24
- @configuration = {} # A hash with config options.
25
-
26
- @backend = Backend::Files.new self
27
- end
28
-
29
- # Get the ids for the given symbol.
30
- #
31
- def ids sym
32
- @inverted[sym] || []
33
- end
34
- # Get a weight for the given symbol.
35
- #
36
- def weight sym
37
- @weights[sym]
38
- end
39
-
40
- # Loads the core index.
41
- #
42
- def load_inverted
43
- self.inverted = @backend.load_inverted
44
- end
45
- # Loads the weights index.
46
- #
47
- def load_weights
48
- self.weights = @backend.load_weights
49
- end
50
- # Loads the similarity index.
51
- #
52
- def load_similarity
53
- self.similarity = @backend.load_similarity
54
- end
55
- # Loads the configuration.
56
- #
57
- def load_configuration
58
- self.configuration = @backend.load_configuration
59
- end
60
-
61
- # Loads the core index.
62
- #
63
- def clear_inverted
64
- self.inverted = {}
65
- end
66
- # Loads the weights index.
67
- #
68
- def clear_weights
69
- self.weights = {}
70
- end
71
- # Loads the similarity index.
72
- #
73
- def clear_similarity
74
- self.similarity = {}
75
- end
76
- # Loads the configuration.
77
- #
78
- def clear_configuration
79
- self.configuration = {}
80
- end
81
-
82
- end
83
-
84
- end
85
-
86
- end
87
-
88
- end
@@ -1,91 +0,0 @@
1
- # encoding: utf-8
2
- #
3
- module Picky
4
-
5
- module Indexed # :nodoc:all
6
-
7
- #
8
- #
9
- module Bundle
10
-
11
- # This is the _actual_ index (based on Redis).
12
- #
13
- # Handles exact/partial index, weights index, and similarity index.
14
- #
15
- class Redis < Base
16
-
17
- def initialize name, category, *args
18
- super name, category, *args
19
-
20
- @backend = Backend::Redis.new self
21
- end
22
-
23
- # Get the ids for the given symbol.
24
- #
25
- # Ids are an array of string values in Redis.
26
- #
27
- def ids sym
28
- @backend.ids sym
29
- end
30
- # Get a weight for the given symbol.
31
- #
32
- # A weight is a string value in Redis.
33
- #
34
- def weight sym
35
- @backend.weight sym
36
- end
37
- # Settings of this bundle can be accessed via [].
38
- #
39
- def [] sym
40
- @backend.setting sym
41
- end
42
-
43
- # Loads the inverted index.
44
- #
45
- def load_inverted
46
- # No loading needed.
47
- end
48
- # Loads the weights index.
49
- #
50
- def load_weights
51
- # No loading needed.
52
- end
53
- # Loads the similarity index.
54
- #
55
- def load_similarity
56
- # No loading needed.
57
- end
58
- # Loads the configuration.
59
- #
60
- def load_configuration
61
- # No loading needed.
62
- end
63
-
64
- # Loads the inverted index.
65
- #
66
- def clear_inverted
67
- # No clearing possible, currently.
68
- end
69
- # Loads the weights index.
70
- #
71
- def clear_weights
72
- # No clearing possible, currently.
73
- end
74
- # Loads the similarity index.
75
- #
76
- def clear_similarity
77
- # No clearing possible, currently.
78
- end
79
- # Loads the configuration.
80
- #
81
- def clear_configuration
82
- # No clearing possible, currently.
83
- end
84
-
85
- end
86
-
87
- end
88
-
89
- end
90
-
91
- end