picky 4.6.3 → 4.6.4

Sign up to get free protection for your applications and to get access to all the features.
Files changed (55) hide show
  1. data/lib/performant.c +4 -4
  2. data/lib/picky/analyzer.rb +6 -3
  3. data/lib/picky/backends/backend.rb +40 -0
  4. data/lib/picky/backends/file/json.rb +4 -0
  5. data/lib/picky/backends/file.rb +1 -25
  6. data/lib/picky/backends/memory/json.rb +4 -0
  7. data/lib/picky/backends/memory.rb +1 -29
  8. data/lib/picky/backends/redis/directly_manipulable.rb +15 -7
  9. data/lib/picky/backends/redis.rb +91 -92
  10. data/lib/picky/backends/sqlite/basic.rb +6 -0
  11. data/lib/picky/bundle.rb +12 -10
  12. data/lib/picky/categories_indexing.rb +0 -13
  13. data/lib/picky/category.rb +24 -21
  14. data/lib/picky/category_indexing.rb +8 -22
  15. data/lib/picky/constants.rb +0 -1
  16. data/lib/picky/generators/aliases.rb +2 -0
  17. data/lib/picky/generators/partial.rb +27 -0
  18. data/lib/picky/generators/similarity.rb +27 -0
  19. data/lib/picky/generators/weights.rb +32 -0
  20. data/lib/picky/helpers/identification.rb +18 -0
  21. data/lib/picky/helpers/indexing.rb +16 -0
  22. data/lib/picky/index.rb +6 -0
  23. data/lib/picky/index_indexing.rb +9 -21
  24. data/lib/picky/indexes_indexing.rb +5 -14
  25. data/lib/picky/loader.rb +204 -199
  26. data/lib/picky/query/indexes.rb +12 -1
  27. data/lib/picky/search.rb +1 -0
  28. data/lib/picky/source.rb +23 -0
  29. data/lib/picky/tokenizer.rb +35 -13
  30. data/spec/functional/facets_spec.rb +1 -1
  31. data/spec/functional/remap_qualifiers_spec.rb +43 -0
  32. data/spec/functional/tokenizer_spec.rb +1 -1
  33. data/spec/lib/api/search/boost_spec.rb +1 -1
  34. data/spec/lib/category_spec.rb +1 -4
  35. data/spec/lib/generators/partial_spec.rb +58 -0
  36. data/spec/lib/generators/similarity_spec.rb +59 -0
  37. data/spec/lib/generators/weights_spec.rb +68 -0
  38. data/spec/lib/index_indexing_spec.rb +2 -4
  39. data/spec/lib/index_spec.rb +6 -0
  40. data/spec/lib/pool_spec.rb +39 -35
  41. data/spec/lib/sinatra_spec.rb +2 -2
  42. data/spec/lib/source_spec.rb +63 -0
  43. data/spec/lib/tokenizer_spec.rb +64 -2
  44. metadata +20 -20
  45. data/lib/picky/api/category/partial.rb +0 -26
  46. data/lib/picky/api/category/similarity.rb +0 -26
  47. data/lib/picky/api/category/weight.rb +0 -28
  48. data/lib/picky/api/source.rb +0 -35
  49. data/lib/picky/api/tokenizer.rb +0 -37
  50. data/lib/picky/deployment.rb +0 -211
  51. data/spec/lib/api/category/partial_spec.rb +0 -49
  52. data/spec/lib/api/category/similarity_spec.rb +0 -50
  53. data/spec/lib/api/category/weight_spec.rb +0 -55
  54. data/spec/lib/api/source_spec.rb +0 -68
  55. data/spec/lib/api/tokenizer_spec.rb +0 -42
@@ -1,26 +0,0 @@
1
- module Picky
2
- module API
3
- module Category
4
-
5
- module Partial
6
-
7
- def extract_partial thing
8
- return Generators::Partial::Default unless thing
9
-
10
- if thing.respond_to? :each_partial
11
- thing
12
- else
13
- raise <<-ERROR
14
- partial options for #{index_name}:#{name} should be either
15
- * for example a Partial::Substring.new(from: m, to: n), Partial::Postfix.new(from: n), Partial::Infix.new(min: m, max: n) etc.
16
- or
17
- * an object that responds to #each_partial(str_or_sym) and yields each partial
18
- ERROR
19
- end
20
- end
21
-
22
- end
23
-
24
- end
25
- end
26
- end
@@ -1,26 +0,0 @@
1
- module Picky
2
- module API
3
- module Category
4
-
5
- module Similarity
6
-
7
- def extract_similarity thing
8
- return Generators::Similarity::Default unless thing
9
-
10
- if thing.respond_to?(:encode) && thing.respond_to?(:prioritize)
11
- thing
12
- else
13
- raise <<-ERROR
14
- similarity options for #{index_name}:#{name} should be either
15
- * for example a Similarity::Phonetic.new(n), Similarity::Metaphone.new(n), Similarity::DoubleMetaphone.new(n) etc.
16
- or
17
- * an object that responds to #encode(text) => encoded_text and #prioritize(array_of_encoded, encoded)
18
- ERROR
19
- end
20
- end
21
-
22
- end
23
-
24
- end
25
- end
26
- end
@@ -1,28 +0,0 @@
1
- module Picky
2
- module API
3
- module Category
4
-
5
- module Weight
6
-
7
- def extract_weight thing
8
- return Generators::Weights::Default unless thing
9
-
10
- if thing.respond_to? :weight_for
11
- thing
12
- elsif thing.respond_to? :to_int
13
- Generators::Weights::Logarithmic.new thing
14
- else
15
- raise <<-ERROR
16
- weight options for #{index_name}:#{name} should be either
17
- * for example a Weights::Logarithmic.new, Weights::Constant.new(int = 0), Weights::Dynamic.new(&block) etc.
18
- or
19
- * an object that responds to #weight_for(amount_of_ids_for_token) => float
20
- ERROR
21
- end
22
- end
23
-
24
- end
25
-
26
- end
27
- end
28
- end
@@ -1,35 +0,0 @@
1
- module Picky
2
- module API
3
- module Source
4
-
5
- def extract_source thing, options = {}
6
- if thing.respond_to?(:each) || thing.respond_to?(:call)
7
- thing
8
- else
9
- return if options[:nil_ok]
10
- if respond_to? :name
11
- if @index
12
- location = " #{@index.name}:#{name}"
13
- else
14
- location = " #{name}"
15
- end
16
- else
17
- location = ''
18
- end
19
- raise ArgumentError.new(<<-ERROR)
20
- The#{location} source should respond to either the method #each or
21
- it can be a lambda/block, returning such a source.
22
- ERROR
23
- end
24
- end
25
-
26
- # Get the actual source if it is wrapped in a time
27
- # capsule, i.e. a block/lambda.
28
- #
29
- def unblock_source
30
- @source.respond_to?(:call) ? @source.call : @source
31
- end
32
-
33
- end
34
- end
35
- end
@@ -1,37 +0,0 @@
1
- module Picky
2
- module API
3
-
4
- module Tokenizer
5
-
6
- def extract_tokenizer thing
7
- return unless thing
8
- if thing.respond_to? :tokenize
9
- thing
10
- else
11
- if thing.respond_to? :[]
12
- Picky::Tokenizer.new thing
13
- else
14
- if respond_to? :name
15
- location = ' for '
16
- if @index
17
- location += "#{@index.name}:#{name}"
18
- else
19
- location += "#{name}"
20
- end
21
- else
22
- location = ''
23
- end
24
- raise <<-ERROR
25
- indexing options#{location} should be either
26
- * a Hash
27
- or
28
- * an object that responds to #tokenize(text) => [[token1, ...], [original1, ...]]
29
- ERROR
30
- end
31
- end
32
- end
33
-
34
- end
35
-
36
- end
37
- end
@@ -1,211 +0,0 @@
1
- require File.expand_path '../constants', __FILE__
2
-
3
- module Picky
4
- module Capistrano
5
-
6
- # Include all
7
- #
8
- module All
9
- def self.extended cap_config
10
-
11
- cap_config.instance_eval do
12
-
13
- # Executes a rake task on the server.
14
- #
15
- # Options:
16
- # * env: The PICKY_ENV. Will not set if set explicitly to false. Default: production.
17
- # * All other options get passed on to the Capistrano run task.
18
- #
19
- def execute_rake_task name, options = {}, &block
20
- env = options.delete :env
21
- env = env == false ? '' : "PICKY_ENV=#{env || 'production'}"
22
- run "cd #{current_path}; rake #{name} #{env}", options, &block
23
- end
24
-
25
- end
26
-
27
- cap_config.extend Standard
28
- cap_config.extend Deploy
29
- cap_config.extend Caching
30
- cap_config.extend Overrides
31
-
32
- end
33
- end
34
-
35
- # Removes unneeded Rails defaults.
36
- #
37
- module Overrides
38
- def self.extended cap_config
39
- cap_config.instance_eval do
40
-
41
- namespace :deploy do
42
- tasks.delete :check
43
- tasks.delete :cold
44
- tasks.delete :migrations
45
- tasks.delete :migrate
46
- tasks.delete :upload
47
-
48
- namespace :web do
49
- tasks.delete :enable
50
- tasks.delete :disable
51
- end
52
- end
53
-
54
- end
55
- end
56
- end
57
-
58
- module Standard
59
- def self.extended cap_config
60
- cap_config.load 'standard'
61
- cap_config.load 'deploy'
62
- end
63
- end
64
-
65
- module Deploy
66
-
67
- def self.extended cap_config
68
- cap_config.instance_eval do
69
-
70
- namespace :deploy do
71
- %w(start stop).each do |action|
72
- desc "#{action} the Servers"
73
- task action.intern, :roles => :app do
74
- execute_rake_task "server:#{action}"
75
- end
76
- end
77
- desc "Restart the Servers sequentially"
78
- task :restart, :roles => :app do
79
- find_servers(:roles => :app).each do |server|
80
- execute_rake_task "server:restart", :hosts => server.host
81
- end
82
- end
83
-
84
- desc 'Hot deploy the code'
85
- task 'hot', :roles => :app do
86
- update
87
- execute_rake_task 'server:usr1', :env => false # No env needed.
88
- end
89
-
90
- desc "Setup a GitHub-style deployment."
91
- task :setup, :roles => :app do
92
- cmd = "git clone #{repository} #{current_path}-clone-cache &&" +
93
- "rm #{current_path} &&" +
94
- "mv #{current_path}-clone-cache #{current_path}"
95
- run cmd
96
- end
97
-
98
- desc "Deploy"
99
- task :default, :roles => :app do
100
- update
101
- restart
102
- end
103
-
104
- desc "Update the deployed code."
105
- task :update_code do # code needs to be updated with all servers
106
- puts "updating code to branch #{branch}"
107
- cmd = "cd #{current_path} &&" +
108
- "git fetch origin &&" +
109
- "(git checkout -f #{branch} || git checkout -b #{branch} origin/#{branch}) &&" +
110
- "git pull;" +
111
- "git branch"
112
- run cmd
113
- symlink
114
- end
115
-
116
- desc "Cleans up the git checkout"
117
- task :cleanup, :roles => :app do
118
- run "cd #{current_path} && git gc --aggressive"
119
- end
120
-
121
- desc "create the symlinks to the shared dirs"
122
- task :symlink do
123
- set :user, 'deploy'
124
- run "rm -rf #{current_path}/log; ln -sf #{shared_path}/log #{current_path}/log"
125
- run "rm -rf #{current_path}/index; ln -sf #{shared_path}/index #{current_path}/index"
126
- # link database-config files
127
- run "ln -sf #{shared_path}/app/db.yml #{current_path}/app/db.yml"
128
- # link unicorn.ru
129
- run "ln -sf #{shared_path}/unicorn.ru #{current_path}/unicorn.ru"
130
- end
131
-
132
- namespace :rollback do
133
- desc "Rollback to last release."
134
- task :default, :roles => :app do
135
- set :branch, branches[-2]
136
- puts "rolling back to branch #{branch}"
137
- deploy.update_code
138
- end
139
-
140
- task :code, :roles => :app do
141
- # implicit
142
- end
143
- end
144
- end
145
-
146
- end
147
- end
148
-
149
- end
150
-
151
- module Caching
152
-
153
- def self.extended cap_config
154
- cap_config.instance_eval do
155
- namespace :cache do
156
- desc "check the index files if they are ready to be used"
157
- task :check, :roles => :cache do
158
- execute_rake_task 'cache:check'
159
- end
160
- end
161
- namespace :cache do
162
- namespace :structure do
163
- desc "create the index cache structure"
164
- task :create, :roles => :app do
165
- execute_rake_task 'cache:structure:create'
166
- end
167
- end
168
- end
169
- namespace :solr do
170
- desc "create the index cache structure"
171
- task :index, :roles => :cache do
172
- execute_rake_task 'solr:index'
173
- end
174
- %w|start stop restart|.collect(&:to_sym).each do |action|
175
- desc "#{action} the solr server"
176
- task action, :roles => :app do
177
- execute_rake_task 'solr:start'
178
- end
179
- end
180
- end
181
- end
182
- end
183
-
184
- end
185
-
186
- module Statistics
187
-
188
- def self.extended cap_config
189
- namespace :statistics do
190
- desc 'Start the statistics server'
191
- task :start, :roles => :statistics do
192
- set :user, 'root'
193
- run "daemonize -c #{current_path} -u deploy -v #{current_path}/script/statistics/start production"
194
- end
195
- desc 'Stop the statistics server'
196
- task :stop, :roles => :statistics do
197
- run "#{current_path}/script/statistics/stop production"
198
- end
199
- desc 'Restart the statistics server'
200
- task :restart, :roles => :statistics do
201
- stop
202
- sleep 2
203
- start
204
- end
205
- end
206
- end
207
-
208
- end
209
-
210
- end
211
- end
@@ -1,49 +0,0 @@
1
- require 'spec_helper'
2
-
3
- describe Picky::API::Category::Partial do
4
- let(:object) do
5
- Class.new do
6
- include Picky::API::Category::Partial
7
-
8
- def index_name
9
- :some_index
10
- end
11
- def name
12
- :some_category
13
- end
14
- end.new
15
- end
16
- context 'extract_partial' do
17
- context 'with nil' do
18
- it 'returns the default' do
19
- object.extract_partial(nil).should == Picky::Partial::Default
20
- end
21
- end
22
- context 'with a partial object' do
23
- let(:partializer) do
24
- Class.new do
25
- def each_partial text
26
- 'tex'
27
- end
28
- end.new
29
- end
30
- it 'yields the partial' do
31
- object.extract_partial(partializer).each_partial('whatevs') do |text|
32
- text.should == 'tex'
33
- end
34
- end
35
- end
36
- context 'invalid weight' do
37
- it 'raises with a nice error message' do
38
- expect {
39
- object.extract_partial Object.new
40
- }.to raise_error(<<-ERROR)
41
- partial options for some_index:some_category should be either
42
- * for example a Partial::Substring.new(from: m, to: n), Partial::Postfix.new(from: n), Partial::Infix.new(min: m, max: n) etc.
43
- or
44
- * an object that responds to #each_partial(str_or_sym) and yields each partial
45
- ERROR
46
- end
47
- end
48
- end
49
- end
@@ -1,50 +0,0 @@
1
- require 'spec_helper'
2
-
3
- describe Picky::API::Category::Similarity do
4
- let(:object) do
5
- Class.new do
6
- include Picky::API::Category::Similarity
7
-
8
- def index_name
9
- :some_index
10
- end
11
- def name
12
- :some_category
13
- end
14
- end.new
15
- end
16
- context 'encode' do
17
- context 'with nil' do
18
- it 'returns the default' do
19
- object.extract_similarity(nil).should == Picky::Similarity::Default
20
- end
21
- end
22
- context 'with a similarity object' do
23
- let(:similarizer) do
24
- Class.new do
25
- def encode text
26
- :encoded
27
- end
28
- def prioritize ary, encoded
29
-
30
- end
31
- end.new
32
- end
33
- it 'returns the encoded string' do
34
- object.extract_similarity(similarizer).encode('whatevs').should == :encoded
35
- end
36
- end
37
- context 'invalid weight' do
38
- it 'raises with a nice error message' do
39
- expect {
40
- object.extract_similarity Object.new
41
- }.to raise_error(<<-ERROR)
42
- similarity options for some_index:some_category should be either
43
- * for example a Similarity::Phonetic.new(n), Similarity::Metaphone.new(n), Similarity::DoubleMetaphone.new(n) etc.
44
- or
45
- * an object that responds to #encode(text) => encoded_text and #prioritize(array_of_encoded, encoded)
46
- ERROR
47
- end
48
- end
49
- end
50
- end
@@ -1,55 +0,0 @@
1
- require 'spec_helper'
2
-
3
- describe Picky::API::Category::Weight do
4
- let(:object) do
5
- Class.new do
6
- include Picky::API::Category::Weight
7
-
8
- def index_name
9
- :some_index
10
- end
11
- def name
12
- :some_category
13
- end
14
- end.new
15
- end
16
- context 'extract_weight' do
17
- context 'with nil' do
18
- it 'returns the default' do
19
- object.extract_weight(nil).should == Picky::Weights::Default
20
- end
21
- end
22
- context 'with a number' do
23
- it 'returns a logarithmic weighter' do
24
- object.extract_weight(7.3).should be_kind_of(Picky::Weights::Logarithmic)
25
- end
26
- it 'returns a logarithmic weighter' do
27
- object.extract_weight(3.14).weight_for(10).should == 5.443 # ln(10) + 3.14 = 2.3025 + 3.14
28
- end
29
- end
30
- context 'with a weight object' do
31
- let(:weighter) do
32
- Class.new do
33
- def weight_for amount
34
- 7.0
35
- end
36
- end.new
37
- end
38
- it 'creates a tokenizer' do
39
- object.extract_weight(weighter).weight_for(21).should == 7.0
40
- end
41
- end
42
- context 'invalid weight' do
43
- it 'raises with a nice error message' do
44
- expect {
45
- object.extract_weight Object.new
46
- }.to raise_error(<<-ERROR)
47
- weight options for some_index:some_category should be either
48
- * for example a Weights::Logarithmic.new, Weights::Constant.new(int = 0), Weights::Dynamic.new(&block) etc.
49
- or
50
- * an object that responds to #weight_for(amount_of_ids_for_token) => float
51
- ERROR
52
- end
53
- end
54
- end
55
- end
@@ -1,68 +0,0 @@
1
- require 'spec_helper'
2
-
3
- describe Picky::API::Source do
4
- let(:object) do
5
- Class.new do
6
- include Picky::API::Source
7
- end.new
8
- end
9
- context 'unblock_source' do
10
- before(:each) do
11
- class << object
12
- def source_with source = nil, &block
13
- @source = source || block
14
- end
15
- end
16
- end
17
- context 'with block' do
18
- it 'unblocks' do
19
- object.source_with do
20
- :some_source
21
- end
22
-
23
- object.unblock_source == :some_source
24
- end
25
- end
26
- context 'with #each' do
27
- it 'takes the source directly' do
28
- object.source_with :some_source
29
-
30
- object.unblock_source == :some_source
31
- end
32
- end
33
- end
34
- context 'extract_source' do
35
- context 'block with source hash' do
36
- it 'extracts a source' do
37
- object.extract_source(Proc.new {}).should be_kind_of(Proc)
38
- end
39
- end
40
- context 'each source' do
41
- let(:source) do
42
- Class.new do
43
- def each
44
-
45
- end
46
- end.new
47
- end
48
- it 'extracts a source' do
49
- object.extract_source(source).should == source
50
- end
51
- end
52
- context 'invalid tokenizer' do
53
- it 'raises with a nice error message' do
54
- expect {
55
- object.extract_source Object.new
56
- }.to raise_error(<<-ERROR)
57
- The source should respond to either the method #each or
58
- it can be a lambda/block, returning such a source.
59
- ERROR
60
- end
61
- end
62
- context 'with nil_ok option' do
63
- it 'simply returns nil back' do
64
- object.extract_source(nil, nil_ok: true).should == nil
65
- end
66
- end
67
- end
68
- end
@@ -1,42 +0,0 @@
1
- require 'spec_helper'
2
-
3
- describe Picky::API::Tokenizer do
4
- let(:object) do
5
- Class.new do
6
- include Picky::API::Tokenizer
7
- end.new
8
- end
9
- context 'extract_tokenizer' do
10
- context 'options hash' do
11
- it 'creates a tokenizer' do
12
- object.extract_tokenizer(indexing: { splits_text_on: /\t/ }).
13
- tokenize("hello\tworld").should == [['hello', 'world'], ['hello', 'world']]
14
- end
15
- end
16
- context 'tokenizer' do
17
- let(:tokenizer) do
18
- Class.new do
19
- def tokenize text
20
- ['unmoved', 'by', 'your', 'texts']
21
- end
22
- end.new
23
- end
24
- it 'creates a tokenizer' do
25
- object.extract_tokenizer(tokenizer).
26
- tokenize("hello\tworld").should == ['unmoved', 'by', 'your', 'texts']
27
- end
28
- end
29
- context 'invalid tokenizer' do
30
- it 'raises with a nice error message' do
31
- expect {
32
- object.extract_tokenizer Object.new
33
- }.to raise_error(<<-ERROR)
34
- indexing options should be either
35
- * a Hash
36
- or
37
- * an object that responds to #tokenize(text) => [[token1, ...], [original1, ...]]
38
- ERROR
39
- end
40
- end
41
- end
42
- end