picky 1.2.4 → 1.3.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (37) hide show
  1. data/lib/picky/adapters/rack/base.rb +23 -0
  2. data/lib/picky/adapters/rack/live_parameters.rb +33 -0
  3. data/lib/picky/adapters/rack/query.rb +59 -0
  4. data/lib/picky/adapters/rack.rb +28 -0
  5. data/lib/picky/alias_instances.rb +2 -0
  6. data/lib/picky/application.rb +9 -8
  7. data/lib/picky/cli.rb +25 -3
  8. data/lib/picky/frontend_adapters/rack.rb +150 -0
  9. data/lib/picky/helpers/measuring.rb +0 -2
  10. data/lib/picky/index_api.rb +1 -1
  11. data/lib/picky/indexed/categories.rb +51 -14
  12. data/lib/picky/indexers/solr.rb +1 -5
  13. data/lib/picky/indexing/indexes.rb +6 -0
  14. data/lib/picky/interfaces/live_parameters.rb +165 -0
  15. data/lib/picky/loader.rb +13 -2
  16. data/lib/picky/query/base.rb +15 -18
  17. data/lib/picky/query/combination.rb +2 -2
  18. data/lib/picky/query/solr.rb +0 -17
  19. data/lib/picky/query/token.rb +14 -27
  20. data/lib/picky/query/weights.rb +13 -1
  21. data/lib/picky/results/base.rb +9 -2
  22. data/spec/lib/adapters/rack/base_spec.rb +24 -0
  23. data/spec/lib/adapters/rack/live_parameters_spec.rb +21 -0
  24. data/spec/lib/adapters/rack/query_spec.rb +33 -0
  25. data/spec/lib/application_spec.rb +27 -8
  26. data/spec/lib/cli_spec.rb +9 -0
  27. data/spec/lib/extensions/symbol_spec.rb +1 -3
  28. data/spec/lib/{routing_spec.rb → frontend_adapters/rack_spec.rb} +69 -66
  29. data/spec/lib/indexed/categories_spec.rb +24 -0
  30. data/spec/lib/interfaces/live_parameters_spec.rb +138 -0
  31. data/spec/lib/query/base_spec.rb +10 -14
  32. data/spec/lib/query/live_spec.rb +1 -30
  33. data/spec/lib/query/token_spec.rb +72 -5
  34. data/spec/lib/query/weights_spec.rb +59 -36
  35. data/spec/lib/results/base_spec.rb +13 -1
  36. metadata +20 -7
  37. data/lib/picky/routing.rb +0 -171
@@ -0,0 +1,165 @@
1
+ # This is very optional.
2
+ # Only load if the user wants it.
3
+ #
4
+ module Interfaces
5
+ # This is an interface that provides the user of
6
+ # Picky with the possibility to change parameters
7
+ # while the Application is running.
8
+ #
9
+ # Important Note: This will only work in Master/Child configurations.
10
+ #
11
+ class LiveParameters
12
+
13
+ def initialize
14
+ @child, @parent = IO.pipe
15
+ start_master_process_thread
16
+ end
17
+
18
+ # This runs a thread that listens to child processes.
19
+ #
20
+ def start_master_process_thread
21
+ # This thread is stopped in the children.
22
+ #
23
+ Thread.new do
24
+ loop do
25
+ sleep 1 # TODO select
26
+ result = @child.gets ';;;'
27
+ pid, configuration_hash = eval result
28
+ next unless Hash === configuration_hash
29
+ next if configuration_hash.empty?
30
+ exclaim "Trying to update MASTER configuration."
31
+ try_updating_configuration_with configuration_hash
32
+ kill_each_worker_except pid
33
+ # TODO rescue on error.
34
+
35
+ end
36
+ end
37
+ end
38
+
39
+ # TODO This needs to be webserver agnostic.
40
+ #
41
+ def worker_pids
42
+ Unicorn::HttpServer::WORKERS.keys
43
+ end
44
+
45
+ # Taken from Unicorn.
46
+ #
47
+ def kill_each_worker_except pid
48
+ worker_pids.each do |wpid|
49
+ next if wpid == pid
50
+ kill_worker :KILL, wpid
51
+ end
52
+ end
53
+ def kill_worker signal, wpid
54
+ Process.kill signal, wpid
55
+ exclaim "Killing worker ##{wpid} with signal #{signal}."
56
+ rescue Errno::ESRCH
57
+ remove_worker wpid
58
+ end
59
+ # TODO This needs to be Webserver agnostic.
60
+ #
61
+ def remove_worker wpid
62
+ worker = Unicorn::HttpServer::WORKERS.delete(wpid) and worker.tmp.close rescue nil
63
+ end
64
+
65
+ # Updates any parameters with the ones given and
66
+ # returns the updated params.
67
+ #
68
+ # The params are a strictly defined hash of:
69
+ # * querying_removes_characters: Regexp
70
+ # * querying_stopwords: Regexp
71
+ # TODO etc.
72
+ #
73
+ # This first tries to update in the child process,
74
+ # and if successful, in the parent process
75
+ #
76
+ def parameters configuration_hash
77
+ close_child
78
+ exclaim "Trying to update worker child configuration." unless configuration_hash.empty?
79
+ try_updating_configuration_with configuration_hash
80
+ write_parent configuration_hash
81
+ extract_configuration
82
+ rescue CouldNotUpdateConfigurationError => e
83
+ # I need to die such that my broken config is never used.
84
+ #
85
+ exclaim "Child process #{Process.pid} performs harakiri because of broken config."
86
+ harakiri
87
+ { e.config_key => :ERROR }
88
+ end
89
+ # Kills itself, but still answering the request honorably.
90
+ #
91
+ def harakiri
92
+ Process.kill :QUIT, Process.pid
93
+ end
94
+ # Write the parent.
95
+ #
96
+ # Note: The ;;; is the end marker for the message.
97
+ #
98
+ def write_parent configuration_hash
99
+ @parent.write "#{[Process.pid, configuration_hash]};;;"
100
+ end
101
+ # Close the child if it isn't yet closed.
102
+ #
103
+ def close_child
104
+ @child.close unless @child.closed?
105
+ end
106
+
107
+ class CouldNotUpdateConfigurationError < StandardError
108
+ attr_reader :config_key
109
+ def initialize config_key, message
110
+ super message
111
+ @config_key = config_key
112
+ end
113
+ end
114
+
115
+ # Tries updating the configuration in the child process or parent process.
116
+ #
117
+ def try_updating_configuration_with configuration_hash
118
+ current_key = nil
119
+ begin
120
+ configuration_hash.each_pair do |key, new_value|
121
+ exclaim " Setting #{key} with #{new_value}."
122
+ current_key = key
123
+ send :"#{key}=", new_value
124
+ end
125
+ rescue StandardError => e
126
+ raise CouldNotUpdateConfigurationError.new current_key, e.message
127
+ end
128
+ end
129
+
130
+ def extract_configuration
131
+ {
132
+ querying_removes_characters: querying_removes_characters,
133
+ querying_stopwords: querying_stopwords,
134
+ querying_splits_text_on: querying_splits_text_on
135
+ }
136
+ end
137
+
138
+ # TODO Move to Interface object.
139
+ #
140
+ def querying_removes_characters
141
+ Tokenizers::Query.default.instance_variable_get(:@removes_characters_regexp).source
142
+ end
143
+ def querying_removes_characters= new_value
144
+ Tokenizers::Query.default.instance_variable_set(:@removes_characters_regexp, %r{#{new_value}})
145
+ end
146
+ def querying_stopwords
147
+ Tokenizers::Query.default.instance_variable_get(:@remove_stopwords_regexp).source
148
+ end
149
+ def querying_stopwords= new_value
150
+ Tokenizers::Query.default.instance_variable_set(:@remove_stopwords_regexp, %r{#{new_value}})
151
+ end
152
+ def querying_splits_text_on
153
+ Tokenizers::Query.default.instance_variable_get(:@splits_text_on_regexp).source
154
+ end
155
+ def querying_splits_text_on= new_value
156
+ Tokenizers::Query.default.instance_variable_set(:@splits_text_on_regexp, %r{#{new_value}})
157
+ end
158
+
159
+ end
160
+
161
+ # Aka.
162
+ #
163
+ ::LiveParameters = LiveParameters
164
+
165
+ end
data/lib/picky/loader.rb CHANGED
@@ -68,7 +68,7 @@ module Loader # :nodoc:all
68
68
 
69
69
  # Finalize the applications.
70
70
  #
71
- # TODO Problem: Reload Routes.
71
+ # TODO Problem: Reload Routes. Throw them all away and do them again?
72
72
  #
73
73
  Application.finalize_apps
74
74
 
@@ -249,9 +249,20 @@ module Loader # :nodoc:all
249
249
  #
250
250
  load_relative 'configuration/index'
251
251
 
252
+ # Interfaces
253
+ #
254
+ load_relative 'interfaces/live_parameters'
255
+
256
+ # Adapters.
257
+ #
258
+ load_relative 'adapters/rack/base'
259
+ load_relative 'adapters/rack/query'
260
+ load_relative 'adapters/rack/live_parameters'
261
+ load_relative 'adapters/rack'
262
+
252
263
  # Application and routing.
253
264
  #
254
- load_relative 'routing'
265
+ load_relative 'frontend_adapters/rack'
255
266
  load_relative 'application'
256
267
 
257
268
  # Load tools.
@@ -23,7 +23,7 @@ module Query
23
23
 
24
24
  include Helpers::Measuring
25
25
 
26
- attr_writer :tokenizer
26
+ attr_writer :tokenizer, :identifiers_to_remove
27
27
  attr_accessor :reduce_to_amount, :weights
28
28
 
29
29
  # Takes:
@@ -43,13 +43,14 @@ module Query
43
43
  @weights = Hash === weights ? Weights.new(weights) : weights
44
44
  end
45
45
 
46
- # Search through this method.
46
+ # This is the main entry point for a query.
47
+ # Use this in specs and also for running queries.
47
48
  #
48
49
  # Parameters:
49
50
  # * text: The search text.
50
51
  # * offset = 0: _optional_ The offset from which position to return the ids. Useful for pagination.
51
52
  #
52
- # Note: The Routing uses this method after unravelling the HTTP request.
53
+ # Note: The Rack adapter calls this method after unravelling the HTTP request.
53
54
  #
54
55
  def search_with_text text, offset = 0
55
56
  search tokenized(text), offset
@@ -75,7 +76,7 @@ module Query
75
76
  # Note: Internal method, use #search_with_text.
76
77
  #
77
78
  def execute tokens, offset
78
- results_from offset, sorted_allocations(tokens)
79
+ result_type.from offset, sorted_allocations(tokens)
79
80
  end
80
81
 
81
82
  # Returns an empty result with default values.
@@ -140,28 +141,24 @@ module Query
140
141
  def reduce allocations # :nodoc:
141
142
  allocations.reduce_to reduce_to_amount if reduce_to_amount
142
143
  end
143
- def remove_identifiers? # :nodoc:
144
- identifiers_to_remove.present?
145
- end
144
+
145
+ #
146
+ #
146
147
  def remove_from allocations # :nodoc:
147
- allocations.remove(identifiers_to_remove) if remove_identifiers?
148
+ allocations.remove identifiers_to_remove
148
149
  end
149
- # Override. TODO No, redesign.
150
+ #
150
151
  #
151
152
  def identifiers_to_remove # :nodoc:
152
153
  @identifiers_to_remove ||= []
153
154
  end
154
155
 
155
- # Packs the sorted allocations into results.
156
- #
157
- # This generates the id intersections. Lots of work going on.
156
+ # Display some nice information for the user.
158
157
  #
159
- # TODO Move to results. result_type.from allocations, offset
160
- #
161
- def results_from offset = 0, allocations = nil # :nodoc:
162
- results = result_type.new offset, allocations
163
- results.prepare!
164
- results
158
+ def to_s
159
+ s = "#{self.class}"
160
+ s << ", weights: #{@weights}" unless @weights.empty?
161
+ s
165
162
  end
166
163
 
167
164
  end
@@ -26,7 +26,7 @@ module Query
26
26
 
27
27
  # Returns the weight of this combination.
28
28
  #
29
- # TODO Really cache?
29
+ # Note: Caching is most oft the time useful.
30
30
  #
31
31
  def weight
32
32
  @weight ||= @bundle.weight(@text)
@@ -34,7 +34,7 @@ module Query
34
34
 
35
35
  # Returns an array of ids for the given text.
36
36
  #
37
- # TODO Really cache?
37
+ # Note: Caching is most oft the time useful.
38
38
  #
39
39
  def ids
40
40
  @ids ||= @bundle.ids(@text)
@@ -13,21 +13,6 @@ module Query
13
13
  super *index_types
14
14
  end
15
15
 
16
- # # This runs the actual search.
17
- # #
18
- # # TODO Remove!
19
- # #
20
- # def search tokens, offset = 0
21
- # results = nil
22
- #
23
- # duration = timed do
24
- # results = execute(tokens, offset) || empty_results # TODO Does not work yet
25
- # end
26
- # results.duration = duration
27
- #
28
- # results
29
- # end
30
-
31
16
  #
32
17
  #
33
18
  def execute tokens, offset = 0
@@ -61,8 +46,6 @@ module Query
61
46
  results.add similar: similar
62
47
  end
63
48
 
64
- # TODO
65
- #
66
49
  class << results
67
50
  def to_log query
68
51
  ?* + super
@@ -69,8 +69,8 @@ module Query
69
69
 
70
70
  # If the text ends with *, partialize it. If with ", don't.
71
71
  #
72
- @@no_partial = /\"$/
73
- @@partial = /\*$/
72
+ @@no_partial = /\"\Z/
73
+ @@partial = /\*\Z/
74
74
  def partialize
75
75
  self.partial = false and return if @text =~ @@no_partial
76
76
  self.partial = true if @text =~ @@partial
@@ -78,8 +78,8 @@ module Query
78
78
 
79
79
  # If the text ends with ~ similarize it. If with ", don't.
80
80
  #
81
- @@no_similar = /\"$/
82
- @@similar = /\~$/
81
+ @@no_similar = /\"\Z/
82
+ @@similar = /\~\Z/
83
83
  def similarize
84
84
  self.similar = false and return if @text =~ @@no_similar
85
85
  self.similar = true if @text =~ @@similar
@@ -118,35 +118,19 @@ module Query
118
118
  def possible_combinations_in type
119
119
  type.possible_combinations self
120
120
  end
121
-
122
- #
121
+
122
+ # Returns a token with the next similar text.
123
123
  #
124
- def from token
125
- new_token = token.dup
126
- new_token.instance_variable_set :@text, @text
127
- new_token.instance_variable_set :@partial, @partial
128
- new_token.instance_variable_set :@original, @original
129
- new_token.instance_variable_set :@qualifier, @qualifier
130
- # TODO
131
- #
132
- # token.instance_variable_set :@similarity, @similarity
133
- new_token
134
- end
135
-
136
- # TODO Rewrite, also next_similar.
124
+ # TODO Rewrite this. It is hard to understand. Also spec performance.
137
125
  #
138
- def next category
139
- token = from self
126
+ def next_similar_token category
127
+ token = self.dup
140
128
  token if token.next_similar category.bundle_for(token)
141
129
  end
142
-
143
130
  # Sets and returns the next similar word.
144
131
  #
145
132
  def next_similar bundle
146
- @text = similarity(bundle).next if similar?
147
- rescue StopIteration => stop_iteration
148
- # reset_similar # TODO
149
- nil # TODO
133
+ @text = (similarity(bundle).shift || return) if similar?
150
134
  end
151
135
  # Lazy similar reader.
152
136
  #
@@ -155,8 +139,11 @@ module Query
155
139
  end
156
140
  # Returns an enumerator that traverses over the similar.
157
141
  #
142
+ # Note: The dup isn't too nice – since it is needed on account of the shift, above.
143
+ # (We avoid a StopIteration exception. Which of both is less evil?)
144
+ #
158
145
  def generate_similarity_for bundle
159
- (bundle.similar(@text) || []).each
146
+ bundle.similar(@text).dup || []
160
147
  end
161
148
 
162
149
  # Generates a solr term from this token.
@@ -7,7 +7,7 @@ module Query
7
7
  #
8
8
  #
9
9
  def initialize weights = {}
10
- @weights_cache = {}
10
+ # @weights_cache = {} # TODO
11
11
  @weights = prepare weights
12
12
  end
13
13
 
@@ -47,5 +47,17 @@ module Query
47
47
  weight_for combinations.map(&:category_name).clustered_uniq_fast
48
48
  end
49
49
 
50
+ # Are there any weights defined?
51
+ #
52
+ def empty?
53
+ @weights.empty?
54
+ end
55
+
56
+ # Prints out a nice representation of the configured weights.
57
+ #
58
+ def to_s
59
+ @weights.to_s
60
+ end
61
+
50
62
  end
51
63
  end
@@ -12,9 +12,16 @@ module Results # :nodoc:all
12
12
 
13
13
  # Takes instances of Query::Allocations as param.
14
14
  #
15
- def initialize offset = 0, allocations = nil
15
+ def initialize offset = 0, allocations = Query::Allocations.new
16
16
  @offset = offset
17
- @allocations = allocations || Query::Allocations.new
17
+ @allocations = allocations # || Query::Allocations.new
18
+ end
19
+ # Create new results and calculate the ids.
20
+ #
21
+ def self.from offset, allocations
22
+ results = new offset, allocations
23
+ results.prepare!
24
+ results
18
25
  end
19
26
 
20
27
  #
@@ -0,0 +1,24 @@
1
+ # encoding: utf-8
2
+ #
3
+ require 'spec_helper'
4
+
5
+ describe Adapters::Rack::Base do
6
+
7
+ before(:each) do
8
+ @adapter = Adapters::Rack::Base.new
9
+ end
10
+
11
+ describe 'respond_with' do
12
+ describe 'by default' do
13
+ it 'uses json' do
14
+ @adapter.respond_with('response').should ==
15
+ [200, { 'Content-Type' => 'application/json', 'Content-Length' => '8' }, ['response']]
16
+ end
17
+ end
18
+ it 'adapts the content length' do
19
+ @adapter.respond_with('123').should ==
20
+ [200, { 'Content-Type' => 'application/json', 'Content-Length' => '3' }, ['123']]
21
+ end
22
+ end
23
+
24
+ end
@@ -0,0 +1,21 @@
1
+ # encoding: utf-8
2
+ #
3
+ require 'spec_helper'
4
+
5
+ describe Adapters::Rack::LiveParameters do
6
+
7
+ before(:each) do
8
+ @live_parameters = stub :live_parameters
9
+ @adapter = Adapters::Rack::LiveParameters.new @live_parameters
10
+ end
11
+
12
+ describe 'to_app' do
13
+ it 'works' do
14
+ lambda { @adapter.to_app }.should_not raise_error
15
+ end
16
+ it 'returns the right thing' do
17
+ @adapter.to_app.should respond_to(:call)
18
+ end
19
+ end
20
+
21
+ end
@@ -0,0 +1,33 @@
1
+ # encoding: utf-8
2
+ #
3
+ require 'spec_helper'
4
+
5
+ describe Adapters::Rack::Query do
6
+
7
+ before(:each) do
8
+ @query = stub :query
9
+ @adapter = Adapters::Rack::Query.new @query
10
+ end
11
+
12
+ describe 'to_app' do
13
+ it 'works' do
14
+ lambda { @adapter.to_app }.should_not raise_error
15
+ end
16
+ it 'returns the right thing' do
17
+ @adapter.to_app.should respond_to(:call)
18
+ end
19
+ end
20
+
21
+ describe 'extracted' do
22
+ it 'extracts the query' do
23
+ @adapter.extracted('query' => 'some_query')[0].should == 'some_query'
24
+ end
25
+ it 'extracts the default offset' do
26
+ @adapter.extracted('query' => 'some_query')[1].should == 0
27
+ end
28
+ it 'extracts a given offset' do
29
+ @adapter.extracted('query' => 'some_query', 'offset' => '123')[1].should == 123
30
+ end
31
+ end
32
+
33
+ end
@@ -10,11 +10,12 @@ describe Application do
10
10
  class MinimalTestApplication < Application
11
11
  books = index :books, Sources::DB.new('SELECT id, title FROM books', :file => 'app/db.yml')
12
12
  books.define_category :title
13
-
14
13
 
15
14
  full = Query::Full.new books
16
15
  live = Query::Live.new books
17
16
 
17
+ rack_adapter.stub! :exclaim # Stopping it from exclaiming.
18
+
18
19
  route %r{^/books/full} => full
19
20
  route %r{^/books/live} => live
20
21
  end
@@ -58,6 +59,8 @@ describe Application do
58
59
  full = Query::Full.new books_index
59
60
  live = Query::Live.new books_index
60
61
 
62
+ rack_adapter.stub! :exclaim # Stopping it from exclaiming.
63
+
61
64
  route %r{^/books/full} => full
62
65
  route %r{^/books/live} => live
63
66
  end
@@ -65,30 +68,46 @@ describe Application do
65
68
  end
66
69
  end
67
70
 
71
+ describe 'finalize' do
72
+ before(:each) do
73
+ Application.stub! :check
74
+ end
75
+ it 'checks if all is ok' do
76
+ Application.should_receive(:check).once.with
77
+
78
+ Application.finalize
79
+ end
80
+ it 'tells the rack adapter to finalize' do
81
+ Application.rack_adapter.should_receive(:finalize).once.with
82
+
83
+ Application.finalize
84
+ end
85
+ end
86
+
68
87
  describe 'delegation' do
69
88
  it "should delegate route" do
70
- Application.routing.should_receive(:route).once.with :path => :query
89
+ Application.rack_adapter.should_receive(:route).once.with :path => :query
71
90
 
72
91
  Application.route :path => :query
73
92
  end
74
93
  end
75
94
 
76
- describe 'routing' do
95
+ describe 'rack_adapter' do
77
96
  it 'should be there' do
78
- lambda { Application.routing }.should_not raise_error
97
+ lambda { Application.rack_adapter }.should_not raise_error
79
98
  end
80
- it "should return a new Routing instance" do
81
- Application.routing.should be_kind_of(Routing)
99
+ it "should return a new FrontendAdapters::Rack instance" do
100
+ Application.rack_adapter.should be_kind_of(FrontendAdapters::Rack)
82
101
  end
83
102
  it "should cache the instance" do
84
- Application.routing.should == Application.routing
103
+ Application.rack_adapter.should == Application.rack_adapter
85
104
  end
86
105
  end
87
106
 
88
107
  describe 'call' do
89
108
  before(:each) do
90
109
  @routes = stub :routes
91
- Application.stub! :routing => @routes
110
+ Application.stub! :rack_adapter => @routes
92
111
  end
93
112
  it 'should delegate' do
94
113
  @routes.should_receive(:call).once.with :env
data/spec/lib/cli_spec.rb CHANGED
@@ -27,6 +27,15 @@ describe Picky::CLI do
27
27
  it 'returns Statistics for stats' do
28
28
  @cli.executor_class_for(:stats).should == [Picky::CLI::Statistics, "logfile, e.g. log/search.log", "port (optional)"]
29
29
  end
30
+ it 'returns Live for live' do
31
+ @cli.executor_class_for(:live).should == [Picky::CLI::Live, "host:port/path (optional, default: localhost:8080/admin)", "port (optional)"]
32
+ end
33
+ end
34
+ end
35
+
36
+ describe Picky::CLI::Live do
37
+ before(:each) do
38
+ @cli = Picky::CLI::Live.new
30
39
  end
31
40
  end
32
41
 
@@ -8,9 +8,7 @@ describe Symbol do
8
8
  @token = (((0..9).to_a)*10).to_s.to_sym
9
9
  end
10
10
  it "should be fast" do
11
- timed do
12
- @token.each_subtoken do |subtoken| end
13
- end.should < 0.0006
11
+ performance_of { @token.each_subtoken { |subtoken| } }.should < 0.00065
14
12
  end
15
13
  end
16
14