picky 1.5.4 → 2.0.0.pre1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/lib/picky/application.rb +7 -29
- data/lib/picky/internals/adapters/rack/query.rb +12 -8
- data/lib/picky/internals/query/indexes.rb +20 -20
- data/lib/picky/loader.rb +6 -10
- data/lib/picky/results.rb +93 -0
- data/lib/picky/search.rb +180 -0
- data/lib/picky/sources/base.rb +5 -3
- data/lib/tasks/todo.rake +8 -0
- data/spec/lib/application_spec.rb +2 -10
- data/spec/lib/character_substituters/west_european_spec.rb +14 -11
- data/spec/lib/internals/adapters/rack/query_spec.rb +8 -2
- data/spec/lib/internals/frontend_adapters/rack_spec.rb +13 -13
- data/spec/lib/{internals/results/base_spec.rb → results_spec.rb} +10 -9
- data/spec/lib/{query/base_spec.rb → search_spec.rb} +39 -39
- metadata +12 -23
- data/lib/picky/internals/results/base.rb +0 -99
- data/lib/picky/internals/results/full.rb +0 -17
- data/lib/picky/internals/results/live.rb +0 -17
- data/lib/picky/query/base.rb +0 -190
- data/lib/picky/query/full.rb +0 -20
- data/lib/picky/query/live.rb +0 -24
- data/spec/lib/internals/results/full_spec.rb +0 -78
- data/spec/lib/internals/results/live_spec.rb +0 -88
- data/spec/lib/query/full_spec.rb +0 -19
- data/spec/lib/query/live_spec.rb +0 -31
data/lib/picky/query/base.rb
DELETED
@@ -1,190 +0,0 @@
|
|
1
|
-
# = Picky Queries
|
2
|
-
#
|
3
|
-
# A Picky Query is an object which:
|
4
|
-
# * holds one or more indexes
|
5
|
-
# * offers an interface to query these indexes.
|
6
|
-
#
|
7
|
-
# You connect URL paths to indexes via a Query.
|
8
|
-
#
|
9
|
-
# We recommend not to use this directly, but connect it to an URL and query through one of these
|
10
|
-
# (Protip: Use "curl 'localhost:8080/query/path?query=exampletext')" in a Terminal.
|
11
|
-
#
|
12
|
-
# There are two flavors of queries:
|
13
|
-
# * Query::Full (Full results with all infos)
|
14
|
-
# * Query::Live (Same as the Full results without result ids. Useful for query result counters.)
|
15
|
-
#
|
16
|
-
module Query
|
17
|
-
|
18
|
-
# The base query class.
|
19
|
-
#
|
20
|
-
# Not directly instantiated. However, its methods are used by its subclasses, Full and Live.
|
21
|
-
#
|
22
|
-
class Base
|
23
|
-
|
24
|
-
include Helpers::Measuring
|
25
|
-
|
26
|
-
attr_reader :indexes
|
27
|
-
attr_writer :tokenizer, :identifiers_to_remove
|
28
|
-
attr_accessor :reduce_to_amount, :weights
|
29
|
-
|
30
|
-
# Takes:
|
31
|
-
# * A number of indexes
|
32
|
-
# * Options hash (optional) with:
|
33
|
-
# * tokenizer: Tokenizers::Query.default by default.
|
34
|
-
# * weights: A hash of weights, or a Query::Weights object.
|
35
|
-
#
|
36
|
-
def initialize *index_definitions
|
37
|
-
options = Hash === index_definitions.last ? index_definitions.pop : {}
|
38
|
-
|
39
|
-
@indexes = Internals::Query::Indexes.new *index_definitions, combinations_type_for(index_definitions)
|
40
|
-
@tokenizer = options[:tokenizer] || Internals::Tokenizers::Query.default
|
41
|
-
weights = options[:weights] || Weights.new
|
42
|
-
@weights = Hash === weights ? Weights.new(weights) : weights
|
43
|
-
end
|
44
|
-
|
45
|
-
# Returns the right combinations strategy for
|
46
|
-
# a number of query indexes.
|
47
|
-
#
|
48
|
-
# Currently it isn't possible using Memory and Redis etc.
|
49
|
-
# indexes in the same query index group.
|
50
|
-
#
|
51
|
-
# Picky will raise a Query::Indexes::DifferentTypesError.
|
52
|
-
#
|
53
|
-
@@mapping = {
|
54
|
-
Index::Memory => Internals::Query::Combinations::Memory,
|
55
|
-
Index::Redis => Internals::Query::Combinations::Redis
|
56
|
-
}
|
57
|
-
def combinations_type_for index_definitions_ary
|
58
|
-
index_types = index_definitions_ary.map(&:class)
|
59
|
-
index_types.uniq!
|
60
|
-
raise_different(index_types) if index_types.size > 1
|
61
|
-
!index_types.empty? && @@mapping[*index_types] || Internals::Query::Combinations::Memory
|
62
|
-
end
|
63
|
-
# Currently it isn't possible using Memory and Redis etc.
|
64
|
-
# indexes in the same query index group.
|
65
|
-
#
|
66
|
-
class DifferentTypesError < StandardError
|
67
|
-
def initialize types
|
68
|
-
@types = types
|
69
|
-
end
|
70
|
-
def to_s
|
71
|
-
"Currently it isn't possible to mix #{@types.join(" and ")} Indexes in the same Query."
|
72
|
-
end
|
73
|
-
end
|
74
|
-
def raise_different index_types
|
75
|
-
raise DifferentTypesError.new(index_types)
|
76
|
-
end
|
77
|
-
|
78
|
-
# This is the main entry point for a query.
|
79
|
-
# Use this in specs and also for running queries.
|
80
|
-
#
|
81
|
-
# Parameters:
|
82
|
-
# * text: The search text.
|
83
|
-
# * offset = 0: _optional_ The offset from which position to return the ids. Useful for pagination.
|
84
|
-
#
|
85
|
-
# Note: The Rack adapter calls this method after unravelling the HTTP request.
|
86
|
-
#
|
87
|
-
def search_with_text text, offset = 0
|
88
|
-
search tokenized(text), offset
|
89
|
-
end
|
90
|
-
|
91
|
-
# Runs the actual search using Query::Tokens.
|
92
|
-
#
|
93
|
-
# Note: Internal method, use #search_with_text.
|
94
|
-
#
|
95
|
-
def search tokens, offset = 0
|
96
|
-
results = nil
|
97
|
-
|
98
|
-
duration = timed do
|
99
|
-
results = execute tokens, offset
|
100
|
-
end
|
101
|
-
results.duration = duration.round 6
|
102
|
-
|
103
|
-
results
|
104
|
-
end
|
105
|
-
|
106
|
-
# Execute a search using Query::Tokens.
|
107
|
-
#
|
108
|
-
# Note: Internal method, use #search_with_text.
|
109
|
-
#
|
110
|
-
def execute tokens, offset
|
111
|
-
result_type.from offset, sorted_allocations(tokens)
|
112
|
-
end
|
113
|
-
|
114
|
-
# Delegates the tokenizing to the query tokenizer.
|
115
|
-
#
|
116
|
-
# Parameters:
|
117
|
-
# * text: The text to tokenize.
|
118
|
-
#
|
119
|
-
def tokenized text
|
120
|
-
@tokenizer.tokenize text
|
121
|
-
end
|
122
|
-
|
123
|
-
# Gets sorted allocations for the tokens.
|
124
|
-
#
|
125
|
-
# This generates the possible allocations, sorted.
|
126
|
-
#
|
127
|
-
# TODO Smallify.
|
128
|
-
#
|
129
|
-
# TODO Rename: allocations
|
130
|
-
#
|
131
|
-
def sorted_allocations tokens # :nodoc:
|
132
|
-
# Get the allocations.
|
133
|
-
#
|
134
|
-
# TODO Pass in reduce_to_amount (aka max_allocations)
|
135
|
-
#
|
136
|
-
# TODO uniq, score, sort in there
|
137
|
-
#
|
138
|
-
allocations = @indexes.allocations_for tokens
|
139
|
-
|
140
|
-
# Callbacks.
|
141
|
-
#
|
142
|
-
# TODO Reduce before sort?
|
143
|
-
#
|
144
|
-
reduce allocations
|
145
|
-
remove_from allocations
|
146
|
-
|
147
|
-
# Remove double allocations.
|
148
|
-
#
|
149
|
-
allocations.uniq
|
150
|
-
|
151
|
-
# Score the allocations using weights as bias.
|
152
|
-
#
|
153
|
-
allocations.calculate_score weights
|
154
|
-
|
155
|
-
# Sort the allocations.
|
156
|
-
# (allocations are sorted according to score, highest to lowest)
|
157
|
-
#
|
158
|
-
allocations.sort
|
159
|
-
|
160
|
-
# Return the allocations.
|
161
|
-
#
|
162
|
-
allocations
|
163
|
-
end
|
164
|
-
def reduce allocations # :nodoc:
|
165
|
-
allocations.reduce_to reduce_to_amount if reduce_to_amount
|
166
|
-
end
|
167
|
-
|
168
|
-
#
|
169
|
-
#
|
170
|
-
def remove_from allocations # :nodoc:
|
171
|
-
allocations.remove identifiers_to_remove
|
172
|
-
end
|
173
|
-
#
|
174
|
-
#
|
175
|
-
def identifiers_to_remove # :nodoc:
|
176
|
-
@identifiers_to_remove ||= []
|
177
|
-
end
|
178
|
-
|
179
|
-
# Display some nice information for the user.
|
180
|
-
#
|
181
|
-
def to_s
|
182
|
-
s = "#{self.class}("
|
183
|
-
s << @indexes.indexes.map(&:name).join(', ')
|
184
|
-
s << ", weights: #{@weights}" unless @weights.empty?
|
185
|
-
s << ")"
|
186
|
-
s
|
187
|
-
end
|
188
|
-
|
189
|
-
end
|
190
|
-
end
|
data/lib/picky/query/full.rb
DELETED
@@ -1,20 +0,0 @@
|
|
1
|
-
module Query
|
2
|
-
|
3
|
-
# This Query class performs full queries.
|
4
|
-
#
|
5
|
-
# It includes in its results:
|
6
|
-
# * A count of results.
|
7
|
-
# * All possible combinations with its weights.
|
8
|
-
# * The top X result ids.
|
9
|
-
#
|
10
|
-
class Full < Base
|
11
|
-
|
12
|
-
# Returns Results::Full as its result type.
|
13
|
-
#
|
14
|
-
def result_type
|
15
|
-
Internals::Results::Full
|
16
|
-
end
|
17
|
-
|
18
|
-
end
|
19
|
-
|
20
|
-
end
|
data/lib/picky/query/live.rb
DELETED
@@ -1,24 +0,0 @@
|
|
1
|
-
module Query
|
2
|
-
|
3
|
-
# This Query class performs live queries.
|
4
|
-
#
|
5
|
-
# It is useful for updating counters, or any job where you don't need the result ids.
|
6
|
-
#
|
7
|
-
# It includes in its results:
|
8
|
-
# * A count of results.
|
9
|
-
# * All possible combinations with its weights.
|
10
|
-
#
|
11
|
-
# But not:
|
12
|
-
# * The top X result ids.
|
13
|
-
#
|
14
|
-
class Live < Base
|
15
|
-
|
16
|
-
# Returns Results::Live as its result type.
|
17
|
-
#
|
18
|
-
def result_type
|
19
|
-
Internals::Results::Live
|
20
|
-
end
|
21
|
-
|
22
|
-
end
|
23
|
-
|
24
|
-
end
|
@@ -1,78 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
describe Internals::Results::Full do
|
4
|
-
|
5
|
-
describe "class accessors" do
|
6
|
-
it "should have accessors for max_results" do
|
7
|
-
max_results = stub :max_results
|
8
|
-
old_results = described_class.max_results
|
9
|
-
described_class.max_results = max_results
|
10
|
-
|
11
|
-
described_class.max_results.should == max_results
|
12
|
-
|
13
|
-
described_class.max_results = old_results
|
14
|
-
end
|
15
|
-
end
|
16
|
-
|
17
|
-
describe 'initialize' do
|
18
|
-
context 'without allocations' do
|
19
|
-
it 'should not fail' do
|
20
|
-
lambda {
|
21
|
-
described_class.new
|
22
|
-
}.should_not raise_error
|
23
|
-
end
|
24
|
-
it 'should set the allocations to allocations that are empty' do
|
25
|
-
described_class.new.instance_variable_get(:@allocations).should be_empty
|
26
|
-
end
|
27
|
-
end
|
28
|
-
context 'with allocations' do
|
29
|
-
it 'should not fail' do
|
30
|
-
lambda {
|
31
|
-
described_class.new :some_allocations
|
32
|
-
}.should_not raise_error
|
33
|
-
end
|
34
|
-
it 'should set the allocations to an empty array' do
|
35
|
-
described_class.new(:unimportant, :some_allocations).instance_variable_get(:@allocations).should == :some_allocations
|
36
|
-
end
|
37
|
-
end
|
38
|
-
end
|
39
|
-
|
40
|
-
describe 'duration' do
|
41
|
-
before(:each) do
|
42
|
-
@results = described_class.new
|
43
|
-
end
|
44
|
-
it 'should return the set duration' do
|
45
|
-
@results.duration = :some_duration
|
46
|
-
|
47
|
-
@results.duration.should == :some_duration
|
48
|
-
end
|
49
|
-
it 'should return 0 as a default' do
|
50
|
-
@results.duration.should == 0
|
51
|
-
end
|
52
|
-
end
|
53
|
-
|
54
|
-
describe 'total' do
|
55
|
-
it 'should delegate to allocations.total' do
|
56
|
-
allocations = stub :allocations
|
57
|
-
results = described_class.new nil, allocations
|
58
|
-
|
59
|
-
allocations.should_receive(:total).once
|
60
|
-
|
61
|
-
results.total
|
62
|
-
end
|
63
|
-
end
|
64
|
-
|
65
|
-
describe 'prepare!' do
|
66
|
-
before(:each) do
|
67
|
-
@results = described_class.new
|
68
|
-
@allocations = stub :allocations
|
69
|
-
@results.stub! :allocations => @allocations
|
70
|
-
end
|
71
|
-
it 'should process' do
|
72
|
-
@allocations.should_receive(:process!).once.with(20, 0).ordered
|
73
|
-
|
74
|
-
@results.prepare!
|
75
|
-
end
|
76
|
-
end
|
77
|
-
|
78
|
-
end
|
@@ -1,88 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
describe Internals::Results::Live do
|
4
|
-
|
5
|
-
it "is the right subclass" do
|
6
|
-
described_class.should < Internals::Results::Base
|
7
|
-
end
|
8
|
-
|
9
|
-
it "logs correctly" do
|
10
|
-
Time.stub! :now => Time.parse('2010-10-25 01:25:07')
|
11
|
-
|
12
|
-
described_class.new.to_log('some query').should == '.|2010-10-25 01:25:07|0.000000|some query | 0| 0| 0|'
|
13
|
-
end
|
14
|
-
|
15
|
-
describe "class accessors" do
|
16
|
-
it "should have accessors for max_results" do
|
17
|
-
max_results = stub :max_results
|
18
|
-
old_results = described_class.max_results
|
19
|
-
described_class.max_results = max_results
|
20
|
-
|
21
|
-
described_class.max_results.should == max_results
|
22
|
-
|
23
|
-
described_class.max_results = old_results
|
24
|
-
end
|
25
|
-
end
|
26
|
-
|
27
|
-
describe 'initialize' do
|
28
|
-
context 'without allocations' do
|
29
|
-
it 'should not fail' do
|
30
|
-
lambda {
|
31
|
-
described_class.new
|
32
|
-
}.should_not raise_error
|
33
|
-
end
|
34
|
-
it 'should set the allocations to allocations that are empty' do
|
35
|
-
described_class.new.instance_variable_get(:@allocations).should be_empty
|
36
|
-
end
|
37
|
-
end
|
38
|
-
context 'with allocations' do
|
39
|
-
it 'should not fail' do
|
40
|
-
lambda {
|
41
|
-
described_class.new :some_allocations
|
42
|
-
}.should_not raise_error
|
43
|
-
end
|
44
|
-
it 'should set the allocations to an empty array' do
|
45
|
-
described_class.new(:unimportant, :some_allocations).instance_variable_get(:@allocations).should == :some_allocations
|
46
|
-
end
|
47
|
-
end
|
48
|
-
end
|
49
|
-
|
50
|
-
describe 'duration' do
|
51
|
-
before(:each) do
|
52
|
-
@results = described_class.new
|
53
|
-
end
|
54
|
-
it 'should return the set duration' do
|
55
|
-
@results.duration = :some_duration
|
56
|
-
|
57
|
-
@results.duration.should == :some_duration
|
58
|
-
end
|
59
|
-
it 'should return 0 as a default' do
|
60
|
-
@results.duration.should == 0
|
61
|
-
end
|
62
|
-
end
|
63
|
-
|
64
|
-
describe 'total' do
|
65
|
-
it 'should delegate to allocations.total' do
|
66
|
-
allocations = stub :allocations
|
67
|
-
results = described_class.new nil, allocations
|
68
|
-
|
69
|
-
allocations.should_receive(:total).once
|
70
|
-
|
71
|
-
results.total
|
72
|
-
end
|
73
|
-
end
|
74
|
-
|
75
|
-
describe 'prepare!' do
|
76
|
-
before(:each) do
|
77
|
-
@results = described_class.new
|
78
|
-
@allocations = stub :allocations
|
79
|
-
@results.stub! :allocations => @allocations
|
80
|
-
end
|
81
|
-
it 'should generate truncate the ids fully' do
|
82
|
-
@allocations.should_receive(:process!).once.with(0, 0).ordered
|
83
|
-
|
84
|
-
@results.prepare!
|
85
|
-
end
|
86
|
-
end
|
87
|
-
|
88
|
-
end
|
data/spec/lib/query/full_spec.rb
DELETED
@@ -1,19 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
describe Query::Full do
|
4
|
-
|
5
|
-
before(:each) do
|
6
|
-
@type = stub :type
|
7
|
-
@index = stub :index, :indexed => @type
|
8
|
-
end
|
9
|
-
|
10
|
-
describe 'result_type' do
|
11
|
-
before(:each) do
|
12
|
-
@query = described_class.new @index
|
13
|
-
end
|
14
|
-
it "should return a specific type" do
|
15
|
-
@query.result_type.should == Internals::Results::Full
|
16
|
-
end
|
17
|
-
end
|
18
|
-
|
19
|
-
end
|
data/spec/lib/query/live_spec.rb
DELETED
@@ -1,31 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
describe Query::Live do
|
4
|
-
|
5
|
-
before(:each) do
|
6
|
-
@indexed = stub :indexed
|
7
|
-
@index = stub :index, :indexed => @indexed
|
8
|
-
end
|
9
|
-
|
10
|
-
describe 'result_type' do
|
11
|
-
before(:each) do
|
12
|
-
@query = described_class.new @index
|
13
|
-
end
|
14
|
-
it "should return a specific type" do
|
15
|
-
@query.result_type.should == Internals::Results::Live
|
16
|
-
end
|
17
|
-
end
|
18
|
-
|
19
|
-
describe "execute" do
|
20
|
-
before(:each) do
|
21
|
-
@query = described_class.new @index
|
22
|
-
end
|
23
|
-
it "should get allocations" do
|
24
|
-
@query.result_type.should_receive :from
|
25
|
-
@query.should_receive(:sorted_allocations).once
|
26
|
-
|
27
|
-
@query.execute 'some_query', 0
|
28
|
-
end
|
29
|
-
end
|
30
|
-
|
31
|
-
end
|