thinking-sphinx 1.4.7 → 1.4.8
Sign up to get free protection for your applications and to get access to all the features.
- data/README.textile +8 -3
- data/features/attribute_transformation.feature +4 -4
- data/features/step_definitions/common_steps.rb +6 -2
- data/lib/thinking_sphinx/active_record.rb +1 -1
- data/lib/thinking_sphinx/adapters/postgresql_adapter.rb +10 -2
- data/lib/thinking_sphinx/attribute.rb +0 -4
- data/lib/thinking_sphinx/configuration.rb +77 -61
- data/lib/thinking_sphinx/context.rb +14 -18
- data/lib/thinking_sphinx/index/builder.rb +64 -69
- data/lib/thinking_sphinx/search.rb +6 -2
- data/lib/thinking_sphinx/tasks.rb +15 -5
- data/lib/thinking_sphinx/version.rb +1 -1
- data/spec/thinking_sphinx/configuration_spec.rb +70 -32
- data/spec/thinking_sphinx/context_spec.rb +7 -5
- metadata +103 -98
data/README.textile
CHANGED
@@ -15,11 +15,11 @@ To quickly see if your system is ready to run the thinking sphinx specs, run the
|
|
15
15
|
To get the spec suite running, you will need to install the ginger gem:
|
16
16
|
|
17
17
|
<pre><code>sudo gem install ginger --source http://gemcutter.org</code></pre>
|
18
|
-
|
18
|
+
|
19
19
|
Then install the cucumber, yard, jeweler and rspec gems. Make sure you have a git install version 1.6.0.0 or higher, otherwise the jeweler gem won't install. Bluecloth is required for some of the yard documentation.
|
20
20
|
|
21
21
|
<pre>
|
22
|
-
sudo gem install bluecloth cucumber yard jeweler rspec
|
22
|
+
sudo gem install bluecloth cucumber yard jeweler rspec
|
23
23
|
</pre>
|
24
24
|
|
25
25
|
Then set up your database:
|
@@ -28,7 +28,7 @@ Then set up your database:
|
|
28
28
|
cp spec/fixtures/database.yml.default spec/fixtures/database.yml &&
|
29
29
|
mysqladmin -u root create thinking_sphinx
|
30
30
|
</pre>
|
31
|
-
|
31
|
+
|
32
32
|
This last step can be done automatically by the contribute.rb script if all dependencies are met.
|
33
33
|
|
34
34
|
Make sure you don't have another Sphinx daemon (searchd) running. If you do, quit it with "rake ts:stop"
|
@@ -197,3 +197,8 @@ Since I first released this library, there's been quite a few people who have su
|
|
197
197
|
* Rémi Prévost
|
198
198
|
* Justin Tanner
|
199
199
|
* Josh Goebel
|
200
|
+
* Ngan Pham
|
201
|
+
* Martin Gordon
|
202
|
+
* Aaron Gibralter
|
203
|
+
* Pete Deffendol
|
204
|
+
* Tony Pitale
|
@@ -2,19 +2,19 @@ Feature: Handle not-quite-supported column types as attributes
|
|
2
2
|
In order for Thinking Sphinx to be more understanding with model structures
|
3
3
|
The plugin
|
4
4
|
Should be able to use translatable columns as attributes
|
5
|
-
|
5
|
+
|
6
6
|
Scenario: Decimals as floats
|
7
7
|
Given Sphinx is running
|
8
8
|
And I am searching on alphas
|
9
9
|
When I filter between 1.0 and 3.0 on cost
|
10
10
|
Then I should get 2 results
|
11
|
-
|
11
|
+
|
12
12
|
Scenario: Dates as Datetimes
|
13
13
|
Given Sphinx is running
|
14
14
|
And I am searching on alphas
|
15
|
-
When I filter between 1 and 3 days ago on created_on
|
15
|
+
When I filter between 1 and 3 days ago on created_on by date
|
16
16
|
Then I should get 2 results
|
17
|
-
|
17
|
+
|
18
18
|
Scenario: Timestamps as Datetimes
|
19
19
|
Given Sphinx is running
|
20
20
|
And I am searching on alphas
|
@@ -95,9 +95,13 @@ When /^I filter between ([\d\.]+) and ([\d\.]+) on (\w+)$/ do |first, last, attr
|
|
95
95
|
end
|
96
96
|
end
|
97
97
|
|
98
|
-
When /^I filter between (\d+) and (\d+) days ago on (\w+)
|
98
|
+
When /^I filter between (\d+) and (\d+) days ago on (\w+)(?: by (date))?$/ do |last, first, attribute, by_date|
|
99
99
|
@results = nil
|
100
|
-
|
100
|
+
|
101
|
+
first, last = first.to_i.days.ago, last.to_i.days.ago
|
102
|
+
first, last = first.to_date, last.to_date if by_date
|
103
|
+
|
104
|
+
@with[attribute.to_sym] = first..last
|
101
105
|
end
|
102
106
|
|
103
107
|
When /^I filter by (\w+) between (\d+) and (\d+)$/ do |attribute, first, last|
|
@@ -20,7 +20,11 @@ module ThinkingSphinx
|
|
20
20
|
end
|
21
21
|
|
22
22
|
def group_concatenate(clause, separator = ' ')
|
23
|
-
|
23
|
+
if connection.raw_connection.server_version >= 80400
|
24
|
+
"array_to_string(array_agg(COALESCE(#{clause}, '0')), '#{separator}')"
|
25
|
+
else
|
26
|
+
"array_to_string(array_accum(COALESCE(#{clause}, '0')), '#{separator}')"
|
27
|
+
end
|
24
28
|
end
|
25
29
|
|
26
30
|
def cast_to_string(clause)
|
@@ -105,7 +109,11 @@ module ThinkingSphinx
|
|
105
109
|
end
|
106
110
|
|
107
111
|
def create_array_accum_function
|
108
|
-
if connection.raw_connection.respond_to?(:server_version) &&
|
112
|
+
if connection.raw_connection.respond_to?(:server_version) &&
|
113
|
+
connection.raw_connection.server_version >= 80400
|
114
|
+
return
|
115
|
+
elsif connection.raw_connection.respond_to?(:server_version) &&
|
116
|
+
connection.raw_connection.server_version > 80200
|
109
117
|
execute <<-SQL
|
110
118
|
CREATE AGGREGATE array_accum (anyelement)
|
111
119
|
(
|
@@ -22,10 +22,6 @@ module ThinkingSphinx
|
|
22
22
|
:wordcount => :sql_attr_str2wordcount
|
23
23
|
}
|
24
24
|
|
25
|
-
if Riddle.loaded_version.to_i > 1
|
26
|
-
SphinxTypeMappings[:string] = :sql_attr_string
|
27
|
-
end
|
28
|
-
|
29
25
|
# To create a new attribute, you'll need to pass in either a single Column
|
30
26
|
# or an array of them, and some (optional) options.
|
31
27
|
#
|
@@ -4,7 +4,7 @@ require 'singleton'
|
|
4
4
|
module ThinkingSphinx
|
5
5
|
# This class both keeps track of the configuration settings for Sphinx and
|
6
6
|
# also generates the resulting file for Sphinx to use.
|
7
|
-
#
|
7
|
+
#
|
8
8
|
# Here are the default settings, relative to RAILS_ROOT where relevant:
|
9
9
|
#
|
10
10
|
# config file:: config/#{environment}.sphinx.conf
|
@@ -39,17 +39,17 @@ module ThinkingSphinx
|
|
39
39
|
# searchd_binary_name, indexer_binary_name.
|
40
40
|
#
|
41
41
|
# I think you've got the idea.
|
42
|
-
#
|
42
|
+
#
|
43
43
|
# Each setting in the YAML file is optional - so only put in the ones you
|
44
44
|
# want to change.
|
45
45
|
#
|
46
46
|
# Keep in mind, if for some particular reason you're using a version of
|
47
47
|
# Sphinx older than 0.9.8 r871 (that's prior to the proper 0.9.8 release),
|
48
48
|
# don't set allow_star to true.
|
49
|
-
#
|
49
|
+
#
|
50
50
|
class Configuration
|
51
51
|
include Singleton
|
52
|
-
|
52
|
+
|
53
53
|
SourceOptions = Riddle::Configuration::SQLSource.settings.map { |setting|
|
54
54
|
setting.to_s
|
55
55
|
} - %w( type sql_query_pre sql_query sql_joined_field sql_file_field
|
@@ -61,29 +61,29 @@ module ThinkingSphinx
|
|
61
61
|
setting.to_s
|
62
62
|
} - %w( source prefix_fields infix_fields )
|
63
63
|
CustomOptions = %w( disable_range use_64_bit )
|
64
|
-
|
64
|
+
|
65
65
|
attr_accessor :searchd_file_path, :allow_star, :app_root,
|
66
66
|
:model_directories, :delayed_job_priority, :indexed_models, :use_64_bit,
|
67
|
-
:touched_reindex_file, :stop_timeout, :version
|
68
|
-
|
67
|
+
:touched_reindex_file, :stop_timeout, :version, :shuffle
|
68
|
+
|
69
69
|
attr_accessor :source_options, :index_options
|
70
|
-
|
70
|
+
|
71
71
|
attr_reader :configuration, :controller
|
72
|
-
|
72
|
+
|
73
73
|
@@environment = nil
|
74
|
-
|
74
|
+
|
75
75
|
# Load in the configuration settings - this will look for config/sphinx.yml
|
76
76
|
# and parse it according to the current environment.
|
77
|
-
#
|
77
|
+
#
|
78
78
|
def initialize(app_root = Dir.pwd)
|
79
79
|
self.reset
|
80
80
|
end
|
81
|
-
|
81
|
+
|
82
82
|
def self.configure(&block)
|
83
83
|
yield instance
|
84
84
|
instance.reset(instance.app_root)
|
85
85
|
end
|
86
|
-
|
86
|
+
|
87
87
|
def reset(custom_app_root=nil)
|
88
88
|
if custom_app_root
|
89
89
|
self.app_root = custom_app_root
|
@@ -93,15 +93,15 @@ module ThinkingSphinx
|
|
93
93
|
self.app_root = RAILS_ROOT if defined?(RAILS_ROOT)
|
94
94
|
self.app_root ||= app_root
|
95
95
|
end
|
96
|
-
|
96
|
+
|
97
97
|
@configuration = Riddle::Configuration.new
|
98
98
|
@configuration.searchd.pid_file = "#{self.app_root}/log/searchd.#{environment}.pid"
|
99
99
|
@configuration.searchd.log = "#{self.app_root}/log/searchd.log"
|
100
100
|
@configuration.searchd.query_log = "#{self.app_root}/log/searchd.query.log"
|
101
|
-
|
101
|
+
|
102
102
|
@controller = Riddle::Controller.new @configuration,
|
103
103
|
"#{self.app_root}/config/#{environment}.sphinx.conf"
|
104
|
-
|
104
|
+
|
105
105
|
self.address = "127.0.0.1"
|
106
106
|
self.port = 9312
|
107
107
|
self.searchd_file_path = "#{self.app_root}/db/sphinx/#{environment}"
|
@@ -111,19 +111,24 @@ module ThinkingSphinx
|
|
111
111
|
Dir.glob("#{app_root}/vendor/plugins/*/app/models/")
|
112
112
|
self.delayed_job_priority = 0
|
113
113
|
self.indexed_models = []
|
114
|
-
|
114
|
+
self.shuffle = true
|
115
|
+
|
115
116
|
self.source_options = {}
|
116
117
|
self.index_options = {
|
117
118
|
:charset_type => "utf-8"
|
118
119
|
}
|
119
|
-
|
120
|
+
|
120
121
|
self.version = nil
|
121
122
|
parse_config
|
122
123
|
self.version ||= @controller.sphinx_version
|
123
|
-
|
124
|
+
|
125
|
+
ThinkingSphinx::Attribute::SphinxTypeMappings.merge!(
|
126
|
+
:string => :sql_attr_string
|
127
|
+
) if Riddle.loaded_version.to_i > 1
|
128
|
+
|
124
129
|
self
|
125
130
|
end
|
126
|
-
|
131
|
+
|
127
132
|
def self.environment
|
128
133
|
@@environment ||= if defined?(Merb)
|
129
134
|
Merb.environment
|
@@ -135,127 +140,127 @@ module ThinkingSphinx
|
|
135
140
|
ENV['RAILS_ENV'] || 'development'
|
136
141
|
end
|
137
142
|
end
|
138
|
-
|
143
|
+
|
139
144
|
def self.reset_environment
|
140
145
|
ThinkingSphinx.mutex.synchronize do
|
141
146
|
@@environment = nil
|
142
147
|
end
|
143
148
|
end
|
144
|
-
|
149
|
+
|
145
150
|
def environment
|
146
151
|
self.class.environment
|
147
152
|
end
|
148
|
-
|
153
|
+
|
149
154
|
def generate
|
150
155
|
@configuration.indexes.clear
|
151
|
-
|
156
|
+
|
152
157
|
ThinkingSphinx.context.indexed_models.each do |model|
|
153
158
|
model = model.constantize
|
154
159
|
model.define_indexes
|
155
160
|
@configuration.indexes.concat model.to_riddle
|
156
|
-
|
161
|
+
|
157
162
|
enforce_common_attribute_types
|
158
163
|
end
|
159
164
|
end
|
160
|
-
|
165
|
+
|
161
166
|
# Generate the config file for Sphinx by using all the settings defined and
|
162
167
|
# looping through all the models with indexes to build the relevant
|
163
168
|
# indexer and searchd configuration, and sources and indexes details.
|
164
169
|
#
|
165
170
|
def build(file_path=nil)
|
166
171
|
file_path ||= "#{self.config_file}"
|
167
|
-
|
172
|
+
|
168
173
|
generate
|
169
|
-
|
174
|
+
|
170
175
|
open(file_path, "w") do |file|
|
171
176
|
file.write @configuration.render
|
172
177
|
end
|
173
178
|
end
|
174
|
-
|
179
|
+
|
175
180
|
def address
|
176
181
|
@address
|
177
182
|
end
|
178
|
-
|
183
|
+
|
179
184
|
def address=(address)
|
180
185
|
@address = address
|
181
186
|
@configuration.searchd.address = address
|
182
187
|
end
|
183
|
-
|
188
|
+
|
184
189
|
def port
|
185
190
|
@port
|
186
191
|
end
|
187
|
-
|
192
|
+
|
188
193
|
def port=(port)
|
189
194
|
@port = port
|
190
195
|
@configuration.searchd.port = port
|
191
196
|
end
|
192
|
-
|
197
|
+
|
193
198
|
def pid_file
|
194
199
|
@configuration.searchd.pid_file
|
195
200
|
end
|
196
|
-
|
201
|
+
|
197
202
|
def pid_file=(pid_file)
|
198
203
|
@configuration.searchd.pid_file = pid_file
|
199
204
|
end
|
200
|
-
|
205
|
+
|
201
206
|
def searchd_log_file
|
202
207
|
@configuration.searchd.log
|
203
208
|
end
|
204
|
-
|
209
|
+
|
205
210
|
def searchd_log_file=(file)
|
206
211
|
@configuration.searchd.log = file
|
207
212
|
end
|
208
|
-
|
213
|
+
|
209
214
|
def query_log_file
|
210
215
|
@configuration.searchd.query_log
|
211
216
|
end
|
212
|
-
|
217
|
+
|
213
218
|
def query_log_file=(file)
|
214
219
|
@configuration.searchd.query_log = file
|
215
220
|
end
|
216
|
-
|
221
|
+
|
217
222
|
def config_file
|
218
223
|
@controller.path
|
219
224
|
end
|
220
|
-
|
225
|
+
|
221
226
|
def config_file=(file)
|
222
227
|
@controller.path = file
|
223
228
|
end
|
224
|
-
|
229
|
+
|
225
230
|
def bin_path
|
226
231
|
@controller.bin_path
|
227
232
|
end
|
228
|
-
|
233
|
+
|
229
234
|
def bin_path=(path)
|
230
235
|
@controller.bin_path = path
|
231
236
|
end
|
232
|
-
|
237
|
+
|
233
238
|
def searchd_binary_name
|
234
239
|
@controller.searchd_binary_name
|
235
240
|
end
|
236
|
-
|
241
|
+
|
237
242
|
def searchd_binary_name=(name)
|
238
243
|
@controller.searchd_binary_name = name
|
239
244
|
end
|
240
|
-
|
245
|
+
|
241
246
|
def indexer_binary_name
|
242
247
|
@controller.indexer_binary_name
|
243
248
|
end
|
244
|
-
|
249
|
+
|
245
250
|
def indexer_binary_name=(name)
|
246
251
|
@controller.indexer_binary_name = name
|
247
252
|
end
|
248
|
-
|
253
|
+
|
249
254
|
attr_accessor :timeout
|
250
255
|
|
251
256
|
def client
|
252
|
-
client = Riddle::Client.new
|
257
|
+
client = Riddle::Client.new shuffled_addresses, port,
|
253
258
|
configuration.searchd.client_key
|
254
259
|
client.max_matches = configuration.searchd.max_matches || 1000
|
255
260
|
client.timeout = timeout || 0
|
256
261
|
client
|
257
262
|
end
|
258
|
-
|
263
|
+
|
259
264
|
def models_by_crc
|
260
265
|
@models_by_crc ||= begin
|
261
266
|
ThinkingSphinx.context.indexed_models.inject({}) do |hash, model|
|
@@ -274,34 +279,34 @@ module ThinkingSphinx
|
|
274
279
|
end
|
275
280
|
|
276
281
|
private
|
277
|
-
|
282
|
+
|
278
283
|
# Parse the config/sphinx.yml file - if it exists - then use the attribute
|
279
284
|
# accessors to set the appropriate values. Nothing too clever.
|
280
|
-
#
|
285
|
+
#
|
281
286
|
def parse_config
|
282
287
|
path = "#{app_root}/config/sphinx.yml"
|
283
288
|
return unless File.exists?(path)
|
284
|
-
|
289
|
+
|
285
290
|
conf = YAML::load(ERB.new(IO.read(path)).result)[environment]
|
286
|
-
|
291
|
+
|
287
292
|
conf.each do |key,value|
|
288
293
|
self.send("#{key}=", value) if self.respond_to?("#{key}=")
|
289
|
-
|
294
|
+
|
290
295
|
set_sphinx_setting self.source_options, key, value, SourceOptions
|
291
296
|
set_sphinx_setting self.index_options, key, value, IndexOptions
|
292
297
|
set_sphinx_setting self.index_options, key, value, CustomOptions
|
293
298
|
set_sphinx_setting @configuration.searchd, key, value
|
294
299
|
set_sphinx_setting @configuration.indexer, key, value
|
295
300
|
end unless conf.nil?
|
296
|
-
|
301
|
+
|
297
302
|
self.bin_path += '/' unless self.bin_path.blank?
|
298
|
-
|
303
|
+
|
299
304
|
if self.allow_star
|
300
305
|
self.index_options[:enable_star] = true
|
301
306
|
self.index_options[:min_prefix_len] = 1
|
302
307
|
end
|
303
308
|
end
|
304
|
-
|
309
|
+
|
305
310
|
def set_sphinx_setting(object, key, value, allowed = {})
|
306
311
|
if object.is_a?(Hash)
|
307
312
|
object[key.to_sym] = value if allowed.include?(key.to_s)
|
@@ -310,26 +315,37 @@ module ThinkingSphinx
|
|
310
315
|
send("#{key}=", value) if self.respond_to?("#{key}")
|
311
316
|
end
|
312
317
|
end
|
313
|
-
|
318
|
+
|
314
319
|
def enforce_common_attribute_types
|
315
320
|
sql_indexes = configuration.indexes.reject { |index|
|
316
321
|
index.is_a? Riddle::Configuration::DistributedIndex
|
317
322
|
}
|
318
|
-
|
323
|
+
|
319
324
|
return unless sql_indexes.any? { |index|
|
320
325
|
index.sources.any? { |source|
|
321
326
|
source.sql_attr_bigint.include? :sphinx_internal_id
|
322
327
|
}
|
323
328
|
}
|
324
|
-
|
329
|
+
|
325
330
|
sql_indexes.each { |index|
|
326
331
|
index.sources.each { |source|
|
327
332
|
next if source.sql_attr_bigint.include? :sphinx_internal_id
|
328
|
-
|
333
|
+
|
329
334
|
source.sql_attr_bigint << :sphinx_internal_id
|
330
335
|
source.sql_attr_uint.delete :sphinx_internal_id
|
331
336
|
}
|
332
337
|
}
|
333
338
|
end
|
339
|
+
|
340
|
+
def shuffled_addresses
|
341
|
+
return address unless shuffle
|
342
|
+
|
343
|
+
addresses = Array(address)
|
344
|
+
if addresses.respond_to?(:shuffle)
|
345
|
+
addresses.shuffle
|
346
|
+
else
|
347
|
+
address.sort_by { rand }
|
348
|
+
end
|
349
|
+
end
|
334
350
|
end
|
335
351
|
end
|