ebeigarts-thinking-sphinx 1.1.21

Sign up to get free protection for your applications and to get access to all the features.
Files changed (91) hide show
  1. data/LICENCE +20 -0
  2. data/README.textile +143 -0
  3. data/lib/thinking_sphinx.rb +217 -0
  4. data/lib/thinking_sphinx/active_record.rb +278 -0
  5. data/lib/thinking_sphinx/active_record/attribute_updates.rb +48 -0
  6. data/lib/thinking_sphinx/active_record/delta.rb +87 -0
  7. data/lib/thinking_sphinx/active_record/has_many_association.rb +29 -0
  8. data/lib/thinking_sphinx/active_record/search.rb +57 -0
  9. data/lib/thinking_sphinx/adapters/abstract_adapter.rb +53 -0
  10. data/lib/thinking_sphinx/adapters/mysql_adapter.rb +54 -0
  11. data/lib/thinking_sphinx/adapters/postgresql_adapter.rb +135 -0
  12. data/lib/thinking_sphinx/association.rb +164 -0
  13. data/lib/thinking_sphinx/attribute.rb +269 -0
  14. data/lib/thinking_sphinx/class_facet.rb +15 -0
  15. data/lib/thinking_sphinx/collection.rb +148 -0
  16. data/lib/thinking_sphinx/configuration.rb +275 -0
  17. data/lib/thinking_sphinx/core/string.rb +15 -0
  18. data/lib/thinking_sphinx/deltas.rb +30 -0
  19. data/lib/thinking_sphinx/deltas/datetime_delta.rb +50 -0
  20. data/lib/thinking_sphinx/deltas/default_delta.rb +68 -0
  21. data/lib/thinking_sphinx/deltas/delayed_delta.rb +27 -0
  22. data/lib/thinking_sphinx/deltas/delayed_delta/delta_job.rb +24 -0
  23. data/lib/thinking_sphinx/deltas/delayed_delta/flag_as_deleted_job.rb +27 -0
  24. data/lib/thinking_sphinx/deltas/delayed_delta/job.rb +26 -0
  25. data/lib/thinking_sphinx/deploy/capistrano.rb +82 -0
  26. data/lib/thinking_sphinx/facet.rb +108 -0
  27. data/lib/thinking_sphinx/facet_collection.rb +59 -0
  28. data/lib/thinking_sphinx/field.rb +82 -0
  29. data/lib/thinking_sphinx/index.rb +99 -0
  30. data/lib/thinking_sphinx/index/builder.rb +287 -0
  31. data/lib/thinking_sphinx/index/faux_column.rb +110 -0
  32. data/lib/thinking_sphinx/property.rb +160 -0
  33. data/lib/thinking_sphinx/rails_additions.rb +136 -0
  34. data/lib/thinking_sphinx/search.rb +727 -0
  35. data/lib/thinking_sphinx/search/facets.rb +104 -0
  36. data/lib/thinking_sphinx/source.rb +175 -0
  37. data/lib/thinking_sphinx/source/internal_properties.rb +46 -0
  38. data/lib/thinking_sphinx/source/sql.rb +126 -0
  39. data/lib/thinking_sphinx/tasks.rb +245 -0
  40. data/rails/init.rb +14 -0
  41. data/spec/unit/thinking_sphinx/active_record/delta_spec.rb +136 -0
  42. data/spec/unit/thinking_sphinx/active_record/has_many_association_spec.rb +53 -0
  43. data/spec/unit/thinking_sphinx/active_record/search_spec.rb +107 -0
  44. data/spec/unit/thinking_sphinx/active_record_spec.rb +329 -0
  45. data/spec/unit/thinking_sphinx/association_spec.rb +246 -0
  46. data/spec/unit/thinking_sphinx/attribute_spec.rb +338 -0
  47. data/spec/unit/thinking_sphinx/collection_spec.rb +15 -0
  48. data/spec/unit/thinking_sphinx/configuration_spec.rb +222 -0
  49. data/spec/unit/thinking_sphinx/core/string_spec.rb +9 -0
  50. data/spec/unit/thinking_sphinx/facet_collection_spec.rb +64 -0
  51. data/spec/unit/thinking_sphinx/facet_spec.rb +302 -0
  52. data/spec/unit/thinking_sphinx/field_spec.rb +154 -0
  53. data/spec/unit/thinking_sphinx/index/builder_spec.rb +355 -0
  54. data/spec/unit/thinking_sphinx/index/faux_column_spec.rb +30 -0
  55. data/spec/unit/thinking_sphinx/index_spec.rb +45 -0
  56. data/spec/unit/thinking_sphinx/rails_additions_spec.rb +191 -0
  57. data/spec/unit/thinking_sphinx/search_spec.rb +228 -0
  58. data/spec/unit/thinking_sphinx/source_spec.rb +217 -0
  59. data/spec/unit/thinking_sphinx_spec.rb +151 -0
  60. data/tasks/distribution.rb +67 -0
  61. data/tasks/rails.rake +1 -0
  62. data/tasks/testing.rb +100 -0
  63. data/vendor/after_commit/LICENSE +20 -0
  64. data/vendor/after_commit/README +16 -0
  65. data/vendor/after_commit/Rakefile +22 -0
  66. data/vendor/after_commit/init.rb +8 -0
  67. data/vendor/after_commit/lib/after_commit.rb +45 -0
  68. data/vendor/after_commit/lib/after_commit/active_record.rb +114 -0
  69. data/vendor/after_commit/lib/after_commit/connection_adapters.rb +103 -0
  70. data/vendor/after_commit/test/after_commit_test.rb +53 -0
  71. data/vendor/delayed_job/lib/delayed/job.rb +251 -0
  72. data/vendor/delayed_job/lib/delayed/message_sending.rb +7 -0
  73. data/vendor/delayed_job/lib/delayed/performable_method.rb +55 -0
  74. data/vendor/delayed_job/lib/delayed/worker.rb +54 -0
  75. data/vendor/riddle/lib/riddle.rb +30 -0
  76. data/vendor/riddle/lib/riddle/client.rb +619 -0
  77. data/vendor/riddle/lib/riddle/client/filter.rb +53 -0
  78. data/vendor/riddle/lib/riddle/client/message.rb +65 -0
  79. data/vendor/riddle/lib/riddle/client/response.rb +84 -0
  80. data/vendor/riddle/lib/riddle/configuration.rb +33 -0
  81. data/vendor/riddle/lib/riddle/configuration/distributed_index.rb +48 -0
  82. data/vendor/riddle/lib/riddle/configuration/index.rb +142 -0
  83. data/vendor/riddle/lib/riddle/configuration/indexer.rb +19 -0
  84. data/vendor/riddle/lib/riddle/configuration/remote_index.rb +17 -0
  85. data/vendor/riddle/lib/riddle/configuration/searchd.rb +25 -0
  86. data/vendor/riddle/lib/riddle/configuration/section.rb +43 -0
  87. data/vendor/riddle/lib/riddle/configuration/source.rb +23 -0
  88. data/vendor/riddle/lib/riddle/configuration/sql_source.rb +34 -0
  89. data/vendor/riddle/lib/riddle/configuration/xml_source.rb +28 -0
  90. data/vendor/riddle/lib/riddle/controller.rb +44 -0
  91. metadata +191 -0
@@ -0,0 +1,82 @@
1
+ module ThinkingSphinx
2
+ # Fields - holding the string data which Sphinx indexes for your searches.
3
+ # This class isn't really useful to you unless you're hacking around with the
4
+ # internals of Thinking Sphinx - but hey, don't let that stop you.
5
+ #
6
+ # One key thing to remember - if you're using the field manually to
7
+ # generate SQL statements, you'll need to set the base model, and all the
8
+ # associations. Which can get messy. Use Index.link!, it really helps.
9
+ #
10
+ class Field < ThinkingSphinx::Property
11
+ attr_accessor :sortable, :infixes, :prefixes
12
+
13
+ # To create a new field, you'll need to pass in either a single Column
14
+ # or an array of them, and some (optional) options. The columns are
15
+ # references to the data that will make up the field.
16
+ #
17
+ # Valid options are:
18
+ # - :as => :alias_name
19
+ # - :sortable => true
20
+ # - :infixes => true
21
+ # - :prefixes => true
22
+ #
23
+ # Alias is only required in three circumstances: when there's
24
+ # another attribute or field with the same name, when the column name is
25
+ # 'id', or when there's more than one column.
26
+ #
27
+ # Sortable defaults to false - but is quite useful when set to true, as
28
+ # it creates an attribute with the same string value (which Sphinx converts
29
+ # to an integer value), which can be sorted by. Thinking Sphinx is smart
30
+ # enough to realise that when you specify fields in sort statements, you
31
+ # mean their respective attributes.
32
+ #
33
+ # If you have partial matching enabled (ie: enable_star), then you can
34
+ # specify certain fields to have their prefixes and infixes indexed. Keep
35
+ # in mind, though, that Sphinx's default is _all_ fields - so once you
36
+ # highlight a particular field, no other fields in the index will have
37
+ # these partial indexes.
38
+ #
39
+ # Here's some examples:
40
+ #
41
+ # Field.new(
42
+ # Column.new(:name)
43
+ # )
44
+ #
45
+ # Field.new(
46
+ # [Column.new(:first_name), Column.new(:last_name)],
47
+ # :as => :name, :sortable => true
48
+ # )
49
+ #
50
+ # Field.new(
51
+ # [Column.new(:posts, :subject), Column.new(:posts, :content)],
52
+ # :as => :posts, :prefixes => true
53
+ # )
54
+ #
55
+ def initialize(source, columns, options = {})
56
+ super
57
+
58
+ @sortable = options[:sortable] || false
59
+ @infixes = options[:infixes] || false
60
+ @prefixes = options[:prefixes] || false
61
+
62
+ source.fields << self
63
+ end
64
+
65
+ # Get the part of the SELECT clause related to this field. Don't forget
66
+ # to set your model and associations first though.
67
+ #
68
+ # This will concatenate strings if there's more than one data source or
69
+ # multiple data values (has_many or has_and_belongs_to_many associations).
70
+ #
71
+ def to_select_sql
72
+ clause = @columns.collect { |column|
73
+ column_with_prefix(column)
74
+ }.join(', ')
75
+
76
+ clause = adapter.concatenate(clause) if concat_ws?
77
+ clause = adapter.group_concatenate(clause) if is_many?
78
+
79
+ "#{adapter.cast_to_string clause } AS #{quote_column(unique_name)}"
80
+ end
81
+ end
82
+ end
@@ -0,0 +1,99 @@
1
+ require 'thinking_sphinx/index/builder'
2
+ require 'thinking_sphinx/index/faux_column'
3
+
4
+ module ThinkingSphinx
5
+ # The Index class is a ruby representation of a Sphinx source (not a Sphinx
6
+ # index - yes, I know it's a little confusing. You'll manage). This is
7
+ # another 'internal' Thinking Sphinx class - if you're using it directly,
8
+ # you either know what you're doing, or messing with things beyond your ken.
9
+ # Enjoy.
10
+ #
11
+ class Index
12
+ attr_accessor :model, :sources, :delta_object
13
+
14
+ # Create a new index instance by passing in the model it is tied to, and
15
+ # a block to build it with (optional but recommended). For documentation
16
+ # on the syntax for inside the block, the Builder class is what you want.
17
+ #
18
+ # Quick Example:
19
+ #
20
+ # Index.new(User) do
21
+ # indexes login, email
22
+ #
23
+ # has created_at
24
+ #
25
+ # set_property :delta => true
26
+ # end
27
+ #
28
+ def initialize(model, &block)
29
+ @model = model
30
+ @sources = []
31
+ @options = {}
32
+ @delta_object = nil
33
+ end
34
+
35
+ def fields
36
+ @sources.collect { |source| source.fields }.flatten
37
+ end
38
+
39
+ def attributes
40
+ @sources.collect { |source| source.attributes }.flatten
41
+ end
42
+
43
+ def name
44
+ self.class.name(@model)
45
+ end
46
+
47
+ def self.name(model)
48
+ model.name.underscore.tr(':/\\', '_')
49
+ end
50
+
51
+ def prefix_fields
52
+ fields.select { |field| field.prefixes }
53
+ end
54
+
55
+ def infix_fields
56
+ fields.select { |field| field.infixes }
57
+ end
58
+
59
+ def local_options
60
+ @options
61
+ end
62
+
63
+ def options
64
+ all_index_options = ThinkingSphinx::Configuration.instance.index_options.clone
65
+ @options.keys.select { |key|
66
+ ThinkingSphinx::Configuration::IndexOptions.include?(key.to_s) ||
67
+ ThinkingSphinx::Configuration::CustomOptions.include?(key.to_s)
68
+ }.each { |key| all_index_options[key.to_sym] = @options[key] }
69
+ all_index_options
70
+ end
71
+
72
+ def delta?
73
+ !@delta_object.nil?
74
+ end
75
+
76
+ private
77
+
78
+ def adapter
79
+ @adapter ||= @model.sphinx_database_adapter
80
+ end
81
+
82
+ def utf8?
83
+ options[:charset_type] == "utf-8"
84
+ end
85
+
86
+ # Does all the magic with the block provided to the base #initialize.
87
+ # Creates a new class subclassed from Builder, and evaluates the block
88
+ # on it, then pulls all relevant settings - fields, attributes, conditions,
89
+ # properties - into the new index.
90
+ #
91
+ def initialize_from_builder(&block)
92
+ #
93
+ end
94
+
95
+ def sql_query_pre_for_delta
96
+ [""]
97
+ end
98
+ end
99
+ end
@@ -0,0 +1,287 @@
1
+ module ThinkingSphinx
2
+ class Index
3
+ # The Builder class is the core for the index definition block processing.
4
+ # There are four methods you really need to pay attention to:
5
+ # - indexes
6
+ # - has
7
+ # - where
8
+ # - set_property/set_properties
9
+ #
10
+ # The first two of these methods allow you to define what data makes up
11
+ # your indexes. #where provides a method to add manual SQL conditions, and
12
+ # set_property allows you to set some settings on a per-index basis. Check
13
+ # out each method's documentation for better ideas of usage.
14
+ #
15
+ class Builder
16
+ instance_methods.grep(/^[^_]/).each { |method|
17
+ next if method.to_s == "instance_eval"
18
+ define_method(method) {
19
+ caller.grep(/irb.completion/).empty? ? method_missing(method) : super
20
+ }
21
+ }
22
+
23
+ def self.generate(model, &block)
24
+ index = ThinkingSphinx::Index.new(model)
25
+ model.sphinx_facets ||= []
26
+
27
+ Builder.new(index, &block) if block_given?
28
+
29
+ index.delta_object = ThinkingSphinx::Deltas.parse index
30
+ index
31
+ end
32
+
33
+ def initialize(index, &block)
34
+ @index = index
35
+ @source = ThinkingSphinx::Source.new(@index)
36
+ @index.sources << @source
37
+ @explicit_source = false
38
+
39
+ self.instance_eval &block
40
+
41
+ if @index.sources.any? { |source|
42
+ source.fields.length == 0
43
+ }
44
+ raise "At least one field is necessary for an index"
45
+ end
46
+ end
47
+
48
+ def define_source(&block)
49
+ if @explicit_source
50
+ @source = ThinkingSphinx::Source.new(@index)
51
+ @index.sources << @source
52
+ else
53
+ @explicit_source = true
54
+ end
55
+
56
+ self.instance_eval &block
57
+ end
58
+
59
+ # This is how you add fields - the strings Sphinx looks at - to your
60
+ # index. Technically, to use this method, you need to pass in some
61
+ # columns and options - but there's some neat method_missing stuff
62
+ # happening, so lets stick to the expected syntax within a define_index
63
+ # block.
64
+ #
65
+ # Expected options are :as, which points to a column alias in symbol
66
+ # form, and :sortable, which indicates whether you want to sort by this
67
+ # field.
68
+ #
69
+ # Adding Single-Column Fields:
70
+ #
71
+ # You can use symbols or methods - and can chain methods together to
72
+ # get access down the associations tree.
73
+ #
74
+ # indexes :id, :as => :my_id
75
+ # indexes :name, :sortable => true
76
+ # indexes first_name, last_name, :sortable => true
77
+ # indexes users.posts.content, :as => :post_content
78
+ # indexes users(:id), :as => :user_ids
79
+ #
80
+ # Keep in mind that if any keywords for Ruby methods - such as id or
81
+ # name - clash with your column names, you need to use the symbol
82
+ # version (see the first, second and last examples above).
83
+ #
84
+ # If you specify multiple columns (example #2), a field will be created
85
+ # for each. Don't use the :as option in this case. If you want to merge
86
+ # those columns together, continue reading.
87
+ #
88
+ # Adding Multi-Column Fields:
89
+ #
90
+ # indexes [first_name, last_name], :as => :name
91
+ # indexes [location, parent.location], :as => :location
92
+ #
93
+ # To combine multiple columns into a single field, you need to wrap
94
+ # them in an Array, as shown by the above examples. There's no
95
+ # limitations on whether they're symbols or methods or what level of
96
+ # associations they come from.
97
+ #
98
+ # Adding SQL Fragment Fields
99
+ #
100
+ # You can also define a field using an SQL fragment, useful for when
101
+ # you would like to index a calculated value.
102
+ #
103
+ # indexes "age < 18", :as => :minor
104
+ #
105
+ def indexes(*args)
106
+ options = args.extract_options!
107
+ args.each do |columns|
108
+ field = Field.new(@source, FauxColumn.coerce(columns), options)
109
+
110
+ add_sort_attribute field, options if field.sortable
111
+ add_facet_attribute field, options if field.faceted
112
+ end
113
+ end
114
+
115
+ # This is the method to add attributes to your index (hence why it is
116
+ # aliased as 'attribute'). The syntax is the same as #indexes, so use
117
+ # that as starting point, but keep in mind the following points.
118
+ #
119
+ # An attribute can have an alias (the :as option), but it is always
120
+ # sortable - so you don't need to explicitly request that. You _can_
121
+ # specify the data type of the attribute (the :type option), but the
122
+ # code's pretty good at figuring that out itself from peering into the
123
+ # database.
124
+ #
125
+ # Attributes are limited to the following types: integers, floats,
126
+ # datetimes (converted to timestamps), booleans and strings. Don't
127
+ # forget that Sphinx converts string attributes to integers, which are
128
+ # useful for sorting, but that's about it.
129
+ #
130
+ # You can also have a collection of integers for multi-value attributes
131
+ # (MVAs). Generally these would be through a has_many relationship,
132
+ # like in this example:
133
+ #
134
+ # has posts(:id), :as => :post_ids
135
+ #
136
+ # This allows you to filter on any of the values tied to a specific
137
+ # record. Might be best to read through the Sphinx documentation to get
138
+ # a better idea of that though.
139
+ #
140
+ # Adding SQL Fragment Attributes
141
+ #
142
+ # You can also define an attribute using an SQL fragment, useful for
143
+ # when you would like to index a calculated value. Don't forget to set
144
+ # the type of the attribute though:
145
+ #
146
+ # has "age < 18", :as => :minor, :type => :boolean
147
+ #
148
+ # If you're creating attributes for latitude and longitude, don't
149
+ # forget that Sphinx expects these values to be in radians.
150
+ #
151
+ def has(*args)
152
+ options = args.extract_options!
153
+ args.each do |columns|
154
+ attribute = Attribute.new(@source, FauxColumn.coerce(columns), options)
155
+
156
+ add_facet_attribute attribute, options if attribute.faceted
157
+ end
158
+ end
159
+
160
+ def facet(*args)
161
+ options = args.extract_options!
162
+ options[:facet] = true
163
+
164
+ args.each do |columns|
165
+ attribute = Attribute.new(@source, FauxColumn.coerce(columns), options)
166
+
167
+ add_facet_attribute attribute, options
168
+ end
169
+ end
170
+
171
+ # Use this method to add some manual SQL conditions for your index
172
+ # request. You can pass in as many strings as you like, they'll get
173
+ # joined together with ANDs later on.
174
+ #
175
+ # where "user_id = 10"
176
+ # where "parent_type = 'Article'", "created_at < NOW()"
177
+ #
178
+ def where(*args)
179
+ @source.conditions += args
180
+ end
181
+
182
+ # Use this method to add some manual SQL strings to the GROUP BY
183
+ # clause. You can pass in as many strings as you'd like, they'll get
184
+ # joined together with commas later on.
185
+ #
186
+ # group_by "lat", "lng"
187
+ #
188
+ def group_by(*args)
189
+ @source.groupings += args
190
+ end
191
+
192
+ # This is what to use to set properties on the index. Chief amongst
193
+ # those is the delta property - to allow automatic updates to your
194
+ # indexes as new models are added and edited - but also you can
195
+ # define search-related properties which will be the defaults for all
196
+ # searches on the model.
197
+ #
198
+ # set_property :delta => true
199
+ # set_property :field_weights => {"name" => 100}
200
+ # set_property :order => "name ASC"
201
+ # set_property :include => :picture
202
+ # set_property :select => 'name'
203
+ #
204
+ # Also, the following two properties are particularly relevant for
205
+ # geo-location searching - latitude_attr and longitude_attr. If your
206
+ # attributes for these two values are named something other than
207
+ # lat/latitude or lon/long/longitude, you can dictate what they are
208
+ # when defining the index, so you don't need to specify them for every
209
+ # geo-related search.
210
+ #
211
+ # set_property :latitude_attr => "lt", :longitude_attr => "lg"
212
+ #
213
+ # Please don't forget to add a boolean field named 'delta' to your
214
+ # model's database table if enabling the delta index for it.
215
+ # Valid options for the delta property are:
216
+ #
217
+ # true
218
+ # false
219
+ # :default
220
+ # :delayed
221
+ # :datetime
222
+ #
223
+ # You can also extend ThinkingSphinx::Deltas::DefaultDelta to implement
224
+ # your own handling for delta indexing.
225
+ #
226
+ def set_property(*args)
227
+ options = args.extract_options!
228
+ options.each do |key, value|
229
+ set_single_property key, value
230
+ end
231
+
232
+ set_single_property args[0], args[1] if args.length == 2
233
+ end
234
+ alias_method :set_properties, :set_property
235
+
236
+ # Handles the generation of new columns for the field and attribute
237
+ # definitions.
238
+ #
239
+ def method_missing(method, *args)
240
+ FauxColumn.new(method, *args)
241
+ end
242
+
243
+ # A method to allow adding fields from associations which have names
244
+ # that clash with method names in the Builder class (ie: properties,
245
+ # fields, attributes).
246
+ #
247
+ # Example: indexes assoc(:properties).column
248
+ #
249
+ def assoc(assoc, *args)
250
+ FauxColumn.new(assoc, *args)
251
+ end
252
+
253
+ private
254
+
255
+ def set_single_property(key, value)
256
+ source_options = ThinkingSphinx::Configuration::SourceOptions
257
+ if source_options.include?(key.to_s)
258
+ @source.options.merge! key => value
259
+ else
260
+ @index.local_options.merge! key => value
261
+ end
262
+ end
263
+
264
+ def add_sort_attribute(field, options)
265
+ add_internal_attribute field, options, "_sort"
266
+ end
267
+
268
+ def add_facet_attribute(property, options)
269
+ add_internal_attribute property, options, "_facet", true
270
+ @index.model.sphinx_facets << property.to_facet
271
+ end
272
+
273
+ def add_internal_attribute(property, options, suffix, crc = false)
274
+ return unless ThinkingSphinx::Facet.translate?(property)
275
+
276
+ Attribute.new(@source,
277
+ property.columns.collect { |col| col.clone },
278
+ options.merge(
279
+ :type => property.is_a?(Field) ? :string : options[:type],
280
+ :as => property.unique_name.to_s.concat(suffix).to_sym,
281
+ :crc => crc
282
+ ).except(:facet)
283
+ )
284
+ end
285
+ end
286
+ end
287
+ end