ebeigarts-thinking-sphinx 1.1.21

Sign up to get free protection for your applications and to get access to all the features.
Files changed (91) hide show
  1. data/LICENCE +20 -0
  2. data/README.textile +143 -0
  3. data/lib/thinking_sphinx.rb +217 -0
  4. data/lib/thinking_sphinx/active_record.rb +278 -0
  5. data/lib/thinking_sphinx/active_record/attribute_updates.rb +48 -0
  6. data/lib/thinking_sphinx/active_record/delta.rb +87 -0
  7. data/lib/thinking_sphinx/active_record/has_many_association.rb +29 -0
  8. data/lib/thinking_sphinx/active_record/search.rb +57 -0
  9. data/lib/thinking_sphinx/adapters/abstract_adapter.rb +53 -0
  10. data/lib/thinking_sphinx/adapters/mysql_adapter.rb +54 -0
  11. data/lib/thinking_sphinx/adapters/postgresql_adapter.rb +135 -0
  12. data/lib/thinking_sphinx/association.rb +164 -0
  13. data/lib/thinking_sphinx/attribute.rb +269 -0
  14. data/lib/thinking_sphinx/class_facet.rb +15 -0
  15. data/lib/thinking_sphinx/collection.rb +148 -0
  16. data/lib/thinking_sphinx/configuration.rb +275 -0
  17. data/lib/thinking_sphinx/core/string.rb +15 -0
  18. data/lib/thinking_sphinx/deltas.rb +30 -0
  19. data/lib/thinking_sphinx/deltas/datetime_delta.rb +50 -0
  20. data/lib/thinking_sphinx/deltas/default_delta.rb +68 -0
  21. data/lib/thinking_sphinx/deltas/delayed_delta.rb +27 -0
  22. data/lib/thinking_sphinx/deltas/delayed_delta/delta_job.rb +24 -0
  23. data/lib/thinking_sphinx/deltas/delayed_delta/flag_as_deleted_job.rb +27 -0
  24. data/lib/thinking_sphinx/deltas/delayed_delta/job.rb +26 -0
  25. data/lib/thinking_sphinx/deploy/capistrano.rb +82 -0
  26. data/lib/thinking_sphinx/facet.rb +108 -0
  27. data/lib/thinking_sphinx/facet_collection.rb +59 -0
  28. data/lib/thinking_sphinx/field.rb +82 -0
  29. data/lib/thinking_sphinx/index.rb +99 -0
  30. data/lib/thinking_sphinx/index/builder.rb +287 -0
  31. data/lib/thinking_sphinx/index/faux_column.rb +110 -0
  32. data/lib/thinking_sphinx/property.rb +160 -0
  33. data/lib/thinking_sphinx/rails_additions.rb +136 -0
  34. data/lib/thinking_sphinx/search.rb +727 -0
  35. data/lib/thinking_sphinx/search/facets.rb +104 -0
  36. data/lib/thinking_sphinx/source.rb +175 -0
  37. data/lib/thinking_sphinx/source/internal_properties.rb +46 -0
  38. data/lib/thinking_sphinx/source/sql.rb +126 -0
  39. data/lib/thinking_sphinx/tasks.rb +245 -0
  40. data/rails/init.rb +14 -0
  41. data/spec/unit/thinking_sphinx/active_record/delta_spec.rb +136 -0
  42. data/spec/unit/thinking_sphinx/active_record/has_many_association_spec.rb +53 -0
  43. data/spec/unit/thinking_sphinx/active_record/search_spec.rb +107 -0
  44. data/spec/unit/thinking_sphinx/active_record_spec.rb +329 -0
  45. data/spec/unit/thinking_sphinx/association_spec.rb +246 -0
  46. data/spec/unit/thinking_sphinx/attribute_spec.rb +338 -0
  47. data/spec/unit/thinking_sphinx/collection_spec.rb +15 -0
  48. data/spec/unit/thinking_sphinx/configuration_spec.rb +222 -0
  49. data/spec/unit/thinking_sphinx/core/string_spec.rb +9 -0
  50. data/spec/unit/thinking_sphinx/facet_collection_spec.rb +64 -0
  51. data/spec/unit/thinking_sphinx/facet_spec.rb +302 -0
  52. data/spec/unit/thinking_sphinx/field_spec.rb +154 -0
  53. data/spec/unit/thinking_sphinx/index/builder_spec.rb +355 -0
  54. data/spec/unit/thinking_sphinx/index/faux_column_spec.rb +30 -0
  55. data/spec/unit/thinking_sphinx/index_spec.rb +45 -0
  56. data/spec/unit/thinking_sphinx/rails_additions_spec.rb +191 -0
  57. data/spec/unit/thinking_sphinx/search_spec.rb +228 -0
  58. data/spec/unit/thinking_sphinx/source_spec.rb +217 -0
  59. data/spec/unit/thinking_sphinx_spec.rb +151 -0
  60. data/tasks/distribution.rb +67 -0
  61. data/tasks/rails.rake +1 -0
  62. data/tasks/testing.rb +100 -0
  63. data/vendor/after_commit/LICENSE +20 -0
  64. data/vendor/after_commit/README +16 -0
  65. data/vendor/after_commit/Rakefile +22 -0
  66. data/vendor/after_commit/init.rb +8 -0
  67. data/vendor/after_commit/lib/after_commit.rb +45 -0
  68. data/vendor/after_commit/lib/after_commit/active_record.rb +114 -0
  69. data/vendor/after_commit/lib/after_commit/connection_adapters.rb +103 -0
  70. data/vendor/after_commit/test/after_commit_test.rb +53 -0
  71. data/vendor/delayed_job/lib/delayed/job.rb +251 -0
  72. data/vendor/delayed_job/lib/delayed/message_sending.rb +7 -0
  73. data/vendor/delayed_job/lib/delayed/performable_method.rb +55 -0
  74. data/vendor/delayed_job/lib/delayed/worker.rb +54 -0
  75. data/vendor/riddle/lib/riddle.rb +30 -0
  76. data/vendor/riddle/lib/riddle/client.rb +619 -0
  77. data/vendor/riddle/lib/riddle/client/filter.rb +53 -0
  78. data/vendor/riddle/lib/riddle/client/message.rb +65 -0
  79. data/vendor/riddle/lib/riddle/client/response.rb +84 -0
  80. data/vendor/riddle/lib/riddle/configuration.rb +33 -0
  81. data/vendor/riddle/lib/riddle/configuration/distributed_index.rb +48 -0
  82. data/vendor/riddle/lib/riddle/configuration/index.rb +142 -0
  83. data/vendor/riddle/lib/riddle/configuration/indexer.rb +19 -0
  84. data/vendor/riddle/lib/riddle/configuration/remote_index.rb +17 -0
  85. data/vendor/riddle/lib/riddle/configuration/searchd.rb +25 -0
  86. data/vendor/riddle/lib/riddle/configuration/section.rb +43 -0
  87. data/vendor/riddle/lib/riddle/configuration/source.rb +23 -0
  88. data/vendor/riddle/lib/riddle/configuration/sql_source.rb +34 -0
  89. data/vendor/riddle/lib/riddle/configuration/xml_source.rb +28 -0
  90. data/vendor/riddle/lib/riddle/controller.rb +44 -0
  91. metadata +191 -0
@@ -0,0 +1,104 @@
1
+ module ThinkingSphinx
2
+ class Search
3
+ module Facets
4
+ # Model.facets *args
5
+ # ThinkingSphinx::Search.facets *args
6
+ # ThinkingSphinx::Search.facets *args, :all_attributes => true
7
+ # ThinkingSphinx::Search.facets *args, :class_facet => false
8
+ #
9
+ def facets(*args)
10
+ options = args.extract_options!
11
+
12
+ if options[:class]
13
+ facets_for_model options[:class], args, options
14
+ else
15
+ facets_for_all_models args, options
16
+ end
17
+ end
18
+
19
+ private
20
+
21
+ def facets_for_model(klass, args, options)
22
+ hash = ThinkingSphinx::FacetCollection.new args + [options]
23
+ options = options.clone.merge! facet_query_options
24
+
25
+ facets = klass.sphinx_facets
26
+ facets = Array(options.delete(:facets)).collect { |name|
27
+ klass.sphinx_facets.detect { |facet| facet.name.to_s == name.to_s }
28
+ }.compact if options[:facets]
29
+
30
+ facets.inject(hash) do |hash, facet|
31
+ unless facet.name == :class && !options[:class_facet]
32
+ options[:group_by] = facet.attribute_name
33
+ hash.add_from_results facet, search(*(args + [options]))
34
+ end
35
+
36
+ hash
37
+ end
38
+ end
39
+
40
+ def facets_for_all_models(args, options)
41
+ options = GlobalFacetOptions.merge(options)
42
+ hash = ThinkingSphinx::FacetCollection.new args + [options]
43
+ options = options.merge! facet_query_options
44
+
45
+ facet_names(options).inject(hash) do |hash, name|
46
+ options[:group_by] = name
47
+ hash.add_from_results name, search(*(args + [options]))
48
+ hash
49
+ end
50
+ end
51
+
52
+ def facet_query_options
53
+ config = ThinkingSphinx::Configuration.instance
54
+ max = config.configuration.searchd.max_matches || 1000
55
+
56
+ {
57
+ :group_function => :attr,
58
+ :limit => max,
59
+ :max_matches => max,
60
+ :page => 1
61
+ }
62
+ end
63
+
64
+ def facet_classes(options)
65
+ options[:classes] || ThinkingSphinx.indexed_models.collect { |model|
66
+ model.constantize
67
+ }
68
+ end
69
+
70
+ def facet_names(options)
71
+ classes = facet_classes(options)
72
+ names = options[:all_attributes] ?
73
+ facet_names_for_all_classes(classes) :
74
+ facet_names_common_to_all_classes(classes)
75
+
76
+ names.delete "class_crc" unless options[:class_facet]
77
+ names
78
+ end
79
+
80
+ def facet_names_for_all_classes(classes)
81
+ all_facets = classes.collect { |klass| klass.sphinx_facets }.flatten
82
+
83
+ all_facets.group_by { |facet|
84
+ facet.name
85
+ }.collect { |name, facets|
86
+ if facets.collect { |facet| facet.type }.uniq.length > 1
87
+ raise "Facet #{name} exists in more than one model with different types"
88
+ end
89
+ facets.first.attribute_name
90
+ }
91
+ end
92
+
93
+ def facet_names_common_to_all_classes(classes)
94
+ facet_names_for_all_classes(classes).select { |name|
95
+ classes.all? { |klass|
96
+ klass.sphinx_facets.detect { |facet|
97
+ facet.attribute_name == name
98
+ }
99
+ }
100
+ }
101
+ end
102
+ end
103
+ end
104
+ end
@@ -0,0 +1,175 @@
1
+ require 'thinking_sphinx/source/internal_properties'
2
+ require 'thinking_sphinx/source/sql'
3
+
4
+ module ThinkingSphinx
5
+ class Source
6
+ include ThinkingSphinx::Source::InternalProperties
7
+ include ThinkingSphinx::Source::SQL
8
+
9
+ attr_accessor :model, :fields, :attributes, :conditions, :groupings,
10
+ :options
11
+ attr_reader :base
12
+
13
+ def initialize(index, options = {})
14
+ @index = index
15
+ @model = index.model
16
+ @fields = []
17
+ @attributes = []
18
+ @conditions = []
19
+ @groupings = []
20
+ @options = options
21
+ @associations = {}
22
+
23
+ @base = ::ActiveRecord::Associations::ClassMethods::JoinDependency.new(
24
+ @model, [], nil
25
+ )
26
+
27
+ unless @model.descends_from_active_record?
28
+ stored_class = @model.store_full_sti_class ? @model.name : @model.name.demodulize
29
+ @conditions << "#{@model.quoted_table_name}.#{quote_column(@model.inheritance_column)} = '#{stored_class}'"
30
+ end
31
+
32
+ add_internal_attributes_and_facets
33
+ end
34
+
35
+ def name
36
+ @model.name.underscore.tr(':/\\', '_')
37
+ end
38
+
39
+ def to_riddle_for_core(offset, index)
40
+ if ThinkingSphinx::Configuration.instance.type == "xml"
41
+ source = Riddle::Configuration::XMLSource.new(
42
+ "#{name}_core_#{index}", "xmlpipe2"
43
+ )
44
+ set_source_xml_settings source, offset
45
+ set_source_attributes source, offset
46
+ set_source_settings source
47
+ source
48
+ else
49
+ source = Riddle::Configuration::SQLSource.new(
50
+ "#{name}_core_#{index}", adapter.sphinx_identifier
51
+ )
52
+ set_source_database_settings source
53
+ set_source_attributes source, offset
54
+ set_source_sql source, offset
55
+ set_source_settings source
56
+ source
57
+ end
58
+ end
59
+
60
+ def to_riddle_for_delta(offset, index)
61
+ if ThinkingSphinx::Configuration.instance.type == "xml"
62
+ source = Riddle::Configuration::XMLSource.new(
63
+ "#{name}_delta_#{index}", "xmlpipe2"
64
+ )
65
+ set_source_xml_settings source, offset
66
+ set_source_attributes source, offset
67
+ source
68
+ else
69
+ source = Riddle::Configuration::SQLSource.new(
70
+ "#{name}_delta_#{index}", adapter.sphinx_identifier
71
+ )
72
+ source.parent = "#{name}_core_#{index}"
73
+ set_source_database_settings source
74
+ set_source_attributes source, offset
75
+ set_source_sql source, offset, true
76
+ source
77
+ end
78
+ end
79
+
80
+ def delta?
81
+ !@index.delta_object.nil?
82
+ end
83
+
84
+ # Gets the association stack for a specific key.
85
+ #
86
+ def association(key)
87
+ @associations[key] ||= Association.children(@model, key)
88
+ end
89
+
90
+ private
91
+
92
+ def adapter
93
+ @adapter ||= @model.sphinx_database_adapter
94
+ end
95
+
96
+ def set_source_database_settings(source)
97
+ config = @model.connection.instance_variable_get(:@config)
98
+ return unless config
99
+
100
+ source.sql_host = config[:host] || "localhost"
101
+ source.sql_user = config[:username] || config[:user] || ""
102
+ source.sql_pass = (config[:password].to_s || "").gsub('#', '\#')
103
+ source.sql_db = config[:database]
104
+ source.sql_port = config[:port]
105
+ source.sql_sock = config[:socket]
106
+ end
107
+
108
+ def set_source_attributes(source, offset)
109
+ attributes.each do |attrib|
110
+ source.send(attrib.type_to_config) << attrib.config_value(offset)
111
+ end
112
+ end
113
+
114
+ def set_source_sql(source, offset, delta = false)
115
+ source.sql_query = to_sql(:offset => offset, :delta => delta).gsub(/\n/, ' ')
116
+ source.sql_query_range = to_sql_query_range(:delta => delta)
117
+ source.sql_query_info = to_sql_query_info(offset)
118
+
119
+ source.sql_query_pre += send(!delta ? :sql_query_pre_for_core : :sql_query_pre_for_delta)
120
+
121
+ if @index.local_options[:group_concat_max_len]
122
+ source.sql_query_pre << "SET SESSION group_concat_max_len = #{@index.local_options[:group_concat_max_len]}"
123
+ end
124
+
125
+ source.sql_query_pre += [adapter.utf8_query_pre].compact if utf8?
126
+ end
127
+
128
+ def set_source_settings(source)
129
+ config = ThinkingSphinx::Configuration.instance
130
+ config.source_options.each do |key, value|
131
+ source.send("#{key}=".to_sym, value)
132
+ end
133
+
134
+ source_options = ThinkingSphinx::Configuration::SourceOptions
135
+ @options.each do |key, value|
136
+ if source_options.include?(key.to_s) && !value.nil?
137
+ source.send("#{key}=".to_sym, value)
138
+ end
139
+ end
140
+ end
141
+
142
+ def set_source_xml_settings(source, offset)
143
+ env = ThinkingSphinx::Configuration.instance.environment
144
+ database = defined?(Database) ? Database : nil
145
+ source.xmlpipe_command = "rake -s ts:xml NAME=#{source.name} OFFSET=#{offset} RAILS_ENV=#{env} MERB_ENV=#{env} DATABASE=#{database}"
146
+ @fields.each do |field|
147
+ source.xmlpipe_field << field.unique_name
148
+ end
149
+ end
150
+
151
+ # Returns all associations used amongst all the fields and attributes.
152
+ # This includes all associations between the model and what the actual
153
+ # columns are from.
154
+ #
155
+ def all_associations
156
+ @all_associations ||= (
157
+ # field associations
158
+ @fields.collect { |field|
159
+ field.associations.values
160
+ }.flatten +
161
+ # attribute associations
162
+ @attributes.collect { |attrib|
163
+ attrib.associations.values if attrib.include_as_association?
164
+ }.compact.flatten
165
+ ).uniq.collect { |assoc|
166
+ # get ancestors as well as column-level associations
167
+ assoc.ancestors
168
+ }.flatten.uniq
169
+ end
170
+
171
+ def utf8?
172
+ @index.options[:charset_type] == "utf-8"
173
+ end
174
+ end
175
+ end
@@ -0,0 +1,46 @@
1
+ module ThinkingSphinx
2
+ class Source
3
+ module InternalProperties
4
+ def add_internal_attributes_and_facets
5
+ add_internal_attribute :sphinx_internal_id, :integer, @model.primary_key.to_sym
6
+ add_internal_attribute :class_crc, :integer, crc_column, true
7
+ add_internal_attribute :subclass_crcs, :multi, subclasses_to_s
8
+ add_internal_attribute :sphinx_deleted, :integer, "0"
9
+
10
+ add_internal_facet :class_crc
11
+ end
12
+
13
+ def add_internal_attribute(name, type, contents, facet = false)
14
+ return unless attribute_by_alias(name).nil?
15
+
16
+ Attribute.new(self,
17
+ ThinkingSphinx::Index::FauxColumn.new(contents),
18
+ :type => type,
19
+ :as => name,
20
+ :facet => facet,
21
+ :admin => true
22
+ )
23
+ end
24
+
25
+ def add_internal_facet(name)
26
+ return unless facet_by_alias(name).nil?
27
+
28
+ @model.sphinx_facets << ClassFacet.new(attribute_by_alias(name))
29
+ end
30
+
31
+ def attribute_by_alias(attr_alias)
32
+ @attributes.detect { |attrib| attrib.alias == attr_alias }
33
+ end
34
+
35
+ def facet_by_alias(name)
36
+ @model.sphinx_facets.detect { |facet| facet.name == name }
37
+ end
38
+
39
+ def subclasses_to_s
40
+ "'" + (@model.send(:subclasses).collect { |klass|
41
+ klass.to_crc32.to_s
42
+ } << @model.to_crc32.to_s).join(",") + "'"
43
+ end
44
+ end
45
+ end
46
+ end
@@ -0,0 +1,126 @@
1
+ module ThinkingSphinx
2
+ class Source
3
+ module SQL
4
+ # Generates the big SQL statement to get the data back for all the fields
5
+ # and attributes, using all the relevant association joins. If you want
6
+ # the version filtered for delta values, send through :delta => true in the
7
+ # options. Won't do much though if the index isn't set up to support a
8
+ # delta sibling.
9
+ #
10
+ # Examples:
11
+ #
12
+ # source.to_sql
13
+ # source.to_sql(:delta => true)
14
+ #
15
+ def to_sql(options={})
16
+ sql = <<-SQL
17
+ SELECT #{ sql_select_clause options[:offset] }
18
+ FROM #{ @model.quoted_table_name }
19
+ #{ all_associations.collect { |assoc| assoc.to_sql }.join(' ') }
20
+ #{ sql_where_clause(options) }
21
+ GROUP BY #{ sql_group_clause }
22
+ SQL
23
+
24
+ sql += " ORDER BY NULL" if adapter.sphinx_identifier == "mysql"
25
+ sql
26
+ end
27
+
28
+ # Simple helper method for the query range SQL - which is a statement that
29
+ # returns minimum and maximum id values. These can be filtered by delta -
30
+ # so pass in :delta => true to get the delta version of the SQL.
31
+ #
32
+ def to_sql_query_range(options={})
33
+ return nil if @index.options[:disable_range]
34
+
35
+ min_statement = adapter.convert_nulls(
36
+ "MIN(#{quote_column(@model.primary_key)})", 1
37
+ )
38
+ max_statement = adapter.convert_nulls(
39
+ "MAX(#{quote_column(@model.primary_key)})", 1
40
+ )
41
+
42
+ sql = "SELECT #{min_statement}, #{max_statement} " +
43
+ "FROM #{@model.quoted_table_name} "
44
+ if self.delta? && !@index.delta_object.clause(@model, options[:delta]).blank?
45
+ sql << "WHERE #{@index.delta_object.clause(@model, options[:delta])}"
46
+ end
47
+
48
+ sql
49
+ end
50
+
51
+ # Simple helper method for the query info SQL - which is a statement that
52
+ # returns the single row for a corresponding id.
53
+ #
54
+ def to_sql_query_info(offset)
55
+ "SELECT * FROM #{@model.quoted_table_name} WHERE " +
56
+ "#{quote_column(@model.primary_key)} = (($id - #{offset}) / #{ThinkingSphinx.indexed_models.size})"
57
+ end
58
+
59
+ def sql_select_clause(offset)
60
+ unique_id_expr = ThinkingSphinx.unique_id_expression(offset)
61
+
62
+ (
63
+ ["#{@model.quoted_table_name}.#{quote_column(@model.primary_key)} #{unique_id_expr} AS #{quote_column(@model.primary_key)} "] +
64
+ @fields.collect { |field| field.to_select_sql } +
65
+ @attributes.collect { |attribute| attribute.to_select_sql }
66
+ ).compact.join(", ")
67
+ end
68
+
69
+ def sql_where_clause(options)
70
+ logic = []
71
+ logic += [
72
+ "#{@model.quoted_table_name}.#{quote_column(@model.primary_key)} >= $start",
73
+ "#{@model.quoted_table_name}.#{quote_column(@model.primary_key)} <= $end"
74
+ ] unless @index.options[:disable_range]
75
+
76
+ if self.delta? && !@index.delta_object.clause(@model, options[:delta]).blank?
77
+ logic << "#{@index.delta_object.clause(@model, options[:delta])}"
78
+ end
79
+
80
+ logic += (@conditions || [])
81
+ logic.empty? ? "" : "WHERE #{logic.join(' AND ')}"
82
+ end
83
+
84
+ def sql_group_clause
85
+ internal_groupings = []
86
+ if @model.column_names.include?(@model.inheritance_column)
87
+ internal_groupings << "#{@model.quoted_table_name}.#{quote_column(@model.inheritance_column)}"
88
+ end
89
+
90
+ (
91
+ ["#{@model.quoted_table_name}.#{quote_column(@model.primary_key)}"] +
92
+ @fields.collect { |field| field.to_group_sql }.compact +
93
+ @attributes.collect { |attribute| attribute.to_group_sql }.compact +
94
+ @groupings + internal_groupings
95
+ ).join(", ")
96
+ end
97
+
98
+ def sql_query_pre_for_core
99
+ if self.delta? && !@index.delta_object.reset_query(@model).blank?
100
+ [@index.delta_object.reset_query(@model)]
101
+ else
102
+ []
103
+ end
104
+ end
105
+
106
+ def sql_query_pre_for_delta
107
+ [""]
108
+ end
109
+
110
+ def quote_column(column)
111
+ @model.connection.quote_column_name(column)
112
+ end
113
+
114
+ def crc_column
115
+ if @model.column_names.include?(@model.inheritance_column)
116
+ adapter.cast_to_unsigned(adapter.convert_nulls(
117
+ adapter.crc(adapter.quote_with_table(@model.inheritance_column), true),
118
+ @model.to_crc32
119
+ ))
120
+ else
121
+ @model.to_crc32.to_s
122
+ end
123
+ end
124
+ end
125
+ end
126
+ end