nixme-thinking-sphinx 0.9.7
Sign up to get free protection for your applications and to get access to all the features.
- data/LICENCE +20 -0
- data/README +52 -0
- data/lib/riddle.rb +22 -0
- data/lib/riddle/client.rb +593 -0
- data/lib/riddle/client/filter.rb +44 -0
- data/lib/riddle/client/message.rb +65 -0
- data/lib/riddle/client/response.rb +84 -0
- data/lib/test.rb +46 -0
- data/lib/thinking_sphinx.rb +82 -0
- data/lib/thinking_sphinx/active_record.rb +138 -0
- data/lib/thinking_sphinx/active_record/delta.rb +90 -0
- data/lib/thinking_sphinx/active_record/has_many_association.rb +29 -0
- data/lib/thinking_sphinx/active_record/search.rb +43 -0
- data/lib/thinking_sphinx/association.rb +140 -0
- data/lib/thinking_sphinx/attribute.rb +282 -0
- data/lib/thinking_sphinx/configuration.rb +277 -0
- data/lib/thinking_sphinx/field.rb +198 -0
- data/lib/thinking_sphinx/index.rb +334 -0
- data/lib/thinking_sphinx/index/builder.rb +212 -0
- data/lib/thinking_sphinx/index/faux_column.rb +97 -0
- data/lib/thinking_sphinx/rails_additions.rb +56 -0
- data/lib/thinking_sphinx/search.rb +455 -0
- data/spec/unit/thinking_sphinx/active_record/delta_spec.rb +185 -0
- data/spec/unit/thinking_sphinx/active_record/has_many_association_spec.rb +53 -0
- data/spec/unit/thinking_sphinx/active_record/search_spec.rb +81 -0
- data/spec/unit/thinking_sphinx/active_record_spec.rb +201 -0
- data/spec/unit/thinking_sphinx/association_spec.rb +247 -0
- data/spec/unit/thinking_sphinx/attribute_spec.rb +356 -0
- data/spec/unit/thinking_sphinx/configuration_spec.rb +476 -0
- data/spec/unit/thinking_sphinx/field_spec.rb +215 -0
- data/spec/unit/thinking_sphinx/index/builder_spec.rb +33 -0
- data/spec/unit/thinking_sphinx/index/faux_column_spec.rb +41 -0
- data/spec/unit/thinking_sphinx/index_spec.rb +230 -0
- data/spec/unit/thinking_sphinx/search_spec.rb +163 -0
- data/spec/unit/thinking_sphinx_spec.rb +107 -0
- data/tasks/thinking_sphinx_tasks.rake +1 -0
- data/tasks/thinking_sphinx_tasks.rb +86 -0
- metadata +90 -0
@@ -0,0 +1,44 @@
|
|
1
|
+
module Riddle
|
2
|
+
class Client
|
3
|
+
# Used for querying Sphinx.
|
4
|
+
class Filter
|
5
|
+
attr_accessor :attribute, :values, :exclude
|
6
|
+
|
7
|
+
# Attribute name, values (which can be an array or a range), and whether
|
8
|
+
# the filter should be exclusive.
|
9
|
+
def initialize(attribute, values, exclude=false)
|
10
|
+
@attribute, @values, @exclude = attribute, values, exclude
|
11
|
+
end
|
12
|
+
|
13
|
+
def exclude?
|
14
|
+
self.exclude
|
15
|
+
end
|
16
|
+
|
17
|
+
# Returns the message for this filter to send to the Sphinx service
|
18
|
+
def query_message
|
19
|
+
message = Message.new
|
20
|
+
|
21
|
+
message.append_string self.attribute
|
22
|
+
case self.values
|
23
|
+
when Range
|
24
|
+
if self.values.first.is_a?(Float) && self.values.last.is_a?(Float)
|
25
|
+
message.append_int FilterTypes[:float_range]
|
26
|
+
message.append_floats self.values.first, self.values.last
|
27
|
+
else
|
28
|
+
message.append_int FilterTypes[:range]
|
29
|
+
message.append_ints self.values.first, self.values.last
|
30
|
+
end
|
31
|
+
when Array
|
32
|
+
message.append_int FilterTypes[:values]
|
33
|
+
message.append_int self.values.length
|
34
|
+
# using to_f is a hack from the php client - to workaround 32bit
|
35
|
+
# signed ints on x32 platforms
|
36
|
+
message.append_ints *self.values.collect { |val| val.to_f }
|
37
|
+
end
|
38
|
+
message.append_int self.exclude? ? 1 : 0
|
39
|
+
|
40
|
+
message.to_s
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
@@ -0,0 +1,65 @@
|
|
1
|
+
module Riddle
|
2
|
+
class Client
|
3
|
+
# This class takes care of the translation of ints, strings and arrays to
|
4
|
+
# the format required by the Sphinx service.
|
5
|
+
class Message
|
6
|
+
def initialize
|
7
|
+
@message = ""
|
8
|
+
@size_method = @message.respond_to?(:bytesize) ? :bytesize : :length
|
9
|
+
end
|
10
|
+
|
11
|
+
# Append raw data (only use if you know what you're doing)
|
12
|
+
def append(*args)
|
13
|
+
return if args.length == 0
|
14
|
+
|
15
|
+
args.each { |arg| @message << arg }
|
16
|
+
end
|
17
|
+
|
18
|
+
# Append a string's length, then the string itself
|
19
|
+
def append_string(str)
|
20
|
+
@message << [str.send(@size_method)].pack('N') + str
|
21
|
+
end
|
22
|
+
|
23
|
+
# Append an integer
|
24
|
+
def append_int(int)
|
25
|
+
@message << [int].pack('N')
|
26
|
+
end
|
27
|
+
|
28
|
+
def append_64bit_int(int)
|
29
|
+
@message << [int >> 32, int & 0xFFFFFFFF].pack('NN')
|
30
|
+
end
|
31
|
+
|
32
|
+
# Append a float
|
33
|
+
def append_float(float)
|
34
|
+
@message << [float].pack('f').unpack('L*').pack("N")
|
35
|
+
end
|
36
|
+
|
37
|
+
# Append multiple integers
|
38
|
+
def append_ints(*ints)
|
39
|
+
ints.each { |int| append_int(int) }
|
40
|
+
end
|
41
|
+
|
42
|
+
def append_64bit_ints(*ints)
|
43
|
+
ints.each { |int| append_64bit_int(int) }
|
44
|
+
end
|
45
|
+
|
46
|
+
# Append multiple floats
|
47
|
+
def append_floats(*floats)
|
48
|
+
floats.each { |float| append_float(float) }
|
49
|
+
end
|
50
|
+
|
51
|
+
# Append an array of strings - first appends the length of the array,
|
52
|
+
# then each item's length and value.
|
53
|
+
def append_array(array)
|
54
|
+
append_int(array.length)
|
55
|
+
|
56
|
+
array.each { |item| append_string(item) }
|
57
|
+
end
|
58
|
+
|
59
|
+
# Returns the entire message
|
60
|
+
def to_s
|
61
|
+
@message
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
@@ -0,0 +1,84 @@
|
|
1
|
+
module Riddle
|
2
|
+
class Client
|
3
|
+
# Used to interrogate responses from the Sphinx daemon. Keep in mind none
|
4
|
+
# of the methods here check whether the data they're grabbing are what the
|
5
|
+
# user expects - it just assumes the user knows what the data stream is
|
6
|
+
# made up of.
|
7
|
+
class Response
|
8
|
+
# Create with the data to interpret
|
9
|
+
def initialize(str)
|
10
|
+
@str = str
|
11
|
+
@marker = 0
|
12
|
+
end
|
13
|
+
|
14
|
+
# Return the next string value in the stream
|
15
|
+
def next
|
16
|
+
len = next_int
|
17
|
+
result = @str[@marker, len]
|
18
|
+
@marker += len
|
19
|
+
|
20
|
+
return result
|
21
|
+
end
|
22
|
+
|
23
|
+
# Return the next integer value from the stream
|
24
|
+
def next_int
|
25
|
+
int = @str[@marker, 4].unpack('N*').first
|
26
|
+
@marker += 4
|
27
|
+
|
28
|
+
return int
|
29
|
+
end
|
30
|
+
|
31
|
+
def next_64bit_int
|
32
|
+
high, low = @str[@marker, 8].unpack('N*N*')[0..1]
|
33
|
+
@marker += 8
|
34
|
+
|
35
|
+
return (high << 32) + low
|
36
|
+
end
|
37
|
+
|
38
|
+
# Return the next float value from the stream
|
39
|
+
def next_float
|
40
|
+
float = @str[@marker, 4].unpack('N*').pack('L').unpack('f*').first
|
41
|
+
@marker += 4
|
42
|
+
|
43
|
+
return float
|
44
|
+
end
|
45
|
+
|
46
|
+
# Returns an array of string items
|
47
|
+
def next_array
|
48
|
+
count = next_int
|
49
|
+
items = []
|
50
|
+
for i in 0...count
|
51
|
+
items << self.next
|
52
|
+
end
|
53
|
+
|
54
|
+
return items
|
55
|
+
end
|
56
|
+
|
57
|
+
# Returns an array of int items
|
58
|
+
def next_int_array
|
59
|
+
count = next_int
|
60
|
+
items = []
|
61
|
+
for i in 0...count
|
62
|
+
items << self.next_int
|
63
|
+
end
|
64
|
+
|
65
|
+
return items
|
66
|
+
end
|
67
|
+
|
68
|
+
def next_float_array
|
69
|
+
count = next_int
|
70
|
+
items = []
|
71
|
+
for i in 0...count
|
72
|
+
items << self.next_float
|
73
|
+
end
|
74
|
+
|
75
|
+
return items
|
76
|
+
end
|
77
|
+
|
78
|
+
# Returns the length of the streamed data
|
79
|
+
def length
|
80
|
+
@str.length
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
84
|
+
end
|
data/lib/test.rb
ADDED
@@ -0,0 +1,46 @@
|
|
1
|
+
require 'thinking_sphinx'
|
2
|
+
|
3
|
+
ActiveRecord::Base.establish_connection(
|
4
|
+
:adapter => 'mysql',
|
5
|
+
:database => 'nullus_development',
|
6
|
+
:username => 'nullus',
|
7
|
+
:password => 'wossname',
|
8
|
+
:host => 'localhost'
|
9
|
+
)
|
10
|
+
ActiveRecord::Base.logger = Logger.new(STDERR)
|
11
|
+
|
12
|
+
class User < ActiveRecord::Base
|
13
|
+
has_many :posts, :foreign_key => "created_by"
|
14
|
+
end
|
15
|
+
|
16
|
+
class Post < ActiveRecord::Base
|
17
|
+
belongs_to :creator, :foreign_key => "created_by", :class_name => "User"
|
18
|
+
belongs_to :updater, :foreign_key => "updated_by", :class_name => "User"
|
19
|
+
belongs_to :topic
|
20
|
+
end
|
21
|
+
|
22
|
+
class Topic < ActiveRecord::Base
|
23
|
+
belongs_to :creator, :foreign_key => "created_by", :class_name => "User"
|
24
|
+
belongs_to :forum
|
25
|
+
has_many :posts
|
26
|
+
end
|
27
|
+
|
28
|
+
class Forum < ActiveRecord::Base
|
29
|
+
belongs_to :creator, :foreign_key => "created_by", :class_name => "User"
|
30
|
+
has_many :topics
|
31
|
+
end
|
32
|
+
|
33
|
+
def index
|
34
|
+
@index ||= ThinkingSphinx::Index.new(Topic) do
|
35
|
+
indexes posts.content, :as => :posts
|
36
|
+
indexes posts.creator.login, :as => :authors
|
37
|
+
|
38
|
+
has :created_at
|
39
|
+
has :id, :as => :topic_id
|
40
|
+
has :forum_id
|
41
|
+
has posts(:id), :as => :post_ids
|
42
|
+
has posts.creator(:id), :as => :user_ids
|
43
|
+
|
44
|
+
where "posts.created_at < NOW()"
|
45
|
+
end
|
46
|
+
end
|
@@ -0,0 +1,82 @@
|
|
1
|
+
require 'active_record'
|
2
|
+
require 'riddle'
|
3
|
+
|
4
|
+
require 'thinking_sphinx/active_record'
|
5
|
+
require 'thinking_sphinx/association'
|
6
|
+
require 'thinking_sphinx/attribute'
|
7
|
+
require 'thinking_sphinx/configuration'
|
8
|
+
require 'thinking_sphinx/field'
|
9
|
+
require 'thinking_sphinx/index'
|
10
|
+
require 'thinking_sphinx/rails_additions'
|
11
|
+
require 'thinking_sphinx/search'
|
12
|
+
|
13
|
+
ActiveRecord::Base.send(:include, ThinkingSphinx::ActiveRecord)
|
14
|
+
|
15
|
+
Merb::Plugins.add_rakefiles(
|
16
|
+
File.join(File.dirname(__FILE__), "..", "tasks", "thinking_sphinx_tasks")
|
17
|
+
) if defined?(Merb)
|
18
|
+
|
19
|
+
module ThinkingSphinx
|
20
|
+
module Version #:nodoc:
|
21
|
+
Major = 0
|
22
|
+
Minor = 9
|
23
|
+
Tiny = 7
|
24
|
+
|
25
|
+
String = [Major, Minor, Tiny].join('.')
|
26
|
+
end
|
27
|
+
|
28
|
+
# A ConnectionError will get thrown when a connection to Sphinx can't be
|
29
|
+
# made.
|
30
|
+
class ConnectionError < StandardError
|
31
|
+
end
|
32
|
+
|
33
|
+
# The collection of indexed models. Keep in mind that Rails lazily loads
|
34
|
+
# its classes, so this may not actually be populated with _all_ the models
|
35
|
+
# that have Sphinx indexes.
|
36
|
+
def self.indexed_models
|
37
|
+
@@indexed_models ||= []
|
38
|
+
end
|
39
|
+
|
40
|
+
# Check if index definition is disabled.
|
41
|
+
#
|
42
|
+
def self.define_indexes?
|
43
|
+
@@define_indexes = true unless defined?(@@define_indexes)
|
44
|
+
@@define_indexes == true
|
45
|
+
end
|
46
|
+
|
47
|
+
# Enable/disable indexes - you may want to do this while migrating data.
|
48
|
+
#
|
49
|
+
# ThinkingSphinx.define_indexes = false
|
50
|
+
#
|
51
|
+
def self.define_indexes=(value)
|
52
|
+
@@define_indexes = value
|
53
|
+
end
|
54
|
+
|
55
|
+
# Check if delta indexing is enabled.
|
56
|
+
#
|
57
|
+
def self.deltas_enabled?
|
58
|
+
@@deltas_enabled = true unless defined?(@@deltas_enabled)
|
59
|
+
@@deltas_enabled == true
|
60
|
+
end
|
61
|
+
|
62
|
+
# Enable/disable all delta indexing.
|
63
|
+
#
|
64
|
+
# ThinkingSphinx.deltas_enabled = false
|
65
|
+
#
|
66
|
+
def self.deltas_enabled=(value)
|
67
|
+
@@deltas_enabled = value
|
68
|
+
end
|
69
|
+
|
70
|
+
# Checks to see if MySQL will allow simplistic GROUP BY statements. If not,
|
71
|
+
# or if not using MySQL, this will return false.
|
72
|
+
#
|
73
|
+
def self.use_group_by_shortcut?
|
74
|
+
::ActiveRecord::ConnectionAdapters.constants.include?("MysqlAdapter") &&
|
75
|
+
::ActiveRecord::Base.connection.is_a?(
|
76
|
+
::ActiveRecord::ConnectionAdapters::MysqlAdapter
|
77
|
+
) &&
|
78
|
+
::ActiveRecord::Base.connection.select_all(
|
79
|
+
"SELECT @@global.sql_mode, @@session.sql_mode;"
|
80
|
+
).all? { |key,value| value.nil? || value[/ONLY_FULL_GROUP_BY/].nil? }
|
81
|
+
end
|
82
|
+
end
|
@@ -0,0 +1,138 @@
|
|
1
|
+
require 'thinking_sphinx/active_record/delta'
|
2
|
+
require 'thinking_sphinx/active_record/search'
|
3
|
+
require 'thinking_sphinx/active_record/has_many_association'
|
4
|
+
|
5
|
+
module ThinkingSphinx
|
6
|
+
# Core additions to ActiveRecord models - define_index for creating indexes
|
7
|
+
# for models. If you want to interrogate the index objects created for the
|
8
|
+
# model, you can use the class-level accessor :indexes.
|
9
|
+
#
|
10
|
+
module ActiveRecord
|
11
|
+
def self.included(base)
|
12
|
+
base.class_eval do
|
13
|
+
class << self
|
14
|
+
attr_accessor :indexes
|
15
|
+
|
16
|
+
# Allows creation of indexes for Sphinx. If you don't do this, there
|
17
|
+
# isn't much point trying to search (or using this plugin at all,
|
18
|
+
# really).
|
19
|
+
#
|
20
|
+
# An example or two:
|
21
|
+
#
|
22
|
+
# define_index
|
23
|
+
# indexes :id, :as => :model_id
|
24
|
+
# indexes name
|
25
|
+
# end
|
26
|
+
#
|
27
|
+
# You can also grab fields from associations - multiple levels deep
|
28
|
+
# if necessary.
|
29
|
+
#
|
30
|
+
# define_index do
|
31
|
+
# indexes tags.name, :as => :tag
|
32
|
+
# indexes articles.content
|
33
|
+
# indexes orders.line_items.product.name, :as => :product
|
34
|
+
# end
|
35
|
+
#
|
36
|
+
# And it will automatically concatenate multiple fields:
|
37
|
+
#
|
38
|
+
# define_index do
|
39
|
+
# indexes [author.first_name, author.last_name], :as => :author
|
40
|
+
# end
|
41
|
+
#
|
42
|
+
# The #indexes method is for fields - if you want attributes, use
|
43
|
+
# #has instead. All the same rules apply - but keep in mind that
|
44
|
+
# attributes are for sorting, grouping and filtering, not searching.
|
45
|
+
#
|
46
|
+
# define_index do
|
47
|
+
# # fields ...
|
48
|
+
#
|
49
|
+
# has created_at, updated_at
|
50
|
+
# end
|
51
|
+
#
|
52
|
+
# One last feature is the delta index. This requires the model to
|
53
|
+
# have a boolean field named 'delta', and is enabled as follows:
|
54
|
+
#
|
55
|
+
# define_index do
|
56
|
+
# # fields ...
|
57
|
+
# # attributes ...
|
58
|
+
#
|
59
|
+
# set_property :delta => true
|
60
|
+
# end
|
61
|
+
#
|
62
|
+
# Check out the more detailed documentation for each of these methods
|
63
|
+
# at ThinkingSphinx::Index::Builder.
|
64
|
+
#
|
65
|
+
def define_index(&block)
|
66
|
+
return unless ThinkingSphinx.define_indexes?
|
67
|
+
|
68
|
+
@indexes ||= []
|
69
|
+
index = Index.new(self, &block)
|
70
|
+
|
71
|
+
@indexes << index
|
72
|
+
unless ThinkingSphinx.indexed_models.include?(self.name)
|
73
|
+
ThinkingSphinx.indexed_models << self.name
|
74
|
+
end
|
75
|
+
|
76
|
+
if index.delta?
|
77
|
+
before_save :toggle_delta
|
78
|
+
after_commit :index_delta
|
79
|
+
end
|
80
|
+
|
81
|
+
after_destroy :toggle_deleted
|
82
|
+
|
83
|
+
index
|
84
|
+
end
|
85
|
+
alias_method :sphinx_index, :define_index
|
86
|
+
|
87
|
+
# Generate a unique CRC value for the model's name, to use to
|
88
|
+
# determine which Sphinx documents belong to which AR records.
|
89
|
+
#
|
90
|
+
# Really only written for internal use - but hey, if it's useful to
|
91
|
+
# you in some other way, awesome.
|
92
|
+
#
|
93
|
+
def to_crc32
|
94
|
+
result = 0xFFFFFFFF
|
95
|
+
self.name.each_byte do |byte|
|
96
|
+
result ^= byte
|
97
|
+
8.times do
|
98
|
+
result = (result >> 1) ^ (0xEDB88320 * (result & 1))
|
99
|
+
end
|
100
|
+
end
|
101
|
+
result ^ 0xFFFFFFFF
|
102
|
+
end
|
103
|
+
end
|
104
|
+
end
|
105
|
+
|
106
|
+
base.send(:include, ThinkingSphinx::ActiveRecord::Delta)
|
107
|
+
base.send(:include, ThinkingSphinx::ActiveRecord::Search)
|
108
|
+
|
109
|
+
::ActiveRecord::Associations::HasManyAssociation.send(
|
110
|
+
:include, ThinkingSphinx::ActiveRecord::HasManyAssociation
|
111
|
+
)
|
112
|
+
::ActiveRecord::Associations::HasManyThroughAssociation.send(
|
113
|
+
:include, ThinkingSphinx::ActiveRecord::HasManyAssociation
|
114
|
+
)
|
115
|
+
end
|
116
|
+
|
117
|
+
def in_core_index?
|
118
|
+
@in_core_index ||= self.class.search_for_id(self.id, "#{self.class.name.downcase}_core")
|
119
|
+
end
|
120
|
+
|
121
|
+
def toggle_deleted
|
122
|
+
config = ThinkingSphinx::Configuration.new
|
123
|
+
client = Riddle::Client.new config.address, config.port
|
124
|
+
|
125
|
+
client.update(
|
126
|
+
"#{self.class.indexes.first.name}_core",
|
127
|
+
['sphinx_deleted'],
|
128
|
+
{self.id => 1}
|
129
|
+
) if self.in_core_index?
|
130
|
+
|
131
|
+
client.update(
|
132
|
+
"#{self.class.indexes.first.name}_delta",
|
133
|
+
['sphinx_deleted'],
|
134
|
+
{self.id => 1}
|
135
|
+
) if self.class.indexes.any? { |index| index.delta? } && self.delta?
|
136
|
+
end
|
137
|
+
end
|
138
|
+
end
|