freelancing-god-thinking-sphinx 0.9.5
Sign up to get free protection for your applications and to get access to all the features.
- data/LICENCE +20 -0
- data/README +25 -0
- data/lib/riddle.rb +22 -0
- data/lib/riddle/client.rb +593 -0
- data/lib/riddle/client/filter.rb +44 -0
- data/lib/riddle/client/message.rb +65 -0
- data/lib/riddle/client/response.rb +84 -0
- data/lib/test.rb +46 -0
- data/lib/thinking_sphinx.rb +79 -0
- data/lib/thinking_sphinx/active_record.rb +115 -0
- data/lib/thinking_sphinx/active_record/delta.rb +86 -0
- data/lib/thinking_sphinx/active_record/has_many_association.rb +29 -0
- data/lib/thinking_sphinx/active_record/search.rb +36 -0
- data/lib/thinking_sphinx/association.rb +140 -0
- data/lib/thinking_sphinx/attribute.rb +279 -0
- data/lib/thinking_sphinx/configuration.rb +275 -0
- data/lib/thinking_sphinx/field.rb +186 -0
- data/lib/thinking_sphinx/index.rb +234 -0
- data/lib/thinking_sphinx/index/builder.rb +197 -0
- data/lib/thinking_sphinx/index/faux_column.rb +97 -0
- data/lib/thinking_sphinx/rails_additions.rb +56 -0
- data/lib/thinking_sphinx/search.rb +413 -0
- data/spec/unit/thinking_sphinx/active_record/delta_spec.rb +184 -0
- data/spec/unit/thinking_sphinx/active_record/has_many_association_spec.rb +53 -0
- data/spec/unit/thinking_sphinx/active_record/search_spec.rb +0 -0
- data/spec/unit/thinking_sphinx/active_record_spec.rb +85 -0
- data/spec/unit/thinking_sphinx/association_spec.rb +0 -0
- data/spec/unit/thinking_sphinx/attribute_spec.rb +73 -0
- data/spec/unit/thinking_sphinx/configuration_spec.rb +7 -0
- data/spec/unit/thinking_sphinx/field_spec.rb +51 -0
- data/spec/unit/thinking_sphinx/index/builder_spec.rb +33 -0
- data/spec/unit/thinking_sphinx/index/faux_column_spec.rb +41 -0
- data/spec/unit/thinking_sphinx/index_spec.rb +5 -0
- data/spec/unit/thinking_sphinx/search_spec.rb +121 -0
- data/spec/unit/thinking_sphinx_spec.rb +82 -0
- data/tasks/thinking_sphinx_tasks.rake +1 -0
- data/tasks/thinking_sphinx_tasks.rb +86 -0
- metadata +90 -0
@@ -0,0 +1,44 @@
|
|
1
|
+
module Riddle
|
2
|
+
class Client
|
3
|
+
# Used for querying Sphinx.
|
4
|
+
class Filter
|
5
|
+
attr_accessor :attribute, :values, :exclude
|
6
|
+
|
7
|
+
# Attribute name, values (which can be an array or a range), and whether
|
8
|
+
# the filter should be exclusive.
|
9
|
+
def initialize(attribute, values, exclude=false)
|
10
|
+
@attribute, @values, @exclude = attribute, values, exclude
|
11
|
+
end
|
12
|
+
|
13
|
+
def exclude?
|
14
|
+
self.exclude
|
15
|
+
end
|
16
|
+
|
17
|
+
# Returns the message for this filter to send to the Sphinx service
|
18
|
+
def query_message
|
19
|
+
message = Message.new
|
20
|
+
|
21
|
+
message.append_string self.attribute
|
22
|
+
case self.values
|
23
|
+
when Range
|
24
|
+
if self.values.first.is_a?(Float) && self.values.last.is_a?(Float)
|
25
|
+
message.append_int FilterTypes[:float_range]
|
26
|
+
message.append_floats self.values.first, self.values.last
|
27
|
+
else
|
28
|
+
message.append_int FilterTypes[:range]
|
29
|
+
message.append_ints self.values.first, self.values.last
|
30
|
+
end
|
31
|
+
when Array
|
32
|
+
message.append_int FilterTypes[:values]
|
33
|
+
message.append_int self.values.length
|
34
|
+
# using to_f is a hack from the php client - to workaround 32bit
|
35
|
+
# signed ints on x32 platforms
|
36
|
+
message.append_ints *self.values.collect { |val| val.to_f }
|
37
|
+
end
|
38
|
+
message.append_int self.exclude? ? 1 : 0
|
39
|
+
|
40
|
+
message.to_s
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
@@ -0,0 +1,65 @@
|
|
1
|
+
module Riddle
|
2
|
+
class Client
|
3
|
+
# This class takes care of the translation of ints, strings and arrays to
|
4
|
+
# the format required by the Sphinx service.
|
5
|
+
class Message
|
6
|
+
def initialize
|
7
|
+
@message = ""
|
8
|
+
@size_method = @message.respond_to?(:bytesize) ? :bytesize : :length
|
9
|
+
end
|
10
|
+
|
11
|
+
# Append raw data (only use if you know what you're doing)
|
12
|
+
def append(*args)
|
13
|
+
return if args.length == 0
|
14
|
+
|
15
|
+
args.each { |arg| @message << arg }
|
16
|
+
end
|
17
|
+
|
18
|
+
# Append a string's length, then the string itself
|
19
|
+
def append_string(str)
|
20
|
+
@message << [str.send(@size_method)].pack('N') + str
|
21
|
+
end
|
22
|
+
|
23
|
+
# Append an integer
|
24
|
+
def append_int(int)
|
25
|
+
@message << [int].pack('N')
|
26
|
+
end
|
27
|
+
|
28
|
+
def append_64bit_int(int)
|
29
|
+
@message << [int >> 32, int & 0xFFFFFFFF].pack('NN')
|
30
|
+
end
|
31
|
+
|
32
|
+
# Append a float
|
33
|
+
def append_float(float)
|
34
|
+
@message << [float].pack('f').unpack('L*').pack("N")
|
35
|
+
end
|
36
|
+
|
37
|
+
# Append multiple integers
|
38
|
+
def append_ints(*ints)
|
39
|
+
ints.each { |int| append_int(int) }
|
40
|
+
end
|
41
|
+
|
42
|
+
def append_64bit_ints(*ints)
|
43
|
+
ints.each { |int| append_64bit_int(int) }
|
44
|
+
end
|
45
|
+
|
46
|
+
# Append multiple floats
|
47
|
+
def append_floats(*floats)
|
48
|
+
floats.each { |float| append_float(float) }
|
49
|
+
end
|
50
|
+
|
51
|
+
# Append an array of strings - first appends the length of the array,
|
52
|
+
# then each item's length and value.
|
53
|
+
def append_array(array)
|
54
|
+
append_int(array.length)
|
55
|
+
|
56
|
+
array.each { |item| append_string(item) }
|
57
|
+
end
|
58
|
+
|
59
|
+
# Returns the entire message
|
60
|
+
def to_s
|
61
|
+
@message
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
@@ -0,0 +1,84 @@
|
|
1
|
+
module Riddle
|
2
|
+
class Client
|
3
|
+
# Used to interrogate responses from the Sphinx daemon. Keep in mind none
|
4
|
+
# of the methods here check whether the data they're grabbing are what the
|
5
|
+
# user expects - it just assumes the user knows what the data stream is
|
6
|
+
# made up of.
|
7
|
+
class Response
|
8
|
+
# Create with the data to interpret
|
9
|
+
def initialize(str)
|
10
|
+
@str = str
|
11
|
+
@marker = 0
|
12
|
+
end
|
13
|
+
|
14
|
+
# Return the next string value in the stream
|
15
|
+
def next
|
16
|
+
len = next_int
|
17
|
+
result = @str[@marker, len]
|
18
|
+
@marker += len
|
19
|
+
|
20
|
+
return result
|
21
|
+
end
|
22
|
+
|
23
|
+
# Return the next integer value from the stream
|
24
|
+
def next_int
|
25
|
+
int = @str[@marker, 4].unpack('N*').first
|
26
|
+
@marker += 4
|
27
|
+
|
28
|
+
return int
|
29
|
+
end
|
30
|
+
|
31
|
+
def next_64bit_int
|
32
|
+
high, low = @str[@marker, 8].unpack('N*N*')[0..1]
|
33
|
+
@marker += 8
|
34
|
+
|
35
|
+
return (high << 32) + low
|
36
|
+
end
|
37
|
+
|
38
|
+
# Return the next float value from the stream
|
39
|
+
def next_float
|
40
|
+
float = @str[@marker, 4].unpack('N*').pack('L').unpack('f*').first
|
41
|
+
@marker += 4
|
42
|
+
|
43
|
+
return float
|
44
|
+
end
|
45
|
+
|
46
|
+
# Returns an array of string items
|
47
|
+
def next_array
|
48
|
+
count = next_int
|
49
|
+
items = []
|
50
|
+
for i in 0...count
|
51
|
+
items << self.next
|
52
|
+
end
|
53
|
+
|
54
|
+
return items
|
55
|
+
end
|
56
|
+
|
57
|
+
# Returns an array of int items
|
58
|
+
def next_int_array
|
59
|
+
count = next_int
|
60
|
+
items = []
|
61
|
+
for i in 0...count
|
62
|
+
items << self.next_int
|
63
|
+
end
|
64
|
+
|
65
|
+
return items
|
66
|
+
end
|
67
|
+
|
68
|
+
def next_float_array
|
69
|
+
count = next_int
|
70
|
+
items = []
|
71
|
+
for i in 0...count
|
72
|
+
items << self.next_float
|
73
|
+
end
|
74
|
+
|
75
|
+
return items
|
76
|
+
end
|
77
|
+
|
78
|
+
# Returns the length of the streamed data
|
79
|
+
def length
|
80
|
+
@str.length
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
84
|
+
end
|
data/lib/test.rb
ADDED
@@ -0,0 +1,46 @@
|
|
1
|
+
require 'thinking_sphinx'
|
2
|
+
|
3
|
+
ActiveRecord::Base.establish_connection(
|
4
|
+
:adapter => 'mysql',
|
5
|
+
:database => 'nullus_development',
|
6
|
+
:username => 'nullus',
|
7
|
+
:password => 'wossname',
|
8
|
+
:host => 'localhost'
|
9
|
+
)
|
10
|
+
ActiveRecord::Base.logger = Logger.new(STDERR)
|
11
|
+
|
12
|
+
class User < ActiveRecord::Base
|
13
|
+
has_many :posts, :foreign_key => "created_by"
|
14
|
+
end
|
15
|
+
|
16
|
+
class Post < ActiveRecord::Base
|
17
|
+
belongs_to :creator, :foreign_key => "created_by", :class_name => "User"
|
18
|
+
belongs_to :updater, :foreign_key => "updated_by", :class_name => "User"
|
19
|
+
belongs_to :topic
|
20
|
+
end
|
21
|
+
|
22
|
+
class Topic < ActiveRecord::Base
|
23
|
+
belongs_to :creator, :foreign_key => "created_by", :class_name => "User"
|
24
|
+
belongs_to :forum
|
25
|
+
has_many :posts
|
26
|
+
end
|
27
|
+
|
28
|
+
class Forum < ActiveRecord::Base
|
29
|
+
belongs_to :creator, :foreign_key => "created_by", :class_name => "User"
|
30
|
+
has_many :topics
|
31
|
+
end
|
32
|
+
|
33
|
+
def index
|
34
|
+
@index ||= ThinkingSphinx::Index.new(Topic) do
|
35
|
+
indexes posts.content, :as => :posts
|
36
|
+
indexes posts.creator.login, :as => :authors
|
37
|
+
|
38
|
+
has :created_at
|
39
|
+
has :id, :as => :topic_id
|
40
|
+
has :forum_id
|
41
|
+
has posts(:id), :as => :post_ids
|
42
|
+
has posts.creator(:id), :as => :user_ids
|
43
|
+
|
44
|
+
where "posts.created_at < NOW()"
|
45
|
+
end
|
46
|
+
end
|
@@ -0,0 +1,79 @@
|
|
1
|
+
require 'active_record'
|
2
|
+
require 'riddle'
|
3
|
+
|
4
|
+
require 'thinking_sphinx/active_record'
|
5
|
+
require 'thinking_sphinx/association'
|
6
|
+
require 'thinking_sphinx/attribute'
|
7
|
+
require 'thinking_sphinx/configuration'
|
8
|
+
require 'thinking_sphinx/field'
|
9
|
+
require 'thinking_sphinx/index'
|
10
|
+
require 'thinking_sphinx/rails_additions'
|
11
|
+
require 'thinking_sphinx/search'
|
12
|
+
|
13
|
+
ActiveRecord::Base.send(:include, ThinkingSphinx::ActiveRecord)
|
14
|
+
|
15
|
+
Merb::Plugins.add_rakefiles(
|
16
|
+
File.join(File.dirname(__FILE__), "..", "tasks", "thinking_sphinx_tasks")
|
17
|
+
) if defined?(Merb)
|
18
|
+
|
19
|
+
module ThinkingSphinx
|
20
|
+
module Version #:nodoc:
|
21
|
+
Major = 0
|
22
|
+
Minor = 9
|
23
|
+
Tiny = 5
|
24
|
+
|
25
|
+
String = [Major, Minor, Tiny].join('.')
|
26
|
+
end
|
27
|
+
|
28
|
+
# A ConnectionError will get thrown when a connection to Sphinx can't be
|
29
|
+
# made.
|
30
|
+
class ConnectionError < StandardError
|
31
|
+
end
|
32
|
+
|
33
|
+
# The collection of indexed models. Keep in mind that Rails lazily loads
|
34
|
+
# its classes, so this may not actually be populated with _all_ the models
|
35
|
+
# that have Sphinx indexes.
|
36
|
+
def self.indexed_models
|
37
|
+
@@indexed_models ||= []
|
38
|
+
end
|
39
|
+
|
40
|
+
# Check if index definition is disabled.
|
41
|
+
#
|
42
|
+
def self.define_indexes?
|
43
|
+
@@define_indexes = true unless defined?(@@define_indexes)
|
44
|
+
@@define_indexes == true
|
45
|
+
end
|
46
|
+
|
47
|
+
# Enable/disable indexes - you may want to do this while migrating data.
|
48
|
+
#
|
49
|
+
# ThinkingSphinx.define_indexes = false
|
50
|
+
#
|
51
|
+
def self.define_indexes=(value)
|
52
|
+
@@define_indexes = value
|
53
|
+
end
|
54
|
+
|
55
|
+
# Check if delta indexing is enabled.
|
56
|
+
#
|
57
|
+
def self.deltas_enabled?
|
58
|
+
@@deltas_enabled = true unless defined?(@@deltas_enabled)
|
59
|
+
@@deltas_enabled == true
|
60
|
+
end
|
61
|
+
|
62
|
+
# Enable/disable all delta indexing.
|
63
|
+
#
|
64
|
+
# ThinkingSphinx.deltas_enabled = false
|
65
|
+
#
|
66
|
+
def self.deltas_enabled=(value)
|
67
|
+
@@deltas_enabled = value
|
68
|
+
end
|
69
|
+
|
70
|
+
def self.use_group_by_shortcut?
|
71
|
+
::ActiveRecord::ConnectionAdapters.constants.include?("MysqlAdapter") &&
|
72
|
+
::ActiveRecord::Base.connection.is_a?(
|
73
|
+
::ActiveRecord::ConnectionAdapters::MysqlAdapter
|
74
|
+
) &&
|
75
|
+
::ActiveRecord::Base.connection.select_all(
|
76
|
+
"SELECT @@global.sql_mode, @@session.sql_mode;"
|
77
|
+
).all? { |key,value| value.nil? || value[/ONLY_FULL_GROUP_BY/].nil? }
|
78
|
+
end
|
79
|
+
end
|
@@ -0,0 +1,115 @@
|
|
1
|
+
require 'thinking_sphinx/active_record/delta'
|
2
|
+
require 'thinking_sphinx/active_record/search'
|
3
|
+
require 'thinking_sphinx/active_record/has_many_association'
|
4
|
+
|
5
|
+
module ThinkingSphinx
|
6
|
+
# Core additions to ActiveRecord models - define_index for creating indexes
|
7
|
+
# for models. If you want to interrogate the index objects created for the
|
8
|
+
# model, you can use the class-level accessor :indexes.
|
9
|
+
#
|
10
|
+
module ActiveRecord
|
11
|
+
def self.included(base)
|
12
|
+
base.class_eval do
|
13
|
+
class << self
|
14
|
+
attr_accessor :indexes
|
15
|
+
|
16
|
+
# Allows creation of indexes for Sphinx. If you don't do this, there
|
17
|
+
# isn't much point trying to search (or using this plugin at all,
|
18
|
+
# really).
|
19
|
+
#
|
20
|
+
# An example or two:
|
21
|
+
#
|
22
|
+
# define_index
|
23
|
+
# indexes :id, :as => :model_id
|
24
|
+
# indexes name
|
25
|
+
# end
|
26
|
+
#
|
27
|
+
# You can also grab fields from associations - multiple levels deep
|
28
|
+
# if necessary.
|
29
|
+
#
|
30
|
+
# define_index do
|
31
|
+
# indexes tags.name, :as => :tag
|
32
|
+
# indexes articles.content
|
33
|
+
# indexes orders.line_items.product.name, :as => :product
|
34
|
+
# end
|
35
|
+
#
|
36
|
+
# And it will automatically concatenate multiple fields:
|
37
|
+
#
|
38
|
+
# define_index do
|
39
|
+
# indexes [author.first_name, author.last_name], :as => :author
|
40
|
+
# end
|
41
|
+
#
|
42
|
+
# The #indexes method is for fields - if you want attributes, use
|
43
|
+
# #has instead. All the same rules apply - but keep in mind that
|
44
|
+
# attributes are for sorting, grouping and filtering, not searching.
|
45
|
+
#
|
46
|
+
# define_index do
|
47
|
+
# # fields ...
|
48
|
+
#
|
49
|
+
# has created_at, updated_at
|
50
|
+
# end
|
51
|
+
#
|
52
|
+
# One last feature is the delta index. This requires the model to
|
53
|
+
# have a boolean field named 'delta', and is enabled as follows:
|
54
|
+
#
|
55
|
+
# define_index do
|
56
|
+
# # fields ...
|
57
|
+
# # attributes ...
|
58
|
+
#
|
59
|
+
# set_property :delta => true
|
60
|
+
# end
|
61
|
+
#
|
62
|
+
# Check out the more detailed documentation for each of these methods
|
63
|
+
# at ThinkingSphinx::Index::Builder.
|
64
|
+
#
|
65
|
+
def define_index(&block)
|
66
|
+
return unless ThinkingSphinx.define_indexes?
|
67
|
+
|
68
|
+
@indexes ||= []
|
69
|
+
index = Index.new(self, &block)
|
70
|
+
|
71
|
+
@indexes << index
|
72
|
+
unless ThinkingSphinx.indexed_models.include?(self.name)
|
73
|
+
ThinkingSphinx.indexed_models << self.name
|
74
|
+
end
|
75
|
+
|
76
|
+
if index.delta?
|
77
|
+
before_save :toggle_delta
|
78
|
+
after_commit :index_delta
|
79
|
+
end
|
80
|
+
|
81
|
+
index
|
82
|
+
end
|
83
|
+
alias_method :sphinx_index, :define_index
|
84
|
+
|
85
|
+
# Generate a unique CRC value for the model's name, to use to
|
86
|
+
# determine which Sphinx documents belong to which AR records.
|
87
|
+
#
|
88
|
+
# Really only written for internal use - but hey, if it's useful to
|
89
|
+
# you in some other way, awesome.
|
90
|
+
#
|
91
|
+
def to_crc32
|
92
|
+
result = 0xFFFFFFFF
|
93
|
+
self.name.each_byte do |byte|
|
94
|
+
result ^= byte
|
95
|
+
8.times do
|
96
|
+
result = (result >> 1) ^ (0xEDB88320 * (result & 1))
|
97
|
+
end
|
98
|
+
end
|
99
|
+
result ^ 0xFFFFFFFF
|
100
|
+
end
|
101
|
+
end
|
102
|
+
end
|
103
|
+
|
104
|
+
base.send(:include, ThinkingSphinx::ActiveRecord::Delta)
|
105
|
+
base.send(:include, ThinkingSphinx::ActiveRecord::Search)
|
106
|
+
|
107
|
+
::ActiveRecord::Associations::HasManyAssociation.send(
|
108
|
+
:include, ThinkingSphinx::ActiveRecord::HasManyAssociation
|
109
|
+
)
|
110
|
+
::ActiveRecord::Associations::HasManyThroughAssociation.send(
|
111
|
+
:include, ThinkingSphinx::ActiveRecord::HasManyAssociation
|
112
|
+
)
|
113
|
+
end
|
114
|
+
end
|
115
|
+
end
|
@@ -0,0 +1,86 @@
|
|
1
|
+
module ThinkingSphinx
|
2
|
+
module ActiveRecord
|
3
|
+
# This module contains all the delta-related code for models. There isn't
|
4
|
+
# really anything you need to call manually in here - except perhaps
|
5
|
+
# index_delta, but not sure what reason why.
|
6
|
+
#
|
7
|
+
module Delta
|
8
|
+
# Code for after_commit callback is written by Eli Miller:
|
9
|
+
# http://elimiller.blogspot.com/2007/06/proper-cache-expiry-with-aftercommit.html
|
10
|
+
# with slight modification from Joost Hietbrink.
|
11
|
+
#
|
12
|
+
def self.included(base)
|
13
|
+
base.class_eval do
|
14
|
+
# The define_callbacks method was added post Rails 2.0.2 - if it
|
15
|
+
# doesn't exist, we define the callback manually
|
16
|
+
#
|
17
|
+
if respond_to?(:define_callbacks)
|
18
|
+
define_callbacks :after_commit
|
19
|
+
else
|
20
|
+
class << self
|
21
|
+
# Handle after_commit callbacks - call all the registered callbacks.
|
22
|
+
#
|
23
|
+
def after_commit(*callbacks, &block)
|
24
|
+
callbacks << block if block_given?
|
25
|
+
write_inheritable_array(:after_commit, callbacks)
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
# Normal boolean save wrapped in a handler for the after_commit
|
31
|
+
# callback.
|
32
|
+
#
|
33
|
+
def save_with_after_commit_callback(*args)
|
34
|
+
value = save_without_after_commit_callback(*args)
|
35
|
+
callback(:after_commit) if value
|
36
|
+
return value
|
37
|
+
end
|
38
|
+
|
39
|
+
alias_method_chain :save, :after_commit_callback
|
40
|
+
|
41
|
+
# Forceful save wrapped in a handler for the after_commit callback.
|
42
|
+
#
|
43
|
+
def save_with_after_commit_callback!(*args)
|
44
|
+
value = save_without_after_commit_callback!(*args)
|
45
|
+
callback(:after_commit) if value
|
46
|
+
return value
|
47
|
+
end
|
48
|
+
|
49
|
+
alias_method_chain :save!, :after_commit_callback
|
50
|
+
|
51
|
+
# Normal destroy wrapped in a handler for the after_commit callback.
|
52
|
+
#
|
53
|
+
def destroy_with_after_commit_callback
|
54
|
+
value = destroy_without_after_commit_callback
|
55
|
+
callback(:after_commit) if value
|
56
|
+
return value
|
57
|
+
end
|
58
|
+
|
59
|
+
alias_method_chain :destroy, :after_commit_callback
|
60
|
+
|
61
|
+
private
|
62
|
+
|
63
|
+
# Set the delta value for the model to be true.
|
64
|
+
def toggle_delta
|
65
|
+
self.delta = true
|
66
|
+
end
|
67
|
+
|
68
|
+
# Build the delta index for the related model. This won't be called
|
69
|
+
# if running in the test environment.
|
70
|
+
#
|
71
|
+
def index_delta
|
72
|
+
if ThinkingSphinx::Configuration.environment == "test" ||
|
73
|
+
!ThinkingSphinx.deltas_enabled?
|
74
|
+
return true
|
75
|
+
end
|
76
|
+
|
77
|
+
configuration = ThinkingSphinx::Configuration.new
|
78
|
+
system "indexer --config #{configuration.config_file} --rotate #{self.class.name.downcase}_delta"
|
79
|
+
|
80
|
+
true
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
84
|
+
end
|
85
|
+
end
|
86
|
+
end
|