thinking-sphinx 1.3.2 → 1.3.3
Sign up to get free protection for your applications and to get access to all the features.
- data/VERSION.yml +1 -1
- data/features/support/env.rb +2 -1
- data/lib/thinking_sphinx.rb +0 -1
- data/lib/thinking_sphinx/0.9.8.rb +3 -0
- data/lib/thinking_sphinx/0.9.9.rb +3 -0
- data/rails/init.rb +1 -1
- data/tasks/distribution.rb +3 -4
- metadata +24 -26
- data/vendor/after_commit/LICENSE +0 -20
- data/vendor/after_commit/README +0 -16
- data/vendor/after_commit/Rakefile +0 -22
- data/vendor/after_commit/init.rb +0 -8
- data/vendor/after_commit/lib/after_commit.rb +0 -45
- data/vendor/after_commit/lib/after_commit/active_record.rb +0 -114
- data/vendor/after_commit/lib/after_commit/connection_adapters.rb +0 -103
- data/vendor/after_commit/test/after_commit_test.rb +0 -53
- data/vendor/riddle/lib/riddle.rb +0 -30
- data/vendor/riddle/lib/riddle/client.rb +0 -635
- data/vendor/riddle/lib/riddle/client/filter.rb +0 -53
- data/vendor/riddle/lib/riddle/client/message.rb +0 -66
- data/vendor/riddle/lib/riddle/client/response.rb +0 -84
- data/vendor/riddle/lib/riddle/configuration.rb +0 -33
- data/vendor/riddle/lib/riddle/configuration/distributed_index.rb +0 -48
- data/vendor/riddle/lib/riddle/configuration/index.rb +0 -142
- data/vendor/riddle/lib/riddle/configuration/indexer.rb +0 -19
- data/vendor/riddle/lib/riddle/configuration/remote_index.rb +0 -17
- data/vendor/riddle/lib/riddle/configuration/searchd.rb +0 -25
- data/vendor/riddle/lib/riddle/configuration/section.rb +0 -43
- data/vendor/riddle/lib/riddle/configuration/source.rb +0 -23
- data/vendor/riddle/lib/riddle/configuration/sql_source.rb +0 -34
- data/vendor/riddle/lib/riddle/configuration/xml_source.rb +0 -28
- data/vendor/riddle/lib/riddle/controller.rb +0 -55
@@ -1,53 +0,0 @@
|
|
1
|
-
module Riddle
|
2
|
-
class Client
|
3
|
-
# Used for querying Sphinx.
|
4
|
-
class Filter
|
5
|
-
attr_accessor :attribute, :values, :exclude
|
6
|
-
|
7
|
-
# Attribute name, values (which can be an array or a range), and whether
|
8
|
-
# the filter should be exclusive.
|
9
|
-
def initialize(attribute, values, exclude=false)
|
10
|
-
@attribute, @values, @exclude = attribute, values, exclude
|
11
|
-
end
|
12
|
-
|
13
|
-
def exclude?
|
14
|
-
self.exclude
|
15
|
-
end
|
16
|
-
|
17
|
-
# Returns the message for this filter to send to the Sphinx service
|
18
|
-
def query_message
|
19
|
-
message = Message.new
|
20
|
-
|
21
|
-
message.append_string self.attribute.to_s
|
22
|
-
case self.values
|
23
|
-
when Range
|
24
|
-
if self.values.first.is_a?(Float) && self.values.last.is_a?(Float)
|
25
|
-
message.append_int FilterTypes[:float_range]
|
26
|
-
message.append_floats self.values.first, self.values.last
|
27
|
-
else
|
28
|
-
message.append_int FilterTypes[:range]
|
29
|
-
message.append_ints self.values.first, self.values.last
|
30
|
-
end
|
31
|
-
when Array
|
32
|
-
message.append_int FilterTypes[:values]
|
33
|
-
message.append_int self.values.length
|
34
|
-
# using to_f is a hack from the php client - to workaround 32bit
|
35
|
-
# signed ints on x32 platforms
|
36
|
-
message.append_ints *self.values.collect { |val|
|
37
|
-
case val
|
38
|
-
when TrueClass
|
39
|
-
1.0
|
40
|
-
when FalseClass
|
41
|
-
0.0
|
42
|
-
else
|
43
|
-
val.to_f
|
44
|
-
end
|
45
|
-
}
|
46
|
-
end
|
47
|
-
message.append_int self.exclude? ? 1 : 0
|
48
|
-
|
49
|
-
message.to_s
|
50
|
-
end
|
51
|
-
end
|
52
|
-
end
|
53
|
-
end
|
@@ -1,66 +0,0 @@
|
|
1
|
-
module Riddle
|
2
|
-
class Client
|
3
|
-
# This class takes care of the translation of ints, strings and arrays to
|
4
|
-
# the format required by the Sphinx service.
|
5
|
-
class Message
|
6
|
-
def initialize
|
7
|
-
@message = ""
|
8
|
-
@size_method = @message.respond_to?(:bytesize) ? :bytesize : :length
|
9
|
-
end
|
10
|
-
|
11
|
-
# Append raw data (only use if you know what you're doing)
|
12
|
-
def append(*args)
|
13
|
-
args.each { |arg| @message << arg }
|
14
|
-
end
|
15
|
-
|
16
|
-
# Append a string's length, then the string itself
|
17
|
-
def append_string(str)
|
18
|
-
string = str.respond_to?(:force_encoding) ?
|
19
|
-
str.dup.force_encoding('ASCII-8BIT') : str
|
20
|
-
|
21
|
-
@message << [string.send(@size_method)].pack('N') + string
|
22
|
-
end
|
23
|
-
|
24
|
-
# Append an integer
|
25
|
-
def append_int(int)
|
26
|
-
@message << [int].pack('N')
|
27
|
-
end
|
28
|
-
|
29
|
-
def append_64bit_int(int)
|
30
|
-
@message << [int >> 32, int & 0xFFFFFFFF].pack('NN')
|
31
|
-
end
|
32
|
-
|
33
|
-
# Append a float
|
34
|
-
def append_float(float)
|
35
|
-
@message << [float].pack('f').unpack('L*').pack("N")
|
36
|
-
end
|
37
|
-
|
38
|
-
# Append multiple integers
|
39
|
-
def append_ints(*ints)
|
40
|
-
ints.each { |int| append_int(int) }
|
41
|
-
end
|
42
|
-
|
43
|
-
def append_64bit_ints(*ints)
|
44
|
-
ints.each { |int| append_64bit_int(int) }
|
45
|
-
end
|
46
|
-
|
47
|
-
# Append multiple floats
|
48
|
-
def append_floats(*floats)
|
49
|
-
floats.each { |float| append_float(float) }
|
50
|
-
end
|
51
|
-
|
52
|
-
# Append an array of strings - first appends the length of the array,
|
53
|
-
# then each item's length and value.
|
54
|
-
def append_array(array)
|
55
|
-
append_int(array.length)
|
56
|
-
|
57
|
-
array.each { |item| append_string(item) }
|
58
|
-
end
|
59
|
-
|
60
|
-
# Returns the entire message
|
61
|
-
def to_s
|
62
|
-
@message
|
63
|
-
end
|
64
|
-
end
|
65
|
-
end
|
66
|
-
end
|
@@ -1,84 +0,0 @@
|
|
1
|
-
module Riddle
|
2
|
-
class Client
|
3
|
-
# Used to interrogate responses from the Sphinx daemon. Keep in mind none
|
4
|
-
# of the methods here check whether the data they're grabbing are what the
|
5
|
-
# user expects - it just assumes the user knows what the data stream is
|
6
|
-
# made up of.
|
7
|
-
class Response
|
8
|
-
# Create with the data to interpret
|
9
|
-
def initialize(str)
|
10
|
-
@str = str
|
11
|
-
@marker = 0
|
12
|
-
end
|
13
|
-
|
14
|
-
# Return the next string value in the stream
|
15
|
-
def next
|
16
|
-
len = next_int
|
17
|
-
result = @str[@marker, len]
|
18
|
-
@marker += len
|
19
|
-
|
20
|
-
return result
|
21
|
-
end
|
22
|
-
|
23
|
-
# Return the next integer value from the stream
|
24
|
-
def next_int
|
25
|
-
int = @str[@marker, 4].unpack('N*').first
|
26
|
-
@marker += 4
|
27
|
-
|
28
|
-
return int
|
29
|
-
end
|
30
|
-
|
31
|
-
def next_64bit_int
|
32
|
-
high, low = @str[@marker, 8].unpack('N*N*')[0..1]
|
33
|
-
@marker += 8
|
34
|
-
|
35
|
-
return (high << 32) + low
|
36
|
-
end
|
37
|
-
|
38
|
-
# Return the next float value from the stream
|
39
|
-
def next_float
|
40
|
-
float = @str[@marker, 4].unpack('N*').pack('L').unpack('f*').first
|
41
|
-
@marker += 4
|
42
|
-
|
43
|
-
return float
|
44
|
-
end
|
45
|
-
|
46
|
-
# Returns an array of string items
|
47
|
-
def next_array
|
48
|
-
count = next_int
|
49
|
-
items = []
|
50
|
-
for i in 0...count
|
51
|
-
items << self.next
|
52
|
-
end
|
53
|
-
|
54
|
-
return items
|
55
|
-
end
|
56
|
-
|
57
|
-
# Returns an array of int items
|
58
|
-
def next_int_array
|
59
|
-
count = next_int
|
60
|
-
items = []
|
61
|
-
for i in 0...count
|
62
|
-
items << self.next_int
|
63
|
-
end
|
64
|
-
|
65
|
-
return items
|
66
|
-
end
|
67
|
-
|
68
|
-
def next_float_array
|
69
|
-
count = next_int
|
70
|
-
items = []
|
71
|
-
for i in 0...count
|
72
|
-
items << self.next_float
|
73
|
-
end
|
74
|
-
|
75
|
-
return items
|
76
|
-
end
|
77
|
-
|
78
|
-
# Returns the length of the streamed data
|
79
|
-
def length
|
80
|
-
@str.length
|
81
|
-
end
|
82
|
-
end
|
83
|
-
end
|
84
|
-
end
|
@@ -1,33 +0,0 @@
|
|
1
|
-
require 'riddle/configuration/section'
|
2
|
-
|
3
|
-
require 'riddle/configuration/distributed_index'
|
4
|
-
require 'riddle/configuration/index'
|
5
|
-
require 'riddle/configuration/indexer'
|
6
|
-
require 'riddle/configuration/remote_index'
|
7
|
-
require 'riddle/configuration/searchd'
|
8
|
-
require 'riddle/configuration/source'
|
9
|
-
require 'riddle/configuration/sql_source'
|
10
|
-
require 'riddle/configuration/xml_source'
|
11
|
-
|
12
|
-
module Riddle
|
13
|
-
class Configuration
|
14
|
-
class ConfigurationError < StandardError #:nodoc:
|
15
|
-
end
|
16
|
-
|
17
|
-
attr_reader :indexes, :searchd
|
18
|
-
attr_accessor :indexer
|
19
|
-
|
20
|
-
def initialize
|
21
|
-
@indexer = Riddle::Configuration::Indexer.new
|
22
|
-
@searchd = Riddle::Configuration::Searchd.new
|
23
|
-
@indexes = []
|
24
|
-
end
|
25
|
-
|
26
|
-
def render
|
27
|
-
(
|
28
|
-
[@indexer.render, @searchd.render] +
|
29
|
-
@indexes.collect { |index| index.render }
|
30
|
-
).join("\n")
|
31
|
-
end
|
32
|
-
end
|
33
|
-
end
|
@@ -1,48 +0,0 @@
|
|
1
|
-
module Riddle
|
2
|
-
class Configuration
|
3
|
-
class DistributedIndex < Riddle::Configuration::Section
|
4
|
-
self.settings = [:type, :local, :agent, :agent_connect_timeout,
|
5
|
-
:agent_query_timeout]
|
6
|
-
|
7
|
-
attr_accessor :name, :local_indexes, :remote_indexes,
|
8
|
-
:agent_connect_timeout, :agent_query_timeout
|
9
|
-
|
10
|
-
def initialize(name)
|
11
|
-
@name = name
|
12
|
-
@local_indexes = []
|
13
|
-
@remote_indexes = []
|
14
|
-
end
|
15
|
-
|
16
|
-
def type
|
17
|
-
"distributed"
|
18
|
-
end
|
19
|
-
|
20
|
-
def local
|
21
|
-
self.local_indexes
|
22
|
-
end
|
23
|
-
|
24
|
-
def agent
|
25
|
-
agents = remote_indexes.collect { |index| index.remote }.uniq
|
26
|
-
agents.collect { |agent|
|
27
|
-
agent + ":" + remote_indexes.select { |index|
|
28
|
-
index.remote == agent
|
29
|
-
}.collect { |index| index.name }.join(",")
|
30
|
-
}
|
31
|
-
end
|
32
|
-
|
33
|
-
def render
|
34
|
-
raise ConfigurationError unless valid?
|
35
|
-
|
36
|
-
(
|
37
|
-
["index #{name}", "{"] +
|
38
|
-
settings_body +
|
39
|
-
["}", ""]
|
40
|
-
).join("\n")
|
41
|
-
end
|
42
|
-
|
43
|
-
def valid?
|
44
|
-
@local_indexes.length > 0 || @remote_indexes.length > 0
|
45
|
-
end
|
46
|
-
end
|
47
|
-
end
|
48
|
-
end
|
@@ -1,142 +0,0 @@
|
|
1
|
-
module Riddle
|
2
|
-
class Configuration
|
3
|
-
class Index < Riddle::Configuration::Section
|
4
|
-
self.settings = [:source, :path, :docinfo, :mlock, :morphology,
|
5
|
-
:stopwords, :wordforms, :exceptions, :min_word_len, :charset_type,
|
6
|
-
:charset_table, :ignore_chars, :min_prefix_len, :min_infix_len,
|
7
|
-
:prefix_fields, :infix_fields, :enable_star, :ngram_len, :ngram_chars,
|
8
|
-
:phrase_boundary, :phrase_boundary_step, :html_strip,
|
9
|
-
:html_index_attrs, :html_remove_elements, :preopen]
|
10
|
-
|
11
|
-
attr_accessor :name, :parent, :sources, :path, :docinfo, :mlock,
|
12
|
-
:morphologies, :stopword_files, :wordform_files, :exception_files,
|
13
|
-
:min_word_len, :charset_type, :charset_table, :ignore_characters,
|
14
|
-
:min_prefix_len, :min_infix_len, :prefix_field_names,
|
15
|
-
:infix_field_names, :enable_star, :ngram_len, :ngram_characters,
|
16
|
-
:phrase_boundaries, :phrase_boundary_step, :html_strip,
|
17
|
-
:html_index_attrs, :html_remove_element_tags, :preopen
|
18
|
-
|
19
|
-
def initialize(name, *sources)
|
20
|
-
@name = name
|
21
|
-
@sources = sources
|
22
|
-
@morphologies = []
|
23
|
-
@stopword_files = []
|
24
|
-
@wordform_files = []
|
25
|
-
@exception_files = []
|
26
|
-
@ignore_characters = []
|
27
|
-
@prefix_field_names = []
|
28
|
-
@infix_field_names = []
|
29
|
-
@ngram_characters = []
|
30
|
-
@phrase_boundaries = []
|
31
|
-
@html_remove_element_tags = []
|
32
|
-
end
|
33
|
-
|
34
|
-
def source
|
35
|
-
@sources.collect { |s| s.name }
|
36
|
-
end
|
37
|
-
|
38
|
-
def morphology
|
39
|
-
nil_join @morphologies, ", "
|
40
|
-
end
|
41
|
-
|
42
|
-
def morphology=(morphology)
|
43
|
-
@morphologies = nil_split morphology, /,\s?/
|
44
|
-
end
|
45
|
-
|
46
|
-
def stopwords
|
47
|
-
nil_join @stopword_files, " "
|
48
|
-
end
|
49
|
-
|
50
|
-
def stopwords=(stopwords)
|
51
|
-
@stopword_files = nil_split stopwords, ' '
|
52
|
-
end
|
53
|
-
|
54
|
-
def wordforms
|
55
|
-
nil_join @wordform_files, " "
|
56
|
-
end
|
57
|
-
|
58
|
-
def wordforms=(wordforms)
|
59
|
-
@wordform_files = nil_split wordforms, ' '
|
60
|
-
end
|
61
|
-
|
62
|
-
def exceptions
|
63
|
-
nil_join @exception_files, " "
|
64
|
-
end
|
65
|
-
|
66
|
-
def exceptions=(exceptions)
|
67
|
-
@exception_files = nil_split exceptions, ' '
|
68
|
-
end
|
69
|
-
|
70
|
-
def ignore_chars
|
71
|
-
nil_join @ignore_characters, ", "
|
72
|
-
end
|
73
|
-
|
74
|
-
def ignore_chars=(ignore_chars)
|
75
|
-
@ignore_characters = nil_split ignore_chars, /,\s?/
|
76
|
-
end
|
77
|
-
|
78
|
-
def prefix_fields
|
79
|
-
nil_join @prefix_field_names, ", "
|
80
|
-
end
|
81
|
-
|
82
|
-
def infix_fields
|
83
|
-
nil_join @infix_field_names, ", "
|
84
|
-
end
|
85
|
-
|
86
|
-
def ngram_chars
|
87
|
-
nil_join @ngram_characters, ", "
|
88
|
-
end
|
89
|
-
|
90
|
-
def ngram_chars=(ngram_chars)
|
91
|
-
@ngram_characters = nil_split ngram_chars, /,\s?/
|
92
|
-
end
|
93
|
-
|
94
|
-
def phrase_boundary
|
95
|
-
nil_join @phrase_boundaries, ", "
|
96
|
-
end
|
97
|
-
|
98
|
-
def phrase_boundary=(phrase_boundary)
|
99
|
-
@phrase_boundaries = nil_split phrase_boundary, /,\s?/
|
100
|
-
end
|
101
|
-
|
102
|
-
def html_remove_elements
|
103
|
-
nil_join @html_remove_element_tags, ", "
|
104
|
-
end
|
105
|
-
|
106
|
-
def html_remove_elements=(html_remove_elements)
|
107
|
-
@html_remove_element_tags = nil_split html_remove_elements, /,\s?/
|
108
|
-
end
|
109
|
-
|
110
|
-
def render
|
111
|
-
raise ConfigurationError, "#{@name} #{@sources.inspect} #{@path} #{@parent}" unless valid?
|
112
|
-
|
113
|
-
inherited_name = "#{name}"
|
114
|
-
inherited_name << " : #{parent}" if parent
|
115
|
-
(
|
116
|
-
@sources.collect { |s| s.render } +
|
117
|
-
["index #{inherited_name}", "{"] +
|
118
|
-
settings_body +
|
119
|
-
["}", ""]
|
120
|
-
).join("\n")
|
121
|
-
end
|
122
|
-
|
123
|
-
def valid?
|
124
|
-
(!@name.nil?) && (!( @sources.length == 0 || @path.nil? ) || !@parent.nil?)
|
125
|
-
end
|
126
|
-
|
127
|
-
private
|
128
|
-
|
129
|
-
def nil_split(string, pattern)
|
130
|
-
(string || "").split(pattern)
|
131
|
-
end
|
132
|
-
|
133
|
-
def nil_join(array, delimiter)
|
134
|
-
if array.length == 0
|
135
|
-
nil
|
136
|
-
else
|
137
|
-
array.join(delimiter)
|
138
|
-
end
|
139
|
-
end
|
140
|
-
end
|
141
|
-
end
|
142
|
-
end
|
@@ -1,19 +0,0 @@
|
|
1
|
-
module Riddle
|
2
|
-
class Configuration
|
3
|
-
class Indexer < Riddle::Configuration::Section
|
4
|
-
self.settings = [:mem_limit, :max_iops, :max_iosize]
|
5
|
-
|
6
|
-
attr_accessor *self.settings
|
7
|
-
|
8
|
-
def render
|
9
|
-
raise ConfigurationError unless valid?
|
10
|
-
|
11
|
-
(
|
12
|
-
["indexer", "{"] +
|
13
|
-
settings_body +
|
14
|
-
["}", ""]
|
15
|
-
).join("\n")
|
16
|
-
end
|
17
|
-
end
|
18
|
-
end
|
19
|
-
end
|