freelancing-god-riddle 0.9.8.1533.1 → 0.9.8.1533.2
Sign up to get free protection for your applications and to get access to all the features.
- data/lib/riddle/client.rb +4 -0
- data/lib/riddle/configuration/distributed_index.rb +48 -0
- data/lib/riddle/configuration/index.rb +134 -0
- data/lib/riddle/configuration/indexer.rb +19 -0
- data/lib/riddle/configuration/remote_index.rb +17 -0
- data/lib/riddle/configuration/searchd.rb +25 -0
- data/lib/riddle/configuration/section.rb +32 -0
- data/lib/riddle/configuration/source.rb +23 -0
- data/lib/riddle/configuration/sql_source.rb +34 -0
- data/lib/riddle/configuration/xml_source.rb +28 -0
- data/lib/riddle/configuration.rb +33 -0
- data/lib/riddle/controller.rb +44 -0
- data/lib/riddle.rb +4 -4
- data/spec/unit/configuration/distributed_index_spec.rb +56 -0
- data/spec/unit/configuration/index_spec.rb +108 -0
- data/spec/unit/configuration/indexer_spec.rb +36 -0
- data/spec/unit/configuration/searchd_spec.rb +48 -0
- data/spec/unit/configuration/source_spec.rb +5 -0
- data/spec/unit/configuration/sql_source_spec.rb +100 -0
- data/spec/unit/configuration/xml_source_spec.rb +51 -0
- data/spec/unit/configuration_spec.rb +25 -0
- metadata +29 -2
data/lib/riddle/client.rb
CHANGED
@@ -0,0 +1,48 @@
|
|
1
|
+
module Riddle
|
2
|
+
class Configuration
|
3
|
+
class DistributedIndex < Riddle::Configuration::Section
|
4
|
+
self.settings = [:type, :local, :agent, :agent_connect_timeout,
|
5
|
+
:agent_query_timeout]
|
6
|
+
|
7
|
+
attr_accessor :name, :local_indexes, :remote_indexes,
|
8
|
+
:agent_connect_timeout, :agent_query_timeout
|
9
|
+
|
10
|
+
def initialize(name)
|
11
|
+
@name = name
|
12
|
+
@local_indexes = []
|
13
|
+
@remote_indexes = []
|
14
|
+
end
|
15
|
+
|
16
|
+
def type
|
17
|
+
"distributed"
|
18
|
+
end
|
19
|
+
|
20
|
+
def local
|
21
|
+
self.local_indexes
|
22
|
+
end
|
23
|
+
|
24
|
+
def agent
|
25
|
+
agents = remote_indexes.collect { |index| index.remote }.uniq
|
26
|
+
agents.collect { |agent|
|
27
|
+
agent + ":" + remote_indexes.select { |index|
|
28
|
+
index.remote == agent
|
29
|
+
}.collect { |index| index.name }.join(",")
|
30
|
+
}
|
31
|
+
end
|
32
|
+
|
33
|
+
def render
|
34
|
+
raise ConfigurationError unless valid?
|
35
|
+
|
36
|
+
(
|
37
|
+
["index #{name}", "{"] +
|
38
|
+
settings_body +
|
39
|
+
["}", ""]
|
40
|
+
).join("\n")
|
41
|
+
end
|
42
|
+
|
43
|
+
def valid?
|
44
|
+
@local_indexes.length > 0 || @remote_indexes.length > 0
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
@@ -0,0 +1,134 @@
|
|
1
|
+
module Riddle
|
2
|
+
class Configuration
|
3
|
+
class Index < Riddle::Configuration::Section
|
4
|
+
self.settings = [:source, :path, :docinfo, :mlock, :morphology,
|
5
|
+
:stopwords, :wordforms, :exceptions, :min_word_len, :charset_type,
|
6
|
+
:charset_table, :ignore_chars, :min_prefix_len, :min_infix_len,
|
7
|
+
:prefix_fields, :infix_fields, :enable_star, :ngram_len, :ngram_chars,
|
8
|
+
:phrase_boundary, :phrase_boundary_step, :html_strip,
|
9
|
+
:html_index_attrs, :html_remove_elements, :preopen]
|
10
|
+
|
11
|
+
attr_accessor :name, :parent, :sources, :path, :docinfo, :mlock,
|
12
|
+
:morphologies, :stopword_files, :wordform_files, :exception_files,
|
13
|
+
:min_word_len, :charset_type, :charset_table, :ignore_characters,
|
14
|
+
:min_prefix_len, :min_infix_len, :prefix_field_names,
|
15
|
+
:infix_field_names, :enable_star, :ngram_len, :ngram_characters,
|
16
|
+
:phrase_boundaries, :phrase_boundary_step, :html_strip,
|
17
|
+
:html_index_attrs, :html_remove_element_tags, :preopen
|
18
|
+
|
19
|
+
def initialize(name, *sources)
|
20
|
+
@name = name
|
21
|
+
@sources = sources
|
22
|
+
@morphologies = []
|
23
|
+
@stopword_files = []
|
24
|
+
@wordform_files = []
|
25
|
+
@exception_files = []
|
26
|
+
@ignore_characters = []
|
27
|
+
@prefix_field_names = []
|
28
|
+
@infix_field_names = []
|
29
|
+
@ngram_characters = []
|
30
|
+
@phrase_boundaries = []
|
31
|
+
@html_remove_element_tags = []
|
32
|
+
end
|
33
|
+
|
34
|
+
def source
|
35
|
+
@sources.collect { |s| s.name }
|
36
|
+
end
|
37
|
+
|
38
|
+
def morphology
|
39
|
+
@morphologies.join(", ")
|
40
|
+
end
|
41
|
+
|
42
|
+
def morphology=(morphology)
|
43
|
+
@morphologies = nil_split morphology, /,\s?/
|
44
|
+
end
|
45
|
+
|
46
|
+
def stopwords
|
47
|
+
@stopword_files.join(" ")
|
48
|
+
end
|
49
|
+
|
50
|
+
def stopwords=(stopwords)
|
51
|
+
@stopword_files = nil_split stopwords, ' '
|
52
|
+
end
|
53
|
+
|
54
|
+
def wordforms
|
55
|
+
@wordform_files.join(" ")
|
56
|
+
end
|
57
|
+
|
58
|
+
def wordforms=(wordforms)
|
59
|
+
@wordform_files = nil_split wordforms, ' '
|
60
|
+
end
|
61
|
+
|
62
|
+
def exceptions
|
63
|
+
@exception_files.join(" ")
|
64
|
+
end
|
65
|
+
|
66
|
+
def exceptions=(exceptions)
|
67
|
+
@exception_files = nil_split exceptions, ' '
|
68
|
+
end
|
69
|
+
|
70
|
+
def ignore_chars
|
71
|
+
@ignore_characters.join(", ")
|
72
|
+
end
|
73
|
+
|
74
|
+
def ignore_chars=(ignore_chars)
|
75
|
+
@ignore_characters = nil_split ignore_chars, /,\s?/
|
76
|
+
end
|
77
|
+
|
78
|
+
def prefix_fields
|
79
|
+
@prefix_field_names.join(", ")
|
80
|
+
end
|
81
|
+
|
82
|
+
def infix_fields
|
83
|
+
@infix_field_names.join(", ")
|
84
|
+
end
|
85
|
+
|
86
|
+
def ngram_chars
|
87
|
+
@ngram_characters.join(", ")
|
88
|
+
end
|
89
|
+
|
90
|
+
def ngram_chars=(ngram_chars)
|
91
|
+
@ngram_characters = nil_split ngram_chars, /,\s?/
|
92
|
+
end
|
93
|
+
|
94
|
+
def phrase_boundary
|
95
|
+
@phrase_boundaries.join(", ")
|
96
|
+
end
|
97
|
+
|
98
|
+
def phrase_boundary=(phrase_boundary)
|
99
|
+
@phrase_boundaries = nil_split phrase_boundary, /,\s?/
|
100
|
+
end
|
101
|
+
|
102
|
+
def html_remove_elements
|
103
|
+
@html_remove_element_tags.join(", ")
|
104
|
+
end
|
105
|
+
|
106
|
+
def html_remove_elements=(html_remove_elements)
|
107
|
+
@html_remove_element_tags = nil_split html_remove_elements, /,\s?/
|
108
|
+
end
|
109
|
+
|
110
|
+
def render
|
111
|
+
raise ConfigurationError, "#{@name} #{@sources.inspect} #{@path} #{@parent}" unless valid?
|
112
|
+
|
113
|
+
inherited_name = "#{name}"
|
114
|
+
inherited_name << " : #{parent}" if parent
|
115
|
+
(
|
116
|
+
@sources.collect { |s| s.render } +
|
117
|
+
["index #{inherited_name}", "{"] +
|
118
|
+
settings_body +
|
119
|
+
["}", ""]
|
120
|
+
).join("\n")
|
121
|
+
end
|
122
|
+
|
123
|
+
def valid?
|
124
|
+
(!@name.nil?) && (!( @sources.length == 0 || @path.nil? ) || !@parent.nil?)
|
125
|
+
end
|
126
|
+
|
127
|
+
private
|
128
|
+
|
129
|
+
def nil_split(string, pattern)
|
130
|
+
(string || "").split(pattern)
|
131
|
+
end
|
132
|
+
end
|
133
|
+
end
|
134
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
module Riddle
|
2
|
+
class Configuration
|
3
|
+
class Indexer < Riddle::Configuration::Section
|
4
|
+
self.settings = [:mem_limit, :max_iops, :max_iosize]
|
5
|
+
|
6
|
+
attr_accessor *self.settings
|
7
|
+
|
8
|
+
def render
|
9
|
+
raise ConfigurationError unless valid?
|
10
|
+
|
11
|
+
(
|
12
|
+
["indexer", "{"] +
|
13
|
+
settings_body +
|
14
|
+
["}", ""]
|
15
|
+
).join("\n")
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
module Riddle
|
2
|
+
class Configuration
|
3
|
+
class RemoteIndex
|
4
|
+
attr_accessor :address, :port, :name
|
5
|
+
|
6
|
+
def initialize(address, port, name)
|
7
|
+
@address = address
|
8
|
+
@port = port
|
9
|
+
@name = name
|
10
|
+
end
|
11
|
+
|
12
|
+
def remote
|
13
|
+
"#{address}:#{port}"
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
module Riddle
|
2
|
+
class Configuration
|
3
|
+
class Searchd < Riddle::Configuration::Section
|
4
|
+
self.settings = [:address, :port, :log, :query_log, :read_timeout,
|
5
|
+
:max_children, :pid_file, :max_matches, :seamless_rotate,
|
6
|
+
:preopen_indexes, :unlink_old]
|
7
|
+
|
8
|
+
attr_accessor *self.settings
|
9
|
+
|
10
|
+
def render
|
11
|
+
raise ConfigurationError unless valid?
|
12
|
+
|
13
|
+
(
|
14
|
+
["searchd", "{"] +
|
15
|
+
settings_body +
|
16
|
+
["}", ""]
|
17
|
+
).join("\n")
|
18
|
+
end
|
19
|
+
|
20
|
+
def valid?
|
21
|
+
!( @port.nil? || @pid_file.nil? )
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
module Riddle
|
2
|
+
class Configuration
|
3
|
+
class Section
|
4
|
+
class << self
|
5
|
+
attr_accessor :settings
|
6
|
+
end
|
7
|
+
|
8
|
+
settings = []
|
9
|
+
|
10
|
+
def valid?
|
11
|
+
true
|
12
|
+
end
|
13
|
+
|
14
|
+
private
|
15
|
+
|
16
|
+
def settings_body
|
17
|
+
self.class.settings.select { |setting|
|
18
|
+
!send(setting).nil?
|
19
|
+
}.collect { |setting|
|
20
|
+
if send(setting) == ""
|
21
|
+
conf = " #{setting} = "
|
22
|
+
else
|
23
|
+
conf = Array(send(setting)).collect { |set|
|
24
|
+
" #{setting} = #{set}"
|
25
|
+
}
|
26
|
+
end
|
27
|
+
conf.length == 0 ? nil : conf
|
28
|
+
}.flatten.compact
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
module Riddle
|
2
|
+
class Configuration
|
3
|
+
class Source < Riddle::Configuration::Section
|
4
|
+
attr_accessor :name, :parent, :type
|
5
|
+
|
6
|
+
def render
|
7
|
+
raise ConfigurationError unless valid?
|
8
|
+
|
9
|
+
inherited_name = "#{name}"
|
10
|
+
inherited_name << " : #{parent}" if parent
|
11
|
+
(
|
12
|
+
["source #{inherited_name}", "{"] +
|
13
|
+
settings_body +
|
14
|
+
["}", ""]
|
15
|
+
).join("\n")
|
16
|
+
end
|
17
|
+
|
18
|
+
def valid?
|
19
|
+
!( @name.nil? || @type.nil? )
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1,34 @@
|
|
1
|
+
module Riddle
|
2
|
+
class Configuration
|
3
|
+
class SQLSource < Riddle::Configuration::Source
|
4
|
+
self.settings = [:type, :sql_host, :sql_user, :sql_pass, :sql_db,
|
5
|
+
:sql_port, :sql_sock, :mysql_connect_flags, :sql_query_pre, :sql_query,
|
6
|
+
:sql_query_range, :sql_range_step, :sql_attr_uint, :sql_attr_bool,
|
7
|
+
:sql_attr_timestamp, :sql_attr_str2ordinal, :sql_attr_float,
|
8
|
+
:sql_attr_multi, :sql_query_post, :sql_query_post_index,
|
9
|
+
:sql_ranged_throttle, :sql_query_info]
|
10
|
+
|
11
|
+
attr_accessor *self.settings
|
12
|
+
|
13
|
+
def initialize(name, type)
|
14
|
+
@name = name
|
15
|
+
@type = type
|
16
|
+
|
17
|
+
@sql_query_pre = []
|
18
|
+
@sql_attr_uint = []
|
19
|
+
@sql_attr_bool = []
|
20
|
+
@sql_attr_timestamp = []
|
21
|
+
@sql_attr_str2ordinal = []
|
22
|
+
@sql_attr_float = []
|
23
|
+
@sql_attr_multi = []
|
24
|
+
@sql_query_post = []
|
25
|
+
@sql_query_post_index = []
|
26
|
+
end
|
27
|
+
|
28
|
+
def valid?
|
29
|
+
super && (!( @sql_host.nil? || @sql_user.nil? || @sql_db.nil? ||
|
30
|
+
@sql_query.nil? ) || !@parent.nil?)
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
module Riddle
|
2
|
+
class Configuration
|
3
|
+
class XMLSource < Riddle::Configuration::Source
|
4
|
+
self.settings = [:type, :xmlpipe_command, :xmlpipe_field,
|
5
|
+
:xmlpipe_attr_uint, :xmlpipe_attr_bool, :xmlpipe_attr_timestamp,
|
6
|
+
:xmlpipe_attr_str2ordinal, :xmlpipe_attr_float, :xmlpipe_attr_multi]
|
7
|
+
|
8
|
+
attr_accessor *self.settings
|
9
|
+
|
10
|
+
def initialize(name, type)
|
11
|
+
@name = name
|
12
|
+
@type = type
|
13
|
+
|
14
|
+
@xmlpipe_field = []
|
15
|
+
@xmlpipe_attr_uint = []
|
16
|
+
@xmlpipe_attr_bool = []
|
17
|
+
@xmlpipe_attr_timestamp = []
|
18
|
+
@xmlpipe_attr_str2ordinal = []
|
19
|
+
@xmlpipe_attr_float = []
|
20
|
+
@xmlpipe_attr_multi = []
|
21
|
+
end
|
22
|
+
|
23
|
+
def valid?
|
24
|
+
super && ( !@xmlpipe_command.nil? || !parent.nil? )
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
require 'riddle/configuration/section'
|
2
|
+
|
3
|
+
require 'riddle/configuration/distributed_index'
|
4
|
+
require 'riddle/configuration/index'
|
5
|
+
require 'riddle/configuration/indexer'
|
6
|
+
require 'riddle/configuration/remote_index'
|
7
|
+
require 'riddle/configuration/searchd'
|
8
|
+
require 'riddle/configuration/source'
|
9
|
+
require 'riddle/configuration/sql_source'
|
10
|
+
require 'riddle/configuration/xml_source'
|
11
|
+
|
12
|
+
module Riddle
|
13
|
+
class Configuration
|
14
|
+
class ConfigurationError < StandardError #:nodoc:
|
15
|
+
end
|
16
|
+
|
17
|
+
attr_reader :indexes, :searchd
|
18
|
+
attr_accessor :indexer
|
19
|
+
|
20
|
+
def initialize
|
21
|
+
@indexer = Riddle::Configuration::Indexer.new
|
22
|
+
@searchd = Riddle::Configuration::Searchd.new
|
23
|
+
@indexes = []
|
24
|
+
end
|
25
|
+
|
26
|
+
def render
|
27
|
+
(
|
28
|
+
[@indexer.render, @searchd.render] +
|
29
|
+
@indexes.collect { |index| index.render }
|
30
|
+
).join("\n")
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,44 @@
|
|
1
|
+
module Riddle
|
2
|
+
class Controller
|
3
|
+
def initialize(configuration, path)
|
4
|
+
@configuration = configuration
|
5
|
+
@path = path
|
6
|
+
end
|
7
|
+
|
8
|
+
def index
|
9
|
+
cmd = "indexer --config #{@path} --all"
|
10
|
+
cmd << " --rotate" if running?
|
11
|
+
`#{cmd}`
|
12
|
+
end
|
13
|
+
|
14
|
+
def start
|
15
|
+
return if running?
|
16
|
+
|
17
|
+
cmd = "searchd --config #{@path}"
|
18
|
+
`#{cmd}`
|
19
|
+
|
20
|
+
sleep(1)
|
21
|
+
|
22
|
+
unless running?
|
23
|
+
puts "Failed to start searchd daemon. Check #{@configuration.searchd.log}."
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
def stop
|
28
|
+
return unless running?
|
29
|
+
`kill #{pid}`
|
30
|
+
end
|
31
|
+
|
32
|
+
def pid
|
33
|
+
if File.exists?("#{@configuration.searchd.pid_file}")
|
34
|
+
`cat #{@configuration.searchd.pid_file}`[/\d+/]
|
35
|
+
else
|
36
|
+
nil
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
def running?
|
41
|
+
pid && `ps #{pid} | wc -l`.to_i > 1
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
data/lib/riddle.rb
CHANGED
@@ -1,9 +1,9 @@
|
|
1
1
|
require 'socket'
|
2
2
|
require 'timeout'
|
3
|
+
|
3
4
|
require 'riddle/client'
|
4
|
-
require 'riddle/
|
5
|
-
require 'riddle/
|
6
|
-
require 'riddle/client/response'
|
5
|
+
require 'riddle/configuration'
|
6
|
+
require 'riddle/controller'
|
7
7
|
|
8
8
|
module Riddle #:nodoc:
|
9
9
|
class ConnectionError < StandardError #:nodoc:
|
@@ -18,7 +18,7 @@ module Riddle #:nodoc:
|
|
18
18
|
Rev = 1533
|
19
19
|
# Release number to mark my own fixes, beyond feature parity with
|
20
20
|
# Sphinx itself.
|
21
|
-
Release =
|
21
|
+
Release = 2
|
22
22
|
|
23
23
|
String = [Major, Minor, Tiny].join('.')
|
24
24
|
GemVersion = [Major, Minor, Tiny, Rev, Release].join('.')
|
@@ -0,0 +1,56 @@
|
|
1
|
+
require 'spec/spec_helper'
|
2
|
+
|
3
|
+
describe Riddle::Configuration::DistributedIndex do
|
4
|
+
it "should not be valid without any indexes" do
|
5
|
+
index = Riddle::Configuration::DistributedIndex.new("dist1")
|
6
|
+
index.should_not be_valid
|
7
|
+
end
|
8
|
+
|
9
|
+
it "should be valid with just local indexes" do
|
10
|
+
index = Riddle::Configuration::DistributedIndex.new("dist1")
|
11
|
+
index.local_indexes << "local_one"
|
12
|
+
index.should be_valid
|
13
|
+
end
|
14
|
+
|
15
|
+
it "should be valid with just remote indexes" do
|
16
|
+
index = Riddle::Configuration::DistributedIndex.new("dist1")
|
17
|
+
index.remote_indexes << Riddle::Configuration::RemoteIndex.new("local", 3312, "remote_one")
|
18
|
+
index.should be_valid
|
19
|
+
end
|
20
|
+
|
21
|
+
it "should be of type 'distributed'" do
|
22
|
+
index = Riddle::Configuration::DistributedIndex.new("dist1")
|
23
|
+
index.type.should == 'distributed'
|
24
|
+
end
|
25
|
+
|
26
|
+
it "should raise a ConfigurationError if rendering when not valid" do
|
27
|
+
index = Riddle::Configuration::DistributedIndex.new("dist1")
|
28
|
+
lambda { index.render }.should raise_error(Riddle::Configuration::ConfigurationError)
|
29
|
+
end
|
30
|
+
|
31
|
+
it "should render correctly if supplied settings are valid" do
|
32
|
+
index = Riddle::Configuration::DistributedIndex.new("dist1")
|
33
|
+
|
34
|
+
index.local_indexes << "test1" << "test1stemmed"
|
35
|
+
index.remote_indexes <<
|
36
|
+
Riddle::Configuration::RemoteIndex.new("localhost", 3313, "remote1") <<
|
37
|
+
Riddle::Configuration::RemoteIndex.new("localhost", 3314, "remote2") <<
|
38
|
+
Riddle::Configuration::RemoteIndex.new("localhost", 3314, "remote3")
|
39
|
+
|
40
|
+
index.agent_connect_timeout = 1000
|
41
|
+
index.agent_query_timeout = 3000
|
42
|
+
|
43
|
+
index.render.should == <<-DISTINDEX
|
44
|
+
index dist1
|
45
|
+
{
|
46
|
+
type = distributed
|
47
|
+
local = test1
|
48
|
+
local = test1stemmed
|
49
|
+
agent = localhost:3313:remote1
|
50
|
+
agent = localhost:3314:remote2,remote3
|
51
|
+
agent_connect_timeout = 1000
|
52
|
+
agent_query_timeout = 3000
|
53
|
+
}
|
54
|
+
DISTINDEX
|
55
|
+
end
|
56
|
+
end
|
@@ -0,0 +1,108 @@
|
|
1
|
+
require 'spec/spec_helper'
|
2
|
+
|
3
|
+
describe Riddle::Configuration::DistributedIndex do
|
4
|
+
it "should be invalid without a name, sources or path if there's no parent" do
|
5
|
+
index = Riddle::Configuration::Index.new(nil)
|
6
|
+
index.should_not be_valid
|
7
|
+
|
8
|
+
index.name = "test1"
|
9
|
+
index.should_not be_valid
|
10
|
+
|
11
|
+
index.sources << Riddle::Configuration::SQLSource.new("source", "mysql")
|
12
|
+
index.should_not be_valid
|
13
|
+
|
14
|
+
index.path = "a/path"
|
15
|
+
index.should be_valid
|
16
|
+
|
17
|
+
index.name = nil
|
18
|
+
index.should_not be_valid
|
19
|
+
|
20
|
+
index.name = "test1"
|
21
|
+
index.sources.clear
|
22
|
+
index.should_not be_valid
|
23
|
+
end
|
24
|
+
|
25
|
+
it "should be invalid without a name but not sources or path if it has a parent" do
|
26
|
+
index = Riddle::Configuration::Index.new(nil)
|
27
|
+
index.should_not be_valid
|
28
|
+
|
29
|
+
index.name = "test1stemmed"
|
30
|
+
index.should_not be_valid
|
31
|
+
|
32
|
+
index.parent = "test1"
|
33
|
+
index.should be_valid
|
34
|
+
end
|
35
|
+
|
36
|
+
it "should raise a ConfigurationError if rendering when not valid" do
|
37
|
+
index = Riddle::Configuration::Index.new("test1")
|
38
|
+
lambda { index.render }.should raise_error(Riddle::Configuration::ConfigurationError)
|
39
|
+
end
|
40
|
+
|
41
|
+
it "should render correctly if supplied settings are valid" do
|
42
|
+
source = Riddle::Configuration::XMLSource.new("src1", "xmlpipe")
|
43
|
+
source.xmlpipe_command = "ls /dev/null"
|
44
|
+
|
45
|
+
index = Riddle::Configuration::Index.new("test1", source)
|
46
|
+
index.path = "/var/data/test1"
|
47
|
+
index.docinfo = "extern"
|
48
|
+
index.mlock = 0
|
49
|
+
index.morphologies << "stem_en" << "stem_ru" << "soundex"
|
50
|
+
index.stopword_files << "/var/data/stopwords.txt" << "/var/data/stopwords2.txt"
|
51
|
+
index.wordform_files << "/var/data/wordforms.txt"
|
52
|
+
index.exception_files << "/var/data/exceptions.txt"
|
53
|
+
index.min_word_len = 1
|
54
|
+
index.charset_type = "utf-8"
|
55
|
+
index.charset_table = "0..9, A..Z->a..z, _, a..z, U+410..U+42F->U+430..U+44F, U+430..U+44F"
|
56
|
+
index.ignore_characters << "U+00AD"
|
57
|
+
index.min_prefix_len = 0
|
58
|
+
index.min_infix_len = 0
|
59
|
+
index.prefix_field_names << "filename"
|
60
|
+
index.infix_field_names << "url" << "domain"
|
61
|
+
index.enable_star = 1
|
62
|
+
index.ngram_len = 1
|
63
|
+
index.ngram_characters << "U+3000..U+2FA1F"
|
64
|
+
index.phrase_boundaries << "." << "?" << "!" << "U+2026"
|
65
|
+
index.phrase_boundary_step = 100
|
66
|
+
index.html_strip = 0
|
67
|
+
index.html_index_attrs = "img=alt,title; a=title"
|
68
|
+
index.html_remove_element_tags << "style" << "script"
|
69
|
+
index.preopen = 1
|
70
|
+
|
71
|
+
index.render.should == <<-INDEX
|
72
|
+
source src1
|
73
|
+
{
|
74
|
+
type = xmlpipe
|
75
|
+
xmlpipe_command = ls /dev/null
|
76
|
+
}
|
77
|
+
|
78
|
+
index test1
|
79
|
+
{
|
80
|
+
source = src1
|
81
|
+
path = /var/data/test1
|
82
|
+
docinfo = extern
|
83
|
+
mlock = 0
|
84
|
+
morphology = stem_en, stem_ru, soundex
|
85
|
+
stopwords = /var/data/stopwords.txt /var/data/stopwords2.txt
|
86
|
+
wordforms = /var/data/wordforms.txt
|
87
|
+
exceptions = /var/data/exceptions.txt
|
88
|
+
min_word_len = 1
|
89
|
+
charset_type = utf-8
|
90
|
+
charset_table = 0..9, A..Z->a..z, _, a..z, U+410..U+42F->U+430..U+44F, U+430..U+44F
|
91
|
+
ignore_chars = U+00AD
|
92
|
+
min_prefix_len = 0
|
93
|
+
min_infix_len = 0
|
94
|
+
prefix_fields = filename
|
95
|
+
infix_fields = url, domain
|
96
|
+
enable_star = 1
|
97
|
+
ngram_len = 1
|
98
|
+
ngram_chars = U+3000..U+2FA1F
|
99
|
+
phrase_boundary = ., ?, !, U+2026
|
100
|
+
phrase_boundary_step = 100
|
101
|
+
html_strip = 0
|
102
|
+
html_index_attrs = img=alt,title; a=title
|
103
|
+
html_remove_elements = style, script
|
104
|
+
preopen = 1
|
105
|
+
}
|
106
|
+
INDEX
|
107
|
+
end
|
108
|
+
end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
require 'spec/spec_helper'
|
2
|
+
|
3
|
+
describe Riddle::Configuration::Indexer do
|
4
|
+
it "should always be valid" do
|
5
|
+
indexer = Riddle::Configuration::Indexer.new
|
6
|
+
indexer.should be_valid
|
7
|
+
end
|
8
|
+
|
9
|
+
it "should support Sphinx's indexer settings" do
|
10
|
+
settings = %w( mem_limit max_iops max_iosize )
|
11
|
+
indexer = Riddle::Configuration::Indexer.new
|
12
|
+
|
13
|
+
settings.each do |setting|
|
14
|
+
indexer.should respond_to(setting.to_sym)
|
15
|
+
indexer.should respond_to("#{setting}=".to_sym)
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
it "should render a correct configuration" do
|
20
|
+
indexer = Riddle::Configuration::Indexer.new
|
21
|
+
|
22
|
+
indexer.render.should == <<-INDEXER
|
23
|
+
indexer
|
24
|
+
{
|
25
|
+
}
|
26
|
+
INDEXER
|
27
|
+
|
28
|
+
indexer.mem_limit = "32M"
|
29
|
+
indexer.render.should == <<-INDEXER
|
30
|
+
indexer
|
31
|
+
{
|
32
|
+
mem_limit = 32M
|
33
|
+
}
|
34
|
+
INDEXER
|
35
|
+
end
|
36
|
+
end
|
@@ -0,0 +1,48 @@
|
|
1
|
+
require 'spec/spec_helper'
|
2
|
+
|
3
|
+
describe Riddle::Configuration::Searchd do
|
4
|
+
it "should be invalid without a port or pid_file" do
|
5
|
+
searchd = Riddle::Configuration::Searchd.new
|
6
|
+
searchd.should_not be_valid
|
7
|
+
|
8
|
+
searchd.port = 3312
|
9
|
+
searchd.should_not be_valid
|
10
|
+
|
11
|
+
searchd.pid_file = "file.pid"
|
12
|
+
searchd.should be_valid
|
13
|
+
|
14
|
+
searchd.port = nil
|
15
|
+
searchd.should_not be_valid
|
16
|
+
end
|
17
|
+
|
18
|
+
it "should raise a ConfigurationError if rendering but not valid" do
|
19
|
+
searchd = Riddle::Configuration::Searchd.new
|
20
|
+
searchd.should_not be_valid
|
21
|
+
lambda { searchd.render }.should raise_error(Riddle::Configuration::ConfigurationError)
|
22
|
+
end
|
23
|
+
|
24
|
+
it "should support Sphinx's searchd settings" do
|
25
|
+
settings = %w( address port log query_log read_timeout max_children
|
26
|
+
pid_file max_matches seamless_rotate preopen_indexes unlink_old )
|
27
|
+
searchd = Riddle::Configuration::Searchd.new
|
28
|
+
|
29
|
+
settings.each do |setting|
|
30
|
+
searchd.should respond_to(setting.to_sym)
|
31
|
+
searchd.should respond_to("#{setting}=".to_sym)
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
it "should render a correct configuration with valid settings" do
|
36
|
+
searchd = Riddle::Configuration::Searchd.new
|
37
|
+
searchd.port = 3312
|
38
|
+
searchd.pid_file = "file.pid"
|
39
|
+
|
40
|
+
searchd.render.should == <<-SEARCHD
|
41
|
+
searchd
|
42
|
+
{
|
43
|
+
port = 3312
|
44
|
+
pid_file = file.pid
|
45
|
+
}
|
46
|
+
SEARCHD
|
47
|
+
end
|
48
|
+
end
|
@@ -0,0 +1,100 @@
|
|
1
|
+
require 'spec/spec_helper'
|
2
|
+
|
3
|
+
describe Riddle::Configuration::SQLSource do
|
4
|
+
it "should be invalid without a host, user, database, and query if there's no parent" do
|
5
|
+
source = Riddle::Configuration::SQLSource.new("src1", "mysql")
|
6
|
+
source.should_not be_valid
|
7
|
+
|
8
|
+
source.sql_host = "localhost"
|
9
|
+
source.sql_user = "test"
|
10
|
+
source.sql_db = "test"
|
11
|
+
source.sql_query = "SELECT * FROM tables"
|
12
|
+
source.should be_valid
|
13
|
+
|
14
|
+
[:name, :type, :sql_host, :sql_user, :sql_db, :sql_query].each do |setting|
|
15
|
+
value = source.send(setting)
|
16
|
+
source.send("#{setting}=".to_sym, nil)
|
17
|
+
source.should_not be_nil
|
18
|
+
source.send("#{setting}=".to_sym, value)
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
it "should be invalid without only a name and type if there is a parent" do
|
23
|
+
source = Riddle::Configuration::SQLSource.new("src1", "mysql")
|
24
|
+
source.should_not be_valid
|
25
|
+
|
26
|
+
source.parent = "sqlparent"
|
27
|
+
source.should be_valid
|
28
|
+
|
29
|
+
source.name = nil
|
30
|
+
source.should_not be_valid
|
31
|
+
|
32
|
+
source.name = "src1"
|
33
|
+
source.type = nil
|
34
|
+
source.should_not be_valid
|
35
|
+
end
|
36
|
+
|
37
|
+
it "should raise a ConfigurationError if rendering when not valid" do
|
38
|
+
source = Riddle::Configuration::SQLSource.new("src1", "mysql")
|
39
|
+
lambda { source.render }.should raise_error(Riddle::Configuration::ConfigurationError)
|
40
|
+
end
|
41
|
+
|
42
|
+
it "should render correctly when valid" do
|
43
|
+
source = Riddle::Configuration::SQLSource.new("src1", "mysql")
|
44
|
+
source.sql_host = "localhost"
|
45
|
+
source.sql_user = "test"
|
46
|
+
source.sql_pass = ""
|
47
|
+
source.sql_db = "test"
|
48
|
+
source.sql_port = 3306
|
49
|
+
source.sql_sock = "/tmp/mysql.sock"
|
50
|
+
source.mysql_connect_flags = 32
|
51
|
+
source.sql_query_pre << "SET NAMES utf8" << "SET SESSION query_cache_type=OFF"
|
52
|
+
source.sql_query = "SELECT id, group_id, UNIX_TIMESTAMP(date_added) AS date_added, title, content FROM documents WHERE id >= $start AND id <= $end"
|
53
|
+
source.sql_query_range = "SELECT MIN(id), MAX(id) FROM documents"
|
54
|
+
source.sql_range_step = 1000
|
55
|
+
source.sql_attr_uint << "author_id" << "forum_id:9" << "group_id"
|
56
|
+
source.sql_attr_bool << "is_deleted"
|
57
|
+
source.sql_attr_timestamp << "posted_ts" << "last_edited_ts" << "date_added"
|
58
|
+
source.sql_attr_str2ordinal << "author_name"
|
59
|
+
source.sql_attr_float << "lat_radians" << "long_radians"
|
60
|
+
source.sql_attr_multi << "uint tag from query; select id, tag FROM tags"
|
61
|
+
source.sql_query_post = ""
|
62
|
+
source.sql_query_post_index = "REPLACE INTO counters (id, val) VALUES ('max_indexed_id', $maxid)"
|
63
|
+
source.sql_ranged_throttle = 0
|
64
|
+
source.sql_query_info = "SELECT * FROM documents WHERE id = $id"
|
65
|
+
|
66
|
+
source.render.should == <<-SQLSOURCE
|
67
|
+
source src1
|
68
|
+
{
|
69
|
+
type = mysql
|
70
|
+
sql_host = localhost
|
71
|
+
sql_user = test
|
72
|
+
sql_pass =
|
73
|
+
sql_db = test
|
74
|
+
sql_port = 3306
|
75
|
+
sql_sock = /tmp/mysql.sock
|
76
|
+
mysql_connect_flags = 32
|
77
|
+
sql_query_pre = SET NAMES utf8
|
78
|
+
sql_query_pre = SET SESSION query_cache_type=OFF
|
79
|
+
sql_query = SELECT id, group_id, UNIX_TIMESTAMP(date_added) AS date_added, title, content FROM documents WHERE id >= $start AND id <= $end
|
80
|
+
sql_query_range = SELECT MIN(id), MAX(id) FROM documents
|
81
|
+
sql_range_step = 1000
|
82
|
+
sql_attr_uint = author_id
|
83
|
+
sql_attr_uint = forum_id:9
|
84
|
+
sql_attr_uint = group_id
|
85
|
+
sql_attr_bool = is_deleted
|
86
|
+
sql_attr_timestamp = posted_ts
|
87
|
+
sql_attr_timestamp = last_edited_ts
|
88
|
+
sql_attr_timestamp = date_added
|
89
|
+
sql_attr_str2ordinal = author_name
|
90
|
+
sql_attr_float = lat_radians
|
91
|
+
sql_attr_float = long_radians
|
92
|
+
sql_attr_multi = uint tag from query; select id, tag FROM tags
|
93
|
+
sql_query_post =
|
94
|
+
sql_query_post_index = REPLACE INTO counters (id, val) VALUES ('max_indexed_id', $maxid)
|
95
|
+
sql_ranged_throttle = 0
|
96
|
+
sql_query_info = SELECT * FROM documents WHERE id = $id
|
97
|
+
}
|
98
|
+
SQLSOURCE
|
99
|
+
end
|
100
|
+
end
|
@@ -0,0 +1,51 @@
|
|
1
|
+
require 'spec/spec_helper'
|
2
|
+
|
3
|
+
describe Riddle::Configuration::XMLSource do
|
4
|
+
it "should be invalid without an xmlpipe command, name and type if there's no parent" do
|
5
|
+
source = Riddle::Configuration::XMLSource.new("xml1", "xmlpipe")
|
6
|
+
source.should_not be_valid
|
7
|
+
|
8
|
+
source.xmlpipe_command = "ls /var/null"
|
9
|
+
source.should be_valid
|
10
|
+
|
11
|
+
source.name = nil
|
12
|
+
source.should_not be_valid
|
13
|
+
|
14
|
+
source.name = "xml1"
|
15
|
+
source.type = nil
|
16
|
+
source.should_not be_valid
|
17
|
+
end
|
18
|
+
|
19
|
+
it "should be invalid without only a name and type if there is a parent" do
|
20
|
+
source = Riddle::Configuration::XMLSource.new("xml1", "xmlpipe")
|
21
|
+
source.should_not be_valid
|
22
|
+
|
23
|
+
source.parent = "xmlparent"
|
24
|
+
source.should be_valid
|
25
|
+
|
26
|
+
source.name = nil
|
27
|
+
source.should_not be_valid
|
28
|
+
|
29
|
+
source.name = "xml1"
|
30
|
+
source.type = nil
|
31
|
+
source.should_not be_valid
|
32
|
+
end
|
33
|
+
|
34
|
+
it "should raise a ConfigurationError if rendering when not valid" do
|
35
|
+
source = Riddle::Configuration::XMLSource.new("xml1", "xmlpipe")
|
36
|
+
lambda { source.render }.should raise_error(Riddle::Configuration::ConfigurationError)
|
37
|
+
end
|
38
|
+
|
39
|
+
it "should render correctly when valid" do
|
40
|
+
source = Riddle::Configuration::XMLSource.new("xml1", "xmlpipe")
|
41
|
+
source.xmlpipe_command = "ls /var/null"
|
42
|
+
|
43
|
+
source.render.should == <<-XMLSOURCE
|
44
|
+
source xml1
|
45
|
+
{
|
46
|
+
type = xmlpipe
|
47
|
+
xmlpipe_command = ls /var/null
|
48
|
+
}
|
49
|
+
XMLSOURCE
|
50
|
+
end
|
51
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
require 'spec/spec_helper'
|
2
|
+
|
3
|
+
describe Riddle::Configuration do
|
4
|
+
it "should render all given indexes and sources, plus the indexer and search sections" do
|
5
|
+
config = Riddle::Configuration.new
|
6
|
+
|
7
|
+
config.searchd.port = 3312
|
8
|
+
config.searchd.pid_file = "file.pid"
|
9
|
+
|
10
|
+
source = Riddle::Configuration::XMLSource.new("src1", "xmlpipe")
|
11
|
+
source.xmlpipe_command = "ls /dev/null"
|
12
|
+
|
13
|
+
index = Riddle::Configuration::Index.new("index1")
|
14
|
+
index.path = "/path/to/index1"
|
15
|
+
index.sources << source
|
16
|
+
|
17
|
+
config.indexes << index
|
18
|
+
generated_conf = config.render
|
19
|
+
|
20
|
+
generated_conf.should match(/index index1/)
|
21
|
+
generated_conf.should match(/source src1/)
|
22
|
+
generated_conf.should match(/indexer/)
|
23
|
+
generated_conf.should match(/searchd/)
|
24
|
+
end
|
25
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: freelancing-god-riddle
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.9.8.1533.
|
4
|
+
version: 0.9.8.1533.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Pat Allan
|
@@ -9,7 +9,7 @@ autorequire:
|
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
11
|
|
12
|
-
date: 2008-11-
|
12
|
+
date: 2008-11-26 00:00:00 -08:00
|
13
13
|
default_executable:
|
14
14
|
dependencies: []
|
15
15
|
|
@@ -26,6 +26,17 @@ files:
|
|
26
26
|
- lib/riddle/client/message.rb
|
27
27
|
- lib/riddle/client/response.rb
|
28
28
|
- lib/riddle/client.rb
|
29
|
+
- lib/riddle/configuration/distributed_index.rb
|
30
|
+
- lib/riddle/configuration/index.rb
|
31
|
+
- lib/riddle/configuration/indexer.rb
|
32
|
+
- lib/riddle/configuration/remote_index.rb
|
33
|
+
- lib/riddle/configuration/searchd.rb
|
34
|
+
- lib/riddle/configuration/section.rb
|
35
|
+
- lib/riddle/configuration/source.rb
|
36
|
+
- lib/riddle/configuration/sql_source.rb
|
37
|
+
- lib/riddle/configuration/xml_source.rb
|
38
|
+
- lib/riddle/configuration.rb
|
39
|
+
- lib/riddle/controller.rb
|
29
40
|
- lib/riddle.rb
|
30
41
|
- lib/tabtab_definitions.rb
|
31
42
|
- MIT-LICENCE
|
@@ -35,6 +46,14 @@ files:
|
|
35
46
|
- spec/functional/search_spec.rb
|
36
47
|
- spec/functional/update_spec.rb
|
37
48
|
- spec/unit/client_spec.rb
|
49
|
+
- spec/unit/configuration/distributed_index_spec.rb
|
50
|
+
- spec/unit/configuration/index_spec.rb
|
51
|
+
- spec/unit/configuration/indexer_spec.rb
|
52
|
+
- spec/unit/configuration/searchd_spec.rb
|
53
|
+
- spec/unit/configuration/source_spec.rb
|
54
|
+
- spec/unit/configuration/sql_source_spec.rb
|
55
|
+
- spec/unit/configuration/xml_source_spec.rb
|
56
|
+
- spec/unit/configuration_spec.rb
|
38
57
|
- spec/unit/filter_spec.rb
|
39
58
|
- spec/unit/message_spec.rb
|
40
59
|
- spec/unit/response_spec.rb
|
@@ -75,6 +94,14 @@ test_files:
|
|
75
94
|
- spec/functional/search_spec.rb
|
76
95
|
- spec/functional/update_spec.rb
|
77
96
|
- spec/unit/client_spec.rb
|
97
|
+
- spec/unit/configuration/distributed_index_spec.rb
|
98
|
+
- spec/unit/configuration/index_spec.rb
|
99
|
+
- spec/unit/configuration/indexer_spec.rb
|
100
|
+
- spec/unit/configuration/searchd_spec.rb
|
101
|
+
- spec/unit/configuration/source_spec.rb
|
102
|
+
- spec/unit/configuration/sql_source_spec.rb
|
103
|
+
- spec/unit/configuration/xml_source_spec.rb
|
104
|
+
- spec/unit/configuration_spec.rb
|
78
105
|
- spec/unit/filter_spec.rb
|
79
106
|
- spec/unit/message_spec.rb
|
80
107
|
- spec/unit/response_spec.rb
|