droonga-engine 1.1.0 → 1.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +2 -1
- data/Rakefile +6 -0
- data/bin/droonga-engine-absorb-data +14 -14
- data/bin/droonga-engine-catalog-generate +24 -12
- data/bin/droonga-engine-catalog-modify +13 -7
- data/bin/droonga-engine-join +8 -8
- data/bin/droonga-engine-set-role +1 -1
- data/bin/droonga-engine-unjoin +2 -2
- data/lib/droonga/address.rb +3 -0
- data/lib/droonga/cluster.rb +16 -10
- data/lib/droonga/command/droonga_engine_service.rb +5 -2
- data/lib/droonga/command/remote_command_base.rb +3 -3
- data/lib/droonga/distributed_command_planner.rb +11 -1
- data/lib/droonga/engine.rb +12 -11
- data/lib/droonga/engine/version.rb +2 -2
- data/lib/droonga/engine_node.rb +28 -28
- data/lib/droonga/engine_state.rb +41 -36
- data/lib/droonga/forward_buffer.rb +21 -10
- data/lib/droonga/node_role.rb +2 -0
- data/lib/droonga/plugins/groonga/select.rb +3 -0
- data/lib/droonga/plugins/search.rb +3 -1
- data/lib/droonga/plugins/search/distributed_search_planner.rb +17 -5
- data/lib/droonga/plugins/system/statistics.rb +1 -0
- data/lib/droonga/searcher.rb +13 -4
- data/test/command/config/single_slice/catalog.json +38 -0
- data/test/command/config/single_slice/droonga-engine.yaml +4 -0
- data/test/command/run-test.rb +3 -2
- data/test/command/suite/catalog/fetch.expected.single_slice +50 -0
- data/test/command/suite/dump/column/index.expected.single_slice +86 -0
- data/test/command/suite/dump/column/scalar.expected.single_slice +52 -0
- data/test/command/suite/dump/column/vector.expected.single_slice +55 -0
- data/test/command/suite/dump/record/scalar.expected.single_slice +52 -0
- data/test/command/suite/dump/record/vector/reference.expected.single_slice +117 -0
- data/test/command/suite/dump/table/array.expected.single_slice +39 -0
- data/test/command/suite/dump/table/double_array_trie.expected.single_slice +40 -0
- data/test/command/suite/dump/table/hash.expected.single_slice +40 -0
- data/test/command/suite/dump/table/patricia_trie.expected.single_slice +40 -0
- data/test/command/suite/message/error/missing-dataset.test +3 -0
- data/test/command/suite/search/condition/query/nonexistent_column.expected.single_slice +26 -0
- data/test/command/suite/search/condition/query/syntax_error.expected.single_slice +26 -0
- data/test/command/suite/search/error/unknown-source.expected.single_slice +28 -0
- data/test/command/suite/search/output/attributes/invalid.expected.single_slice +24 -0
- data/test/command/suite/system/absorb-data/records.catalog.json.single_slice +44 -0
- data/test/command/suite/system/absorb-data/records.expected.single_slice +32 -0
- data/test/command/suite/system/statistics/object/count/per-volume/empty.test +1 -0
- data/test/command/suite/system/statistics/object/count/record.expected.single_slice +11 -0
- data/test/command/suite/system/statistics/object/count/schema.expected.single_slice +11 -0
- data/test/unit/catalog/test_generator.rb +3 -2
- data/test/unit/helper.rb +2 -1
- data/test/unit/helper/stub_serf.rb +28 -0
- data/test/unit/plugins/system/statistics/test_object_count.rb +135 -0
- data/test/unit/plugins/system/statistics/test_object_count_per_volume.rb +149 -0
- data/test/unit/plugins/test_basic.rb +0 -406
- data/test/unit/test_address.rb +111 -10
- data/test/unit/test_cluster.rb +232 -0
- data/test/unit/test_differ.rb +49 -0
- data/test/unit/test_engine_node.rb +556 -0
- data/test/unit/test_engine_state.rb +151 -0
- data/test/unit/test_forward_buffer.rb +106 -0
- data/test/unit/test_node_name.rb +160 -0
- data/test/unit/test_node_role.rb +53 -0
- data/test/unit/test_reducer.rb +525 -0
- metadata +111 -49
@@ -15,8 +15,12 @@
|
|
15
15
|
# License along with this library; if not, write to the Free Software
|
16
16
|
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
17
17
|
|
18
|
+
require "droonga/loggable"
|
19
|
+
|
18
20
|
module Droonga
|
19
21
|
class DistributedCommandPlanner
|
22
|
+
include Loggable
|
23
|
+
|
20
24
|
attr_accessor :key
|
21
25
|
|
22
26
|
REDUCE_SUM = "sum"
|
@@ -38,7 +42,9 @@ module Droonga
|
|
38
42
|
end
|
39
43
|
|
40
44
|
def plan
|
41
|
-
unified_reducers + unified_gatherers + [fixed_processor]
|
45
|
+
steps = unified_reducers + unified_gatherers + [fixed_processor]
|
46
|
+
logger.debug("distribution plan", :steps => steps)
|
47
|
+
steps
|
42
48
|
end
|
43
49
|
|
44
50
|
def reduce(params=nil)
|
@@ -173,5 +179,9 @@ module Droonga
|
|
173
179
|
def plan_errors_handling
|
174
180
|
reduce("errors"=> REDUCE_SUM)
|
175
181
|
end
|
182
|
+
|
183
|
+
def log_tag
|
184
|
+
"distributed_command_planner"
|
185
|
+
end
|
176
186
|
end
|
177
187
|
end
|
data/lib/droonga/engine.rb
CHANGED
@@ -36,20 +36,21 @@ module Droonga
|
|
36
36
|
|
37
37
|
attr_reader :cluster
|
38
38
|
|
39
|
-
def initialize(
|
40
|
-
@name = name
|
41
|
-
@internal_name = internal_name
|
42
|
-
@loop = loop
|
39
|
+
def initialize(params={})
|
40
|
+
@name = params[:name]
|
41
|
+
@internal_name = params[:internal_name]
|
42
|
+
@loop = params[:loop]
|
43
43
|
@catalog = load_catalog
|
44
|
-
@state = EngineState.new(loop
|
45
|
-
|
46
|
-
:
|
44
|
+
@state = EngineState.new(:loop => @loop,
|
45
|
+
:name => @name,
|
46
|
+
:internal_name => @internal_name,
|
47
|
+
:catalog => @catalog,
|
47
48
|
:internal_connection_lifetime =>
|
48
|
-
|
49
|
-
@cluster = Cluster.new(loop,
|
50
|
-
:catalog
|
49
|
+
params[:internal_connection_lifetime])
|
50
|
+
@cluster = Cluster.new(:loop => @loop,
|
51
|
+
:catalog => @catalog,
|
51
52
|
:internal_connection_lifetime =>
|
52
|
-
|
53
|
+
params[:internal_connection_lifetime])
|
53
54
|
|
54
55
|
@dispatcher = create_dispatcher
|
55
56
|
@cluster.on_change = lambda do
|
@@ -1,4 +1,4 @@
|
|
1
|
-
# Copyright (C)
|
1
|
+
# Copyright (C) 2015 Droonga Project
|
2
2
|
#
|
3
3
|
# This library is free software; you can redistribute it and/or
|
4
4
|
# modify it under the terms of the GNU Lesser General Public
|
@@ -15,6 +15,6 @@
|
|
15
15
|
|
16
16
|
module Droonga
|
17
17
|
class Engine
|
18
|
-
VERSION = "1.1.
|
18
|
+
VERSION = "1.1.1"
|
19
19
|
end
|
20
20
|
end
|
data/lib/droonga/engine_node.rb
CHANGED
@@ -30,24 +30,19 @@ module Droonga
|
|
30
30
|
|
31
31
|
attr_reader :name
|
32
32
|
|
33
|
-
def initialize(
|
34
|
-
@loop
|
35
|
-
@name = name
|
36
|
-
@state = state
|
33
|
+
def initialize(params)
|
34
|
+
@loop = params[:loop]
|
35
|
+
@name = params[:name]
|
36
|
+
@state = params[:state]
|
37
37
|
logger.trace("initialize: start")
|
38
38
|
|
39
|
-
@buffer =
|
40
|
-
boundary_timestamp = accept_messages_newer_than_timestamp
|
41
|
-
@buffer.process_messages_newer_than(boundary_timestamp)
|
42
|
-
@buffer.on_forward = lambda do |message, destination|
|
43
|
-
output(message, destination)
|
44
|
-
end
|
39
|
+
@buffer = create_buffer
|
45
40
|
|
46
41
|
@node_name = NodeName.parse(@name)
|
47
42
|
|
48
43
|
@sender = nil
|
49
44
|
@auto_close_timer = nil
|
50
|
-
@auto_close_timeout =
|
45
|
+
@auto_close_timeout = params[:auto_close_timeout] ||
|
51
46
|
DEFAULT_AUTO_CLOSE_TIMEOUT_SECONDS
|
52
47
|
|
53
48
|
logger.trace("initialize: done")
|
@@ -112,19 +107,20 @@ module Droonga
|
|
112
107
|
|
113
108
|
def role
|
114
109
|
if @state
|
115
|
-
@state["role"]
|
110
|
+
NodeRole.normalize(@state["role"])
|
116
111
|
else
|
117
112
|
NodeRole::SERVICE_PROVIDER
|
118
113
|
end
|
119
114
|
end
|
120
115
|
|
121
116
|
def live?
|
122
|
-
@state.nil? or
|
117
|
+
@state.nil? or
|
118
|
+
@state["live"] == true
|
123
119
|
end
|
124
120
|
|
125
121
|
def forwardable?
|
126
122
|
return false unless live?
|
127
|
-
role ==
|
123
|
+
role == sender_role
|
128
124
|
end
|
129
125
|
|
130
126
|
def readable?
|
@@ -133,7 +129,7 @@ module Droonga
|
|
133
129
|
end
|
134
130
|
|
135
131
|
def writable?
|
136
|
-
case
|
132
|
+
case sender_role
|
137
133
|
when NodeRole::SERVICE_PROVIDER
|
138
134
|
true
|
139
135
|
when NodeRole::ABSORB_SOURCE
|
@@ -181,18 +177,21 @@ module Droonga
|
|
181
177
|
end
|
182
178
|
|
183
179
|
private
|
184
|
-
def
|
185
|
-
|
186
|
-
|
180
|
+
def sender_role
|
181
|
+
NodeRole.mine
|
182
|
+
end
|
183
|
+
|
184
|
+
def create_buffer
|
185
|
+
buffer = ForwardBuffer.new(@name)
|
186
|
+
boundary_timestamp = accept_messages_newer_than_timestamp
|
187
|
+
buffer.process_messages_newer_than(boundary_timestamp)
|
188
|
+
buffer.on_forward = lambda do |message, destination|
|
189
|
+
output(message, destination)
|
187
190
|
end
|
188
|
-
|
189
|
-
:host => $1,
|
190
|
-
:port => $2,
|
191
|
-
:tag => $3,
|
192
|
-
}
|
191
|
+
buffer
|
193
192
|
end
|
194
193
|
|
195
|
-
def
|
194
|
+
def have_unprocessed_messages_in_other_node?
|
196
195
|
@state and @state["have_unprocessed_messages"]
|
197
196
|
end
|
198
197
|
|
@@ -222,12 +221,13 @@ module Droonga
|
|
222
221
|
end
|
223
222
|
|
224
223
|
def complete_service_provider?
|
225
|
-
service_provider? and
|
224
|
+
service_provider? and
|
225
|
+
not have_unprocessed_messages_in_other_node?
|
226
226
|
end
|
227
227
|
|
228
228
|
def really_writable?
|
229
229
|
return false unless writable?
|
230
|
-
case
|
230
|
+
case sender_role
|
231
231
|
when NodeRole::SERVICE_PROVIDER
|
232
232
|
service_provider?
|
233
233
|
when NodeRole::ABSORB_SOURCE
|
@@ -249,14 +249,14 @@ module Droonga
|
|
249
249
|
command = destination["type"]
|
250
250
|
receiver = destination["to"]
|
251
251
|
arguments = destination["arguments"]
|
252
|
-
parsed_receiver =
|
252
|
+
parsed_receiver = NodeName.parse(receiver)
|
253
253
|
|
254
254
|
override_message = {
|
255
255
|
"type" => command,
|
256
256
|
}
|
257
257
|
override_message["arguments"] = arguments if arguments
|
258
258
|
message = message.merge(override_message)
|
259
|
-
output_tag = "#{parsed_receiver
|
259
|
+
output_tag = "#{parsed_receiver.tag}.message"
|
260
260
|
log_info = "<#{receiver}>:<#{output_tag}>"
|
261
261
|
logger.trace("forward: start: #{log_info}")
|
262
262
|
sender.send(output_tag, message)
|
data/lib/droonga/engine_state.rb
CHANGED
@@ -17,6 +17,7 @@ require "English"
|
|
17
17
|
|
18
18
|
require "droonga/loggable"
|
19
19
|
require "droonga/deferrable"
|
20
|
+
require "droonga/address"
|
20
21
|
require "droonga/event_loop"
|
21
22
|
require "droonga/forwarder"
|
22
23
|
require "droonga/replier"
|
@@ -37,17 +38,15 @@ module Droonga
|
|
37
38
|
attr_accessor :catalog
|
38
39
|
attr_accessor :on_finish
|
39
40
|
|
40
|
-
def initialize(
|
41
|
-
@loop = loop
|
42
|
-
@name = name
|
43
|
-
@internal_name = internal_name
|
41
|
+
def initialize(params)
|
42
|
+
@loop = params[:loop]
|
43
|
+
@name = params[:name]
|
44
|
+
@internal_name = params[:internal_name]
|
44
45
|
@internal_connection_lifetime = params[:internal_connection_lifetime]
|
45
46
|
@sessions = {}
|
46
47
|
@current_id = 0
|
47
|
-
@forwarder =
|
48
|
-
|
49
|
-
@internal_connection_lifetime)
|
50
|
-
@replier = Replier.new(@forwarder)
|
48
|
+
@forwarder = create_forwarder
|
49
|
+
@replier = create_replier
|
51
50
|
@on_finish = nil
|
52
51
|
@catalog = params[:catalog]
|
53
52
|
end
|
@@ -68,52 +67,48 @@ module Droonga
|
|
68
67
|
route.start_with?(@name) or route.start_with?(@internal_name)
|
69
68
|
end
|
70
69
|
|
71
|
-
FARM_PATH_MATCHER = /\A[^:]+:\d+\/[^.]+/
|
72
|
-
|
73
70
|
def internal_route(route)
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
71
|
+
name = Address.parse(route).node
|
72
|
+
if name == @name
|
73
|
+
route.sub(name, @internal_name)
|
74
|
+
else
|
75
|
+
route
|
79
76
|
end
|
77
|
+
rescue ArgumentError
|
80
78
|
route
|
81
79
|
end
|
82
80
|
|
83
81
|
def public_route(route)
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
82
|
+
name = Address.parse(route).node
|
83
|
+
if name == @internal_name
|
84
|
+
route.sub(name, @name)
|
85
|
+
else
|
86
|
+
route
|
89
87
|
end
|
88
|
+
rescue ArgumentError
|
90
89
|
route
|
91
90
|
end
|
92
91
|
|
93
92
|
def internal_farm_path(route)
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
@internal_name
|
98
|
-
else
|
99
|
-
name
|
100
|
-
end
|
93
|
+
name = Address.parse(route).node
|
94
|
+
if name == @name
|
95
|
+
@internal_name
|
101
96
|
else
|
102
|
-
|
97
|
+
name
|
103
98
|
end
|
99
|
+
rescue ArgumentError
|
100
|
+
route
|
104
101
|
end
|
105
102
|
|
106
103
|
def public_farm_path(route)
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
@name
|
111
|
-
else
|
112
|
-
name
|
113
|
-
end
|
104
|
+
name = Address.parse(route).node
|
105
|
+
if name == @internal_name
|
106
|
+
@name
|
114
107
|
else
|
115
|
-
|
108
|
+
name
|
116
109
|
end
|
110
|
+
rescue ArgumentError
|
111
|
+
route
|
117
112
|
end
|
118
113
|
|
119
114
|
def generate_id
|
@@ -154,6 +149,16 @@ module Droonga
|
|
154
149
|
end
|
155
150
|
|
156
151
|
private
|
152
|
+
def create_forwarder
|
153
|
+
Forwarder.new(@loop,
|
154
|
+
:auto_close_timeout =>
|
155
|
+
@internal_connection_lifetime)
|
156
|
+
end
|
157
|
+
|
158
|
+
def create_replier
|
159
|
+
Replier.new(@forwarder)
|
160
|
+
end
|
161
|
+
|
157
162
|
def log_tag
|
158
163
|
"engine_state"
|
159
164
|
end
|
@@ -41,16 +41,12 @@ module Droonga
|
|
41
41
|
@unpacker = MessagePack::Unpacker.new
|
42
42
|
|
43
43
|
@target = node_name
|
44
|
-
@
|
45
|
-
|
46
|
-
dirname = node_name.gsub("/", ":")
|
47
|
-
@data_directory = Path.intentional_buffer + dirname
|
48
|
-
FileUtils.mkdir_p(@data_directory.to_s)
|
44
|
+
@process_messages_newer_than_timestamp = nil
|
49
45
|
end
|
50
46
|
|
51
47
|
def add(message, destination)
|
52
48
|
logger.trace("add: start")
|
53
|
-
|
49
|
+
serf.set_have_unprocessed_messages_for(@target)
|
54
50
|
buffered_message = {
|
55
51
|
"message" => message,
|
56
52
|
"destination" => destination,
|
@@ -81,18 +77,18 @@ module Droonga
|
|
81
77
|
# (ex. messages generated by Dispatcher)
|
82
78
|
@process_messages_newer_than_timestamp = nil
|
83
79
|
end
|
84
|
-
|
80
|
+
serf.reset_have_unprocessed_messages_for(@target)
|
85
81
|
logger.trace("start_forward: done")
|
86
82
|
end
|
87
83
|
|
88
84
|
def buffered_messages
|
89
|
-
Pathname.glob("#{
|
85
|
+
Pathname.glob("#{data_directory}/*#{SUFFIX}").sort_by do |path|
|
90
86
|
path
|
91
87
|
end
|
92
88
|
end
|
93
89
|
|
94
90
|
def empty?
|
95
|
-
|
91
|
+
data_directory.children.empty?
|
96
92
|
end
|
97
93
|
|
98
94
|
def process_messages_newer_than(timestamp)
|
@@ -100,6 +96,21 @@ module Droonga
|
|
100
96
|
end
|
101
97
|
|
102
98
|
private
|
99
|
+
def data_directory
|
100
|
+
@data_directory ||= resolve_data_directory
|
101
|
+
end
|
102
|
+
|
103
|
+
def resolve_data_directory
|
104
|
+
dirname = @target.gsub("/", ":")
|
105
|
+
data_directory = Path.intentional_buffer + dirname
|
106
|
+
FileUtils.mkdir_p(data_directory.to_s)
|
107
|
+
data_directory
|
108
|
+
end
|
109
|
+
|
110
|
+
def serf
|
111
|
+
@serf ||= Serf.new(ENV["DROONGA_ENGINE_NAME"] || @target)
|
112
|
+
end
|
113
|
+
|
103
114
|
def forward(buffered_message_path)
|
104
115
|
logger.trace("forward: start (#{buffered_message_path})")
|
105
116
|
|
@@ -146,7 +157,7 @@ module Droonga
|
|
146
157
|
|
147
158
|
def create_buffered_message_path(time_stamp=Time.now)
|
148
159
|
basename = Timestamp.stringify(time_stamp)
|
149
|
-
Path.unique_file_path(
|
160
|
+
Path.unique_file_path(data_directory, basename, SUFFIX)
|
150
161
|
end
|
151
162
|
|
152
163
|
def on_forward(message, destination)
|
data/lib/droonga/node_role.rb
CHANGED
@@ -256,6 +256,9 @@ module Droonga
|
|
256
256
|
converter = RequestConverter.new
|
257
257
|
select_request = input_message.body
|
258
258
|
search_request = converter.convert(select_request)
|
259
|
+
logger.debug("Conversion of select:",
|
260
|
+
:select => select_request,
|
261
|
+
:search => search_request)
|
259
262
|
input_message.type = "search"
|
260
263
|
input_message.body = search_request
|
261
264
|
end
|
@@ -69,7 +69,9 @@ module Droonga
|
|
69
69
|
# I have to apply it at first, before "limit" and "offset" are applied.
|
70
70
|
count_mapper = elements["count"]
|
71
71
|
if count_mapper
|
72
|
-
if count_mapper
|
72
|
+
if count_mapper.is_a?(String)
|
73
|
+
value["count"] = value[count_mapper]
|
74
|
+
elsif count_mapper["no_output"]
|
73
75
|
value.delete("count")
|
74
76
|
else
|
75
77
|
value["count"] = value[count_mapper["target"]].size
|
@@ -39,9 +39,11 @@ module Droonga
|
|
39
39
|
|
40
40
|
ensure_unifiable!
|
41
41
|
|
42
|
+
logger.debug("from", :queries => @request)
|
42
43
|
@queries.each do |input_name, query|
|
43
44
|
transform_query(input_name, query)
|
44
45
|
end
|
46
|
+
logger.debug("to", :queries => @request)
|
45
47
|
|
46
48
|
broadcast(:body => @request)
|
47
49
|
|
@@ -99,6 +101,10 @@ module Droonga
|
|
99
101
|
end
|
100
102
|
end
|
101
103
|
|
104
|
+
def log_tag
|
105
|
+
"distributed_search_planner"
|
106
|
+
end
|
107
|
+
|
102
108
|
class QueryTransformer
|
103
109
|
attr_reader :reducers, :mappers
|
104
110
|
|
@@ -154,6 +160,8 @@ module Droonga
|
|
154
160
|
@records_offset = final_offset
|
155
161
|
@records_limit = final_limit
|
156
162
|
|
163
|
+
return unless @dataset.sliced?
|
164
|
+
|
157
165
|
updated_sort_limit = nil
|
158
166
|
updated_output_limit = nil
|
159
167
|
if final_limit == UNLIMITED
|
@@ -174,6 +182,7 @@ module Droonga
|
|
174
182
|
end
|
175
183
|
|
176
184
|
def calculate_sort_offset!
|
185
|
+
return unless @dataset.sliced?
|
177
186
|
# Offset for workers must be zero, because we have to apply "limit" and
|
178
187
|
# "offset" on the last gathering phase instead of each reducing phase.
|
179
188
|
if rich_sort?
|
@@ -206,17 +215,18 @@ module Droonga
|
|
206
215
|
end
|
207
216
|
|
208
217
|
def calculate_output_offset!
|
218
|
+
return unless @dataset.sliced?
|
209
219
|
@output["offset"] = 0 if have_records? and @output["offset"]
|
210
220
|
end
|
211
221
|
|
212
222
|
def final_offset
|
213
|
-
return
|
223
|
+
return 0 unless @dataset.sliced?
|
214
224
|
|
215
225
|
@original_sort_offset + @original_output_offset
|
216
226
|
end
|
217
227
|
|
218
228
|
def final_limit
|
219
|
-
return
|
229
|
+
return UNLIMITED unless @dataset.sliced?
|
220
230
|
|
221
231
|
if @original_sort_limit == UNLIMITED and
|
222
232
|
@original_output_limit == UNLIMITED
|
@@ -256,10 +266,12 @@ module Droonga
|
|
256
266
|
@query["sortBy"]["limit"] = UNLIMITED
|
257
267
|
end
|
258
268
|
@output["limit"] = UNLIMITED
|
269
|
+
mapper = {
|
270
|
+
"target" => "records",
|
271
|
+
}
|
272
|
+
else
|
273
|
+
mapper = "count"
|
259
274
|
end
|
260
|
-
mapper = {
|
261
|
-
"target" => "records",
|
262
|
-
}
|
263
275
|
unless @output["elements"].include?("records")
|
264
276
|
@records_limit = UNLIMITED if @dataset.sliced?
|
265
277
|
@output["elements"] << "records"
|