droonga-engine 1.1.0 → 1.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +2 -1
  3. data/Rakefile +6 -0
  4. data/bin/droonga-engine-absorb-data +14 -14
  5. data/bin/droonga-engine-catalog-generate +24 -12
  6. data/bin/droonga-engine-catalog-modify +13 -7
  7. data/bin/droonga-engine-join +8 -8
  8. data/bin/droonga-engine-set-role +1 -1
  9. data/bin/droonga-engine-unjoin +2 -2
  10. data/lib/droonga/address.rb +3 -0
  11. data/lib/droonga/cluster.rb +16 -10
  12. data/lib/droonga/command/droonga_engine_service.rb +5 -2
  13. data/lib/droonga/command/remote_command_base.rb +3 -3
  14. data/lib/droonga/distributed_command_planner.rb +11 -1
  15. data/lib/droonga/engine.rb +12 -11
  16. data/lib/droonga/engine/version.rb +2 -2
  17. data/lib/droonga/engine_node.rb +28 -28
  18. data/lib/droonga/engine_state.rb +41 -36
  19. data/lib/droonga/forward_buffer.rb +21 -10
  20. data/lib/droonga/node_role.rb +2 -0
  21. data/lib/droonga/plugins/groonga/select.rb +3 -0
  22. data/lib/droonga/plugins/search.rb +3 -1
  23. data/lib/droonga/plugins/search/distributed_search_planner.rb +17 -5
  24. data/lib/droonga/plugins/system/statistics.rb +1 -0
  25. data/lib/droonga/searcher.rb +13 -4
  26. data/test/command/config/single_slice/catalog.json +38 -0
  27. data/test/command/config/single_slice/droonga-engine.yaml +4 -0
  28. data/test/command/run-test.rb +3 -2
  29. data/test/command/suite/catalog/fetch.expected.single_slice +50 -0
  30. data/test/command/suite/dump/column/index.expected.single_slice +86 -0
  31. data/test/command/suite/dump/column/scalar.expected.single_slice +52 -0
  32. data/test/command/suite/dump/column/vector.expected.single_slice +55 -0
  33. data/test/command/suite/dump/record/scalar.expected.single_slice +52 -0
  34. data/test/command/suite/dump/record/vector/reference.expected.single_slice +117 -0
  35. data/test/command/suite/dump/table/array.expected.single_slice +39 -0
  36. data/test/command/suite/dump/table/double_array_trie.expected.single_slice +40 -0
  37. data/test/command/suite/dump/table/hash.expected.single_slice +40 -0
  38. data/test/command/suite/dump/table/patricia_trie.expected.single_slice +40 -0
  39. data/test/command/suite/message/error/missing-dataset.test +3 -0
  40. data/test/command/suite/search/condition/query/nonexistent_column.expected.single_slice +26 -0
  41. data/test/command/suite/search/condition/query/syntax_error.expected.single_slice +26 -0
  42. data/test/command/suite/search/error/unknown-source.expected.single_slice +28 -0
  43. data/test/command/suite/search/output/attributes/invalid.expected.single_slice +24 -0
  44. data/test/command/suite/system/absorb-data/records.catalog.json.single_slice +44 -0
  45. data/test/command/suite/system/absorb-data/records.expected.single_slice +32 -0
  46. data/test/command/suite/system/statistics/object/count/per-volume/empty.test +1 -0
  47. data/test/command/suite/system/statistics/object/count/record.expected.single_slice +11 -0
  48. data/test/command/suite/system/statistics/object/count/schema.expected.single_slice +11 -0
  49. data/test/unit/catalog/test_generator.rb +3 -2
  50. data/test/unit/helper.rb +2 -1
  51. data/test/unit/helper/stub_serf.rb +28 -0
  52. data/test/unit/plugins/system/statistics/test_object_count.rb +135 -0
  53. data/test/unit/plugins/system/statistics/test_object_count_per_volume.rb +149 -0
  54. data/test/unit/plugins/test_basic.rb +0 -406
  55. data/test/unit/test_address.rb +111 -10
  56. data/test/unit/test_cluster.rb +232 -0
  57. data/test/unit/test_differ.rb +49 -0
  58. data/test/unit/test_engine_node.rb +556 -0
  59. data/test/unit/test_engine_state.rb +151 -0
  60. data/test/unit/test_forward_buffer.rb +106 -0
  61. data/test/unit/test_node_name.rb +160 -0
  62. data/test/unit/test_node_role.rb +53 -0
  63. data/test/unit/test_reducer.rb +525 -0
  64. metadata +111 -49
@@ -15,8 +15,12 @@
15
15
  # License along with this library; if not, write to the Free Software
16
16
  # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
 
18
+ require "droonga/loggable"
19
+
18
20
  module Droonga
19
21
  class DistributedCommandPlanner
22
+ include Loggable
23
+
20
24
  attr_accessor :key
21
25
 
22
26
  REDUCE_SUM = "sum"
@@ -38,7 +42,9 @@ module Droonga
38
42
  end
39
43
 
40
44
  def plan
41
- unified_reducers + unified_gatherers + [fixed_processor]
45
+ steps = unified_reducers + unified_gatherers + [fixed_processor]
46
+ logger.debug("distribution plan", :steps => steps)
47
+ steps
42
48
  end
43
49
 
44
50
  def reduce(params=nil)
@@ -173,5 +179,9 @@ module Droonga
173
179
  def plan_errors_handling
174
180
  reduce("errors"=> REDUCE_SUM)
175
181
  end
182
+
183
+ def log_tag
184
+ "distributed_command_planner"
185
+ end
176
186
  end
177
187
  end
@@ -36,20 +36,21 @@ module Droonga
36
36
 
37
37
  attr_reader :cluster
38
38
 
39
- def initialize(loop, name, internal_name, options={})
40
- @name = name
41
- @internal_name = internal_name
42
- @loop = loop
39
+ def initialize(params={})
40
+ @name = params[:name]
41
+ @internal_name = params[:internal_name]
42
+ @loop = params[:loop]
43
43
  @catalog = load_catalog
44
- @state = EngineState.new(loop, name,
45
- internal_name,
46
- :catalog => @catalog,
44
+ @state = EngineState.new(:loop => @loop,
45
+ :name => @name,
46
+ :internal_name => @internal_name,
47
+ :catalog => @catalog,
47
48
  :internal_connection_lifetime =>
48
- options[:internal_connection_lifetime])
49
- @cluster = Cluster.new(loop,
50
- :catalog => @catalog,
49
+ params[:internal_connection_lifetime])
50
+ @cluster = Cluster.new(:loop => @loop,
51
+ :catalog => @catalog,
51
52
  :internal_connection_lifetime =>
52
- options[:internal_connection_lifetime])
53
+ params[:internal_connection_lifetime])
53
54
 
54
55
  @dispatcher = create_dispatcher
55
56
  @cluster.on_change = lambda do
@@ -1,4 +1,4 @@
1
- # Copyright (C) 2014 Droonga Project
1
+ # Copyright (C) 2015 Droonga Project
2
2
  #
3
3
  # This library is free software; you can redistribute it and/or
4
4
  # modify it under the terms of the GNU Lesser General Public
@@ -15,6 +15,6 @@
15
15
 
16
16
  module Droonga
17
17
  class Engine
18
- VERSION = "1.1.0"
18
+ VERSION = "1.1.1"
19
19
  end
20
20
  end
@@ -30,24 +30,19 @@ module Droonga
30
30
 
31
31
  attr_reader :name
32
32
 
33
- def initialize(loop, name, state, options={})
34
- @loop = loop
35
- @name = name
36
- @state = state
33
+ def initialize(params)
34
+ @loop = params[:loop]
35
+ @name = params[:name]
36
+ @state = params[:state]
37
37
  logger.trace("initialize: start")
38
38
 
39
- @buffer = ForwardBuffer.new(name)
40
- boundary_timestamp = accept_messages_newer_than_timestamp
41
- @buffer.process_messages_newer_than(boundary_timestamp)
42
- @buffer.on_forward = lambda do |message, destination|
43
- output(message, destination)
44
- end
39
+ @buffer = create_buffer
45
40
 
46
41
  @node_name = NodeName.parse(@name)
47
42
 
48
43
  @sender = nil
49
44
  @auto_close_timer = nil
50
- @auto_close_timeout = options[:auto_close_timeout] ||
45
+ @auto_close_timeout = params[:auto_close_timeout] ||
51
46
  DEFAULT_AUTO_CLOSE_TIMEOUT_SECONDS
52
47
 
53
48
  logger.trace("initialize: done")
@@ -112,19 +107,20 @@ module Droonga
112
107
 
113
108
  def role
114
109
  if @state
115
- @state["role"]
110
+ NodeRole.normalize(@state["role"])
116
111
  else
117
112
  NodeRole::SERVICE_PROVIDER
118
113
  end
119
114
  end
120
115
 
121
116
  def live?
122
- @state.nil? or @state["live"]
117
+ @state.nil? or
118
+ @state["live"] == true
123
119
  end
124
120
 
125
121
  def forwardable?
126
122
  return false unless live?
127
- role == NodeRole.mine
123
+ role == sender_role
128
124
  end
129
125
 
130
126
  def readable?
@@ -133,7 +129,7 @@ module Droonga
133
129
  end
134
130
 
135
131
  def writable?
136
- case NodeRole.mine
132
+ case sender_role
137
133
  when NodeRole::SERVICE_PROVIDER
138
134
  true
139
135
  when NodeRole::ABSORB_SOURCE
@@ -181,18 +177,21 @@ module Droonga
181
177
  end
182
178
 
183
179
  private
184
- def parse_node_name(name)
185
- unless name =~ /\A(.*):(\d+)\/([^.]+)\z/
186
- raise "name format: hostname:port/tag"
180
+ def sender_role
181
+ NodeRole.mine
182
+ end
183
+
184
+ def create_buffer
185
+ buffer = ForwardBuffer.new(@name)
186
+ boundary_timestamp = accept_messages_newer_than_timestamp
187
+ buffer.process_messages_newer_than(boundary_timestamp)
188
+ buffer.on_forward = lambda do |message, destination|
189
+ output(message, destination)
187
190
  end
188
- {
189
- :host => $1,
190
- :port => $2,
191
- :tag => $3,
192
- }
191
+ buffer
193
192
  end
194
193
 
195
- def have_unprocessed_messages?
194
+ def have_unprocessed_messages_in_other_node?
196
195
  @state and @state["have_unprocessed_messages"]
197
196
  end
198
197
 
@@ -222,12 +221,13 @@ module Droonga
222
221
  end
223
222
 
224
223
  def complete_service_provider?
225
- service_provider? and not have_unprocessed_messages?
224
+ service_provider? and
225
+ not have_unprocessed_messages_in_other_node?
226
226
  end
227
227
 
228
228
  def really_writable?
229
229
  return false unless writable?
230
- case NodeRole.mine
230
+ case sender_role
231
231
  when NodeRole::SERVICE_PROVIDER
232
232
  service_provider?
233
233
  when NodeRole::ABSORB_SOURCE
@@ -249,14 +249,14 @@ module Droonga
249
249
  command = destination["type"]
250
250
  receiver = destination["to"]
251
251
  arguments = destination["arguments"]
252
- parsed_receiver = parse_node_name(receiver)
252
+ parsed_receiver = NodeName.parse(receiver)
253
253
 
254
254
  override_message = {
255
255
  "type" => command,
256
256
  }
257
257
  override_message["arguments"] = arguments if arguments
258
258
  message = message.merge(override_message)
259
- output_tag = "#{parsed_receiver[:tag]}.message"
259
+ output_tag = "#{parsed_receiver.tag}.message"
260
260
  log_info = "<#{receiver}>:<#{output_tag}>"
261
261
  logger.trace("forward: start: #{log_info}")
262
262
  sender.send(output_tag, message)
@@ -17,6 +17,7 @@ require "English"
17
17
 
18
18
  require "droonga/loggable"
19
19
  require "droonga/deferrable"
20
+ require "droonga/address"
20
21
  require "droonga/event_loop"
21
22
  require "droonga/forwarder"
22
23
  require "droonga/replier"
@@ -37,17 +38,15 @@ module Droonga
37
38
  attr_accessor :catalog
38
39
  attr_accessor :on_finish
39
40
 
40
- def initialize(loop, name, internal_name, params)
41
- @loop = loop
42
- @name = name
43
- @internal_name = internal_name
41
+ def initialize(params)
42
+ @loop = params[:loop]
43
+ @name = params[:name]
44
+ @internal_name = params[:internal_name]
44
45
  @internal_connection_lifetime = params[:internal_connection_lifetime]
45
46
  @sessions = {}
46
47
  @current_id = 0
47
- @forwarder = Forwarder.new(@loop,
48
- :auto_close_timeout =>
49
- @internal_connection_lifetime)
50
- @replier = Replier.new(@forwarder)
48
+ @forwarder = create_forwarder
49
+ @replier = create_replier
51
50
  @on_finish = nil
52
51
  @catalog = params[:catalog]
53
52
  end
@@ -68,52 +67,48 @@ module Droonga
68
67
  route.start_with?(@name) or route.start_with?(@internal_name)
69
68
  end
70
69
 
71
- FARM_PATH_MATCHER = /\A[^:]+:\d+\/[^.]+/
72
-
73
70
  def internal_route(route)
74
- if FARM_PATH_MATCHER =~ route
75
- name = $MATCH
76
- if name == @name or name == @internal_name
77
- return route.sub(name, @internal_name)
78
- end
71
+ name = Address.parse(route).node
72
+ if name == @name
73
+ route.sub(name, @internal_name)
74
+ else
75
+ route
79
76
  end
77
+ rescue ArgumentError
80
78
  route
81
79
  end
82
80
 
83
81
  def public_route(route)
84
- if FARM_PATH_MATCHER =~ route
85
- name = $MATCH
86
- if name == @internal_name
87
- return route.sub(name, @name)
88
- end
82
+ name = Address.parse(route).node
83
+ if name == @internal_name
84
+ route.sub(name, @name)
85
+ else
86
+ route
89
87
  end
88
+ rescue ArgumentError
90
89
  route
91
90
  end
92
91
 
93
92
  def internal_farm_path(route)
94
- if FARM_PATH_MATCHER =~ route
95
- name = $MATCH
96
- if name == @name or name == @internal_name
97
- @internal_name
98
- else
99
- name
100
- end
93
+ name = Address.parse(route).node
94
+ if name == @name
95
+ @internal_name
101
96
  else
102
- route
97
+ name
103
98
  end
99
+ rescue ArgumentError
100
+ route
104
101
  end
105
102
 
106
103
  def public_farm_path(route)
107
- if FARM_PATH_MATCHER =~ route
108
- name = $MATCH
109
- if name == @internal_name
110
- @name
111
- else
112
- name
113
- end
104
+ name = Address.parse(route).node
105
+ if name == @internal_name
106
+ @name
114
107
  else
115
- route
108
+ name
116
109
  end
110
+ rescue ArgumentError
111
+ route
117
112
  end
118
113
 
119
114
  def generate_id
@@ -154,6 +149,16 @@ module Droonga
154
149
  end
155
150
 
156
151
  private
152
+ def create_forwarder
153
+ Forwarder.new(@loop,
154
+ :auto_close_timeout =>
155
+ @internal_connection_lifetime)
156
+ end
157
+
158
+ def create_replier
159
+ Replier.new(@forwarder)
160
+ end
161
+
157
162
  def log_tag
158
163
  "engine_state"
159
164
  end
@@ -41,16 +41,12 @@ module Droonga
41
41
  @unpacker = MessagePack::Unpacker.new
42
42
 
43
43
  @target = node_name
44
- @serf = Serf.new(ENV["DROONGA_ENGINE_NAME"])
45
-
46
- dirname = node_name.gsub("/", ":")
47
- @data_directory = Path.intentional_buffer + dirname
48
- FileUtils.mkdir_p(@data_directory.to_s)
44
+ @process_messages_newer_than_timestamp = nil
49
45
  end
50
46
 
51
47
  def add(message, destination)
52
48
  logger.trace("add: start")
53
- @serf.set_have_unprocessed_messages_for(@target)
49
+ serf.set_have_unprocessed_messages_for(@target)
54
50
  buffered_message = {
55
51
  "message" => message,
56
52
  "destination" => destination,
@@ -81,18 +77,18 @@ module Droonga
81
77
  # (ex. messages generated by Dispatcher)
82
78
  @process_messages_newer_than_timestamp = nil
83
79
  end
84
- @serf.reset_have_unprocessed_messages_for(@target)
80
+ serf.reset_have_unprocessed_messages_for(@target)
85
81
  logger.trace("start_forward: done")
86
82
  end
87
83
 
88
84
  def buffered_messages
89
- Pathname.glob("#{@data_directory}/*#{SUFFIX}").sort_by do |path|
85
+ Pathname.glob("#{data_directory}/*#{SUFFIX}").sort_by do |path|
90
86
  path
91
87
  end
92
88
  end
93
89
 
94
90
  def empty?
95
- @data_directory.children.empty?
91
+ data_directory.children.empty?
96
92
  end
97
93
 
98
94
  def process_messages_newer_than(timestamp)
@@ -100,6 +96,21 @@ module Droonga
100
96
  end
101
97
 
102
98
  private
99
+ def data_directory
100
+ @data_directory ||= resolve_data_directory
101
+ end
102
+
103
+ def resolve_data_directory
104
+ dirname = @target.gsub("/", ":")
105
+ data_directory = Path.intentional_buffer + dirname
106
+ FileUtils.mkdir_p(data_directory.to_s)
107
+ data_directory
108
+ end
109
+
110
+ def serf
111
+ @serf ||= Serf.new(ENV["DROONGA_ENGINE_NAME"] || @target)
112
+ end
113
+
103
114
  def forward(buffered_message_path)
104
115
  logger.trace("forward: start (#{buffered_message_path})")
105
116
 
@@ -146,7 +157,7 @@ module Droonga
146
157
 
147
158
  def create_buffered_message_path(time_stamp=Time.now)
148
159
  basename = Timestamp.stringify(time_stamp)
149
- Path.unique_file_path(@data_directory, basename, SUFFIX)
160
+ Path.unique_file_path(data_directory, basename, SUFFIX)
150
161
  end
151
162
 
152
163
  def on_forward(message, destination)
@@ -26,6 +26,8 @@ module Droonga
26
26
 
27
27
  ANY = "any".downcase
28
28
 
29
+ #XXX ANY is not a valid role for a node. It is used
30
+ # just for checking acceptability of messages.
29
31
  ROLES = [
30
32
  SERVICE_PROVIDER,
31
33
  ABSORB_SOURCE,
@@ -256,6 +256,9 @@ module Droonga
256
256
  converter = RequestConverter.new
257
257
  select_request = input_message.body
258
258
  search_request = converter.convert(select_request)
259
+ logger.debug("Conversion of select:",
260
+ :select => select_request,
261
+ :search => search_request)
259
262
  input_message.type = "search"
260
263
  input_message.body = search_request
261
264
  end
@@ -69,7 +69,9 @@ module Droonga
69
69
  # I have to apply it at first, before "limit" and "offset" are applied.
70
70
  count_mapper = elements["count"]
71
71
  if count_mapper
72
- if count_mapper["no_output"]
72
+ if count_mapper.is_a?(String)
73
+ value["count"] = value[count_mapper]
74
+ elsif count_mapper["no_output"]
73
75
  value.delete("count")
74
76
  else
75
77
  value["count"] = value[count_mapper["target"]].size
@@ -39,9 +39,11 @@ module Droonga
39
39
 
40
40
  ensure_unifiable!
41
41
 
42
+ logger.debug("from", :queries => @request)
42
43
  @queries.each do |input_name, query|
43
44
  transform_query(input_name, query)
44
45
  end
46
+ logger.debug("to", :queries => @request)
45
47
 
46
48
  broadcast(:body => @request)
47
49
 
@@ -99,6 +101,10 @@ module Droonga
99
101
  end
100
102
  end
101
103
 
104
+ def log_tag
105
+ "distributed_search_planner"
106
+ end
107
+
102
108
  class QueryTransformer
103
109
  attr_reader :reducers, :mappers
104
110
 
@@ -154,6 +160,8 @@ module Droonga
154
160
  @records_offset = final_offset
155
161
  @records_limit = final_limit
156
162
 
163
+ return unless @dataset.sliced?
164
+
157
165
  updated_sort_limit = nil
158
166
  updated_output_limit = nil
159
167
  if final_limit == UNLIMITED
@@ -174,6 +182,7 @@ module Droonga
174
182
  end
175
183
 
176
184
  def calculate_sort_offset!
185
+ return unless @dataset.sliced?
177
186
  # Offset for workers must be zero, because we have to apply "limit" and
178
187
  # "offset" on the last gathering phase instead of each reducing phase.
179
188
  if rich_sort?
@@ -206,17 +215,18 @@ module Droonga
206
215
  end
207
216
 
208
217
  def calculate_output_offset!
218
+ return unless @dataset.sliced?
209
219
  @output["offset"] = 0 if have_records? and @output["offset"]
210
220
  end
211
221
 
212
222
  def final_offset
213
- return @original_output_offset unless @dataset.sliced?
223
+ return 0 unless @dataset.sliced?
214
224
 
215
225
  @original_sort_offset + @original_output_offset
216
226
  end
217
227
 
218
228
  def final_limit
219
- return @original_output_limit unless @dataset.sliced?
229
+ return UNLIMITED unless @dataset.sliced?
220
230
 
221
231
  if @original_sort_limit == UNLIMITED and
222
232
  @original_output_limit == UNLIMITED
@@ -256,10 +266,12 @@ module Droonga
256
266
  @query["sortBy"]["limit"] = UNLIMITED
257
267
  end
258
268
  @output["limit"] = UNLIMITED
269
+ mapper = {
270
+ "target" => "records",
271
+ }
272
+ else
273
+ mapper = "count"
259
274
  end
260
- mapper = {
261
- "target" => "records",
262
- }
263
275
  unless @output["elements"].include?("records")
264
276
  @records_limit = UNLIMITED if @dataset.sliced?
265
277
  @output["elements"] << "records"