droonga-engine 1.0.3 → 1.0.4

Sign up to get free protection for your applications and to get access to all the features.
Files changed (44) hide show
  1. checksums.yaml +4 -4
  2. data/Gemfile +7 -0
  3. data/bin/droonga-engine-absorb-data +82 -0
  4. data/bin/droonga-engine-catalog-generate +16 -13
  5. data/bin/droonga-engine-catalog-modify +108 -0
  6. data/bin/droonga-engine-join +115 -0
  7. data/bin/droonga-engine-unjoin +90 -0
  8. data/doc/text/news.md +8 -0
  9. data/droonga-engine.gemspec +2 -1
  10. data/lib/droonga/buffered_tcp_socket.rb +132 -0
  11. data/lib/droonga/catalog_generator.rb +87 -4
  12. data/lib/droonga/command/droonga_engine.rb +27 -7
  13. data/lib/droonga/command/droonga_engine_service.rb +3 -2
  14. data/lib/droonga/command/serf_event_handler.rb +211 -14
  15. data/lib/droonga/data_absorber.rb +55 -0
  16. data/lib/droonga/dispatcher.rb +25 -11
  17. data/lib/droonga/engine/version.rb +1 -1
  18. data/lib/droonga/engine.rb +24 -24
  19. data/lib/droonga/engine_state.rb +23 -0
  20. data/lib/droonga/{catalog_observer.rb → file_observer.rb} +12 -7
  21. data/lib/droonga/fluent_message_sender.rb +24 -37
  22. data/lib/droonga/forwarder.rb +30 -5
  23. data/lib/droonga/handler_messenger.rb +3 -2
  24. data/lib/droonga/handler_runner.rb +29 -16
  25. data/lib/droonga/job_pusher.rb +12 -0
  26. data/lib/droonga/line_buffer.rb +42 -0
  27. data/lib/droonga/logger.rb +10 -6
  28. data/lib/droonga/path.rb +16 -0
  29. data/lib/droonga/plugins/search/distributed_search_planner.rb +1 -1
  30. data/lib/droonga/plugins/system.rb +50 -0
  31. data/lib/droonga/processor.rb +9 -4
  32. data/lib/droonga/safe_file_writer.rb +39 -0
  33. data/lib/droonga/serf.rb +212 -14
  34. data/lib/droonga/test/stub_handler_messenger.rb +3 -0
  35. data/lib/droonga/worker.rb +6 -1
  36. data/test/command/config/default/catalog.json +1 -1
  37. data/test/command/config/version1/catalog.json +2 -2
  38. data/test/command/suite/system/status.expected +12 -0
  39. data/test/command/suite/system/status.test +5 -0
  40. data/test/unit/plugins/system/test_status.rb +79 -0
  41. data/test/unit/test_catalog_generator.rb +1 -1
  42. data/test/unit/test_line_buffer.rb +62 -0
  43. metadata +46 -12
  44. data/lib/droonga/live_nodes_list_observer.rb +0 -72
@@ -0,0 +1,39 @@
1
+ # Copyright (C) 2013-2014 Droonga Project
2
+ #
3
+ # This library is free software; you can redistribute it and/or
4
+ # modify it under the terms of the GNU Lesser General Public
5
+ # License version 2.1 as published by the Free Software Foundation.
6
+ #
7
+ # This library is distributed in the hope that it will be useful,
8
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
9
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
10
+ # Lesser General Public License for more details.
11
+ #
12
+ # You should have received a copy of the GNU Lesser General Public
13
+ # License along with this library; if not, write to the Free Software
14
+ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
15
+
16
+ require "pathname"
17
+ require "fileutils"
18
+ require "tempfile"
19
+
20
+ module Droonga
21
+ class SafeFileWriter
22
+ class << self
23
+ def write(path, contents=nil)
24
+ # Don't output the file directly to prevent loading of incomplete file!
25
+ path = Pathname(path).expand_path
26
+ FileUtils.mkdir_p(path.dirname.to_s)
27
+ Tempfile.open(path.basename.to_s, path.dirname.to_s, "w") do |output|
28
+ if block_given?
29
+ yield(output)
30
+ else
31
+ output.write(contents)
32
+ end
33
+ output.flush
34
+ File.rename(output.path, path.to_s)
35
+ end
36
+ end
37
+ end
38
+ end
39
+ end
data/lib/droonga/serf.rb CHANGED
@@ -13,25 +13,67 @@
13
13
  # License along with this library; if not, write to the Free Software
14
14
  # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
15
15
 
16
+ require "English"
17
+
18
+ require "json"
19
+ require "coolio"
20
+
16
21
  require "droonga/path"
17
22
  require "droonga/loggable"
18
23
  require "droonga/catalog_loader"
19
24
  require "droonga/serf_downloader"
25
+ require "droonga/line_buffer"
20
26
 
21
27
  module Droonga
22
28
  class Serf
29
+ ROLE = {
30
+ :default => {
31
+ :port => 7946,
32
+ },
33
+ :source => {
34
+ :port => 7947,
35
+ },
36
+ :destination => {
37
+ :port => 7948,
38
+ },
39
+ }
40
+
23
41
  class << self
24
42
  def path
25
43
  Droonga::Path.base + "serf"
26
44
  end
45
+
46
+ def status_file
47
+ Droonga::Path.state + "status_file"
48
+ end
49
+
50
+ def load_status
51
+ if status_file.exist?
52
+ contents = status_file.read
53
+ unless contents.empty?
54
+ return JSON.parse(contents, :symbolize_names => true)
55
+ end
56
+ end
57
+ {}
58
+ end
59
+
60
+ def send_event(name, event, payload)
61
+ new(nil, name).send_event(event, payload)
62
+ end
63
+
64
+ def send_query(name, query, payload)
65
+ new(nil, name).send_query(query, payload)
66
+ end
27
67
  end
28
68
 
29
69
  include Loggable
30
70
 
31
71
  def initialize(loop, name)
72
+ # TODO: Don't allow nil for loop. It reduces nil checks and
73
+ # simplifies source code.
32
74
  @loop = loop
33
75
  @name = name
34
- @pid = nil
76
+ @agent = nil
35
77
  end
36
78
 
37
79
  def start
@@ -43,29 +85,35 @@ module Droonga
43
85
  detect_other_hosts.each do |other_host|
44
86
  retry_joins.push("-retry-join", other_host)
45
87
  end
46
- @pid = run("agent",
47
- "-node", @name,
48
- "-bind", extract_host(@name),
49
- "-event-handler", "droonga-engine-serf-event-handler",
50
- *retry_joins)
88
+ @agent = run("agent",
89
+ "-node", @name,
90
+ "-bind", "#{extract_host(@name)}:#{port}",
91
+ "-event-handler", "droonga-engine-serf-event-handler",
92
+ "-log-level", log_level,
93
+ *retry_joins)
51
94
  logger.trace("start: done")
52
95
  end
53
96
 
54
97
  def running?
55
- not @pid.nil?
98
+ @agent and @agent.running?
56
99
  end
57
100
 
58
101
  def shutdown
59
102
  logger.trace("shutdown: start")
60
- Process.waitpid(run("leave"))
61
- Process.waitpid(@pid)
62
- @pid = nil
103
+ run("leave").shutdown
104
+ @agent.shutdown
105
+ @agent = nil
63
106
  logger.trace("shutdown: done")
64
107
  end
65
108
 
66
- def restart
67
- shutdown
68
- start
109
+ def send_event(event, payload)
110
+ ensure_serf
111
+ run("event", event, JSON.generate(payload)).shutdown
112
+ end
113
+
114
+ def send_query(query, payload)
115
+ ensure_serf
116
+ run("query", query, JSON.generate(payload)).shutdown
69
117
  end
70
118
 
71
119
  private
@@ -90,17 +138,51 @@ module Droonga
90
138
  end
91
139
 
92
140
  def run(command, *options)
93
- spawn(@serf, command, "-rpc-addr", rpc_address, *options)
141
+ process = SerfProcess.new(@loop, @serf, command,
142
+ "-rpc-addr", rpc_address,
143
+ *options)
144
+ process.start
145
+ process
94
146
  end
95
147
 
96
148
  def extract_host(node_name)
97
149
  node_name.split(":").first
98
150
  end
99
151
 
152
+ def log_level
153
+ level = Logger::Level.default
154
+ case level
155
+ when "trace", "debug", "info", "warn"
156
+ level
157
+ when "error", "fatal"
158
+ "err"
159
+ else
160
+ level # Or error?
161
+ end
162
+ end
163
+
100
164
  def rpc_address
101
165
  "#{extract_host(@name)}:7373"
102
166
  end
103
167
 
168
+ def status
169
+ @status ||= self.class.load_status
170
+ end
171
+
172
+ def role
173
+ if status[:role]
174
+ role = status[:role].to_sym
175
+ if self.class::ROLE.key?(role)
176
+ return role
177
+ end
178
+ end
179
+ :default
180
+ end
181
+
182
+ def port
183
+ self.class::ROLE[role][:port]
184
+ end
185
+
104
186
  def detect_other_hosts
105
187
  loader = CatalogLoader.new(Path.catalog.to_s)
106
188
  catalog = loader.load
@@ -115,5 +197,121 @@ module Droonga
115
197
  def log_tag
116
198
  "serf"
117
199
  end
200
+
201
+ class SerfProcess
202
+ include Loggable
203
+
204
+ def initialize(loop, serf, command, *options)
205
+ @loop = loop
206
+ @serf = serf
207
+ @command = command
208
+ @options = options
209
+ @pid = nil
210
+ end
211
+
212
+ def start
213
+ capture_output do |output_write, error_write|
214
+ env = {}
215
+ spawn_options = {
216
+ :out => output_write,
217
+ :err => error_write,
218
+ }
219
+ @pid = spawn(env, @serf, @command, *@options, spawn_options)
220
+ end
221
+ end
222
+
223
+ def shutdown
224
+ return if @pid.nil?
225
+ Process.waitpid(@pid)
226
+ @output_io.close
227
+ @error_io.close
228
+ @pid = nil
229
+ end
230
+
231
+ def running?
232
+ not @pid.nil?
233
+ end
234
+
235
+ private
236
+ def capture_output
237
+ result = nil
238
+ output_read, output_write = IO.pipe
239
+ error_read, error_write = IO.pipe
240
+
241
+ begin
242
+ result = yield(output_write, error_write)
243
+ rescue
244
+ output_read.close unless output_read.closed?
245
+ output_write.close unless output_write.closed?
246
+ error_read.close unless error_read.closed?
247
+ error_write.close unless error_write.closed?
248
+ raise
249
+ end
250
+
251
+ output_line_buffer = LineBuffer.new
252
+ on_read_output = lambda do |data|
253
+ on_standard_output(output_line_buffer, data)
254
+ end
255
+ @output_io = Coolio::IO.new(output_read)
256
+ @output_io.on_read do |data|
257
+ on_read_output.call(data)
258
+ end
259
+ # TODO: Don't allow nil for loop. It reduces nil checks and
260
+ # simplifies source code.
261
+ @loop.attach(@output_io) if @loop
262
+
263
+ error_line_buffer = LineBuffer.new
264
+ on_read_error = lambda do |data|
265
+ on_error_output(error_line_buffer, data)
266
+ end
267
+ @error_io = Coolio::IO.new(error_read)
268
+ @error_io.on_read do |data|
269
+ on_read_error.call(data)
270
+ end
271
+ # TODO: Don't allow nil for loop. It reduces nil checks and
272
+ # simplifies source code.
273
+ @loop.attach(@error_io) if @loop
274
+
275
+ result
276
+ end
277
+
278
+ def on_standard_output(line_buffer, data)
279
+ line_buffer.feed(data) do |line|
280
+ line = line.chomp
281
+ case line
282
+ when /\A==> /
283
+ content = $POSTMATCH
284
+ logger.info(content)
285
+ when /\A /
286
+ content = $POSTMATCH
287
+ case content
288
+ when /\A(\d{4})\/(\d{2})\/(\d{2}) (\d{2}):(\d{2}):(\d{2}) \[(\w+)\] /
289
+ year, month, day = $1, $2, $3
290
+ hour, minute, second = $4, $5, $6
291
+ level = $7
292
+ content = $POSTMATCH
293
+ logger.send(level.downcase, content)
294
+ else
295
+ logger.info(content)
296
+ end
297
+ else
298
+ logger.info(line)
299
+ end
300
+ end
301
+ end
302
+
303
+ def on_error_output(line_buffer, data)
304
+ line_buffer.feed(data) do |line|
305
+ line = line.chomp
306
+ logger.error(line.gsub(/\A==> /, ""))
307
+ end
308
+ end
309
+
310
+ def log_tag
311
+ tag = "serf"
312
+ tag << "[#{@pid}]" if @pid
313
+ tag
314
+ end
315
+ end
118
316
  end
119
317
  end
@@ -17,9 +17,12 @@ module Droonga
17
17
  module Test
18
18
  class StubHandlerMessenger
19
19
  attr_reader :values, :messages
20
+ attr_accessor :engine_state
21
+
20
22
  def initialize
21
23
  @values = []
22
24
  @messages = []
25
+ @engine_state = nil
23
26
  end
24
27
 
25
28
  def emit(value)
@@ -22,8 +22,11 @@ module Droonga
22
22
  def initialize
23
23
  @raw_loop = Coolio::Loop.new
24
24
  @loop = EventLoop.new(@raw_loop)
25
+ @forwarder = Forwarder.new(@loop)
25
26
  @handler_runner = HandlerRunner.new(@loop,
26
- config.merge(:dispatcher => nil))
27
+ config.merge(:dispatcher => nil,
28
+ :engine_state => nil,
29
+ :forwarder => @forwarder))
27
30
  receive_socket_path = config[:job_receive_socket_path]
28
31
  @job_receiver = JobReceiver.new(@loop, receive_socket_path) do |message|
29
32
  process(message)
@@ -32,10 +35,12 @@ module Droonga
32
35
 
33
36
  def run
34
37
  Droonga.logger.trace("#{log_tag}: run: start")
38
+ @forwarder.start
35
39
  @handler_runner.start
36
40
  @job_receiver.start
37
41
  @raw_loop.run
38
42
  @handler_runner.shutdown
43
+ @forwarder.shutdown
39
44
  Droonga.logger.trace("#{log_tag}: run: done")
40
45
  end
41
46
 
@@ -4,7 +4,7 @@
4
4
  "datasets": {
5
5
  "Default": {
6
6
  "nWorkers": 4,
7
- "plugins": ["groonga", "crud", "search", "dump"],
7
+ "plugins": ["groonga", "crud", "search", "dump", "system"],
8
8
  "replicas": [
9
9
  {
10
10
  "dimension": "_key",
@@ -11,7 +11,7 @@
11
11
  "datasets": {
12
12
  "Default": {
13
13
  "workers": 4,
14
- "plugins": ["groonga", "crud", "search"],
14
+ "plugins": ["groonga", "crud", "search", "dump", "system"],
15
15
  "number_of_replicas": 2,
16
16
  "number_of_partitions": 3,
17
17
  "partition_key": "_key",
@@ -48,7 +48,7 @@
48
48
  },
49
49
  "Watch": {
50
50
  "workers": 4,
51
- "plugins": ["groonga", "watch", "search", "crud"],
51
+ "plugins": ["groonga", "watch", "search", "crud", "dump", "system"],
52
52
  "number_of_replicas": 1,
53
53
  "number_of_partitions": 1,
54
54
  "partition_key": "_key",
@@ -0,0 +1,12 @@
1
+ {
2
+ "inReplyTo": "request-id",
3
+ "statusCode": 200,
4
+ "type": "system.status.result",
5
+ "body": {
6
+ "nodes": {
7
+ "127.0.0.1:23003/droonga": {
8
+ "live": true
9
+ }
10
+ }
11
+ }
12
+ }
@@ -0,0 +1,5 @@
1
+ {
2
+ "type": "system.status",
3
+ "dataset": "Default",
4
+ "body": {}
5
+ }
@@ -0,0 +1,79 @@
1
+ # Copyright (C) 2013-2014 Droonga Project
2
+ #
3
+ # This library is free software; you can redistribute it and/or
4
+ # modify it under the terms of the GNU Lesser General Public
5
+ # License version 2.1 as published by the Free Software Foundation.
6
+ #
7
+ # This library is distributed in the hope that it will be useful,
8
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
9
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
10
+ # Lesser General Public License for more details.
11
+ #
12
+ # You should have received a copy of the GNU Lesser General Public
13
+ # License along with this library; if not, write to the Free Software
14
+ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
15
+
16
+ require "droonga/plugins/system"
17
+
18
+ class SystemStatusHandlerTest < Test::Unit::TestCase
19
+ def setup
20
+ setup_handler
21
+ end
22
+
23
+ def teardown
24
+ teardown_handler
25
+ end
26
+
27
+ private
28
+ def setup_handler
29
+ @worker = StubWorker.new
30
+ @messenger = Droonga::Test::StubHandlerMessenger.new
31
+ @messenger.engine_state = StubEngineState.new
32
+ @loop = nil
33
+ @handler = Droonga::Plugins::System::StatusHandler.new("name",
34
+ @worker.context,
35
+ @messenger,
36
+ @loop)
37
+ end
38
+
39
+ def teardown_handler
40
+ @handler = nil
41
+ end
42
+
43
+ def process(request)
44
+ message = Droonga::Test::StubHandlerMessage.new(request)
45
+ @handler.handle(message)
46
+ end
47
+
48
+ class StubEngineState
49
+ def all_nodes
50
+ [
51
+ "127.0.0.1:10031/droonga",
52
+ "127.0.0.1:10032/droonga",
53
+ ]
54
+ end
55
+
56
+ def live_nodes
57
+ [
58
+ "127.0.0.1:10031/droonga",
59
+ ]
60
+ end
61
+ end
62
+
63
+ public
64
+ def test_request
65
+ request = {}
66
+ response = process(request)
67
+ status = {
68
+ "nodes" => {
69
+ "127.0.0.1:10031/droonga" => {
70
+ "live" => true,
71
+ },
72
+ "127.0.0.1:10032/droonga" => {
73
+ "live" => false,
74
+ },
75
+ },
76
+ }
77
+ assert_equal(status, response)
78
+ end
79
+ end
@@ -55,7 +55,7 @@ class CatalogGeneratorTest < Test::Unit::TestCase
55
55
  @generator.add_dataset("Droonga", {})
56
56
  dataset = {
57
57
  "nWorkers" => 4,
58
- "plugins" => ["groonga", "search", "crud", "dump"],
58
+ "plugins" => ["groonga", "search", "crud", "dump", "system"],
59
59
  "schema" => {},
60
60
  "replicas" => [
61
61
  {
@@ -0,0 +1,62 @@
1
+ # Copyright (C) 2014 Droonga Project
2
+ #
3
+ # This library is free software; you can redistribute it and/or
4
+ # modify it under the terms of the GNU Lesser General Public
5
+ # License version 2.1 as published by the Free Software Foundation.
6
+ #
7
+ # This library is distributed in the hope that it will be useful,
8
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
9
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
10
+ # Lesser General Public License for more details.
11
+ #
12
+ # You should have received a copy of the GNU Lesser General Public
13
+ # License along with this library; if not, write to the Free Software
14
+ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
15
+
16
+ require "droonga/line_buffer"
17
+
18
+ class LineBufferTest < Test::Unit::TestCase
19
+ def setup
20
+ @line_buffer = Droonga::LineBuffer.new
21
+ end
22
+
23
+ def feed(data)
24
+ lines = []
25
+ @line_buffer.feed(data) do |line|
26
+ lines << line
27
+ end
28
+ lines
29
+ end
30
+
31
+ class NoBufferTest < self
32
+ def test_no_new_line
33
+ assert_equal([], feed("a"))
34
+ end
35
+
36
+ def test_one_new_line
37
+ assert_equal(["a\n"], feed("a\n"))
38
+ end
39
+
40
+ def test_multiple_new_lines
41
+ assert_equal(["a\n", "b\n"], feed("a\nb\n"))
42
+ end
43
+ end
44
+
45
+ class BufferedTest < self
46
+ def test_one_line
47
+ assert_equal([], feed("a"))
48
+ assert_equal(["a\n"], feed("\n"))
49
+ end
50
+
51
+ def test_multiple_lines
52
+ assert_equal([], feed("a"))
53
+ assert_equal(["a\n", "b\n"], feed("\nb\n"))
54
+ end
55
+
56
+ def test_multiple_buffered
57
+ assert_equal([], feed("a"))
58
+ assert_equal(["a\n"], feed("\nb"))
59
+ assert_equal(["b\n"], feed("\n"))
60
+ end
61
+ end
62
+ end