fluent-plugin-droonga 0.7.0 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (163) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +1 -4
  3. data/benchmark/watch/benchmark-notify.rb +2 -2
  4. data/benchmark/watch/benchmark-scan.rb +3 -0
  5. data/benchmark/watch/fluentd.conf +0 -1
  6. data/fluent-plugin-droonga.gemspec +2 -3
  7. data/lib/droonga/catalog.rb +10 -124
  8. data/lib/droonga/catalog/base.rb +140 -0
  9. data/lib/droonga/catalog/version1.rb +23 -0
  10. data/lib/droonga/catalog_loader.rb +33 -0
  11. data/lib/droonga/collector.rb +2 -71
  12. data/lib/droonga/collector_plugin.rb +2 -34
  13. data/lib/droonga/dispatcher.rb +141 -196
  14. data/lib/droonga/distribution_planner.rb +76 -0
  15. data/lib/droonga/distributor.rb +5 -7
  16. data/lib/droonga/distributor_plugin.rb +23 -15
  17. data/lib/droonga/engine.rb +2 -2
  18. data/lib/droonga/event_loop.rb +46 -0
  19. data/lib/droonga/farm.rb +9 -5
  20. data/lib/droonga/fluent_message_sender.rb +84 -0
  21. data/lib/droonga/forwarder.rb +43 -53
  22. data/lib/droonga/handler.rb +20 -68
  23. data/lib/droonga/handler_message.rb +61 -0
  24. data/lib/droonga/handler_messenger.rb +92 -0
  25. data/lib/droonga/handler_plugin.rb +10 -12
  26. data/lib/droonga/input_adapter.rb +52 -0
  27. data/lib/droonga/{adapter.rb → input_adapter_plugin.rb} +7 -13
  28. data/lib/droonga/input_message.rb +11 -11
  29. data/lib/droonga/logger.rb +4 -3
  30. data/lib/droonga/message_pack_packer.rb +62 -0
  31. data/lib/droonga/message_processing_error.rb +54 -0
  32. data/lib/droonga/message_pusher.rb +60 -0
  33. data/lib/droonga/message_receiver.rb +61 -0
  34. data/lib/droonga/output_adapter.rb +53 -0
  35. data/lib/droonga/{adapter_plugin.rb → output_adapter_plugin.rb} +3 -21
  36. data/lib/droonga/output_message.rb +37 -0
  37. data/lib/droonga/partition.rb +27 -5
  38. data/lib/droonga/pluggable.rb +9 -4
  39. data/lib/droonga/plugin.rb +12 -3
  40. data/lib/droonga/plugin/collector/basic.rb +91 -18
  41. data/lib/droonga/plugin/distributor/crud.rb +9 -9
  42. data/lib/droonga/plugin/distributor/distributed_search_planner.rb +401 -0
  43. data/lib/droonga/plugin/distributor/groonga.rb +5 -5
  44. data/lib/droonga/plugin/distributor/search.rb +4 -246
  45. data/lib/droonga/plugin/distributor/watch.rb +11 -6
  46. data/lib/droonga/plugin/handler/add.rb +69 -7
  47. data/lib/droonga/plugin/handler/groonga.rb +6 -6
  48. data/lib/droonga/plugin/handler/search.rb +5 -3
  49. data/lib/droonga/plugin/handler/watch.rb +19 -13
  50. data/lib/droonga/plugin/{adapter → input_adapter}/groonga.rb +5 -11
  51. data/lib/droonga/plugin/{adapter → input_adapter}/groonga/select.rb +2 -36
  52. data/lib/droonga/plugin/output_adapter/groonga.rb +30 -0
  53. data/lib/droonga/plugin/output_adapter/groonga/select.rb +54 -0
  54. data/lib/droonga/plugin_loader.rb +2 -2
  55. data/lib/droonga/processor.rb +21 -23
  56. data/lib/droonga/replier.rb +40 -0
  57. data/lib/droonga/searcher.rb +298 -174
  58. data/lib/droonga/server.rb +0 -67
  59. data/lib/droonga/session.rb +85 -0
  60. data/lib/droonga/test.rb +21 -0
  61. data/lib/droonga/test/stub_distributor.rb +31 -0
  62. data/lib/droonga/test/stub_handler.rb +37 -0
  63. data/lib/droonga/test/stub_handler_message.rb +35 -0
  64. data/lib/droonga/test/stub_handler_messenger.rb +34 -0
  65. data/lib/droonga/time_formatter.rb +37 -0
  66. data/lib/droonga/watcher.rb +1 -0
  67. data/lib/droonga/worker.rb +16 -19
  68. data/lib/fluent/plugin/out_droonga.rb +9 -9
  69. data/lib/groonga_command_converter.rb +5 -5
  70. data/sample/cluster/catalog.json +1 -1
  71. data/test/command/config/default/catalog.json +19 -1
  72. data/test/command/fixture/event.jsons +41 -0
  73. data/test/command/fixture/user-table.jsons +9 -0
  74. data/test/command/run-test.rb +2 -2
  75. data/test/command/suite/add/error/invalid-integer.expected +20 -0
  76. data/test/command/suite/add/error/invalid-integer.test +12 -0
  77. data/test/command/suite/add/error/invalid-time.expected +20 -0
  78. data/test/command/suite/add/error/invalid-time.test +12 -0
  79. data/test/command/suite/add/error/missing-key.expected +13 -0
  80. data/test/command/suite/add/error/missing-key.test +16 -0
  81. data/test/command/suite/add/error/missing-table.expected +13 -0
  82. data/test/command/suite/add/error/missing-table.test +16 -0
  83. data/test/command/suite/add/error/unknown-column.expected +20 -0
  84. data/test/command/suite/add/error/unknown-column.test +12 -0
  85. data/test/command/suite/add/error/unknown-table.expected +13 -0
  86. data/test/command/suite/add/error/unknown-table.test +17 -0
  87. data/test/command/suite/add/minimum.expected +1 -3
  88. data/test/command/suite/add/with-values.expected +1 -3
  89. data/test/command/suite/add/without-key.expected +1 -3
  90. data/test/command/suite/message/error/missing-dataset.expected +13 -0
  91. data/test/command/suite/message/error/missing-dataset.test +5 -0
  92. data/test/command/suite/message/error/unknown-command.expected +13 -0
  93. data/test/command/suite/message/error/unknown-command.test +6 -0
  94. data/test/command/suite/message/error/unknown-dataset.expected +13 -0
  95. data/test/command/suite/message/error/unknown-dataset.test +6 -0
  96. data/test/command/suite/search/{array-attribute-label.expected → attributes/array.expected} +0 -0
  97. data/test/command/suite/search/{array-attribute-label.test → attributes/array.test} +0 -0
  98. data/test/command/suite/search/{hash-attribute-label.expected → attributes/hash.expected} +0 -0
  99. data/test/command/suite/search/{hash-attribute-label.test → attributes/hash.test} +0 -0
  100. data/test/command/suite/search/{condition-nested.expected → condition/nested.expected} +0 -0
  101. data/test/command/suite/search/{condition-nested.test → condition/nested.test} +0 -0
  102. data/test/command/suite/search/{condition-query.expected → condition/query.expected} +0 -0
  103. data/test/command/suite/search/{condition-query.test → condition/query.test} +0 -0
  104. data/test/command/suite/search/{condition-script.expected → condition/script.expected} +0 -0
  105. data/test/command/suite/search/{condition-script.test → condition/script.test} +0 -0
  106. data/test/command/suite/search/error/cyclic-source.expected +18 -0
  107. data/test/command/suite/search/error/cyclic-source.test +12 -0
  108. data/test/command/suite/search/error/deeply-cyclic-source.expected +21 -0
  109. data/test/command/suite/search/error/deeply-cyclic-source.test +15 -0
  110. data/test/command/suite/search/error/missing-source-parameter.expected +17 -0
  111. data/test/command/suite/search/error/missing-source-parameter.test +11 -0
  112. data/test/command/suite/search/error/unknown-source.expected +18 -0
  113. data/test/command/suite/search/error/unknown-source.test +12 -0
  114. data/test/command/suite/search/{minimum.expected → group/count.expected} +2 -1
  115. data/test/command/suite/search/{minimum.test → group/count.test} +5 -3
  116. data/test/command/suite/search/group/limit.expected +19 -0
  117. data/test/command/suite/search/group/limit.test +20 -0
  118. data/test/command/suite/search/group/string.expected +36 -0
  119. data/test/command/suite/search/group/string.test +44 -0
  120. data/test/command/suite/search/{chained-queries.expected → multiple/chained.expected} +0 -0
  121. data/test/command/suite/search/{chained-queries.test → multiple/chained.test} +0 -0
  122. data/test/command/suite/search/{multiple-queries.expected → multiple/parallel.expected} +0 -0
  123. data/test/command/suite/search/{multiple-queries.test → multiple/parallel.test} +0 -0
  124. data/test/command/suite/search/{output-range.expected → range/only-output.expected} +0 -0
  125. data/test/command/suite/search/{output-range.test → range/only-output.test} +0 -0
  126. data/test/command/suite/search/{sort-range.expected → range/only-sort.expected} +0 -0
  127. data/test/command/suite/search/{sort-range.test → range/only-sort.test} +0 -0
  128. data/test/command/suite/search/{sort-and-output-range.expected → range/sort-and-output.expected} +0 -0
  129. data/test/command/suite/search/{sort-and-output-range.test → range/sort-and-output.test} +0 -0
  130. data/test/command/suite/search/range/too-large-output-offset.expected +16 -0
  131. data/test/command/suite/search/range/too-large-output-offset.test +25 -0
  132. data/test/command/suite/search/range/too-large-sort-offset.expected +16 -0
  133. data/test/command/suite/search/range/too-large-sort-offset.test +28 -0
  134. data/test/command/suite/search/response/records/value/time.expected +24 -0
  135. data/test/command/suite/search/response/records/value/time.test +24 -0
  136. data/test/command/suite/search/sort/default-offset-limit.expected +43 -0
  137. data/test/command/suite/search/sort/default-offset-limit.test +26 -0
  138. data/test/command/suite/search/{sort-with-invisible-column.expected → sort/invisible-column.expected} +0 -0
  139. data/test/command/suite/search/{sort-with-invisible-column.test → sort/invisible-column.test} +0 -0
  140. data/test/command/suite/watch/subscribe.expected +12 -0
  141. data/test/command/suite/watch/subscribe.test +9 -0
  142. data/test/command/suite/watch/unsubscribe.expected +12 -0
  143. data/test/command/suite/watch/unsubscribe.test +9 -0
  144. data/test/unit/{test_catalog.rb → catalog/test_version1.rb} +12 -4
  145. data/test/unit/fixtures/{catalog.json → catalog/version1.json} +0 -0
  146. data/test/unit/helper.rb +2 -0
  147. data/test/unit/plugin/collector/test_basic.rb +289 -33
  148. data/test/unit/plugin/distributor/test_search.rb +176 -861
  149. data/test/unit/plugin/distributor/test_search_planner.rb +1102 -0
  150. data/test/unit/plugin/handler/groonga/test_column_create.rb +17 -13
  151. data/test/unit/plugin/handler/groonga/test_table_create.rb +10 -10
  152. data/test/unit/plugin/handler/test_add.rb +74 -11
  153. data/test/unit/plugin/handler/test_groonga.rb +15 -1
  154. data/test/unit/plugin/handler/test_search.rb +33 -17
  155. data/test/unit/plugin/handler/test_watch.rb +43 -27
  156. data/test/unit/run-test.rb +2 -0
  157. data/test/unit/test_message_pack_packer.rb +51 -0
  158. data/test/unit/test_time_formatter.rb +29 -0
  159. metadata +208 -110
  160. data/lib/droonga/job_queue.rb +0 -87
  161. data/lib/droonga/job_queue_schema.rb +0 -65
  162. data/test/unit/test_adapter.rb +0 -51
  163. data/test/unit/test_job_queue_schema.rb +0 -45
@@ -47,8 +47,8 @@ module Droonga
47
47
  end
48
48
 
49
49
  command "watch.subscribe" => :subscribe
50
- def subscribe(request)
51
- subscriber, condition, query, route = parse_request(request)
50
+ def subscribe(message, messenger)
51
+ subscriber, condition, query, route = parse_message(message)
52
52
  normalized_request = {
53
53
  :subscriber => subscriber,
54
54
  :condition => condition,
@@ -56,40 +56,46 @@ module Droonga
56
56
  :route => route,
57
57
  }
58
58
  @watcher.subscribe(normalized_request)
59
- emit([true])
59
+ messenger.emit([true])
60
60
  end
61
61
 
62
62
  command "watch.unsubscribe" => :unsubscribe
63
- def unsubscribe(request)
64
- subscriber, condition, query, route = parse_request(request)
63
+ def unsubscribe(message, messenger)
64
+ subscriber, condition, query, route = parse_message(message)
65
65
  normalized_request = {
66
66
  :subscriber => subscriber,
67
67
  :condition => condition,
68
68
  :query => query,
69
69
  }
70
70
  @watcher.unsubscribe(normalized_request)
71
- emit([true])
71
+ messenger.emit([true])
72
72
  end
73
73
 
74
74
  command "watch.feed" => :feed
75
- def feed(request)
75
+ def feed(message, messenger)
76
+ request = message.request
76
77
  @watcher.feed(:targets => request["targets"]) do |route, subscribers|
77
- message = request # return request itself
78
- envelope["to"] = subscribers
79
- post(message, "to" => route, "type" => "watch.notification")
78
+ notification_message = {
79
+ "to" => subscribers,
80
+ "body" => request,
81
+ }
82
+ notification_message = message.raw.merge(notification_message)
83
+ messenger.forward(notification_message,
84
+ "to" => route, "type" => "watch.notification")
80
85
  end
81
86
  end
82
87
 
83
88
  command "watch.sweep" => :sweep
84
- def sweep(request)
89
+ def sweep(message, messenger)
85
90
  @sweeper.sweep_expired_subscribers
86
91
  end
87
92
 
88
93
  private
89
- def parse_request(request)
94
+ def parse_message(message)
95
+ request = message.request
90
96
  subscriber = request["subscriber"]
91
97
  condition = request["condition"]
92
- route = request["route"] || envelope["from"]
98
+ route = request["route"] || message["from"]
93
99
  query = condition && condition.to_json
94
100
  [subscriber, condition, query, route]
95
101
  end
@@ -13,28 +13,22 @@
13
13
  # License along with this library; if not, write to the Free Software
14
14
  # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
15
15
 
16
- require "droonga/adapter_plugin"
16
+ require "droonga/input_adapter_plugin"
17
17
 
18
18
  module Droonga
19
- class GroongaAdapter < Droonga::AdapterPlugin
20
- repository.register("select", self)
19
+ class GroongaInputAdapter < Droonga::InputAdapterPlugin
20
+ repository.register("groonga", self)
21
21
 
22
22
  command :select
23
23
  def select(input_message)
24
24
  command = Select.new
25
25
  select_request = input_message.body
26
- search_request = command.convert_request(select_request)
26
+ search_request = command.convert(select_request)
27
27
  input_message.add_route("select_response")
28
28
  input_message.command = "search"
29
29
  input_message.body = search_request
30
30
  end
31
-
32
- command :select_response
33
- def select_response(search_response)
34
- command = Select.new
35
- emit(command.convert_response(search_response))
36
- end
37
31
  end
38
32
  end
39
33
 
40
- require "droonga/plugin/adapter/groonga/select"
34
+ require "droonga/plugin/input_adapter/groonga/select"
@@ -14,9 +14,9 @@
14
14
  # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
15
15
 
16
16
  module Droonga
17
- class GroongaAdapter
17
+ class GroongaInputAdapter
18
18
  class Select
19
- def convert_request(select_request)
19
+ def convert(select_request)
20
20
  table = select_request["table"]
21
21
  result_name = table + "_result"
22
22
  match_columns = select_request["match_columns"]
@@ -58,40 +58,6 @@ module Droonga
58
58
  end
59
59
  search_request
60
60
  end
61
-
62
- def convert_response(search_response)
63
- select_responses = search_response.collect do |key, value|
64
- status_code = 0
65
-
66
- start_time = value["startTime"]
67
- start_time_in_unix_time = if start_time
68
- Time.parse(start_time).to_f
69
- else
70
- Time.now.to_f
71
- end
72
- elapsed_time = value["elapsedTime"] || 0
73
- count = value["count"]
74
-
75
- attributes = value["attributes"] || []
76
- converted_attributes = attributes.collect do |attribute|
77
- name = attribute["name"]
78
- type = attribute["type"]
79
- [name, type]
80
- end
81
-
82
- header = [status_code, start_time_in_unix_time, elapsed_time]
83
- records = value["records"]
84
- if records.empty?
85
- results = [[count], converted_attributes]
86
- else
87
- results = [[count], converted_attributes, records]
88
- end
89
- body = [results]
90
-
91
- [header, body]
92
- end
93
- select_responses.first
94
- end
95
61
  end
96
62
  end
97
63
  end
@@ -0,0 +1,30 @@
1
+ # Copyright (C) 2013 Droonga Project
2
+ #
3
+ # This library is free software; you can redistribute it and/or
4
+ # modify it under the terms of the GNU Lesser General Public
5
+ # License version 2.1 as published by the Free Software Foundation.
6
+ #
7
+ # This library is distributed in the hope that it will be useful,
8
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
9
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
10
+ # Lesser General Public License for more details.
11
+ #
12
+ # You should have received a copy of the GNU Lesser General Public
13
+ # License along with this library; if not, write to the Free Software
14
+ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
15
+
16
+ require "droonga/output_adapter_plugin"
17
+
18
+ module Droonga
19
+ class GroongaOutputAdapter < Droonga::OutputAdapterPlugin
20
+ repository.register("groonga", self)
21
+
22
+ command :select_response
23
+ def select_response(output_message)
24
+ command = Select.new
25
+ output_message.body = command.convert(output_message.body)
26
+ end
27
+ end
28
+ end
29
+
30
+ require "droonga/plugin/output_adapter/groonga/select"
@@ -0,0 +1,54 @@
1
+ # Copyright (C) 2013 Droonga Project
2
+ #
3
+ # This library is free software; you can redistribute it and/or
4
+ # modify it under the terms of the GNU Lesser General Public
5
+ # License version 2.1 as published by the Free Software Foundation.
6
+ #
7
+ # This library is distributed in the hope that it will be useful,
8
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
9
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
10
+ # Lesser General Public License for more details.
11
+ #
12
+ # You should have received a copy of the GNU Lesser General Public
13
+ # License along with this library; if not, write to the Free Software
14
+ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
15
+
16
+ module Droonga
17
+ class GroongaOutputAdapter
18
+ class Select
19
+ def convert(search_response)
20
+ select_responses = search_response.collect do |key, value|
21
+ status_code = 0
22
+
23
+ start_time = value["startTime"]
24
+ start_time_in_unix_time = if start_time
25
+ Time.parse(start_time).to_f
26
+ else
27
+ Time.now.to_f
28
+ end
29
+ elapsed_time = value["elapsedTime"] || 0
30
+ count = value["count"]
31
+
32
+ attributes = value["attributes"] || []
33
+ converted_attributes = attributes.collect do |attribute|
34
+ name = attribute["name"]
35
+ type = attribute["type"]
36
+ [name, type]
37
+ end
38
+
39
+ header = [status_code, start_time_in_unix_time, elapsed_time]
40
+ records = value["records"]
41
+ if records.empty?
42
+ results = [[count], converted_attributes]
43
+ else
44
+ results = [[count], converted_attributes, records]
45
+ end
46
+ body = [results]
47
+
48
+ [header, body]
49
+ end
50
+ select_responses.first
51
+ end
52
+ end
53
+ end
54
+ end
@@ -25,8 +25,8 @@ module Droonga
25
25
  type = File.basename(type_path)
26
26
  Dir.glob("#{type_path}/*.rb") do |path|
27
27
  name = File.basename(path, ".rb")
28
- plugin = new(type, name)
29
- plugin.load
28
+ loader = new(type, name)
29
+ loader.load
30
30
  end
31
31
  end
32
32
  end
@@ -15,50 +15,48 @@
15
15
  # License along with this library; if not, write to the Free Software
16
16
  # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
17
 
18
- require "droonga/job_queue"
19
18
  require "droonga/handler"
20
19
 
21
20
  module Droonga
22
21
  class Processor
23
- def initialize(options={})
22
+ def initialize(loop, message_pusher, options={})
23
+ @loop = loop
24
+ @message_pusher = message_pusher
24
25
  @options = options
25
- @database_name = @options[:database]
26
- @queue_name = @options[:options] || "DroongaQueue"
27
26
  @n_workers = @options[:n_workers] || 0
28
27
  end
29
28
 
30
29
  def start
31
- Droonga::JobQueue.ensure_schema(@database_name,
32
- @queue_name)
33
- @job_queue = JobQueue.open(@database_name, @queue_name)
34
- @handler = Handler.new(@options)
30
+ @handler = Handler.new(@loop, @options)
31
+ @handler.start
35
32
  end
36
33
 
37
34
  def shutdown
38
- $log.trace("processor: shutdown: start")
35
+ $log.trace("#{log_tag}: shutdown: start")
39
36
  @handler.shutdown
40
- @job_queue.close
41
- $log.trace("processor: shutdown: done")
37
+ $log.trace("#{log_tag}: shutdown: done")
42
38
  end
43
39
 
44
- def process(envelope, synchronous=nil)
45
- $log.trace("proessor: process: start")
46
- reply_to = envelope["replyTo"]
47
- command = envelope["type"]
40
+ def process(message)
41
+ $log.trace("#{log_tag}: process: start")
42
+ command = message["type"]
48
43
  if @handler.processable?(command)
49
- $log.trace("proessor: process: handlable: #{command}")
50
- if synchronous.nil?
51
- synchronous = @handler.prefer_synchronous?(command)
52
- end
44
+ $log.trace("#{log_tag}: process: handlable: #{command}")
45
+ synchronous = @handler.prefer_synchronous?(command)
53
46
  if @n_workers.zero? or synchronous
54
- @handler.process(envelope)
47
+ @handler.process(message)
55
48
  else
56
- @job_queue.push_message(envelope)
49
+ @message_pusher.push(message)
57
50
  end
58
51
  else
59
- $log.trace("proessor: process: ignore #{command}")
52
+ $log.trace("#{log_tag}: process: ignore #{command}")
60
53
  end
61
- $log.trace("proessor: process: done")
54
+ $log.trace("#{log_tag}: process: done")
55
+ end
56
+
57
+ private
58
+ def log_tag
59
+ "processor"
62
60
  end
63
61
  end
64
62
  end
@@ -0,0 +1,40 @@
1
+ # Copyright (C) 2013 Droonga Project
2
+ #
3
+ # This library is free software; you can redistribute it and/or
4
+ # modify it under the terms of the GNU Lesser General Public
5
+ # License version 2.1 as published by the Free Software Foundation.
6
+ #
7
+ # This library is distributed in the hope that it will be useful,
8
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
9
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
10
+ # Lesser General Public License for more details.
11
+ #
12
+ # You should have received a copy of the GNU Lesser General Public
13
+ # License along with this library; if not, write to the Free Software
14
+ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
15
+
16
+ module Droonga
17
+ class Replier
18
+ def initialize(forwarder)
19
+ @forwarder = forwarder
20
+ end
21
+
22
+ def reply(message)
23
+ $log.trace("#{log_tag}: reply: start")
24
+ destination = message["replyTo"]
25
+ reply_message = {
26
+ "inReplyTo" => message["id"],
27
+ "statusCode" => message["statusCode"] || 200,
28
+ "type" => destination["type"],
29
+ "body" => message["body"],
30
+ }
31
+ @forwarder.forward(reply_message, destination)
32
+ $log.trace("#{log_tag}: reply: done")
33
+ end
34
+
35
+ private
36
+ def log_tag
37
+ "[#{Process.ppid}][#{Process.pid}] replier"
38
+ end
39
+ end
40
+ end
@@ -19,16 +19,30 @@ require "English"
19
19
  require "tsort"
20
20
  require "groonga"
21
21
 
22
+ require "droonga/time_formatter"
23
+
22
24
  module Droonga
23
25
  class Searcher
24
- class Error < StandardError
26
+ class MissingSourceParameter < BadRequest
27
+ def initialize(query, queries)
28
+ super("The query #{query.inspect} has no source. " +
29
+ "Query must have a valid source.",
30
+ queries)
31
+ end
32
+ end
33
+
34
+ class UnknownSource < NotFound
35
+ def initialize(source, queries)
36
+ super("The source #{source.inspect} does not exist. " +
37
+ "It must be a name of an existing table or another query.",
38
+ queries)
39
+ end
25
40
  end
26
41
 
27
- class UndefinedSourceError < Error
28
- attr_reader :name
29
- def initialize(name)
30
- @name = name
31
- super("undefined source was used: <#{name}>")
42
+ class CyclicSource < BadRequest
43
+ def initialize(queries)
44
+ super("There is cyclic reference of sources.",
45
+ queries)
32
46
  end
33
47
  end
34
48
 
@@ -54,11 +68,7 @@ module Droonga
54
68
  return {}
55
69
  end
56
70
  $log.trace("#{log_tag}: process_queries: sort: start")
57
- query_sorter = QuerySorter.new
58
- queries.each do |name, query|
59
- query_sorter.add(name, [query["source"]])
60
- end
61
- sorted_queries = query_sorter.tsort
71
+ sorted_queries = QuerySorter.sort(queries)
62
72
  $log.trace("#{log_tag}: process_queries: sort: done")
63
73
  outputs = {}
64
74
  results = {}
@@ -66,21 +76,22 @@ module Droonga
66
76
  if queries[name]
67
77
  $log.trace("#{log_tag}: process_queries: search: start",
68
78
  :name => name)
69
- searcher = QuerySearcher.new(@context, queries[name])
70
- results[name] = searcher.search(results)
79
+ search_request = SearchRequest.new(@context, queries[name], results)
80
+ search_result = QuerySearcher.search(search_request)
81
+ results[name] = search_result.records
71
82
  $log.trace("#{log_tag}: process_queries: search: done",
72
83
  :name => name)
73
- if searcher.need_output?
84
+ if search_request.need_output?
74
85
  $log.trace("#{log_tag}: process_queries: format: start",
75
86
  :name => name)
76
- outputs[name] = searcher.format
87
+ outputs[name] = ResultFormatter.format(search_request, search_result)
77
88
  $log.trace("#{log_tag}: process_queries: format: done",
78
89
  :name => name)
79
90
  end
80
91
  elsif @context[name]
81
92
  results[name] = @context[name]
82
93
  else
83
- raise UndefinedSourceError.new(name)
94
+ raise UnknownSource.new(name, queries)
84
95
  end
85
96
  end
86
97
  $log.trace("#{log_tag}: process_queries: done")
@@ -93,6 +104,29 @@ module Droonga
93
104
 
94
105
  class QuerySorter
95
106
  include TSort
107
+
108
+ class << self
109
+ def sort(queries)
110
+ query_sorter = new
111
+ queries.each do |name, query|
112
+ source = query["source"]
113
+ raise MissingSourceParameter.new(name, queries) unless source
114
+ raise CyclicSource.new(queries) if name == source
115
+ query_sorter.add(name, [source])
116
+ end
117
+ begin
118
+ sorted_queries = query_sorter.tsort
119
+ rescue TSort::Cyclic
120
+ raise CyclicSource.new(queries)
121
+ end
122
+ sorted_queries
123
+ end
124
+
125
+ def validate_dependencies(queries)
126
+ sort(queries)
127
+ end
128
+ end
129
+
96
130
  def initialize()
97
131
  @queries = {}
98
132
  end
@@ -112,102 +146,132 @@ module Droonga
112
146
  end
113
147
  end
114
148
 
115
- class QuerySearcher
116
- def initialize(context, query)
149
+ class SearchRequest
150
+ attr_reader :context, :query, :resolved_results
151
+
152
+ def initialize(context, query, resolved_results)
117
153
  @context = context
118
154
  @query = query
119
- @result = nil
120
- @condition = nil
121
- @start_time = nil
155
+ @resolved_results = resolved_results
122
156
  end
123
157
 
124
- def search(results)
125
- search_query(results)
158
+ def need_output?
159
+ @query.has_key?("output")
126
160
  end
127
161
 
128
- def need_output?
129
- @result and @query.has_key?("output")
162
+ def output
163
+ @query["output"]
130
164
  end
131
165
 
132
- def format
133
- formatted_result = {}
134
- format_count(formatted_result)
135
- format_attributes(formatted_result)
136
- format_records(formatted_result)
137
- if need_element_output?("startTime")
138
- formatted_result["startTime"] = @start_time.iso8601
139
- end
140
- if need_element_output?("elapsedTime")
141
- formatted_result["elapsedTime"] = Time.now.to_f - @start_time.to_f
166
+ def complex_output?
167
+ output["format"] == "complex"
168
+ end
169
+
170
+ def source
171
+ source_name = @query["source"]
172
+ @resolved_results[source_name]
173
+ end
174
+ end
175
+
176
+ class SearchResult
177
+ attr_accessor :start_time, :end_time, :condition, :records, :count
178
+
179
+ def initialize
180
+ @start_time = nil
181
+ @end_time = nil
182
+ @condition = nil
183
+ @records = nil
184
+ @count = nil
185
+ end
186
+ end
187
+
188
+ class QuerySearcher
189
+ OPERATOR_CONVERSION_TABLE = {
190
+ "||" => Groonga::Operator::OR,
191
+ "&&" => Groonga::Operator::AND,
192
+ "-" => Groonga::Operator::AND_NOT,
193
+ }.freeze
194
+
195
+ class << self
196
+ def search(search_request)
197
+ new(search_request).search
142
198
  end
143
- formatted_result
199
+ end
200
+
201
+ def initialize(search_request)
202
+ @result = SearchResult.new
203
+ @request = search_request
204
+ end
205
+
206
+ def search
207
+ search_query!
208
+ @result
144
209
  end
145
210
 
146
211
  private
147
- def parseCondition(source, expression, condition)
148
- if condition.is_a? String
212
+ def parse_condition(source, expression, condition)
213
+ case condition
214
+ when String
149
215
  expression.parse(condition, :syntax => :script)
150
- elsif condition.is_a? Hash
151
- options = {}
152
- if condition["matchTo"]
153
- matchTo = Groonga::Expression.new(context: @context)
154
- matchTo.define_variable(:domain => source)
155
- match_columns = condition["matchTo"]
156
- match_columns = match_columns.join(",") if match_columns.is_a?(Array)
157
- matchTo.parse(match_columns, :syntax => :script)
158
- options[:default_column] = matchTo
159
- end
160
- if condition["query"]
161
- options[:syntax] = :query
162
- if condition["defaultOperator"]
163
- case condition["defaultOperator"]
164
- when "||"
165
- options[:default_operator] = Groonga::Operator::OR
166
- when "&&"
167
- options[:default_operator] = Groonga::Operator::AND
168
- when "-"
169
- options[:default_operator] = Groonga::Operator::BUT
170
- else
171
- raise "undefined operator assigned #{condition["default_operator"]}"
172
- end
173
- end
174
- if condition["allowPragma"]
175
- options[:allow_pragma] = true
176
- end
177
- if condition["allowColumn"]
178
- options[:allow_column] = true
179
- end
180
- expression.parse(condition["query"], options)
181
- elsif condition["script"]
182
- # "script" is ignored when "query" is also assigned.
183
- options[:syntax] = :script
184
- if condition["allowUpdate"]
185
- options[:allow_update] = true
216
+ when Hash
217
+ parse_condition_hash(source, expression, condition)
218
+ when Array
219
+ parse_condition_array(source, expression, condition)
220
+ else
221
+ raise "unacceptable object #{condition.inspect} assigned"
222
+ end
223
+ end
224
+
225
+ def parse_condition_hash(source, expression, condition)
226
+ options = {}
227
+ if condition["matchTo"]
228
+ matchTo = Groonga::Expression.new(context: @request.context)
229
+ matchTo.define_variable(:domain => source)
230
+ match_columns = condition["matchTo"]
231
+ match_columns = match_columns.join(",") if match_columns.is_a?(Array)
232
+ matchTo.parse(match_columns, :syntax => :script)
233
+ options[:default_column] = matchTo
234
+ end
235
+ if condition["query"]
236
+ options[:syntax] = :query
237
+ if condition["defaultOperator"]
238
+ default_operator_string = condition["defaultOperator"]
239
+ default_operator = OPERATOR_CONVERSION_TABLE[default_operator_string]
240
+ unless default_operator
241
+ raise "undefined operator assigned #{default_operator_string}"
186
242
  end
187
- expression.parse(condition["script"], options)
188
- else
189
- raise "neither 'query' nor 'script' assigned in #{condition.inspect}"
243
+ options[:default_operator] = default_operator
190
244
  end
191
- elsif condition.is_a? Array
192
- case condition[0]
193
- when "||"
194
- operator = Groonga::Operator::OR
195
- when "&&"
196
- operator = Groonga::Operator::AND
197
- when "-"
198
- operator = Groonga::Operator::BUT
199
- else
200
- raise "undefined operator assigned #{condition[0]}"
245
+ if condition["allowPragma"]
246
+ options[:allow_pragma] = true
201
247
  end
202
- if condition[1]
203
- parseCondition(source, expression, condition[1])
248
+ if condition["allowColumn"]
249
+ options[:allow_column] = true
204
250
  end
205
- condition[2..-1].each do |element|
206
- parseCondition(source, expression, element)
207
- expression.append_operation(operator, 2)
251
+ expression.parse(condition["query"], options)
252
+ elsif condition["script"]
253
+ # "script" is ignored when "query" is also assigned.
254
+ options[:syntax] = :script
255
+ if condition["allowUpdate"]
256
+ options[:allow_update] = true
208
257
  end
258
+ expression.parse(condition["script"], options)
209
259
  else
210
- raise "unacceptable object #{condition.inspect} assigned"
260
+ raise "neither 'query' nor 'script' assigned in #{condition.inspect}"
261
+ end
262
+ end
263
+
264
+ def parse_condition_array(source, expression, condition)
265
+ operator = OPERATOR_CONVERSION_TABLE[condition[0]]
266
+ unless operator
267
+ raise "undefined operator assigned #{condition[0]}"
268
+ end
269
+ if condition[1]
270
+ parse_condition(source, expression, condition[1])
271
+ end
272
+ condition[2..-1].each do |element|
273
+ parse_condition(source, expression, element)
274
+ expression.append_operation(operator, 2)
211
275
  end
212
276
  end
213
277
 
@@ -221,113 +285,169 @@ module Droonga
221
285
  end
222
286
  end
223
287
 
224
- def search_query(results)
288
+ def search_query!
225
289
  $log.trace("#{log_tag}: search_query: start")
226
- @start_time = Time.now
227
- @result = source = results[@query["source"]]
228
- condition = @query["condition"]
229
- if condition
230
- expression = Groonga::Expression.new(context: @context)
231
- expression.define_variable(:domain => source)
232
- parseCondition(source, expression, condition)
233
- $log.trace("#{log_tag}: search_query: select: start",
234
- :condition => condition)
235
- @result = source.select(expression)
236
- $log.trace("#{log_tag}: search_query: select: done")
237
- @condition = expression
290
+
291
+ @result.start_time = Time.now
292
+
293
+ @records = @request.source
294
+
295
+ condition = @request.query["condition"]
296
+ apply_condition!(condition) if condition
297
+
298
+ group_by = @request.query["groupBy"]
299
+ apply_group_by!(group_by) if group_by
300
+
301
+ @result.count = @records.size
302
+
303
+ sort_by = @request.query["sortBy"]
304
+ apply_sort_by!(sort_by) if sort_by
305
+
306
+ $log.trace("#{log_tag}: search_query: done")
307
+ @result.records = @records
308
+ @result.end_time = Time.now
309
+ end
310
+
311
+ def apply_condition!(condition)
312
+ expression = Groonga::Expression.new(context: @request.context)
313
+ expression.define_variable(:domain => @records)
314
+ parse_condition(@records, expression, condition)
315
+ $log.trace("#{log_tag}: search_query: select: start",
316
+ :condition => condition)
317
+ @records = @records.select(expression)
318
+ $log.trace("#{log_tag}: search_query: select: done")
319
+ @result.condition = expression
320
+ end
321
+
322
+ def apply_group_by!(group_by)
323
+ $log.trace("#{log_tag}: search_query: group: start",
324
+ :by => group_by)
325
+ case group_by
326
+ when String
327
+ @records = @records.group(group_by)
328
+ when Hash
329
+ key = group_by["key"]
330
+ max_n_sub_records = group_by["maxNSubRecords"]
331
+ @records = @records.group(key, :max_n_sub_records => max_n_sub_records)
332
+ else
333
+ raise '"groupBy" parameter must be a Hash or a String'
238
334
  end
239
- group_by = @query["groupBy"]
240
- if group_by
241
- $log.trace("#{log_tag}: search_query: group: start",
242
- :by => group_by)
243
- if group_by.is_a? String
244
- @result = @result.group(group_by)
245
- elsif group_by.is_a? Hash
246
- key = group_by["key"]
247
- max_n_sub_records = group_by["maxNSubRecords"]
248
- @result = @result.group(key, :max_n_sub_records => max_n_sub_records)
249
- else
250
- raise '"groupBy" parameter must be a Hash or a String'
251
- end
252
- $log.trace("#{log_tag}: search_query: group: done",
253
- :by => group_by)
335
+ $log.trace("#{log_tag}: search_query: group: done",
336
+ :by => group_by)
337
+ end
338
+
339
+ def apply_sort_by!(sort_by)
340
+ $log.trace("#{log_tag}: search_query: sort: start",
341
+ :by => sort_by)
342
+ case sort_by
343
+ when Array
344
+ keys = parse_order_keys(sort_by)
345
+ offset = 0
346
+ limit = -1
347
+ when Hash
348
+ keys = parse_order_keys(sort_by["keys"])
349
+ offset = sort_by["offset"]
350
+ limit = sort_by["limit"]
351
+ else
352
+ raise '"sortBy" parameter must be a Hash or an Array'
254
353
  end
255
- @count = @result.size
256
- sort_by = @query["sortBy"]
257
- if sort_by
258
- $log.trace("#{log_tag}: search_query: sort: start",
259
- :by => sort_by)
260
- if sort_by.is_a? Array
261
- keys = parse_order_keys(sort_by)
262
- offset = 0
263
- limit = -1
264
- elsif sort_by.is_a? Hash
265
- keys = parse_order_keys(sort_by["keys"])
266
- offset = sort_by["offset"]
267
- limit = sort_by["limit"]
268
- else
269
- raise '"sortBy" parameter must be a Hash or an Array'
270
- end
271
- @result = @result.sort(keys, :offset => offset, :limit => limit)
272
- $log.trace("#{log_tag}: search_query: sort: done",
273
- :by => sort_by)
354
+ @records = @records.sort(keys, :offset => offset, :limit => limit)
355
+ $log.trace("#{log_tag}: search_query: sort: done",
356
+ :by => sort_by)
357
+ end
358
+
359
+ def log_tag
360
+ "[#{Process.ppid}][#{Process.pid}] query_searcher"
361
+ end
362
+ end
363
+
364
+ class ResultFormatter
365
+ class << self
366
+ def format(search_request, search_result)
367
+ new(search_request, search_result).format
274
368
  end
275
- $log.trace("#{log_tag}: search_query: done")
276
- @result
277
369
  end
278
370
 
279
- def need_element_output?(element)
280
- params = @query["output"]
371
+ def initialize(search_request, search_result)
372
+ @request = search_request
373
+ @result = search_result
374
+ end
375
+
376
+ def format
377
+ formatted_result = {}
378
+
379
+ output_elements.each do |name|
380
+ value = format_element(name)
381
+ next if value.nil?
382
+ formatted_result[name] = value
383
+ end
281
384
 
282
- elements = params["elements"]
283
- return false if elements.nil?
385
+ formatted_result
386
+ end
284
387
 
285
- elements.include?(element)
388
+ private
389
+ def format_element(name)
390
+ case name
391
+ when "count"
392
+ format_count
393
+ when "attributes"
394
+ format_attributes
395
+ when "records"
396
+ format_records
397
+ when "startTime"
398
+ format_start_time
399
+ when "elapsedTime"
400
+ format_elapsed_time
401
+ else
402
+ nil
403
+ end
286
404
  end
287
405
 
288
- def format_count(formatted_result)
289
- return unless need_element_output?("count")
290
- formatted_result["count"] = @count
406
+ def output_elements
407
+ @request.output["elements"] || []
291
408
  end
292
409
 
293
- def format_attributes(formatted_result)
294
- return unless need_element_output?("attributes")
410
+ def format_count
411
+ @result.count
412
+ end
295
413
 
414
+ def format_attributes
296
415
  # XXX IMPLEMENT ME!!!
297
416
  attributes = nil
298
- if @query["output"]["format"] == "complex"
299
- # should convert columnst to an object like:
417
+ if @request.complex_output?
418
+ # should convert columns to an object like:
300
419
  # {"_id" => {"type" => "UInt32", "vector" => false}}
301
420
  attributes = {}
302
421
  else
303
- # should convert columnst to an object like:
422
+ # should convert columns to an object like:
304
423
  # [{"name" => "_id", "type" => "UInt32", "vector" => false}]
305
424
  attributes = []
306
425
  end
307
426
 
308
- formatted_result["attributes"] = attributes
427
+ attributes
309
428
  end
310
429
 
311
- def format_records(formatted_result)
312
- return unless need_element_output?("records")
313
-
314
- params = @query["output"]
430
+ def format_records
431
+ params = @request.output
315
432
 
316
433
  attributes = params["attributes"]
317
434
  target_attributes = normalize_target_attributes(attributes)
318
435
  offset = params["offset"] || 0
319
436
  limit = params["limit"] || 10
320
- @result.open_cursor(:offset => offset, :limit => limit) do |cursor|
321
- if params["format"] == "complex"
322
- formatted_result["records"] = cursor.collect do |record|
437
+ formatted_records = nil
438
+ @result.records.open_cursor(:offset => offset, :limit => limit) do |cursor|
439
+ if @request.complex_output?
440
+ formatted_records = cursor.collect do |record|
323
441
  complex_record(target_attributes, record)
324
442
  end
325
443
  else
326
- formatted_result["records"] = cursor.collect do |record|
444
+ formatted_records = cursor.collect do |record|
327
445
  simple_record(target_attributes, record)
328
446
  end
329
447
  end
330
448
  end
449
+
450
+ formatted_records
331
451
  end
332
452
 
333
453
  def complex_record(attributes, record)
@@ -346,7 +466,7 @@ module Droonga
346
466
 
347
467
  def record_value(record, attribute)
348
468
  if attribute[:source] == "_subrecs"
349
- if @query["output"]["format"] == "complex"
469
+ if @request.complex_output?
350
470
  record.sub_records.collect do |sub_record|
351
471
  target_attributes = resolve_attributes(attribute, sub_record)
352
472
  complex_record(target_attributes, sub_record)
@@ -382,11 +502,7 @@ module Droonga
382
502
  return attribute[:target_attributes]
383
503
  end
384
504
 
385
- def accessor_name?(source)
386
- /\A[a-zA-Z\#@$_][a-zA-Z\d\#@$_\-.]*\z/ === source
387
- end
388
-
389
- def normalize_target_attributes(attributes, domain = @result)
505
+ def normalize_target_attributes(attributes, domain = @result.records)
390
506
  attributes.collect do |attribute|
391
507
  if attribute.is_a?(String)
392
508
  attribute = {
@@ -398,12 +514,12 @@ module Droonga
398
514
  expression = nil
399
515
  variable = nil
400
516
  else
401
- expression = Groonga::Expression.new(context: @context)
517
+ expression = Groonga::Expression.new(context: @request.context)
402
518
  variable = expression.define_variable(domain: domain)
403
519
  expression.parse(source, syntax: :script)
404
520
  condition = expression.define_variable(name: "$condition",
405
521
  reference: true)
406
- condition.value = @condition
522
+ condition.value = @result.condition
407
523
  source = nil
408
524
  end
409
525
  {
@@ -416,8 +532,16 @@ module Droonga
416
532
  end
417
533
  end
418
534
 
419
- def log_tag
420
- "[#{Process.ppid}][#{Process.pid}] query_searcher"
535
+ def accessor_name?(source)
536
+ /\A[a-zA-Z\#@$_][a-zA-Z\d\#@$_\-.]*\z/ === source
537
+ end
538
+
539
+ def format_start_time
540
+ TimeFormatter.format(@result.start_time)
541
+ end
542
+
543
+ def format_elapsed_time
544
+ @result.end_time.to_f - @result.start_time.to_f
421
545
  end
422
546
  end
423
547
  end