fluent-plugin-droonga 0.7.0 → 0.8.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (163) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +1 -4
  3. data/benchmark/watch/benchmark-notify.rb +2 -2
  4. data/benchmark/watch/benchmark-scan.rb +3 -0
  5. data/benchmark/watch/fluentd.conf +0 -1
  6. data/fluent-plugin-droonga.gemspec +2 -3
  7. data/lib/droonga/catalog.rb +10 -124
  8. data/lib/droonga/catalog/base.rb +140 -0
  9. data/lib/droonga/catalog/version1.rb +23 -0
  10. data/lib/droonga/catalog_loader.rb +33 -0
  11. data/lib/droonga/collector.rb +2 -71
  12. data/lib/droonga/collector_plugin.rb +2 -34
  13. data/lib/droonga/dispatcher.rb +141 -196
  14. data/lib/droonga/distribution_planner.rb +76 -0
  15. data/lib/droonga/distributor.rb +5 -7
  16. data/lib/droonga/distributor_plugin.rb +23 -15
  17. data/lib/droonga/engine.rb +2 -2
  18. data/lib/droonga/event_loop.rb +46 -0
  19. data/lib/droonga/farm.rb +9 -5
  20. data/lib/droonga/fluent_message_sender.rb +84 -0
  21. data/lib/droonga/forwarder.rb +43 -53
  22. data/lib/droonga/handler.rb +20 -68
  23. data/lib/droonga/handler_message.rb +61 -0
  24. data/lib/droonga/handler_messenger.rb +92 -0
  25. data/lib/droonga/handler_plugin.rb +10 -12
  26. data/lib/droonga/input_adapter.rb +52 -0
  27. data/lib/droonga/{adapter.rb → input_adapter_plugin.rb} +7 -13
  28. data/lib/droonga/input_message.rb +11 -11
  29. data/lib/droonga/logger.rb +4 -3
  30. data/lib/droonga/message_pack_packer.rb +62 -0
  31. data/lib/droonga/message_processing_error.rb +54 -0
  32. data/lib/droonga/message_pusher.rb +60 -0
  33. data/lib/droonga/message_receiver.rb +61 -0
  34. data/lib/droonga/output_adapter.rb +53 -0
  35. data/lib/droonga/{adapter_plugin.rb → output_adapter_plugin.rb} +3 -21
  36. data/lib/droonga/output_message.rb +37 -0
  37. data/lib/droonga/partition.rb +27 -5
  38. data/lib/droonga/pluggable.rb +9 -4
  39. data/lib/droonga/plugin.rb +12 -3
  40. data/lib/droonga/plugin/collector/basic.rb +91 -18
  41. data/lib/droonga/plugin/distributor/crud.rb +9 -9
  42. data/lib/droonga/plugin/distributor/distributed_search_planner.rb +401 -0
  43. data/lib/droonga/plugin/distributor/groonga.rb +5 -5
  44. data/lib/droonga/plugin/distributor/search.rb +4 -246
  45. data/lib/droonga/plugin/distributor/watch.rb +11 -6
  46. data/lib/droonga/plugin/handler/add.rb +69 -7
  47. data/lib/droonga/plugin/handler/groonga.rb +6 -6
  48. data/lib/droonga/plugin/handler/search.rb +5 -3
  49. data/lib/droonga/plugin/handler/watch.rb +19 -13
  50. data/lib/droonga/plugin/{adapter → input_adapter}/groonga.rb +5 -11
  51. data/lib/droonga/plugin/{adapter → input_adapter}/groonga/select.rb +2 -36
  52. data/lib/droonga/plugin/output_adapter/groonga.rb +30 -0
  53. data/lib/droonga/plugin/output_adapter/groonga/select.rb +54 -0
  54. data/lib/droonga/plugin_loader.rb +2 -2
  55. data/lib/droonga/processor.rb +21 -23
  56. data/lib/droonga/replier.rb +40 -0
  57. data/lib/droonga/searcher.rb +298 -174
  58. data/lib/droonga/server.rb +0 -67
  59. data/lib/droonga/session.rb +85 -0
  60. data/lib/droonga/test.rb +21 -0
  61. data/lib/droonga/test/stub_distributor.rb +31 -0
  62. data/lib/droonga/test/stub_handler.rb +37 -0
  63. data/lib/droonga/test/stub_handler_message.rb +35 -0
  64. data/lib/droonga/test/stub_handler_messenger.rb +34 -0
  65. data/lib/droonga/time_formatter.rb +37 -0
  66. data/lib/droonga/watcher.rb +1 -0
  67. data/lib/droonga/worker.rb +16 -19
  68. data/lib/fluent/plugin/out_droonga.rb +9 -9
  69. data/lib/groonga_command_converter.rb +5 -5
  70. data/sample/cluster/catalog.json +1 -1
  71. data/test/command/config/default/catalog.json +19 -1
  72. data/test/command/fixture/event.jsons +41 -0
  73. data/test/command/fixture/user-table.jsons +9 -0
  74. data/test/command/run-test.rb +2 -2
  75. data/test/command/suite/add/error/invalid-integer.expected +20 -0
  76. data/test/command/suite/add/error/invalid-integer.test +12 -0
  77. data/test/command/suite/add/error/invalid-time.expected +20 -0
  78. data/test/command/suite/add/error/invalid-time.test +12 -0
  79. data/test/command/suite/add/error/missing-key.expected +13 -0
  80. data/test/command/suite/add/error/missing-key.test +16 -0
  81. data/test/command/suite/add/error/missing-table.expected +13 -0
  82. data/test/command/suite/add/error/missing-table.test +16 -0
  83. data/test/command/suite/add/error/unknown-column.expected +20 -0
  84. data/test/command/suite/add/error/unknown-column.test +12 -0
  85. data/test/command/suite/add/error/unknown-table.expected +13 -0
  86. data/test/command/suite/add/error/unknown-table.test +17 -0
  87. data/test/command/suite/add/minimum.expected +1 -3
  88. data/test/command/suite/add/with-values.expected +1 -3
  89. data/test/command/suite/add/without-key.expected +1 -3
  90. data/test/command/suite/message/error/missing-dataset.expected +13 -0
  91. data/test/command/suite/message/error/missing-dataset.test +5 -0
  92. data/test/command/suite/message/error/unknown-command.expected +13 -0
  93. data/test/command/suite/message/error/unknown-command.test +6 -0
  94. data/test/command/suite/message/error/unknown-dataset.expected +13 -0
  95. data/test/command/suite/message/error/unknown-dataset.test +6 -0
  96. data/test/command/suite/search/{array-attribute-label.expected → attributes/array.expected} +0 -0
  97. data/test/command/suite/search/{array-attribute-label.test → attributes/array.test} +0 -0
  98. data/test/command/suite/search/{hash-attribute-label.expected → attributes/hash.expected} +0 -0
  99. data/test/command/suite/search/{hash-attribute-label.test → attributes/hash.test} +0 -0
  100. data/test/command/suite/search/{condition-nested.expected → condition/nested.expected} +0 -0
  101. data/test/command/suite/search/{condition-nested.test → condition/nested.test} +0 -0
  102. data/test/command/suite/search/{condition-query.expected → condition/query.expected} +0 -0
  103. data/test/command/suite/search/{condition-query.test → condition/query.test} +0 -0
  104. data/test/command/suite/search/{condition-script.expected → condition/script.expected} +0 -0
  105. data/test/command/suite/search/{condition-script.test → condition/script.test} +0 -0
  106. data/test/command/suite/search/error/cyclic-source.expected +18 -0
  107. data/test/command/suite/search/error/cyclic-source.test +12 -0
  108. data/test/command/suite/search/error/deeply-cyclic-source.expected +21 -0
  109. data/test/command/suite/search/error/deeply-cyclic-source.test +15 -0
  110. data/test/command/suite/search/error/missing-source-parameter.expected +17 -0
  111. data/test/command/suite/search/error/missing-source-parameter.test +11 -0
  112. data/test/command/suite/search/error/unknown-source.expected +18 -0
  113. data/test/command/suite/search/error/unknown-source.test +12 -0
  114. data/test/command/suite/search/{minimum.expected → group/count.expected} +2 -1
  115. data/test/command/suite/search/{minimum.test → group/count.test} +5 -3
  116. data/test/command/suite/search/group/limit.expected +19 -0
  117. data/test/command/suite/search/group/limit.test +20 -0
  118. data/test/command/suite/search/group/string.expected +36 -0
  119. data/test/command/suite/search/group/string.test +44 -0
  120. data/test/command/suite/search/{chained-queries.expected → multiple/chained.expected} +0 -0
  121. data/test/command/suite/search/{chained-queries.test → multiple/chained.test} +0 -0
  122. data/test/command/suite/search/{multiple-queries.expected → multiple/parallel.expected} +0 -0
  123. data/test/command/suite/search/{multiple-queries.test → multiple/parallel.test} +0 -0
  124. data/test/command/suite/search/{output-range.expected → range/only-output.expected} +0 -0
  125. data/test/command/suite/search/{output-range.test → range/only-output.test} +0 -0
  126. data/test/command/suite/search/{sort-range.expected → range/only-sort.expected} +0 -0
  127. data/test/command/suite/search/{sort-range.test → range/only-sort.test} +0 -0
  128. data/test/command/suite/search/{sort-and-output-range.expected → range/sort-and-output.expected} +0 -0
  129. data/test/command/suite/search/{sort-and-output-range.test → range/sort-and-output.test} +0 -0
  130. data/test/command/suite/search/range/too-large-output-offset.expected +16 -0
  131. data/test/command/suite/search/range/too-large-output-offset.test +25 -0
  132. data/test/command/suite/search/range/too-large-sort-offset.expected +16 -0
  133. data/test/command/suite/search/range/too-large-sort-offset.test +28 -0
  134. data/test/command/suite/search/response/records/value/time.expected +24 -0
  135. data/test/command/suite/search/response/records/value/time.test +24 -0
  136. data/test/command/suite/search/sort/default-offset-limit.expected +43 -0
  137. data/test/command/suite/search/sort/default-offset-limit.test +26 -0
  138. data/test/command/suite/search/{sort-with-invisible-column.expected → sort/invisible-column.expected} +0 -0
  139. data/test/command/suite/search/{sort-with-invisible-column.test → sort/invisible-column.test} +0 -0
  140. data/test/command/suite/watch/subscribe.expected +12 -0
  141. data/test/command/suite/watch/subscribe.test +9 -0
  142. data/test/command/suite/watch/unsubscribe.expected +12 -0
  143. data/test/command/suite/watch/unsubscribe.test +9 -0
  144. data/test/unit/{test_catalog.rb → catalog/test_version1.rb} +12 -4
  145. data/test/unit/fixtures/{catalog.json → catalog/version1.json} +0 -0
  146. data/test/unit/helper.rb +2 -0
  147. data/test/unit/plugin/collector/test_basic.rb +289 -33
  148. data/test/unit/plugin/distributor/test_search.rb +176 -861
  149. data/test/unit/plugin/distributor/test_search_planner.rb +1102 -0
  150. data/test/unit/plugin/handler/groonga/test_column_create.rb +17 -13
  151. data/test/unit/plugin/handler/groonga/test_table_create.rb +10 -10
  152. data/test/unit/plugin/handler/test_add.rb +74 -11
  153. data/test/unit/plugin/handler/test_groonga.rb +15 -1
  154. data/test/unit/plugin/handler/test_search.rb +33 -17
  155. data/test/unit/plugin/handler/test_watch.rb +43 -27
  156. data/test/unit/run-test.rb +2 -0
  157. data/test/unit/test_message_pack_packer.rb +51 -0
  158. data/test/unit/test_time_formatter.rb +29 -0
  159. metadata +208 -110
  160. data/lib/droonga/job_queue.rb +0 -87
  161. data/lib/droonga/job_queue_schema.rb +0 -65
  162. data/test/unit/test_adapter.rb +0 -51
  163. data/test/unit/test_job_queue_schema.rb +0 -45
@@ -1,87 +0,0 @@
1
- # -*- coding: utf-8 -*-
2
- #
3
- # Copyright (C) 2013 Droonga Project
4
- #
5
- # This library is free software; you can redistribute it and/or
6
- # modify it under the terms of the GNU Lesser General Public
7
- # License version 2.1 as published by the Free Software Foundation.
8
- #
9
- # This library is distributed in the hope that it will be useful,
10
- # but WITHOUT ANY WARRANTY; without even the implied warranty of
11
- # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12
- # Lesser General Public License for more details.
13
- #
14
- # You should have received a copy of the GNU Lesser General Public
15
- # License along with this library; if not, write to the Free Software
16
- # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
-
18
- require "droonga/job_queue_schema"
19
- require "msgpack"
20
-
21
- module Droonga
22
- class JobQueue
23
- class << self
24
- def ensure_schema(database_path, queue_name)
25
- schema = JobQueueSchema.new(database_path, queue_name)
26
- schema.ensure_created
27
- end
28
-
29
- def open(database_path, queue_name)
30
- job_queue = new(database_path, queue_name)
31
- job_queue.open
32
- job_queue
33
- end
34
- end
35
-
36
- def initialize(database_path, queue_name)
37
- @database_path = database_path
38
- @queue_name = queue_name
39
- end
40
-
41
- def open
42
- @context = Groonga::Context.new
43
- @database = @context.open_database(@database_path)
44
- @context.encoding = :none
45
-
46
- @queue = @context[@queue_name]
47
- end
48
-
49
- def push_message(message)
50
- $log.trace("#{log_tag}: push_message: start")
51
- packed_message = message.to_msgpack
52
- @queue.push do |record|
53
- record.message = packed_message
54
- end
55
- $log.trace("#{log_tag}: push_message: done")
56
- end
57
-
58
- def pull_message
59
- packed_message = nil
60
- @queue.pull do |record|
61
- if record
62
- packed_message = record.message
63
- record.delete
64
- end
65
- end
66
- return nil unless packed_message
67
- MessagePack.unpack(packed_message)
68
- end
69
-
70
- def unblock
71
- @queue.unblock
72
- end
73
-
74
- def close
75
- @queue = nil
76
- if @database
77
- @database.close
78
- @context.close
79
- @database = @context = nil
80
- end
81
- end
82
-
83
- def log_tag
84
- "[#{Process.ppid}][#{Process.pid}] job_queue"
85
- end
86
- end
87
- end
@@ -1,65 +0,0 @@
1
- # -*- coding: utf-8 -*-
2
- #
3
- # Copyright (C) 2013 Droonga Project
4
- #
5
- # This library is free software; you can redistribute it and/or
6
- # modify it under the terms of the GNU Lesser General Public
7
- # License version 2.1 as published by the Free Software Foundation.
8
- #
9
- # This library is distributed in the hope that it will be useful,
10
- # but WITHOUT ANY WARRANTY; without even the implied warranty of
11
- # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12
- # Lesser General Public License for more details.
13
- #
14
- # You should have received a copy of the GNU Lesser General Public
15
- # License along with this library; if not, write to the Free Software
16
- # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
-
18
- require "fileutils"
19
-
20
- require "groonga"
21
-
22
- module Droonga
23
- class JobQueueSchema
24
- def initialize(database_path, queue_name)
25
- @database_path = database_path
26
- @queue_name = queue_name
27
- end
28
-
29
- def ensure_created
30
- ensure_database
31
- ensure_queue
32
- end
33
-
34
- private
35
- def ensure_database
36
- return if File.exist?(@database_path)
37
- FileUtils.mkdir_p(File.dirname(@database_path))
38
- create_context do |context|
39
- context.create_database(@database_path) do
40
- end
41
- end
42
- end
43
-
44
- def ensure_queue
45
- create_context do |context|
46
- context.open_database(@database_path) do
47
- Groonga::Schema.define(:context => context) do |schema|
48
- schema.create_table(@queue_name, :type => :array) do |table|
49
- table.text("message")
50
- end
51
- end
52
- end
53
- end
54
- end
55
-
56
- def create_context
57
- context = Groonga::Context.new
58
- begin
59
- yield(context)
60
- ensure
61
- context.close
62
- end
63
- end
64
- end
65
- end
@@ -1,51 +0,0 @@
1
- # Copyright (C) 2013 Droonga Project
2
- #
3
- # This library is free software; you can redistribute it and/or
4
- # modify it under the terms of the GNU Lesser General Public
5
- # License version 2.1 as published by the Free Software Foundation.
6
- #
7
- # This library is distributed in the hope that it will be useful,
8
- # but WITHOUT ANY WARRANTY; without even the implied warranty of
9
- # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
10
- # Lesser General Public License for more details.
11
- #
12
- # You should have received a copy of the GNU Lesser General Public
13
- # License along with this library; if not, write to the Free Software
14
- # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
15
-
16
- require "droonga/adapter"
17
-
18
- class AdapterTest < Test::Unit::TestCase
19
- class AdaptTest < self
20
- # class GroongaAdapter < Droonga::Adapter
21
- # command :select
22
- # def select(request)
23
- # post(:search) do |response|
24
- # # do nothing
25
- # end
26
- # :selected
27
- # end
28
- # end
29
-
30
- def setup
31
- omit("Pending")
32
- @dispatcher = Object.new
33
- @groonga_adapter = GroongaAdapter.new(@dispatcher)
34
- end
35
-
36
- def test_called
37
- omit("Pending")
38
- request = nil
39
- stub(@dispatcher).post
40
- assert_equal(:selected, @groonga_adapter.adapt(:select, request))
41
- end
42
-
43
- def test_post
44
- omit("Pending")
45
- request = nil
46
- response = nil
47
- mock(@dispatcher).post(:search).yields(response)
48
- @groonga_adapter.adapt(:select, request)
49
- end
50
- end
51
- end
@@ -1,45 +0,0 @@
1
- # Copyright (C) 2013 Droonga Project
2
- #
3
- # This library is free software; you can redistribute it and/or
4
- # modify it under the terms of the GNU Lesser General Public
5
- # License version 2.1 as published by the Free Software Foundation.
6
- #
7
- # This library is distributed in the hope that it will be useful,
8
- # but WITHOUT ANY WARRANTY; without even the implied warranty of
9
- # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
10
- # Lesser General Public License for more details.
11
- #
12
- # You should have received a copy of the GNU Lesser General Public
13
- # License along with this library; if not, write to the Free Software
14
- # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
15
-
16
- require "helper"
17
-
18
- require "droonga/job_queue_schema"
19
-
20
- class JobQueueSchemaTest < Test::Unit::TestCase
21
- def setup
22
- @database_path = @temporary_directory + "droonga/db"
23
- @queue_name = "DroongaQueue"
24
- end
25
-
26
- def test_ensure_created
27
- schema = Droonga::JobQueueSchema.new(@database_path.to_s, @queue_name)
28
-
29
- assert_not_predicate(@database_path, :exist?)
30
- schema.ensure_created
31
- assert_predicate(@database_path, :exist?)
32
-
33
- context = Groonga::Context.new
34
- dumped_commands = nil
35
- context.open_database(@database_path.to_s) do |database|
36
- dumped_commands = Groonga::DatabaseDumper.dump(:context => context,
37
- :database => database)
38
- end
39
- context.close
40
- assert_equal(<<-SCHEMA, dumped_commands)
41
- table_create #{@queue_name} TABLE_NO_KEY
42
- column_create #{@queue_name} message COLUMN_SCALAR Text
43
- SCHEMA
44
- end
45
- end