fluent-plugin-droonga 0.7.0 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (163) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +1 -4
  3. data/benchmark/watch/benchmark-notify.rb +2 -2
  4. data/benchmark/watch/benchmark-scan.rb +3 -0
  5. data/benchmark/watch/fluentd.conf +0 -1
  6. data/fluent-plugin-droonga.gemspec +2 -3
  7. data/lib/droonga/catalog.rb +10 -124
  8. data/lib/droonga/catalog/base.rb +140 -0
  9. data/lib/droonga/catalog/version1.rb +23 -0
  10. data/lib/droonga/catalog_loader.rb +33 -0
  11. data/lib/droonga/collector.rb +2 -71
  12. data/lib/droonga/collector_plugin.rb +2 -34
  13. data/lib/droonga/dispatcher.rb +141 -196
  14. data/lib/droonga/distribution_planner.rb +76 -0
  15. data/lib/droonga/distributor.rb +5 -7
  16. data/lib/droonga/distributor_plugin.rb +23 -15
  17. data/lib/droonga/engine.rb +2 -2
  18. data/lib/droonga/event_loop.rb +46 -0
  19. data/lib/droonga/farm.rb +9 -5
  20. data/lib/droonga/fluent_message_sender.rb +84 -0
  21. data/lib/droonga/forwarder.rb +43 -53
  22. data/lib/droonga/handler.rb +20 -68
  23. data/lib/droonga/handler_message.rb +61 -0
  24. data/lib/droonga/handler_messenger.rb +92 -0
  25. data/lib/droonga/handler_plugin.rb +10 -12
  26. data/lib/droonga/input_adapter.rb +52 -0
  27. data/lib/droonga/{adapter.rb → input_adapter_plugin.rb} +7 -13
  28. data/lib/droonga/input_message.rb +11 -11
  29. data/lib/droonga/logger.rb +4 -3
  30. data/lib/droonga/message_pack_packer.rb +62 -0
  31. data/lib/droonga/message_processing_error.rb +54 -0
  32. data/lib/droonga/message_pusher.rb +60 -0
  33. data/lib/droonga/message_receiver.rb +61 -0
  34. data/lib/droonga/output_adapter.rb +53 -0
  35. data/lib/droonga/{adapter_plugin.rb → output_adapter_plugin.rb} +3 -21
  36. data/lib/droonga/output_message.rb +37 -0
  37. data/lib/droonga/partition.rb +27 -5
  38. data/lib/droonga/pluggable.rb +9 -4
  39. data/lib/droonga/plugin.rb +12 -3
  40. data/lib/droonga/plugin/collector/basic.rb +91 -18
  41. data/lib/droonga/plugin/distributor/crud.rb +9 -9
  42. data/lib/droonga/plugin/distributor/distributed_search_planner.rb +401 -0
  43. data/lib/droonga/plugin/distributor/groonga.rb +5 -5
  44. data/lib/droonga/plugin/distributor/search.rb +4 -246
  45. data/lib/droonga/plugin/distributor/watch.rb +11 -6
  46. data/lib/droonga/plugin/handler/add.rb +69 -7
  47. data/lib/droonga/plugin/handler/groonga.rb +6 -6
  48. data/lib/droonga/plugin/handler/search.rb +5 -3
  49. data/lib/droonga/plugin/handler/watch.rb +19 -13
  50. data/lib/droonga/plugin/{adapter → input_adapter}/groonga.rb +5 -11
  51. data/lib/droonga/plugin/{adapter → input_adapter}/groonga/select.rb +2 -36
  52. data/lib/droonga/plugin/output_adapter/groonga.rb +30 -0
  53. data/lib/droonga/plugin/output_adapter/groonga/select.rb +54 -0
  54. data/lib/droonga/plugin_loader.rb +2 -2
  55. data/lib/droonga/processor.rb +21 -23
  56. data/lib/droonga/replier.rb +40 -0
  57. data/lib/droonga/searcher.rb +298 -174
  58. data/lib/droonga/server.rb +0 -67
  59. data/lib/droonga/session.rb +85 -0
  60. data/lib/droonga/test.rb +21 -0
  61. data/lib/droonga/test/stub_distributor.rb +31 -0
  62. data/lib/droonga/test/stub_handler.rb +37 -0
  63. data/lib/droonga/test/stub_handler_message.rb +35 -0
  64. data/lib/droonga/test/stub_handler_messenger.rb +34 -0
  65. data/lib/droonga/time_formatter.rb +37 -0
  66. data/lib/droonga/watcher.rb +1 -0
  67. data/lib/droonga/worker.rb +16 -19
  68. data/lib/fluent/plugin/out_droonga.rb +9 -9
  69. data/lib/groonga_command_converter.rb +5 -5
  70. data/sample/cluster/catalog.json +1 -1
  71. data/test/command/config/default/catalog.json +19 -1
  72. data/test/command/fixture/event.jsons +41 -0
  73. data/test/command/fixture/user-table.jsons +9 -0
  74. data/test/command/run-test.rb +2 -2
  75. data/test/command/suite/add/error/invalid-integer.expected +20 -0
  76. data/test/command/suite/add/error/invalid-integer.test +12 -0
  77. data/test/command/suite/add/error/invalid-time.expected +20 -0
  78. data/test/command/suite/add/error/invalid-time.test +12 -0
  79. data/test/command/suite/add/error/missing-key.expected +13 -0
  80. data/test/command/suite/add/error/missing-key.test +16 -0
  81. data/test/command/suite/add/error/missing-table.expected +13 -0
  82. data/test/command/suite/add/error/missing-table.test +16 -0
  83. data/test/command/suite/add/error/unknown-column.expected +20 -0
  84. data/test/command/suite/add/error/unknown-column.test +12 -0
  85. data/test/command/suite/add/error/unknown-table.expected +13 -0
  86. data/test/command/suite/add/error/unknown-table.test +17 -0
  87. data/test/command/suite/add/minimum.expected +1 -3
  88. data/test/command/suite/add/with-values.expected +1 -3
  89. data/test/command/suite/add/without-key.expected +1 -3
  90. data/test/command/suite/message/error/missing-dataset.expected +13 -0
  91. data/test/command/suite/message/error/missing-dataset.test +5 -0
  92. data/test/command/suite/message/error/unknown-command.expected +13 -0
  93. data/test/command/suite/message/error/unknown-command.test +6 -0
  94. data/test/command/suite/message/error/unknown-dataset.expected +13 -0
  95. data/test/command/suite/message/error/unknown-dataset.test +6 -0
  96. data/test/command/suite/search/{array-attribute-label.expected → attributes/array.expected} +0 -0
  97. data/test/command/suite/search/{array-attribute-label.test → attributes/array.test} +0 -0
  98. data/test/command/suite/search/{hash-attribute-label.expected → attributes/hash.expected} +0 -0
  99. data/test/command/suite/search/{hash-attribute-label.test → attributes/hash.test} +0 -0
  100. data/test/command/suite/search/{condition-nested.expected → condition/nested.expected} +0 -0
  101. data/test/command/suite/search/{condition-nested.test → condition/nested.test} +0 -0
  102. data/test/command/suite/search/{condition-query.expected → condition/query.expected} +0 -0
  103. data/test/command/suite/search/{condition-query.test → condition/query.test} +0 -0
  104. data/test/command/suite/search/{condition-script.expected → condition/script.expected} +0 -0
  105. data/test/command/suite/search/{condition-script.test → condition/script.test} +0 -0
  106. data/test/command/suite/search/error/cyclic-source.expected +18 -0
  107. data/test/command/suite/search/error/cyclic-source.test +12 -0
  108. data/test/command/suite/search/error/deeply-cyclic-source.expected +21 -0
  109. data/test/command/suite/search/error/deeply-cyclic-source.test +15 -0
  110. data/test/command/suite/search/error/missing-source-parameter.expected +17 -0
  111. data/test/command/suite/search/error/missing-source-parameter.test +11 -0
  112. data/test/command/suite/search/error/unknown-source.expected +18 -0
  113. data/test/command/suite/search/error/unknown-source.test +12 -0
  114. data/test/command/suite/search/{minimum.expected → group/count.expected} +2 -1
  115. data/test/command/suite/search/{minimum.test → group/count.test} +5 -3
  116. data/test/command/suite/search/group/limit.expected +19 -0
  117. data/test/command/suite/search/group/limit.test +20 -0
  118. data/test/command/suite/search/group/string.expected +36 -0
  119. data/test/command/suite/search/group/string.test +44 -0
  120. data/test/command/suite/search/{chained-queries.expected → multiple/chained.expected} +0 -0
  121. data/test/command/suite/search/{chained-queries.test → multiple/chained.test} +0 -0
  122. data/test/command/suite/search/{multiple-queries.expected → multiple/parallel.expected} +0 -0
  123. data/test/command/suite/search/{multiple-queries.test → multiple/parallel.test} +0 -0
  124. data/test/command/suite/search/{output-range.expected → range/only-output.expected} +0 -0
  125. data/test/command/suite/search/{output-range.test → range/only-output.test} +0 -0
  126. data/test/command/suite/search/{sort-range.expected → range/only-sort.expected} +0 -0
  127. data/test/command/suite/search/{sort-range.test → range/only-sort.test} +0 -0
  128. data/test/command/suite/search/{sort-and-output-range.expected → range/sort-and-output.expected} +0 -0
  129. data/test/command/suite/search/{sort-and-output-range.test → range/sort-and-output.test} +0 -0
  130. data/test/command/suite/search/range/too-large-output-offset.expected +16 -0
  131. data/test/command/suite/search/range/too-large-output-offset.test +25 -0
  132. data/test/command/suite/search/range/too-large-sort-offset.expected +16 -0
  133. data/test/command/suite/search/range/too-large-sort-offset.test +28 -0
  134. data/test/command/suite/search/response/records/value/time.expected +24 -0
  135. data/test/command/suite/search/response/records/value/time.test +24 -0
  136. data/test/command/suite/search/sort/default-offset-limit.expected +43 -0
  137. data/test/command/suite/search/sort/default-offset-limit.test +26 -0
  138. data/test/command/suite/search/{sort-with-invisible-column.expected → sort/invisible-column.expected} +0 -0
  139. data/test/command/suite/search/{sort-with-invisible-column.test → sort/invisible-column.test} +0 -0
  140. data/test/command/suite/watch/subscribe.expected +12 -0
  141. data/test/command/suite/watch/subscribe.test +9 -0
  142. data/test/command/suite/watch/unsubscribe.expected +12 -0
  143. data/test/command/suite/watch/unsubscribe.test +9 -0
  144. data/test/unit/{test_catalog.rb → catalog/test_version1.rb} +12 -4
  145. data/test/unit/fixtures/{catalog.json → catalog/version1.json} +0 -0
  146. data/test/unit/helper.rb +2 -0
  147. data/test/unit/plugin/collector/test_basic.rb +289 -33
  148. data/test/unit/plugin/distributor/test_search.rb +176 -861
  149. data/test/unit/plugin/distributor/test_search_planner.rb +1102 -0
  150. data/test/unit/plugin/handler/groonga/test_column_create.rb +17 -13
  151. data/test/unit/plugin/handler/groonga/test_table_create.rb +10 -10
  152. data/test/unit/plugin/handler/test_add.rb +74 -11
  153. data/test/unit/plugin/handler/test_groonga.rb +15 -1
  154. data/test/unit/plugin/handler/test_search.rb +33 -17
  155. data/test/unit/plugin/handler/test_watch.rb +43 -27
  156. data/test/unit/run-test.rb +2 -0
  157. data/test/unit/test_message_pack_packer.rb +51 -0
  158. data/test/unit/test_time_formatter.rb +29 -0
  159. metadata +208 -110
  160. data/lib/droonga/job_queue.rb +0 -87
  161. data/lib/droonga/job_queue_schema.rb +0 -65
  162. data/test/unit/test_adapter.rb +0 -51
  163. data/test/unit/test_job_queue_schema.rb +0 -45
@@ -1,87 +0,0 @@
1
- # -*- coding: utf-8 -*-
2
- #
3
- # Copyright (C) 2013 Droonga Project
4
- #
5
- # This library is free software; you can redistribute it and/or
6
- # modify it under the terms of the GNU Lesser General Public
7
- # License version 2.1 as published by the Free Software Foundation.
8
- #
9
- # This library is distributed in the hope that it will be useful,
10
- # but WITHOUT ANY WARRANTY; without even the implied warranty of
11
- # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12
- # Lesser General Public License for more details.
13
- #
14
- # You should have received a copy of the GNU Lesser General Public
15
- # License along with this library; if not, write to the Free Software
16
- # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
-
18
- require "droonga/job_queue_schema"
19
- require "msgpack"
20
-
21
- module Droonga
22
- class JobQueue
23
- class << self
24
- def ensure_schema(database_path, queue_name)
25
- schema = JobQueueSchema.new(database_path, queue_name)
26
- schema.ensure_created
27
- end
28
-
29
- def open(database_path, queue_name)
30
- job_queue = new(database_path, queue_name)
31
- job_queue.open
32
- job_queue
33
- end
34
- end
35
-
36
- def initialize(database_path, queue_name)
37
- @database_path = database_path
38
- @queue_name = queue_name
39
- end
40
-
41
- def open
42
- @context = Groonga::Context.new
43
- @database = @context.open_database(@database_path)
44
- @context.encoding = :none
45
-
46
- @queue = @context[@queue_name]
47
- end
48
-
49
- def push_message(message)
50
- $log.trace("#{log_tag}: push_message: start")
51
- packed_message = message.to_msgpack
52
- @queue.push do |record|
53
- record.message = packed_message
54
- end
55
- $log.trace("#{log_tag}: push_message: done")
56
- end
57
-
58
- def pull_message
59
- packed_message = nil
60
- @queue.pull do |record|
61
- if record
62
- packed_message = record.message
63
- record.delete
64
- end
65
- end
66
- return nil unless packed_message
67
- MessagePack.unpack(packed_message)
68
- end
69
-
70
- def unblock
71
- @queue.unblock
72
- end
73
-
74
- def close
75
- @queue = nil
76
- if @database
77
- @database.close
78
- @context.close
79
- @database = @context = nil
80
- end
81
- end
82
-
83
- def log_tag
84
- "[#{Process.ppid}][#{Process.pid}] job_queue"
85
- end
86
- end
87
- end
@@ -1,65 +0,0 @@
1
- # -*- coding: utf-8 -*-
2
- #
3
- # Copyright (C) 2013 Droonga Project
4
- #
5
- # This library is free software; you can redistribute it and/or
6
- # modify it under the terms of the GNU Lesser General Public
7
- # License version 2.1 as published by the Free Software Foundation.
8
- #
9
- # This library is distributed in the hope that it will be useful,
10
- # but WITHOUT ANY WARRANTY; without even the implied warranty of
11
- # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12
- # Lesser General Public License for more details.
13
- #
14
- # You should have received a copy of the GNU Lesser General Public
15
- # License along with this library; if not, write to the Free Software
16
- # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
-
18
- require "fileutils"
19
-
20
- require "groonga"
21
-
22
- module Droonga
23
- class JobQueueSchema
24
- def initialize(database_path, queue_name)
25
- @database_path = database_path
26
- @queue_name = queue_name
27
- end
28
-
29
- def ensure_created
30
- ensure_database
31
- ensure_queue
32
- end
33
-
34
- private
35
- def ensure_database
36
- return if File.exist?(@database_path)
37
- FileUtils.mkdir_p(File.dirname(@database_path))
38
- create_context do |context|
39
- context.create_database(@database_path) do
40
- end
41
- end
42
- end
43
-
44
- def ensure_queue
45
- create_context do |context|
46
- context.open_database(@database_path) do
47
- Groonga::Schema.define(:context => context) do |schema|
48
- schema.create_table(@queue_name, :type => :array) do |table|
49
- table.text("message")
50
- end
51
- end
52
- end
53
- end
54
- end
55
-
56
- def create_context
57
- context = Groonga::Context.new
58
- begin
59
- yield(context)
60
- ensure
61
- context.close
62
- end
63
- end
64
- end
65
- end
@@ -1,51 +0,0 @@
1
- # Copyright (C) 2013 Droonga Project
2
- #
3
- # This library is free software; you can redistribute it and/or
4
- # modify it under the terms of the GNU Lesser General Public
5
- # License version 2.1 as published by the Free Software Foundation.
6
- #
7
- # This library is distributed in the hope that it will be useful,
8
- # but WITHOUT ANY WARRANTY; without even the implied warranty of
9
- # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
10
- # Lesser General Public License for more details.
11
- #
12
- # You should have received a copy of the GNU Lesser General Public
13
- # License along with this library; if not, write to the Free Software
14
- # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
15
-
16
- require "droonga/adapter"
17
-
18
- class AdapterTest < Test::Unit::TestCase
19
- class AdaptTest < self
20
- # class GroongaAdapter < Droonga::Adapter
21
- # command :select
22
- # def select(request)
23
- # post(:search) do |response|
24
- # # do nothing
25
- # end
26
- # :selected
27
- # end
28
- # end
29
-
30
- def setup
31
- omit("Pending")
32
- @dispatcher = Object.new
33
- @groonga_adapter = GroongaAdapter.new(@dispatcher)
34
- end
35
-
36
- def test_called
37
- omit("Pending")
38
- request = nil
39
- stub(@dispatcher).post
40
- assert_equal(:selected, @groonga_adapter.adapt(:select, request))
41
- end
42
-
43
- def test_post
44
- omit("Pending")
45
- request = nil
46
- response = nil
47
- mock(@dispatcher).post(:search).yields(response)
48
- @groonga_adapter.adapt(:select, request)
49
- end
50
- end
51
- end
@@ -1,45 +0,0 @@
1
- # Copyright (C) 2013 Droonga Project
2
- #
3
- # This library is free software; you can redistribute it and/or
4
- # modify it under the terms of the GNU Lesser General Public
5
- # License version 2.1 as published by the Free Software Foundation.
6
- #
7
- # This library is distributed in the hope that it will be useful,
8
- # but WITHOUT ANY WARRANTY; without even the implied warranty of
9
- # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
10
- # Lesser General Public License for more details.
11
- #
12
- # You should have received a copy of the GNU Lesser General Public
13
- # License along with this library; if not, write to the Free Software
14
- # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
15
-
16
- require "helper"
17
-
18
- require "droonga/job_queue_schema"
19
-
20
- class JobQueueSchemaTest < Test::Unit::TestCase
21
- def setup
22
- @database_path = @temporary_directory + "droonga/db"
23
- @queue_name = "DroongaQueue"
24
- end
25
-
26
- def test_ensure_created
27
- schema = Droonga::JobQueueSchema.new(@database_path.to_s, @queue_name)
28
-
29
- assert_not_predicate(@database_path, :exist?)
30
- schema.ensure_created
31
- assert_predicate(@database_path, :exist?)
32
-
33
- context = Groonga::Context.new
34
- dumped_commands = nil
35
- context.open_database(@database_path.to_s) do |database|
36
- dumped_commands = Groonga::DatabaseDumper.dump(:context => context,
37
- :database => database)
38
- end
39
- context.close
40
- assert_equal(<<-SCHEMA, dumped_commands)
41
- table_create #{@queue_name} TABLE_NO_KEY
42
- column_create #{@queue_name} message COLUMN_SCALAR Text
43
- SCHEMA
44
- end
45
- end