fluent-plugin-droonga 0.9.0 → 0.9.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (122) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +3 -0
  3. data/Gemfile +8 -1
  4. data/fluent-plugin-droonga.gemspec +2 -2
  5. data/lib/droonga/adapter.rb +39 -0
  6. data/lib/droonga/adapter_runner.rb +99 -0
  7. data/lib/droonga/catalog/base.rb +11 -11
  8. data/lib/droonga/catalog/dataset.rb +54 -0
  9. data/lib/droonga/catalog/version1.rb +1 -1
  10. data/lib/droonga/collector.rb +5 -7
  11. data/lib/droonga/collector_plugin.rb +7 -7
  12. data/lib/droonga/command.rb +36 -0
  13. data/lib/droonga/{plugin/input_adapter/crud.rb → command_repository.rb} +14 -8
  14. data/lib/droonga/dispatcher.rb +86 -54
  15. data/lib/droonga/distributed_command_planner.rb +183 -0
  16. data/lib/droonga/distributor.rb +43 -17
  17. data/lib/droonga/handler.rb +13 -72
  18. data/lib/droonga/handler_message.rb +5 -5
  19. data/lib/droonga/handler_messenger.rb +4 -1
  20. data/lib/droonga/handler_plugin.rb +2 -2
  21. data/lib/droonga/handler_runner.rb +104 -0
  22. data/lib/droonga/input_message.rb +4 -4
  23. data/lib/droonga/legacy_pluggable.rb +66 -0
  24. data/lib/droonga/{input_adapter.rb → legacy_plugin.rb} +27 -22
  25. data/lib/droonga/{plugin_repository.rb → legacy_plugin_repository.rb} +2 -4
  26. data/lib/droonga/message_matcher.rb +101 -0
  27. data/lib/droonga/{input_adapter_plugin.rb → planner.rb} +14 -10
  28. data/lib/droonga/planner_plugin.rb +54 -0
  29. data/lib/droonga/pluggable.rb +9 -45
  30. data/lib/droonga/plugin.rb +9 -33
  31. data/lib/droonga/plugin/collector/basic.rb +2 -0
  32. data/lib/droonga/plugin/collector/search.rb +31 -37
  33. data/lib/droonga/plugin/{handler/groonga/table_remove.rb → metadata/adapter_message.rb} +23 -18
  34. data/lib/droonga/plugin/{handler/search.rb → metadata/handler_action.rb} +19 -15
  35. data/lib/droonga/plugin/metadata/input_message.rb +39 -0
  36. data/lib/droonga/plugin/planner/crud.rb +49 -0
  37. data/lib/droonga/plugin/{distributor → planner}/distributed_search_planner.rb +62 -70
  38. data/lib/droonga/plugin/{distributor → planner}/groonga.rb +11 -32
  39. data/lib/droonga/plugin/{distributor → planner}/search.rb +5 -5
  40. data/lib/droonga/plugin/{distributor → planner}/watch.rb +15 -6
  41. data/lib/droonga/plugin_loader.rb +10 -0
  42. data/lib/droonga/plugin_registerable.rb +34 -10
  43. data/lib/droonga/plugin_registry.rb +58 -0
  44. data/lib/droonga/plugins/crud.rb +124 -0
  45. data/lib/droonga/plugins/error.rb +50 -0
  46. data/lib/droonga/{output_adapter_plugin.rb → plugins/groonga.rb} +9 -13
  47. data/lib/droonga/plugins/groonga/column_create.rb +123 -0
  48. data/lib/droonga/plugins/groonga/generic_command.rb +65 -0
  49. data/lib/droonga/{plugin/output_adapter/groonga.rb → plugins/groonga/generic_response.rb} +16 -15
  50. data/lib/droonga/plugins/groonga/select.rb +124 -0
  51. data/lib/droonga/plugins/groonga/table_create.rb +106 -0
  52. data/lib/droonga/plugins/groonga/table_remove.rb +57 -0
  53. data/lib/droonga/plugins/search.rb +40 -0
  54. data/lib/droonga/plugins/watch.rb +156 -0
  55. data/lib/droonga/processor.rb +8 -10
  56. data/lib/droonga/searcher.rb +14 -4
  57. data/lib/droonga/searcher/mecab_filter.rb +67 -0
  58. data/lib/droonga/session.rb +5 -5
  59. data/lib/droonga/test.rb +1 -1
  60. data/lib/droonga/test/stub_handler_message.rb +1 -1
  61. data/lib/droonga/test/{stub_distributor.rb → stub_planner.rb} +1 -1
  62. data/lib/droonga/worker.rb +7 -8
  63. data/lib/fluent/plugin/out_droonga.rb +0 -1
  64. data/sample/cluster/catalog.json +2 -4
  65. data/sample/mecab_filter/data.grn +7 -0
  66. data/sample/mecab_filter/ddl.grn +7 -0
  67. data/sample/mecab_filter/search_with_mecab_filter.json +21 -0
  68. data/sample/mecab_filter/search_without_mecab_filter.json +21 -0
  69. data/test/command/config/default/catalog.json +2 -5
  70. data/test/command/suite/search/error/no-query.expected +13 -0
  71. data/test/command/suite/search/error/no-query.test +7 -0
  72. data/test/command/suite/search/error/unknown-source.expected +26 -0
  73. data/test/command/suite/watch/subscribe.expected +3 -3
  74. data/test/command/suite/watch/unsubscribe.expected +3 -3
  75. data/test/unit/catalog/test_dataset.rb +385 -0
  76. data/test/unit/catalog/test_version1.rb +111 -45
  77. data/test/unit/fixtures/catalog/version1.json +0 -3
  78. data/test/unit/helper.rb +2 -1
  79. data/test/unit/helper/distributed_search_planner_helper.rb +83 -0
  80. data/test/unit/plugin/collector/test_basic.rb +233 -376
  81. data/test/unit/plugin/collector/test_search.rb +8 -17
  82. data/test/unit/plugin/planner/search_planner/test_basic.rb +120 -0
  83. data/test/unit/plugin/planner/search_planner/test_group_by.rb +573 -0
  84. data/test/unit/plugin/planner/search_planner/test_output.rb +388 -0
  85. data/test/unit/plugin/planner/search_planner/test_sort_by.rb +938 -0
  86. data/test/unit/plugin/{distributor → planner}/test_search.rb +20 -75
  87. data/test/unit/{plugin/handler → plugins/crud}/test_add.rb +11 -11
  88. data/test/unit/plugins/groonga/select/test_adapter_input.rb +213 -0
  89. data/test/unit/{plugin/output_adapter/groonga/test_select.rb → plugins/groonga/select/test_adapter_output.rb} +12 -13
  90. data/test/unit/{plugin/handler → plugins}/groonga/test_column_create.rb +20 -5
  91. data/test/unit/{plugin/handler → plugins}/groonga/test_table_create.rb +5 -0
  92. data/test/unit/{plugin/handler → plugins}/groonga/test_table_remove.rb +8 -1
  93. data/test/unit/{plugin/handler → plugins}/test_groonga.rb +5 -5
  94. data/test/unit/{plugin/handler → plugins}/test_search.rb +21 -5
  95. data/test/unit/{plugin/handler → plugins}/test_watch.rb +29 -10
  96. data/{lib/droonga/command_mapper.rb → test/unit/test_command_repository.rb} +16 -22
  97. data/test/unit/{test_plugin.rb → test_legacy_plugin.rb} +3 -3
  98. data/test/unit/{test_plugin_repository.rb → test_legacy_plugin_repository.rb} +3 -3
  99. data/test/unit/test_message_matcher.rb +137 -0
  100. metadata +86 -66
  101. data/bin/grn2jsons +0 -82
  102. data/lib/droonga/distribution_planner.rb +0 -76
  103. data/lib/droonga/distributor_plugin.rb +0 -95
  104. data/lib/droonga/output_adapter.rb +0 -53
  105. data/lib/droonga/plugin/collector/groonga.rb +0 -83
  106. data/lib/droonga/plugin/distributor/crud.rb +0 -84
  107. data/lib/droonga/plugin/handler/add.rb +0 -109
  108. data/lib/droonga/plugin/handler/forward.rb +0 -75
  109. data/lib/droonga/plugin/handler/groonga.rb +0 -99
  110. data/lib/droonga/plugin/handler/groonga/column_create.rb +0 -106
  111. data/lib/droonga/plugin/handler/groonga/table_create.rb +0 -91
  112. data/lib/droonga/plugin/handler/watch.rb +0 -108
  113. data/lib/droonga/plugin/input_adapter/groonga.rb +0 -49
  114. data/lib/droonga/plugin/input_adapter/groonga/select.rb +0 -63
  115. data/lib/droonga/plugin/output_adapter/crud.rb +0 -51
  116. data/lib/droonga/plugin/output_adapter/groonga/select.rb +0 -54
  117. data/lib/groonga_command_converter.rb +0 -143
  118. data/sample/fluentd.conf +0 -8
  119. data/test/unit/plugin/distributor/test_search_planner.rb +0 -1102
  120. data/test/unit/plugin/input_adapter/groonga/test_select.rb +0 -248
  121. data/test/unit/test_command_mapper.rb +0 -44
  122. data/test/unit/test_groonga_command_converter.rb +0 -242
@@ -1,106 +0,0 @@
1
- # -*- coding: utf-8 -*-
2
- #
3
- # Copyright (C) 2013 Droonga Project
4
- #
5
- # This library is free software; you can redistribute it and/or
6
- # modify it under the terms of the GNU Lesser General Public
7
- # License version 2.1 as published by the Free Software Foundation.
8
- #
9
- # This library is distributed in the hope that it will be useful,
10
- # but WITHOUT ANY WARRANTY; without even the implied warranty of
11
- # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12
- # Lesser General Public License for more details.
13
- #
14
- # You should have received a copy of the GNU Lesser General Public
15
- # License along with this library; if not, write to the Free Software
16
- # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
-
18
- require "groonga"
19
- require "groonga/command/column-create"
20
-
21
- module Droonga
22
- class GroongaHandler
23
- class ColumnCreate < Command
24
- def process_request(request)
25
- command_class = Groonga::Command.find("column_create")
26
- @command = command_class.new("column_create", request)
27
-
28
- table_name = @command["table"]
29
- if table_name.nil? || @context[table_name].nil?
30
- raise CommandError.new(:status => Status::INVALID_ARGUMENT,
31
- :message => "table doesn't exist: <#{table_name.to_s}>",
32
- :result => false)
33
- end
34
-
35
- if @command.column_index?
36
- define_index
37
- else
38
- define_column
39
- end
40
- end
41
-
42
- private
43
- def define_column
44
- table_name = @command["table"]
45
- column_name = @command["name"]
46
- column_type = @command["type"]
47
-
48
- options = create_column_options
49
- Groonga::Schema.define(:context => @context) do |schema|
50
- schema.change_table(table_name) do |table|
51
- table.column(column_name, column_type, options)
52
- end
53
- end
54
- true
55
- end
56
-
57
- def create_column_options
58
- options = {}
59
- create_column_options_flags(options)
60
- options
61
- end
62
-
63
- def create_column_options_flags(options)
64
- options[:type] = :scalar
65
- if @command.column_scalar?
66
- options[:type] = :scalar
67
- elsif @command.column_vector?
68
- options[:type] = :vector
69
- end
70
- options
71
- end
72
-
73
- def define_index
74
- table_name = @command["table"]
75
- target_table = @command["type"]
76
- target_column = @command["source"]
77
-
78
- options = create_index_options
79
- Groonga::Schema.define(:context => @context) do |schema|
80
- schema.change_table(table_name) do |table|
81
- table.index("#{target_table}.#{target_column}", options)
82
- end
83
- end
84
- true
85
- end
86
-
87
- def create_index_options
88
- options = {}
89
- create_index_options_name(options)
90
- create_index_options_flags(options)
91
- options
92
- end
93
-
94
- def create_index_options_name(options)
95
- options[:name] = @command["name"]
96
- end
97
-
98
- def create_index_options_flags(options)
99
- options[:with_section] = true if @command.with_section?
100
- options[:with_weight] = true if @command.with_weight?
101
- options[:with_position] = true if @command.with_position?
102
- options
103
- end
104
- end
105
- end
106
- end
@@ -1,91 +0,0 @@
1
- # -*- coding: utf-8 -*-
2
- #
3
- # Copyright (C) 2013 Droonga Project
4
- #
5
- # This library is free software; you can redistribute it and/or
6
- # modify it under the terms of the GNU Lesser General Public
7
- # License version 2.1 as published by the Free Software Foundation.
8
- #
9
- # This library is distributed in the hope that it will be useful,
10
- # but WITHOUT ANY WARRANTY; without even the implied warranty of
11
- # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12
- # Lesser General Public License for more details.
13
- #
14
- # You should have received a copy of the GNU Lesser General Public
15
- # License along with this library; if not, write to the Free Software
16
- # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
-
18
- require "groonga"
19
- require "groonga/command/table-create"
20
-
21
- module Droonga
22
- class GroongaHandler
23
- class TableCreate < Command
24
- def process_request(request)
25
- command_class = Groonga::Command.find("table_create")
26
- @command = command_class.new("table_create", request)
27
-
28
- name = @command["name"]
29
- unless name
30
- raise CommandError.new(:status => Status::INVALID_ARGUMENT,
31
- :message => "Should not create anonymous table",
32
- :result => false)
33
- end
34
-
35
- options = parse_command
36
- Groonga::Schema.define(:context => @context) do |schema|
37
- schema.create_table(name, options)
38
- end
39
- true
40
- end
41
-
42
- private
43
- def parse_command
44
- options = {}
45
- parse_flags(options)
46
- parse_key_type(options)
47
- parse_value_type(options)
48
- parse_default_tokenizer(options)
49
- parse_normalizer(options)
50
- options
51
- end
52
-
53
- def parse_flags(options)
54
- options[:type] = :hash
55
- if @command.table_no_key?
56
- options[:type] = :array
57
- elsif @command.table_hash_key?
58
- options[:type] = :hash
59
- elsif @command.table_pat_key?
60
- options[:type] = :patricia_trie
61
- elsif @command.table_dat_key?
62
- options[:type] = :double_array_trie
63
- end
64
- if @command.key_with_sis? and @command.table_pat_key?
65
- options[:key_with_sis] = true
66
- end
67
- options
68
- end
69
-
70
- def parse_key_type(options)
71
- options[:key_type] = @command["key_type"] if @command["key_type"]
72
- options
73
- end
74
-
75
- def parse_value_type(options)
76
- options[:value_type] = @command["value_type"] if @command["value_type"]
77
- options
78
- end
79
-
80
- def parse_default_tokenizer(options)
81
- options[:default_tokenizer] = @command["default_tokenizer"] if @command["default_tokenizer"]
82
- options
83
- end
84
-
85
- def parse_normalizer(options)
86
- options[:normalizer] = @command["normalizer"] if @command["normalizer"]
87
- options
88
- end
89
- end
90
- end
91
- end
@@ -1,108 +0,0 @@
1
- # -*- coding: utf-8 -*-
2
- #
3
- # Copyright (C) 2013 Droonga Project
4
- #
5
- # This library is free software; you can redistribute it and/or
6
- # modify it under the terms of the GNU Lesser General Public
7
- # License version 2.1 as published by the Free Software Foundation.
8
- #
9
- # This library is distributed in the hope that it will be useful,
10
- # but WITHOUT ANY WARRANTY; without even the implied warranty of
11
- # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12
- # Lesser General Public License for more details.
13
- #
14
- # You should have received a copy of the GNU Lesser General Public
15
- # License along with this library; if not, write to the Free Software
16
- # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
-
18
- require "droonga/watcher"
19
- require "droonga/sweeper"
20
- require "droonga/watch_schema"
21
- require "droonga/handler_plugin"
22
-
23
- module Droonga
24
- class WatchHandler < Droonga::HandlerPlugin
25
- repository.register("watch", self)
26
-
27
- def initialize(*args)
28
- super
29
-
30
- # XXX just workaround. This must be re-written.
31
- # When secondary and later processes opens the database,
32
- # creation processes of tables by the first process is
33
- # not finished yet. Then secondary and others tries to
34
- # create tables and raises errors. To avoid such a problem,
35
- # the creation processes of tables is disabled on workers.
36
- if $0 !~ /\AServer/
37
- ensure_schema_created
38
- else
39
- until @context["Keyword"]
40
- sleep 0.1
41
- end
42
- sleep 1
43
- end
44
-
45
- @watcher = Watcher.new(@context)
46
- @sweeper = Sweeper.new(@context)
47
- end
48
-
49
- command "watch.subscribe" => :subscribe
50
- def subscribe(message, messenger)
51
- subscriber, condition, query, route = parse_message(message)
52
- normalized_request = {
53
- :subscriber => subscriber,
54
- :condition => condition,
55
- :query => query,
56
- :route => route,
57
- }
58
- @watcher.subscribe(normalized_request)
59
- messenger.emit([true])
60
- end
61
-
62
- command "watch.unsubscribe" => :unsubscribe
63
- def unsubscribe(message, messenger)
64
- subscriber, condition, query, route = parse_message(message)
65
- normalized_request = {
66
- :subscriber => subscriber,
67
- :condition => condition,
68
- :query => query,
69
- }
70
- @watcher.unsubscribe(normalized_request)
71
- messenger.emit([true])
72
- end
73
-
74
- command "watch.feed" => :feed
75
- def feed(message, messenger)
76
- request = message.request
77
- @watcher.feed(:targets => request["targets"]) do |route, subscribers|
78
- published_message = {
79
- "to" => subscribers,
80
- "body" => request,
81
- }
82
- published_message = message.raw.merge(published_message)
83
- messenger.forward(published_message,
84
- "to" => route, "type" => "watch.publish")
85
- end
86
- end
87
-
88
- command "watch.sweep" => :sweep
89
- def sweep(message, messenger)
90
- @sweeper.sweep_expired_subscribers
91
- end
92
-
93
- private
94
- def parse_message(message)
95
- request = message.request
96
- subscriber = request["subscriber"]
97
- condition = request["condition"]
98
- route = request["route"] || message["from"]
99
- query = condition && condition.to_json
100
- [subscriber, condition, query, route]
101
- end
102
-
103
- def ensure_schema_created
104
- schema = WatchSchema.new(@context)
105
- schema.ensure_created
106
- end
107
- end
108
- end
@@ -1,49 +0,0 @@
1
- # Copyright (C) 2013-2014 Droonga Project
2
- #
3
- # This library is free software; you can redistribute it and/or
4
- # modify it under the terms of the GNU Lesser General Public
5
- # License version 2.1 as published by the Free Software Foundation.
6
- #
7
- # This library is distributed in the hope that it will be useful,
8
- # but WITHOUT ANY WARRANTY; without even the implied warranty of
9
- # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
10
- # Lesser General Public License for more details.
11
- #
12
- # You should have received a copy of the GNU Lesser General Public
13
- # License along with this library; if not, write to the Free Software
14
- # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
15
-
16
- require "droonga/input_adapter_plugin"
17
-
18
- module Droonga
19
- class GroongaInputAdapter < Droonga::InputAdapterPlugin
20
- repository.register("groonga", self)
21
-
22
- command :select
23
- def select(input_message)
24
- command = Select.new
25
- select_request = input_message.body
26
- search_request = command.convert(select_request)
27
- input_message.add_route("select_response")
28
- input_message.command = "search"
29
- input_message.body = search_request
30
- end
31
-
32
- command :table_create
33
- def table_create(input_message)
34
- input_message.add_route("groonga_generic_response")
35
- end
36
-
37
- command :table_remove
38
- def table_remove(input_message)
39
- input_message.add_route("groonga_generic_response")
40
- end
41
-
42
- command :column_create
43
- def column_create(input_message)
44
- input_message.add_route("groonga_generic_response")
45
- end
46
- end
47
- end
48
-
49
- require "droonga/plugin/input_adapter/groonga/select"
@@ -1,63 +0,0 @@
1
- # Copyright (C) 2013 Droonga Project
2
- #
3
- # This library is free software; you can redistribute it and/or
4
- # modify it under the terms of the GNU Lesser General Public
5
- # License version 2.1 as published by the Free Software Foundation.
6
- #
7
- # This library is distributed in the hope that it will be useful,
8
- # but WITHOUT ANY WARRANTY; without even the implied warranty of
9
- # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
10
- # Lesser General Public License for more details.
11
- #
12
- # You should have received a copy of the GNU Lesser General Public
13
- # License along with this library; if not, write to the Free Software
14
- # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
15
-
16
- module Droonga
17
- class GroongaInputAdapter
18
- class Select
19
- def convert(select_request)
20
- table = select_request["table"]
21
- result_name = table + "_result"
22
- match_columns = select_request["match_columns"]
23
- match_to = match_columns ? match_columns.split(/ *\|\| */) : []
24
- query = select_request["query"]
25
- output_columns = select_request["output_columns"] || ""
26
- attributes = output_columns.split(/, */)
27
- offset = (select_request["offset"] || "0").to_i
28
- limit = (select_request["limit"] || "10").to_i
29
-
30
- search_request = {
31
- "queries" => {
32
- result_name => {
33
- "source" => table,
34
- "output" => {
35
- "elements" => [
36
- "startTime",
37
- "elapsedTime",
38
- "count",
39
- "attributes",
40
- "records",
41
- ],
42
- "attributes" => attributes,
43
- "offset" => offset,
44
- "limit" => limit,
45
- },
46
- }
47
- }
48
- }
49
- if query
50
- condition = {
51
- "query" => query,
52
- "matchTo"=> match_to,
53
- "defaultOperator"=> "&&",
54
- "allowPragma"=> false,
55
- "allowColumn"=> true,
56
- }
57
- search_request["queries"][result_name]["condition"] = condition
58
- end
59
- search_request
60
- end
61
- end
62
- end
63
- end
@@ -1,51 +0,0 @@
1
- # Copyright (C) 2013-2014 Droonga Project
2
- #
3
- # This library is free software; you can redistribute it and/or
4
- # modify it under the terms of the GNU Lesser General Public
5
- # License version 2.1 as published by the Free Software Foundation.
6
- #
7
- # This library is distributed in the hope that it will be useful,
8
- # but WITHOUT ANY WARRANTY; without even the implied warranty of
9
- # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
10
- # Lesser General Public License for more details.
11
- #
12
- # You should have received a copy of the GNU Lesser General Public
13
- # License along with this library; if not, write to the Free Software
14
- # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
15
-
16
- require "droonga/output_adapter_plugin"
17
-
18
- module Droonga
19
- class CRUDOutputAdapter < Droonga::OutputAdapterPlugin
20
- repository.register("crud", self)
21
-
22
- command :crud_generic_response
23
- def crud_generic_response(output_message)
24
- if output_message.body.include?("errors")
25
- errors = output_message.body["errors"]
26
- if errors && !errors.empty?
27
- output_message.errors = errors
28
-
29
- status_codes = []
30
- errors.values.each do |error|
31
- status_codes << error["statusCode"]
32
- end
33
- status_codes = status_codes.uniq
34
- if status_codes.size == 1
35
- output_message.status_code = status_codes.first
36
- else
37
- output_message.status_code = MessageProcessingError::STATUS_CODE
38
- end
39
-
40
- output_message.body = errors.values.first["body"]
41
- end
42
- end
43
- if output_message.body.include?("success")
44
- success = output_message.body["success"]
45
- unless success.nil?
46
- output_message.body = output_message.body["success"]
47
- end
48
- end
49
- end
50
- end
51
- end