fluent-plugin-droonga 0.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +1 -0
  3. data/.travis.yml +7 -0
  4. data/Gemfile +40 -0
  5. data/LICENSE.txt +14 -0
  6. data/README.md +18 -0
  7. data/Rakefile +25 -0
  8. data/benchmark/benchmark.rb +123 -0
  9. data/benchmark/utils.rb +243 -0
  10. data/benchmark/watch/benchmark-notify.rb +143 -0
  11. data/benchmark/watch/benchmark-notify.sh +19 -0
  12. data/benchmark/watch/benchmark-publish.rb +120 -0
  13. data/benchmark/watch/benchmark-scan.rb +210 -0
  14. data/benchmark/watch/catalog.json +32 -0
  15. data/benchmark/watch/fluentd.conf +12 -0
  16. data/bin/grn2jsons +85 -0
  17. data/fluent-plugin-droonga.gemspec +41 -0
  18. data/lib/droonga/adapter.rb +156 -0
  19. data/lib/droonga/catalog.rb +153 -0
  20. data/lib/droonga/command_mapper.rb +45 -0
  21. data/lib/droonga/engine.rb +83 -0
  22. data/lib/droonga/executor.rb +289 -0
  23. data/lib/droonga/handler.rb +140 -0
  24. data/lib/droonga/handler_plugin.rb +35 -0
  25. data/lib/droonga/job_queue.rb +83 -0
  26. data/lib/droonga/job_queue_schema.rb +65 -0
  27. data/lib/droonga/logger.rb +34 -0
  28. data/lib/droonga/plugin.rb +41 -0
  29. data/lib/droonga/plugin/adapter/groonga/select.rb +88 -0
  30. data/lib/droonga/plugin/adapter_groonga.rb +40 -0
  31. data/lib/droonga/plugin/handler/groonga/column_create.rb +103 -0
  32. data/lib/droonga/plugin/handler/groonga/table_create.rb +100 -0
  33. data/lib/droonga/plugin/handler_add.rb +44 -0
  34. data/lib/droonga/plugin/handler_forward.rb +70 -0
  35. data/lib/droonga/plugin/handler_groonga.rb +52 -0
  36. data/lib/droonga/plugin/handler_proxy.rb +82 -0
  37. data/lib/droonga/plugin/handler_search.rb +33 -0
  38. data/lib/droonga/plugin/handler_watch.rb +102 -0
  39. data/lib/droonga/proxy.rb +371 -0
  40. data/lib/droonga/searcher.rb +415 -0
  41. data/lib/droonga/server.rb +112 -0
  42. data/lib/droonga/sweeper.rb +42 -0
  43. data/lib/droonga/watch_schema.rb +88 -0
  44. data/lib/droonga/watcher.rb +256 -0
  45. data/lib/droonga/worker.rb +51 -0
  46. data/lib/fluent/plugin/out_droonga.rb +56 -0
  47. data/lib/groonga_command_converter.rb +137 -0
  48. data/sample/cluster/catalog.json +43 -0
  49. data/sample/cluster/fluentd.conf +12 -0
  50. data/sample/fluentd.conf +8 -0
  51. data/test/fixtures/catalog.json +43 -0
  52. data/test/fixtures/document.grn +23 -0
  53. data/test/helper.rb +24 -0
  54. data/test/helper/fixture.rb +28 -0
  55. data/test/helper/sandbox.rb +73 -0
  56. data/test/helper/stub_worker.rb +27 -0
  57. data/test/helper/watch_helper.rb +35 -0
  58. data/test/plugin/adapter/groonga/test_select.rb +176 -0
  59. data/test/plugin/handler/groonga/test_column_create.rb +127 -0
  60. data/test/plugin/handler/groonga/test_table_create.rb +140 -0
  61. data/test/plugin/handler/test_handler_add.rb +135 -0
  62. data/test/plugin/handler/test_handler_groonga.rb +64 -0
  63. data/test/plugin/handler/test_handler_search.rb +512 -0
  64. data/test/plugin/handler/test_handler_watch.rb +168 -0
  65. data/test/run-test.rb +55 -0
  66. data/test/test_adapter.rb +48 -0
  67. data/test/test_catalog.rb +59 -0
  68. data/test/test_command_mapper.rb +44 -0
  69. data/test/test_groonga_command_converter.rb +242 -0
  70. data/test/test_handler.rb +53 -0
  71. data/test/test_job_queue_schema.rb +45 -0
  72. data/test/test_output.rb +99 -0
  73. data/test/test_sweeper.rb +95 -0
  74. data/test/test_watch_schema.rb +57 -0
  75. data/test/test_watcher.rb +336 -0
  76. data/test/test_worker.rb +144 -0
  77. metadata +299 -0
@@ -0,0 +1,32 @@
1
+ {
2
+ "effective_date": "2013-09-01T00:00:00Z",
3
+ "zones": ["localhost:23003/droonga"],
4
+ "farms": {
5
+ "localhost:23003/droonga": {
6
+ "device": ".",
7
+ "capacity": 10
8
+ }
9
+ },
10
+ "datasets": {
11
+ "Watch": {
12
+ "workers": 0,
13
+ "plugins": ["search", "groonga", "add", "watch"],
14
+ "number_of_replicas": 1,
15
+ "number_of_partitions": 1,
16
+ "partition_key": "_key",
17
+ "date_range": "infinity",
18
+ "ring": {
19
+ "localhost:23041": {
20
+ "weight": 50,
21
+ "partitions": {
22
+ "2013-09-01": [
23
+ "localhost:23003/droonga.watch"
24
+ ]
25
+ }
26
+ }
27
+ }
28
+ }
29
+ },
30
+ "options": {
31
+ }
32
+ }
@@ -0,0 +1,12 @@
1
+ <source>
2
+ type forward
3
+ port 23003
4
+ </source>
5
+ <match droonga.message>
6
+ name localhost:23003/droonga
7
+ type droonga
8
+ proxy true
9
+ </match>
10
+ <match output.message>
11
+ type stdout
12
+ </match>
data/bin/grn2jsons ADDED
@@ -0,0 +1,85 @@
1
+ #!/usr/bin/env ruby
2
+ # -*- coding: utf-8 -*-
3
+ #
4
+ # Copyright (C) 2013 droonga project
5
+ #
6
+ # This library is free software; you can redistribute it and/or
7
+ # modify it under the terms of the GNU Lesser General Public
8
+ # License version 2.1 as published by the Free Software Foundation.
9
+ #
10
+ # This library is distributed in the hope that it will be useful,
11
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
12
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13
+ # Lesser General Public License for more details.
14
+ #
15
+ # You should have received a copy of the GNU Lesser General Public
16
+ # License along with this library; if not, write to the Free Software
17
+ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
18
+
19
+ require "groonga_command_converter"
20
+ require "json"
21
+ require "ostruct"
22
+ require "optparse"
23
+
24
+ options = OpenStruct.new
25
+ option_parser = OptionParser.new do |parser|
26
+ parser.on("-i=ID", "--id=ID",
27
+ "base id. (optional)") do |id|
28
+ options.id = id
29
+ end
30
+
31
+ parser.on("-d=DATE", "--date=DATE",
32
+ "date. (optional)") do |date|
33
+ options.date = date
34
+ end
35
+
36
+ parser.on("-r=REPLYTO", "--reply-to=REPLYTO",
37
+ "value of replyTo field.") do |reply_to|
38
+ options.reply_to = reply_to
39
+ end
40
+
41
+ parser.on("-s=DATASET", "--dataset=DATASET",
42
+ "dataset.") do |dataset|
43
+ options.dataset = dataset
44
+ end
45
+ end
46
+ args = option_parser.parse!(ARGV)
47
+
48
+ if options.reply_to.nil?
49
+ raise "You must specify the value of the \"replyTo\" field by --reply-to option."
50
+ end
51
+ if options.dataset.nil?
52
+ raise "You must specify the name of the dataset by --dataset option."
53
+ end
54
+
55
+ convert_options = {
56
+ :id => options.id,
57
+ :date => options.date,
58
+ :reply_to => options.reply_to,
59
+ :dataset => options.dataset,
60
+ }
61
+ converter = Droonga::GroongaCommandConverter.new(convert_options)
62
+
63
+ source_file = args[0]
64
+ result_file = args[1]
65
+
66
+ input = nil
67
+ if source_file.nil?
68
+ input = STDIN.read
69
+ else
70
+ input = File.read(source_file)
71
+ end
72
+
73
+ result_file = args[1]
74
+
75
+ if result_file.nil?
76
+ converter.convert(input) do |command|
77
+ puts(JSON.generate(command))
78
+ end
79
+ else
80
+ File.open("w", result_file) do |file|
81
+ converter.convert(input) do |command|
82
+ file.puts(JSON.generate(command))
83
+ end
84
+ end
85
+ end
@@ -0,0 +1,41 @@
1
+ # -*- mode: ruby; coding: utf-8 -*-
2
+ #
3
+ # Copyright (C) 2013 droonga project
4
+ #
5
+ # This library is free software; you can redistribute it and/or
6
+ # modify it under the terms of the GNU Lesser General Public
7
+ # License version 2.1 as published by the Free Software Foundation.
8
+ #
9
+ # This library is distributed in the hope that it will be useful,
10
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
11
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12
+ # Lesser General Public License for more details.
13
+ #
14
+ # You should have received a copy of the GNU Lesser General Public
15
+ # License along with this library; if not, write to the Free Software
16
+ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
+
18
+ Gem::Specification.new do |gem|
19
+ gem.name = "fluent-plugin-droonga"
20
+ gem.version = "0.0.2"
21
+ gem.authors = ["Droonga Project"]
22
+ gem.email = ["droonga@groonga.org"]
23
+ gem.description = "Droonga(distributed groonga) plugin for Fluent event collector"
24
+ gem.summary = gem.description
25
+ gem.homepage = "https://github.com/droonga/fluent-plugin-droonga"
26
+ gem.files = `git ls-files`.split($/)
27
+ gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
28
+ gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
29
+ gem.require_paths = ["lib"]
30
+ gem.add_dependency "fluentd"
31
+ gem.add_dependency "rroonga", ">= 3.0.3"
32
+ gem.add_dependency "groonga-command-parser"
33
+ gem.add_dependency "fluent-logger"
34
+ gem.add_dependency "serverengine"
35
+ gem.add_development_dependency "rake"
36
+ gem.add_development_dependency "bundler"
37
+ gem.add_development_dependency "droonga-client"
38
+ gem.add_development_dependency "test-unit"
39
+ gem.add_development_dependency "test-unit-notify"
40
+ gem.add_development_dependency "test-unit-rr"
41
+ end
@@ -0,0 +1,156 @@
1
+ # -*- coding: utf-8 -*-
2
+ #
3
+ # Copyright (C) 2013 droonga project
4
+ #
5
+ # This library is free software; you can redistribute it and/or
6
+ # modify it under the terms of the GNU Lesser General Public
7
+ # License version 2.1 as published by the Free Software Foundation.
8
+ #
9
+ # This library is distributed in the hope that it will be useful,
10
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
11
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12
+ # Lesser General Public License for more details.
13
+ #
14
+ # You should have received a copy of the GNU Lesser General Public
15
+ # License along with this library; if not, write to the Free Software
16
+ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
+
18
+ require "droonga/handler"
19
+
20
+ module Droonga
21
+ class Adapter < Droonga::Handler
22
+ def scatter_all(request, key)
23
+ message = [{
24
+ "command"=> envelope["type"],
25
+ "dataset"=> envelope["dataset"],
26
+ "body"=> request,
27
+ "key"=> key,
28
+ "type"=> "scatter",
29
+ "replica"=> "all",
30
+ "post"=> true
31
+ }]
32
+ post(message, "proxy")
33
+ end
34
+
35
+ def broadcast_all(request)
36
+ message = [{
37
+ "command"=> envelope["type"],
38
+ "dataset"=> envelope["dataset"],
39
+ "body"=> request,
40
+ "type"=> "broadcast",
41
+ "replica"=> "all",
42
+ "post"=> true
43
+ }]
44
+ post(message, "proxy")
45
+ end
46
+
47
+ def prefer_synchronous?(command)
48
+ return true
49
+ end
50
+ end
51
+
52
+ class BasicAdapter < Adapter
53
+ Droonga::HandlerPlugin.register("adapter", self)
54
+
55
+ command :table_create
56
+ def table_create(request)
57
+ broadcast_all(request)
58
+ end
59
+
60
+ command :column_create
61
+ def column_create(request)
62
+ broadcast_all(request)
63
+ end
64
+
65
+ command "watch.feed" => :feed
66
+ def feed(request)
67
+ broadcast_all(request)
68
+ end
69
+
70
+ command "watch.subscribe" => :subscribe
71
+ def subscribe(request)
72
+ broadcast_all(request)
73
+ end
74
+
75
+ command "watch.unsubscribe" => :unsubscribe
76
+ def unsubscribe(request)
77
+ broadcast_all(request)
78
+ end
79
+
80
+ command :add
81
+ def add(request)
82
+ # TODO: update events must be serialized in the primary node of replicas.
83
+ key = request["key"] || rand.to_s
84
+ scatter_all(request, key)
85
+ end
86
+
87
+ command :update
88
+ def update(request)
89
+ # TODO: update events must be serialized in the primary node of replicas.
90
+ key = request["key"] || rand.to_s
91
+ scatter_all(request, key)
92
+ end
93
+
94
+ command :reset
95
+ def add(request)
96
+ # TODO: update events must be serialized in the primary node of replicas.
97
+ key = request["key"] || rand.to_s
98
+ scatter_all(request, key)
99
+ end
100
+
101
+ command :search
102
+ def search(request)
103
+ message = []
104
+ input_names = []
105
+ output_names = []
106
+ name_mapper = {}
107
+ request["queries"].each do |input_name, query|
108
+ output = query["output"]
109
+ next unless output
110
+ input_names << input_name
111
+ output_name = input_name + "_reduced"
112
+ output_names << output_name
113
+ name_mapper[output_name] = input_name
114
+ # TODO: offset & limit must be arranged here.
115
+ elements = {}
116
+ output["elements"].each do |element|
117
+ case element
118
+ when "count"
119
+ elements[element] = ["sum"]
120
+ when "records"
121
+ # TODO: must take "sortBy" section into account.
122
+ elements[element] = ["sort", "<"]
123
+ end
124
+ end
125
+ reducer = {
126
+ "inputs"=> [input_name],
127
+ "outputs"=> [output_name],
128
+ "type"=> "reduce",
129
+ "body"=> {
130
+ input_name=> {
131
+ output_name=> elements
132
+ }
133
+ }
134
+ }
135
+ message << reducer
136
+ end
137
+ gatherer = {
138
+ "inputs"=> output_names,
139
+ "type"=> "gather",
140
+ "body"=> name_mapper,
141
+ "post"=> true
142
+ }
143
+ message << gatherer
144
+ searcher = {
145
+ "dataset"=> envelope["dataset"] || request["dataset"],
146
+ "outputs"=> input_names,
147
+ "type"=> "broadcast",
148
+ "command"=> "search",
149
+ "replica"=> "random",
150
+ "body"=> request
151
+ }
152
+ message.push(searcher)
153
+ post(message, "proxy")
154
+ end
155
+ end
156
+ end
@@ -0,0 +1,153 @@
1
+ # -*- coding: utf-8 -*-
2
+ #
3
+ # Copyright (C) 2013 droonga project
4
+ #
5
+ # This library is free software; you can redistribute it and/or
6
+ # modify it under the terms of the GNU Lesser General Public
7
+ # License version 2.1 as published by the Free Software Foundation.
8
+ #
9
+ # This library is distributed in the hope that it will be useful,
10
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
11
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12
+ # Lesser General Public License for more details.
13
+ #
14
+ # You should have received a copy of the GNU Lesser General Public
15
+ # License along with this library; if not, write to the Free Software
16
+ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
+
18
+ require "digest/sha1"
19
+ require "zlib"
20
+
21
+ module Droonga
22
+ class << self
23
+ def catalog
24
+ @catalog ||= Catalog.new
25
+ end
26
+ end
27
+
28
+ class Catalog
29
+ CATALOG_FILE_PATH = "catalog.json"
30
+
31
+ attr_reader :path
32
+
33
+ def initialize(path=nil)
34
+ @path = path || default_path
35
+
36
+ open(@path) do |file|
37
+ @catalog = JSON.parse(file.read)
38
+ end
39
+ @catalog["datasets"].each do |name, dataset|
40
+ number_of_partitions = dataset["number_of_partitions"]
41
+ next if number_of_partitions < 2
42
+ total_weight = dataset["ring"].reduce do |a, b|
43
+ a[1]["weight"] + b[1]["weight"]
44
+ end
45
+ continuum = []
46
+ dataset["ring"].each do |key, value|
47
+ points = number_of_partitions * 160 * value["weight"] / total_weight
48
+ points.times do |point|
49
+ hash = Digest::SHA1.hexdigest("#{key}:#{point}")
50
+ continuum << [hash[0..7].to_i(16), key]
51
+ end
52
+ end
53
+ dataset["continuum"] = continuum.sort do |a, b| a[0] - b[0]; end
54
+ end
55
+ @options = @catalog["options"] || {}
56
+ end
57
+
58
+ def base_path
59
+ @base_path ||= File.dirname(@path)
60
+ end
61
+
62
+ def option(name)
63
+ @options[name]
64
+ end
65
+
66
+ def get_engines(name)
67
+ device = @catalog["farms"][name]["device"]
68
+ pattern = Regexp.new("^#{name}\.")
69
+ results = {}
70
+ @catalog["datasets"].each do |key, dataset|
71
+ workers = dataset["workers"]
72
+ plugins = dataset["plugins"]
73
+ dataset["ring"].each do |key, part|
74
+ part["partitions"].each do |range, partitions|
75
+ partitions.each do |partition|
76
+ if partition =~ pattern
77
+ path = File.join([device, $POSTMATCH, "db"])
78
+ path = File.expand_path(path, base_path)
79
+ options = {
80
+ :database => path,
81
+ :n_workers => workers,
82
+ :handlers => plugins
83
+ }
84
+ results[partition] = options
85
+ end
86
+ end
87
+ end
88
+ end
89
+ end
90
+ return results
91
+ end
92
+
93
+ def get_routes(name, args)
94
+ routes = []
95
+ dataset = dataset(name)
96
+ return routes unless dataset
97
+ case args["type"]
98
+ when "broadcast"
99
+ dataset["ring"].each do |key, partition|
100
+ select_range_and_replicas(partition, args, routes)
101
+ end
102
+ when "scatter"
103
+ name = get_partition(dataset, args["key"])
104
+ partition = dataset["ring"][name]
105
+ select_range_and_replicas(partition, args, routes)
106
+ end
107
+ return routes
108
+ end
109
+
110
+ def get_partition(dataset, key)
111
+ continuum = dataset["continuum"]
112
+ return dataset["ring"].keys[0] unless continuum
113
+ hash = Zlib.crc32(key)
114
+ min = 0
115
+ max = continuum.size - 1
116
+ while (min < max) do
117
+ index = (min + max) / 2
118
+ value, key = continuum[index]
119
+ return key if value == hash
120
+ if value > hash
121
+ max = index
122
+ else
123
+ min = index + 1
124
+ end
125
+ end
126
+ return continuum[max][1]
127
+ end
128
+
129
+ def dataset(name)
130
+ @catalog["datasets"][name]
131
+ end
132
+
133
+ def select_range_and_replicas(partition, args, routes)
134
+ date_range = args["date_range"] || 0..-1
135
+ partition["partitions"].sort[date_range].each do |time, replicas|
136
+ case args["replica"]
137
+ when "top"
138
+ routes << replicas[0]
139
+ when "random"
140
+ routes << replicas[rand(replicas.size)]
141
+ when "all"
142
+ routes.concat(replicas)
143
+ end
144
+ end
145
+ end
146
+
147
+ private
148
+ def default_path
149
+ path = ENV["DROONGA_CATALOG"] || Catalog::CATALOG_FILE_PATH
150
+ File.expand_path(path)
151
+ end
152
+ end
153
+ end