fluent-plugin-droonga 0.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +1 -0
  3. data/.travis.yml +7 -0
  4. data/Gemfile +40 -0
  5. data/LICENSE.txt +14 -0
  6. data/README.md +18 -0
  7. data/Rakefile +25 -0
  8. data/benchmark/benchmark.rb +123 -0
  9. data/benchmark/utils.rb +243 -0
  10. data/benchmark/watch/benchmark-notify.rb +143 -0
  11. data/benchmark/watch/benchmark-notify.sh +19 -0
  12. data/benchmark/watch/benchmark-publish.rb +120 -0
  13. data/benchmark/watch/benchmark-scan.rb +210 -0
  14. data/benchmark/watch/catalog.json +32 -0
  15. data/benchmark/watch/fluentd.conf +12 -0
  16. data/bin/grn2jsons +85 -0
  17. data/fluent-plugin-droonga.gemspec +41 -0
  18. data/lib/droonga/adapter.rb +156 -0
  19. data/lib/droonga/catalog.rb +153 -0
  20. data/lib/droonga/command_mapper.rb +45 -0
  21. data/lib/droonga/engine.rb +83 -0
  22. data/lib/droonga/executor.rb +289 -0
  23. data/lib/droonga/handler.rb +140 -0
  24. data/lib/droonga/handler_plugin.rb +35 -0
  25. data/lib/droonga/job_queue.rb +83 -0
  26. data/lib/droonga/job_queue_schema.rb +65 -0
  27. data/lib/droonga/logger.rb +34 -0
  28. data/lib/droonga/plugin.rb +41 -0
  29. data/lib/droonga/plugin/adapter/groonga/select.rb +88 -0
  30. data/lib/droonga/plugin/adapter_groonga.rb +40 -0
  31. data/lib/droonga/plugin/handler/groonga/column_create.rb +103 -0
  32. data/lib/droonga/plugin/handler/groonga/table_create.rb +100 -0
  33. data/lib/droonga/plugin/handler_add.rb +44 -0
  34. data/lib/droonga/plugin/handler_forward.rb +70 -0
  35. data/lib/droonga/plugin/handler_groonga.rb +52 -0
  36. data/lib/droonga/plugin/handler_proxy.rb +82 -0
  37. data/lib/droonga/plugin/handler_search.rb +33 -0
  38. data/lib/droonga/plugin/handler_watch.rb +102 -0
  39. data/lib/droonga/proxy.rb +371 -0
  40. data/lib/droonga/searcher.rb +415 -0
  41. data/lib/droonga/server.rb +112 -0
  42. data/lib/droonga/sweeper.rb +42 -0
  43. data/lib/droonga/watch_schema.rb +88 -0
  44. data/lib/droonga/watcher.rb +256 -0
  45. data/lib/droonga/worker.rb +51 -0
  46. data/lib/fluent/plugin/out_droonga.rb +56 -0
  47. data/lib/groonga_command_converter.rb +137 -0
  48. data/sample/cluster/catalog.json +43 -0
  49. data/sample/cluster/fluentd.conf +12 -0
  50. data/sample/fluentd.conf +8 -0
  51. data/test/fixtures/catalog.json +43 -0
  52. data/test/fixtures/document.grn +23 -0
  53. data/test/helper.rb +24 -0
  54. data/test/helper/fixture.rb +28 -0
  55. data/test/helper/sandbox.rb +73 -0
  56. data/test/helper/stub_worker.rb +27 -0
  57. data/test/helper/watch_helper.rb +35 -0
  58. data/test/plugin/adapter/groonga/test_select.rb +176 -0
  59. data/test/plugin/handler/groonga/test_column_create.rb +127 -0
  60. data/test/plugin/handler/groonga/test_table_create.rb +140 -0
  61. data/test/plugin/handler/test_handler_add.rb +135 -0
  62. data/test/plugin/handler/test_handler_groonga.rb +64 -0
  63. data/test/plugin/handler/test_handler_search.rb +512 -0
  64. data/test/plugin/handler/test_handler_watch.rb +168 -0
  65. data/test/run-test.rb +55 -0
  66. data/test/test_adapter.rb +48 -0
  67. data/test/test_catalog.rb +59 -0
  68. data/test/test_command_mapper.rb +44 -0
  69. data/test/test_groonga_command_converter.rb +242 -0
  70. data/test/test_handler.rb +53 -0
  71. data/test/test_job_queue_schema.rb +45 -0
  72. data/test/test_output.rb +99 -0
  73. data/test/test_sweeper.rb +95 -0
  74. data/test/test_watch_schema.rb +57 -0
  75. data/test/test_watcher.rb +336 -0
  76. data/test/test_worker.rb +144 -0
  77. metadata +299 -0
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 99aed806731b4f63bdefc707bddd4fb9c6f1d105
4
+ data.tar.gz: 0a4ad0ab5c237b44a81656a489744aac4b264a64
5
+ SHA512:
6
+ metadata.gz: dbeb3a6142fe82e708355ef091ecb2789aa2f18058ac5a9b66089828c91bbfc1237b8f4c5b0998e7fee55a8188f321ac394a547a1608a2e4b15bd6036d24b8e5
7
+ data.tar.gz: 1db4e2fe92124dbd2d259dad398fb54868214e339319286ca2ce2c3bd7128b7b76b4cde19b5c6f6f262c6a703142a7a8ebd2b2c80021b381cf024d9c61f558ca
data/.gitignore ADDED
@@ -0,0 +1 @@
1
+ /Gemfile.lock
data/.travis.yml ADDED
@@ -0,0 +1,7 @@
1
+ notifications:
2
+ recipients:
3
+ - groonga-commit@lists.sourceforge.jp
4
+ rvm:
5
+ - 1.9.3
6
+ before_install:
7
+ - curl https://raw.github.com/groonga/groonga/master/data/travis/setup.sh | sh
data/Gemfile ADDED
@@ -0,0 +1,40 @@
1
+ # Copyright (C) 2013 droonga project
2
+ #
3
+ # This library is free software; you can redistribute it and/or
4
+ # modify it under the terms of the GNU Lesser General Public
5
+ # License version 2.1 as published by the Free Software Foundation.
6
+ #
7
+ # This library is distributed in the hope that it will be useful,
8
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
9
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
10
+ # Lesser General Public License for more details.
11
+ #
12
+ # You should have received a copy of the GNU Lesser General Public
13
+ # License along with this library; if not, write to the Free Software
14
+ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
15
+
16
+ source "https://rubygems.org"
17
+
18
+ gemspec
19
+
20
+ parent_dir = File.join(File.dirname(__FILE__), "..")
21
+ local_rroonga_path = File.join(parent_dir, "rroonga")
22
+ local_groonga_command_path = File.join(parent_dir, "groonga-command")
23
+ local_groonga_command_parser_path = File.join(parent_dir,
24
+ "groonga-command-parser")
25
+ if File.exist?(local_rroonga_path)
26
+ gem "rroonga", :path => local_rroonga_path
27
+ gem "groonga-command", :path => local_groonga_command_path
28
+ gem "groonga-command-parser", :path => local_groonga_command_parser_path
29
+ elsif ENV["TRAVIS"] == "true"
30
+ require_unreleased_gems = false
31
+ if require_unreleased_gems
32
+ gem "rroonga", :git => "git://github.com/ranguba/rroonga.git"
33
+ gem "groonga-command",
34
+ :git => "git://github.com/groonga/groonga-command.git"
35
+ gem "groonga-command-parser",
36
+ :git => "git://github.com/groonga/groonga-command-parser.git"
37
+ end
38
+ end
39
+
40
+ gem "droonga-client", github: "droonga/droonga-client-ruby"
data/LICENSE.txt ADDED
@@ -0,0 +1,14 @@
1
+ Copyright (C) 2013 droonga project
2
+
3
+ This library is free software; you can redistribute it and/or
4
+ modify it under the terms of the GNU Lesser General Public
5
+ License version 2.1 as published by the Free Software Foundation.
6
+
7
+ This library is distributed in the hope that it will be useful,
8
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
9
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
10
+ Lesser General Public License for more details.
11
+
12
+ You should have received a copy of the GNU Lesser General Public
13
+ License along with this library; if not, write to the Free Software
14
+ Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
data/README.md ADDED
@@ -0,0 +1,18 @@
1
+ # Fluent::Plugin::Droonga
2
+
3
+ TODO: Write a gem description
4
+
5
+ ## What's "droonga"?
6
+
7
+ Droonga is a distributed fulltext search engine, named from "distributed-groonga".
8
+ This package provides droonga plugin for Fluent event collector.
9
+
10
+ ## Usage
11
+
12
+ TODO: Write usage.
13
+
14
+ ## Copyright
15
+
16
+ * Copyright (c) 2013 droonga project
17
+ * License
18
+ * GNU Lesser General Public License version 2.1
data/Rakefile ADDED
@@ -0,0 +1,25 @@
1
+ # -*- ruby -*-
2
+ #
3
+ # Copyright (C) 2013 droonga project
4
+ #
5
+ # This library is free software; you can redistribute it and/or
6
+ # modify it under the terms of the GNU Lesser General Public
7
+ # License version 2.1 as published by the Free Software Foundation.
8
+ #
9
+ # This library is distributed in the hope that it will be useful,
10
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
11
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12
+ # Lesser General Public License for more details.
13
+ #
14
+ # You should have received a copy of the GNU Lesser General Public
15
+ # License along with this library; if not, write to the Free Software
16
+ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
+
18
+ require "bundler/gem_tasks"
19
+
20
+ desc "Run test"
21
+ task :test do
22
+ ruby(File.join(File.dirname(__FILE__), "test", "run-test.rb"))
23
+ end
24
+
25
+ task :default => :test
@@ -0,0 +1,123 @@
1
+ # -*- coding: utf-8 -*-
2
+ #
3
+ # Copyright (C) 2013 droonga project
4
+ #
5
+ # This library is free software; you can redistribute it and/or
6
+ # modify it under the terms of the GNU Lesser General Public
7
+ # License version 2.1 as published by the Free Software Foundation.
8
+ #
9
+ # This library is distributed in the hope that it will be useful,
10
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
11
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12
+ # Lesser General Public License for more details.
13
+ #
14
+ # You should have received a copy of the GNU Lesser General Public
15
+ # License along with this library; if not, write to the Free Software
16
+ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
+
18
+ require "droonga/client"
19
+
20
+ module Droonga
21
+ class Benchmark
22
+ class Terms
23
+ class << self
24
+ def generate
25
+ new.to_enum(:each)
26
+ end
27
+ end
28
+
29
+ FIRST_INITIAL_LETTER = "㐀"
30
+ SUFFIX = "あいうえおかきくけこ"
31
+ def each
32
+ initial_letter = FIRST_INITIAL_LETTER
33
+ while true do
34
+ yield "#{initial_letter}#{SUFFIX}"
35
+ initial_letter.succ!
36
+ end
37
+ end
38
+ end
39
+
40
+ TERMS_STEP = 1000
41
+
42
+ N_TARGETS = 1000
43
+ TARGET_PADDING = "パディング"
44
+ TARGET_TITLE_SIZE = 100
45
+ TARGET_BODY_SIZE = 1000
46
+
47
+ def initialize(params)
48
+ @params = params
49
+ @terms = Terms.generate
50
+ @client = Droonga::Client.new(tag: @params[:tag], port: @params[:port])
51
+ end
52
+
53
+ def run
54
+ prepare_watching_subscribers(TERMS_STEP)
55
+ populate_feeds(0.1)
56
+ start_at = Time.now
57
+ @feeds.each do |feed|
58
+ @client.send(feed)
59
+ end
60
+ end_at = Time.now
61
+ end
62
+
63
+ def prepare_watching_subscribers(step)
64
+ step.times do
65
+ term = @terms.next
66
+ add_subscriber(term)
67
+ @watching_terms << term
68
+ end
69
+ end
70
+
71
+ def add_subscriber(term)
72
+ subscribe_envelope = envelope_to_subscribe(term)
73
+ @client.connection.send_receive(subscribe_envelope)
74
+ end
75
+
76
+ def envelope_to_subscribe(term, route=nil)
77
+ message = {
78
+ "id" => Time.now.to_f.to_s,
79
+ "date" => Time.now,
80
+ "statusCode" => 200,
81
+ "type" => "watch.subscribe",
82
+ "body" => {
83
+ "condition" => term,
84
+ "subscriber" => term,
85
+ },
86
+ }
87
+ unless route.nil?
88
+ message["body"]["route"] = route
89
+ end
90
+ message
91
+ end
92
+
93
+ def populate_feeds(incidence)
94
+ @feeds = []
95
+
96
+ n_matched_targets = (N_TARGETS.to_f * incidence).to_i
97
+ n_unmatched_targets = (N_TARGETS - n_matched_targets)
98
+
99
+ n_matched_targets.times do
100
+ @feeds << envelope_to_feed(@watching_terms.sample(1))
101
+ end
102
+
103
+ n_unmatched_targets.times do
104
+ @feeds << envelope_to_feed(@terms.next)
105
+ end
106
+ end
107
+
108
+ def envelope_to_feed(term)
109
+ {
110
+ "id" => Time.now.to_f.to_s,
111
+ "date" => Time.now,
112
+ "statusCode" => 200,
113
+ "type" => "watch.feed",
114
+ "body" => {
115
+ "targets" => {
116
+ "title" => TARGET_PADDING * (TARGET_TITLE_SIZE / TARGET_PADDING.size),
117
+ "body" => term + (TARGET_PADDING * (TARGET_BODY_SIZE / TARGET_PADDING.size)),
118
+ },
119
+ },
120
+ }
121
+ end
122
+ end
123
+ end
@@ -0,0 +1,243 @@
1
+ # -*- coding: utf-8 -*-
2
+ #
3
+ # Copyright (C) 2013 droonga project
4
+ #
5
+ # This library is free software; you can redistribute it and/or
6
+ # modify it under the terms of the GNU Lesser General Public
7
+ # License version 2.1 as published by the Free Software Foundation.
8
+ #
9
+ # This library is distributed in the hope that it will be useful,
10
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
11
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12
+ # Lesser General Public License for more details.
13
+ #
14
+ # You should have received a copy of the GNU Lesser General Public
15
+ # License along with this library; if not, write to the Free Software
16
+ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
+
18
+ require "json"
19
+ require "thread"
20
+ require "droonga/watch_schema"
21
+
22
+ module DroongaBenchmark
23
+ WATCH_DATASET = "Watch"
24
+
25
+ class WatchDatabase
26
+ attr_reader :context
27
+
28
+ def initialize
29
+ @database_dir = "/tmp/watch-benchmark"
30
+ @database_path = "#{@database_dir}/db"
31
+ FileUtils.rm_rf(@database_dir)
32
+ FileUtils.mkdir_p(@database_dir)
33
+
34
+ Groonga::Database.create(:path => @database_path)
35
+
36
+ @context = Groonga::Context.new
37
+ @context.open_database(@database_path)
38
+ schema = Droonga::WatchSchema.new(@context)
39
+ schema.ensure_created
40
+ end
41
+
42
+ =begin
43
+ def subscribe_to(keywords)
44
+ @context.send("load --table Query")
45
+ @context.send("[")
46
+ keywords.each do |keyword|
47
+ @context.send("{'_key':'#{keyword}'," +
48
+ "'keywords':['#{keyword}']},")
49
+ end
50
+ @context.send("]")
51
+
52
+ @context.send("load --table Subscriber")
53
+ @context.send("[")
54
+ keywords.each do |keyword|
55
+ @context.send("{'_key':'subscriber for #{keyword}'," +
56
+ "'subscriptions':['#{keyword}']," +
57
+ "'route':'0.0.0.0:0/benchamrk'},")
58
+ end
59
+ @context.send("]")
60
+ end
61
+ =end
62
+
63
+ def subscribe_to(keywords)
64
+ queries = []
65
+ subscribers = []
66
+ keywords.each do |keyword|
67
+ queries << {:_key => keyword,
68
+ :keywords => [keyword]}
69
+ subscribers << {:_key => "subscriber for #{keyword}",
70
+ :subscriptions => [keyword],
71
+ :route => "0.0.0.0:0/benchamrk"}
72
+ end
73
+
74
+ command_load_queries = [
75
+ "load --table Query",
76
+ JSON.generate(queries)
77
+ ]
78
+ command_load_subscribers = [
79
+ "load --table Subscriber",
80
+ JSON.generate(subscribers)
81
+ ]
82
+
83
+ @context.restore(command_load_queries.join("\n"))
84
+ @context.restore(command_load_subscribers.join("\n"))
85
+ end
86
+
87
+ def subscribe(keyword)
88
+ queries = @context["Query"]
89
+ query = queries.add(keyword, :keywords => [keyword])
90
+
91
+ subscribers = @context["Subscriber"]
92
+ subscribers.add("subscriber for #{keyword}",
93
+ :subscriptions => [query],
94
+ :route => "0.0.0.0:0/benchamrk")
95
+ end
96
+ end
97
+
98
+ class KeywordsGenerator
99
+ class << self
100
+ def generate(n_keywords)
101
+ new.generate(n_keywords)
102
+ end
103
+ end
104
+
105
+ def initialize
106
+ @generator = to_enum(:each)
107
+ end
108
+
109
+ def generate(n_keywords)
110
+ keywords = []
111
+ n_keywords.times do
112
+ keywords << @generator.next
113
+ end
114
+ keywords
115
+ end
116
+
117
+ def next
118
+ @generator.next
119
+ end
120
+
121
+ FIRST_INITIAL_LETTER = "㐀"
122
+ SUFFIX = "あいうえおかきくけこ"
123
+ def each
124
+ initial_letter = FIRST_INITIAL_LETTER
125
+ while true do
126
+ yield "#{initial_letter}#{SUFFIX}"
127
+ initial_letter.succ!
128
+ end
129
+ end
130
+ end
131
+
132
+ class TargetsGenerator
133
+ class << self
134
+ def generate(n_keywords, params)
135
+ new(params).generate(n_keywords)
136
+ end
137
+ end
138
+
139
+ PADDING = "パディング"
140
+ SIZE = 1000
141
+
142
+ def initialize(params)
143
+ @keywords = params[:keywords]
144
+ @incidence = params[:incidence]
145
+ @matched_keywords = params[:matched_keywords] || 1
146
+ end
147
+
148
+ def generate(n_targets)
149
+ targets = []
150
+
151
+ n_matched_targets = (n_targets.to_f * @incidence).to_i
152
+ n_unmatched_targets = (n_targets - n_matched_targets)
153
+
154
+ n_matched_targets.times do
155
+ targets << generate_target(@keywords.sample(@matched_keywords))
156
+ end
157
+
158
+ n_unmatched_targets.times do
159
+ targets << generate_target
160
+ end
161
+
162
+ targets
163
+ end
164
+
165
+ def generate_target(keywords=[])
166
+ (PADDING * (SIZE / PADDING.size)) + keywords.join("/")
167
+ end
168
+ end
169
+
170
+ class MessageCreator
171
+ class << self
172
+ def envelope_to_subscribe(keyword, route=nil)
173
+ message = {
174
+ "id" => Time.now.to_f.to_s,
175
+ "dataset" => WATCH_DATASET,
176
+ "date" => Time.now,
177
+ "statusCode" => 200,
178
+ "type" => "watch.subscribe",
179
+ "body" => {
180
+ "condition" => keyword,
181
+ "subscriber" => "subscriber for #{keyword}",
182
+ },
183
+ }
184
+ message
185
+ end
186
+
187
+ def envelope_to_feed(keyword)
188
+ {
189
+ "id" => Time.now.to_f.to_s,
190
+ "dataset" => WATCH_DATASET,
191
+ "date" => Time.now,
192
+ "statusCode" => 200,
193
+ "type" => "watch.feed",
194
+ "body" => {
195
+ "targets" => {
196
+ "keyword" => keyword,
197
+ },
198
+ },
199
+ }
200
+ end
201
+ end
202
+ end
203
+
204
+ class MessageReceiver
205
+ def initialize(options={})
206
+ default_options = {
207
+ :host => "0.0.0.0",
208
+ :port => 0,
209
+ }
210
+ options = default_options.merge(options)
211
+ @socket = TCPServer.new(options[:host], options[:port])
212
+ start
213
+ end
214
+
215
+ def close
216
+ @socket.close
217
+ end
218
+
219
+ def host
220
+ @socket.addr[3]
221
+ end
222
+
223
+ def port
224
+ @socket.addr[1]
225
+ end
226
+
227
+ def new_message
228
+ @messages.pop
229
+ end
230
+
231
+ def start
232
+ @messages = Queue.new
233
+ Thread.new do
234
+ client = @socket.accept
235
+ unpacker = MessagePack::Unpacker.new(client)
236
+ unpacker.each do |object|
237
+ @messages.push(object)
238
+ end
239
+ client.close
240
+ end
241
+ end
242
+ end
243
+ end