fluent-plugin-droonga 0.0.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (77) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +1 -0
  3. data/.travis.yml +7 -0
  4. data/Gemfile +40 -0
  5. data/LICENSE.txt +14 -0
  6. data/README.md +18 -0
  7. data/Rakefile +25 -0
  8. data/benchmark/benchmark.rb +123 -0
  9. data/benchmark/utils.rb +243 -0
  10. data/benchmark/watch/benchmark-notify.rb +143 -0
  11. data/benchmark/watch/benchmark-notify.sh +19 -0
  12. data/benchmark/watch/benchmark-publish.rb +120 -0
  13. data/benchmark/watch/benchmark-scan.rb +210 -0
  14. data/benchmark/watch/catalog.json +32 -0
  15. data/benchmark/watch/fluentd.conf +12 -0
  16. data/bin/grn2jsons +85 -0
  17. data/fluent-plugin-droonga.gemspec +41 -0
  18. data/lib/droonga/adapter.rb +156 -0
  19. data/lib/droonga/catalog.rb +153 -0
  20. data/lib/droonga/command_mapper.rb +45 -0
  21. data/lib/droonga/engine.rb +83 -0
  22. data/lib/droonga/executor.rb +289 -0
  23. data/lib/droonga/handler.rb +140 -0
  24. data/lib/droonga/handler_plugin.rb +35 -0
  25. data/lib/droonga/job_queue.rb +83 -0
  26. data/lib/droonga/job_queue_schema.rb +65 -0
  27. data/lib/droonga/logger.rb +34 -0
  28. data/lib/droonga/plugin.rb +41 -0
  29. data/lib/droonga/plugin/adapter/groonga/select.rb +88 -0
  30. data/lib/droonga/plugin/adapter_groonga.rb +40 -0
  31. data/lib/droonga/plugin/handler/groonga/column_create.rb +103 -0
  32. data/lib/droonga/plugin/handler/groonga/table_create.rb +100 -0
  33. data/lib/droonga/plugin/handler_add.rb +44 -0
  34. data/lib/droonga/plugin/handler_forward.rb +70 -0
  35. data/lib/droonga/plugin/handler_groonga.rb +52 -0
  36. data/lib/droonga/plugin/handler_proxy.rb +82 -0
  37. data/lib/droonga/plugin/handler_search.rb +33 -0
  38. data/lib/droonga/plugin/handler_watch.rb +102 -0
  39. data/lib/droonga/proxy.rb +371 -0
  40. data/lib/droonga/searcher.rb +415 -0
  41. data/lib/droonga/server.rb +112 -0
  42. data/lib/droonga/sweeper.rb +42 -0
  43. data/lib/droonga/watch_schema.rb +88 -0
  44. data/lib/droonga/watcher.rb +256 -0
  45. data/lib/droonga/worker.rb +51 -0
  46. data/lib/fluent/plugin/out_droonga.rb +56 -0
  47. data/lib/groonga_command_converter.rb +137 -0
  48. data/sample/cluster/catalog.json +43 -0
  49. data/sample/cluster/fluentd.conf +12 -0
  50. data/sample/fluentd.conf +8 -0
  51. data/test/fixtures/catalog.json +43 -0
  52. data/test/fixtures/document.grn +23 -0
  53. data/test/helper.rb +24 -0
  54. data/test/helper/fixture.rb +28 -0
  55. data/test/helper/sandbox.rb +73 -0
  56. data/test/helper/stub_worker.rb +27 -0
  57. data/test/helper/watch_helper.rb +35 -0
  58. data/test/plugin/adapter/groonga/test_select.rb +176 -0
  59. data/test/plugin/handler/groonga/test_column_create.rb +127 -0
  60. data/test/plugin/handler/groonga/test_table_create.rb +140 -0
  61. data/test/plugin/handler/test_handler_add.rb +135 -0
  62. data/test/plugin/handler/test_handler_groonga.rb +64 -0
  63. data/test/plugin/handler/test_handler_search.rb +512 -0
  64. data/test/plugin/handler/test_handler_watch.rb +168 -0
  65. data/test/run-test.rb +55 -0
  66. data/test/test_adapter.rb +48 -0
  67. data/test/test_catalog.rb +59 -0
  68. data/test/test_command_mapper.rb +44 -0
  69. data/test/test_groonga_command_converter.rb +242 -0
  70. data/test/test_handler.rb +53 -0
  71. data/test/test_job_queue_schema.rb +45 -0
  72. data/test/test_output.rb +99 -0
  73. data/test/test_sweeper.rb +95 -0
  74. data/test/test_watch_schema.rb +57 -0
  75. data/test/test_watcher.rb +336 -0
  76. data/test/test_worker.rb +144 -0
  77. metadata +299 -0
@@ -0,0 +1,135 @@
1
+ # Copyright (C) 2013 droonga project
2
+ #
3
+ # This library is free software; you can redistribute it and/or
4
+ # modify it under the terms of the GNU Lesser General Public
5
+ # License version 2.1 as published by the Free Software Foundation.
6
+ #
7
+ # This library is distributed in the hope that it will be useful,
8
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
9
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
10
+ # Lesser General Public License for more details.
11
+ #
12
+ # You should have received a copy of the GNU Lesser General Public
13
+ # License along with this library; if not, write to the Free Software
14
+ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
15
+
16
+ require "droonga/plugin/handler_add"
17
+
18
+ class AddHandlerTest < Test::Unit::TestCase
19
+ def setup
20
+ setup_database
21
+ setup_schema
22
+ setup_handler
23
+ end
24
+
25
+ def teardown
26
+ teardown_handler
27
+ teardown_database
28
+ end
29
+
30
+ private
31
+ def setup_database
32
+ FileUtils.rm_rf(@database_path.dirname.to_s)
33
+ FileUtils.mkdir_p(@database_path.dirname.to_s)
34
+ @database = Groonga::Database.create(:path => @database_path.to_s)
35
+ end
36
+
37
+ def setup_schema
38
+ end
39
+
40
+ def teardown_database
41
+ @database.close
42
+ @database = nil
43
+ FileUtils.rm_rf(@database_path.dirname.to_s)
44
+ end
45
+
46
+ def setup_handler
47
+ @worker = StubWorker.new
48
+ @handler = Droonga::AddHandler.new(@worker)
49
+ end
50
+
51
+ def teardown_handler
52
+ @handler = nil
53
+ end
54
+
55
+ public
56
+ class HasKeyTest < self
57
+ def setup_schema
58
+ Groonga::Schema.define do |schema|
59
+ schema.create_table("Users",
60
+ :type => :hash,
61
+ :key_type => :short_text) do |table|
62
+ table.short_text("country")
63
+ end
64
+ end
65
+ end
66
+
67
+ def test_empty_values
68
+ request = {
69
+ "table" => "Users",
70
+ "key" => "mori",
71
+ "values" => {},
72
+ }
73
+ mock(@handler).emit([true])
74
+ @handler.add(request)
75
+ table = @worker.context["Users"]
76
+ assert_equal(["mori"], table.collect(&:key))
77
+ end
78
+
79
+ def test_values
80
+ request = {
81
+ "table" => "Users",
82
+ "key" => "asami",
83
+ "values" => {"country" => "japan"},
84
+ }
85
+ mock(@handler).emit([true])
86
+ @handler.add(request)
87
+ table = @worker.context["Users"]
88
+ assert_equal(["japan"], table.collect(&:country))
89
+ end
90
+ end
91
+
92
+ class NoKeyTest < self
93
+ def setup_schema
94
+ Groonga::Schema.define do |schema|
95
+ schema.create_table("Books",
96
+ :type => :array) do |table|
97
+ table.short_text("title")
98
+ end
99
+ end
100
+ end
101
+
102
+ def test_empty_values
103
+ request = {
104
+ "table" => "Books",
105
+ "values" => {},
106
+ }
107
+ mock(@handler).emit([true])
108
+ @handler.add(request)
109
+ table = @worker.context["Books"]
110
+ assert_equal([nil], table.collect(&:title))
111
+ end
112
+
113
+ def test_with_values
114
+ request = {
115
+ "table" => "Books",
116
+ "values" => {"title" => "CSS"},
117
+ }
118
+ mock(@handler).emit([true])
119
+ @handler.add(request)
120
+ table = @worker.context["Books"]
121
+ assert_equal(["CSS"], table.collect(&:title))
122
+ end
123
+ end
124
+
125
+ class FailureTest < self
126
+ def test_nonexistent_table
127
+ request = {
128
+ "table" => "Nonexistent",
129
+ "values" => {},
130
+ }
131
+ mock(@handler).emit([false])
132
+ @handler.add(request)
133
+ end
134
+ end
135
+ end
@@ -0,0 +1,64 @@
1
+ # Copyright (C) 2013 droonga project
2
+ #
3
+ # This library is free software; you can redistribute it and/or
4
+ # modify it under the terms of the GNU Lesser General Public
5
+ # License version 2.1 as published by the Free Software Foundation.
6
+ #
7
+ # This library is distributed in the hope that it will be useful,
8
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
9
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
10
+ # Lesser General Public License for more details.
11
+ #
12
+ # You should have received a copy of the GNU Lesser General Public
13
+ # License along with this library; if not, write to the Free Software
14
+ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
15
+
16
+ require "droonga/plugin/handler_groonga"
17
+
18
+ class GroongaHandlerTest < Test::Unit::TestCase
19
+ def setup
20
+ setup_database
21
+ setup_handler
22
+ end
23
+
24
+ def teardown
25
+ teardown_handler
26
+ teardown_database
27
+ end
28
+
29
+ private
30
+ def setup_database
31
+ FileUtils.rm_rf(@database_path.dirname.to_s)
32
+ FileUtils.mkdir_p(@database_path.dirname.to_s)
33
+ @database = Groonga::Database.create(:path => @database_path.to_s)
34
+ end
35
+
36
+ def teardown_database
37
+ @database.close
38
+ @database = nil
39
+ FileUtils.rm_rf(@database_path.dirname.to_s)
40
+ end
41
+
42
+ def setup_handler
43
+ @worker = StubWorker.new
44
+ @handler = Droonga::GroongaHandler.new(@worker)
45
+ end
46
+
47
+ def teardown_handler
48
+ @handler = nil
49
+ end
50
+
51
+ private
52
+ def dump
53
+ database_dumper = Groonga::DatabaseDumper.new(:database => @database)
54
+ database_dumper.dump
55
+ end
56
+
57
+ NORMALIZED_START_TIME = Time.parse("2013-07-11T16:04:28+0900").to_i
58
+ NORMALIZED_ELAPSED_TIME = 1
59
+ def normalize_header(header)
60
+ start_time = NORMALIZED_START_TIME
61
+ elapsed_time = NORMALIZED_ELAPSED_TIME
62
+ [header[0], start_time, elapsed_time]
63
+ end
64
+ end
@@ -0,0 +1,512 @@
1
+ # Copyright (C) 2013 droonga project
2
+ #
3
+ # This library is free software; you can redistribute it and/or
4
+ # modify it under the terms of the GNU Lesser General Public
5
+ # License version 2.1 as published by the Free Software Foundation.
6
+ #
7
+ # This library is distributed in the hope that it will be useful,
8
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
9
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
10
+ # Lesser General Public License for more details.
11
+ #
12
+ # You should have received a copy of the GNU Lesser General Public
13
+ # License along with this library; if not, write to the Free Software
14
+ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
15
+
16
+ require "droonga/plugin/handler_search"
17
+
18
+ class SearchHandlerTest < Test::Unit::TestCase
19
+ def setup
20
+ setup_database
21
+ setup_handler
22
+ end
23
+
24
+ def teardown
25
+ teardown_handler
26
+ teardown_database
27
+ end
28
+
29
+ private
30
+ def setup_database
31
+ restore(fixture_data("document.grn"))
32
+ @database = Groonga::Database.open(@database_path.to_s)
33
+ end
34
+
35
+ def teardown_database
36
+ @database.close
37
+ @database = nil
38
+ end
39
+
40
+ def setup_handler
41
+ @worker = StubWorker.new
42
+ @handler = Droonga::SearchHandler.new(@worker)
43
+ end
44
+
45
+ def teardown_handler
46
+ @handler = nil
47
+ end
48
+
49
+ def search(request)
50
+ @handler.search(request)
51
+ normalize_result_set(@worker.body)
52
+ end
53
+
54
+ def normalize_result_set(result_set)
55
+ result_set.each do |name, result|
56
+ result["startTime"] = start_time if result["startTime"]
57
+ result["elapsedTime"] = elapsed_time if result["elapsedTime"]
58
+ end
59
+ result_set
60
+ end
61
+
62
+ def start_time
63
+ "2013-01-31T14:34:47+09:00"
64
+ end
65
+
66
+ def elapsed_time
67
+ 0.01
68
+ end
69
+
70
+ def assert_search(expected, request)
71
+ assert_equal(expected, search(request))
72
+ end
73
+
74
+ class NoParameterTest < self
75
+ def test_empty
76
+ assert_search({}, {})
77
+ end
78
+ end
79
+
80
+ class QueriesTest < self
81
+ def test_empty
82
+ assert_search({}, {"queries" => {}})
83
+ end
84
+ end
85
+
86
+ class HashQueryTest < self
87
+ def test_string_matchTo
88
+ request = base_request
89
+ request["queries"]["sections-result"]["condition"] = {
90
+ "query" => "Groonga",
91
+ "matchTo" => "title"
92
+ }
93
+ assert_search({
94
+ "sections-result" => {
95
+ "records" => [
96
+ { "title" => "Groonga overview" },
97
+ ],
98
+ },
99
+ },
100
+ request)
101
+ end
102
+
103
+ def test_array_matchTo
104
+ request = base_request
105
+ request["queries"]["sections-result"]["condition"] = {
106
+ "query" => "Groonga",
107
+ "matchTo" => ["title"]
108
+ }
109
+ assert_search({
110
+ "sections-result" => {
111
+ "records" => [
112
+ { "title" => "Groonga overview" },
113
+ ],
114
+ },
115
+ },
116
+ request)
117
+ end
118
+
119
+ def base_request
120
+ {
121
+ "queries" => {
122
+ "sections-result" => {
123
+ "source" => "Sections",
124
+ "output" => {
125
+ "elements" => [
126
+ "records",
127
+ ],
128
+ "format" => "complex",
129
+ "limit" => 1,
130
+ "attributes" => ["title"],
131
+ },
132
+ },
133
+ },
134
+ }
135
+ end
136
+ end
137
+
138
+ class SourceTest < self
139
+ def test_non_existent
140
+ assert_raise(Droonga::Searcher::UndefinedSourceError) do
141
+ search({
142
+ "queries" => {
143
+ "non-existent-result" => {
144
+ "source" => "non-existent",
145
+ },
146
+ },
147
+ })
148
+ end
149
+ end
150
+
151
+ def test_existent
152
+ assert_search({
153
+ "sections-result" => {},
154
+ },
155
+ {
156
+ "queries" => {
157
+ "sections-result" => {
158
+ "source" => "Sections",
159
+ "output" => {},
160
+ },
161
+ },
162
+ })
163
+ end
164
+ end
165
+
166
+ class OutputTest < self
167
+ def test_count
168
+ assert_search({
169
+ "sections-result" => {
170
+ "count" => 9,
171
+ },
172
+ },
173
+ {
174
+ "queries" => {
175
+ "sections-result" => {
176
+ "source" => "Sections",
177
+ "output" => {
178
+ "elements" => [
179
+ "count",
180
+ ],
181
+ },
182
+ },
183
+ },
184
+ })
185
+ end
186
+
187
+ def test_elapsed_time
188
+ assert_search({
189
+ "sections-result" => {
190
+ "startTime" => start_time,
191
+ "elapsedTime" => elapsed_time,
192
+ },
193
+ },
194
+ {
195
+ "queries" => {
196
+ "sections-result" => {
197
+ "source" => "Sections",
198
+ "output" => {
199
+ "elements" => [
200
+ "startTime",
201
+ "elapsedTime",
202
+ ],
203
+ },
204
+ },
205
+ },
206
+ })
207
+ end
208
+
209
+ class AttributesTest < self
210
+ def test_source_only
211
+ expected = {
212
+ "sections-result" => {
213
+ "records" => [
214
+ {
215
+ "_key" => "1.1",
216
+ "title" => "Groonga overview",
217
+ },
218
+ {
219
+ "_key" => "1.2",
220
+ "title" => "Full text search and Instant update",
221
+ },
222
+ {
223
+ "_key" => "1.3",
224
+ "title" => "Column store and aggregate query",
225
+ },
226
+ ],
227
+ },
228
+ }
229
+ request = {
230
+ "queries" => {
231
+ "sections-result" => {
232
+ "source" => "Sections",
233
+ "output" => {
234
+ "elements" => [
235
+ "records",
236
+ ],
237
+ "format" => "complex",
238
+ "limit" => 3,
239
+ "attributes" => ["_key", "title"],
240
+ },
241
+ },
242
+ },
243
+ }
244
+ assert_search(expected, request)
245
+ end
246
+
247
+ def test_label
248
+ expected = {
249
+ "sections-result" => {
250
+ "records" => [
251
+ {
252
+ "key" => "1.1",
253
+ "title" => "Groonga overview",
254
+ },
255
+ {
256
+ "key" => "1.2",
257
+ "title" => "Full text search and Instant update",
258
+ },
259
+ {
260
+ "key" => "1.3",
261
+ "title" => "Column store and aggregate query",
262
+ },
263
+ ],
264
+ },
265
+ }
266
+ request = {
267
+ "queries" => {
268
+ "sections-result" => {
269
+ "source" => "Sections",
270
+ "output" => {
271
+ "elements" => [
272
+ "records",
273
+ ],
274
+ "format" => "complex",
275
+ "limit" => 3,
276
+ "attributes" => [
277
+ {
278
+ "label" => "key",
279
+ "source" => "_key",
280
+ },
281
+ "title",
282
+ ],
283
+ },
284
+ },
285
+ },
286
+ }
287
+ assert_search(expected, request)
288
+ end
289
+
290
+ def test_static_value
291
+ expected = {
292
+ "sections-result" => {
293
+ "records" => [
294
+ {
295
+ "single_quote_string" => "string value",
296
+ "double_quote_string" => "string value",
297
+ "integer" => 29,
298
+ "complex_negative_number" => -29.29,
299
+ },
300
+ {
301
+ "single_quote_string" => "string value",
302
+ "double_quote_string" => "string value",
303
+ "integer" => 29,
304
+ "complex_negative_number" => -29.29,
305
+ },
306
+ ],
307
+ },
308
+ }
309
+ request = {
310
+ "queries" => {
311
+ "sections-result" => {
312
+ "source" => "Sections",
313
+ "output" => {
314
+ "elements" => [
315
+ "records",
316
+ ],
317
+ "format" => "complex",
318
+ "limit" => 2,
319
+ "attributes" => [
320
+ {
321
+ "label" => "single_quote_string",
322
+ "source" => "'string value'",
323
+ },
324
+ {
325
+ "label" => "double_quote_string",
326
+ "source" => '"string value"',
327
+ },
328
+ {
329
+ "label" => "integer",
330
+ "source" => "29",
331
+ },
332
+ {
333
+ "label" => "complex_negative_number",
334
+ "source" => "-29.29",
335
+ },
336
+ ],
337
+ },
338
+ },
339
+ },
340
+ }
341
+ assert_search(expected, request)
342
+ end
343
+
344
+ def test_expression
345
+ expected = {
346
+ "sections-result" => {
347
+ "records" => [
348
+ {
349
+ "formatted title" => "<Groonga overview>",
350
+ "title" => "Groonga overview",
351
+ },
352
+ ],
353
+ },
354
+ }
355
+ request = {
356
+ "queries" => {
357
+ "sections-result" => {
358
+ "source" => "Sections",
359
+ "output" => {
360
+ "elements" => [
361
+ "records",
362
+ ],
363
+ "format" => "complex",
364
+ "limit" => 1,
365
+ "attributes" => [
366
+ "title",
367
+ {
368
+ "label" => "formatted title",
369
+ "source" => "'<' + title + '>'",
370
+ },
371
+ ],
372
+ },
373
+ },
374
+ },
375
+ }
376
+ assert_search(expected, request)
377
+ end
378
+
379
+ def test_snippet_html
380
+ expected = {
381
+ "sections-result" => {
382
+ "records" => [
383
+ {
384
+ "title" => "Groonga overview",
385
+ "snippet" => [
386
+ "<span class=\"keyword\">Groonga</span> overview",
387
+ ],
388
+ },
389
+ ],
390
+ },
391
+ }
392
+ request = {
393
+ "queries" => {
394
+ "sections-result" => {
395
+ "source" => "Sections",
396
+ "condition" => {
397
+ "query" => "Groonga",
398
+ "matchTo" => ["title"],
399
+ },
400
+ "output" => {
401
+ "elements" => [
402
+ "records",
403
+ ],
404
+ "format" => "complex",
405
+ "limit" => 1,
406
+ "attributes" => [
407
+ "title",
408
+ {
409
+ "label" => "snippet",
410
+ "source" => "snippet_html(title)",
411
+ },
412
+ ],
413
+ },
414
+ },
415
+ },
416
+ }
417
+ assert_search(expected, request)
418
+ end
419
+ end
420
+
421
+ class FormatTest < self
422
+ def test_complex
423
+ request = {
424
+ "queries" => {
425
+ "sections-result" => {
426
+ "source" => "Sections",
427
+ "output" => {
428
+ "elements" => [
429
+ "records",
430
+ ],
431
+ "format" => "complex",
432
+ "limit" => 3,
433
+ "attributes" => ["_key", "title"],
434
+ },
435
+ },
436
+ },
437
+ }
438
+ assert_search(complex_result, request)
439
+ end
440
+
441
+ def test_simple
442
+ request = {
443
+ "queries" => {
444
+ "sections-result" => {
445
+ "source" => "Sections",
446
+ "output" => {
447
+ "elements" => [
448
+ "records",
449
+ ],
450
+ "format" => "simple",
451
+ "limit" => 3,
452
+ "attributes" => ["_key", "title"],
453
+ },
454
+ },
455
+ },
456
+ }
457
+ assert_search(simple_result, request)
458
+ end
459
+
460
+ def test_default
461
+ request = {
462
+ "queries" => {
463
+ "sections-result" => {
464
+ "source" => "Sections",
465
+ "output" => {
466
+ "elements" => [
467
+ "records",
468
+ ],
469
+ "limit" => 3,
470
+ "attributes" => ["_key", "title"],
471
+ },
472
+ },
473
+ },
474
+ }
475
+ assert_search(simple_result, request)
476
+ end
477
+
478
+ def complex_result
479
+ {
480
+ "sections-result" => {
481
+ "records" => [
482
+ {
483
+ "_key" => "1.1",
484
+ "title" => "Groonga overview",
485
+ },
486
+ {
487
+ "_key" => "1.2",
488
+ "title" => "Full text search and Instant update",
489
+ },
490
+ {
491
+ "_key" => "1.3",
492
+ "title" => "Column store and aggregate query",
493
+ },
494
+ ],
495
+ },
496
+ }
497
+ end
498
+
499
+ def simple_result
500
+ {
501
+ "sections-result" => {
502
+ "records" => [
503
+ ["1.1", "Groonga overview"],
504
+ ["1.2", "Full text search and Instant update"],
505
+ ["1.3", "Column store and aggregate query"],
506
+ ],
507
+ },
508
+ }
509
+ end
510
+ end
511
+ end
512
+ end