fluent-plugin-droonga 0.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +1 -0
  3. data/.travis.yml +7 -0
  4. data/Gemfile +40 -0
  5. data/LICENSE.txt +14 -0
  6. data/README.md +18 -0
  7. data/Rakefile +25 -0
  8. data/benchmark/benchmark.rb +123 -0
  9. data/benchmark/utils.rb +243 -0
  10. data/benchmark/watch/benchmark-notify.rb +143 -0
  11. data/benchmark/watch/benchmark-notify.sh +19 -0
  12. data/benchmark/watch/benchmark-publish.rb +120 -0
  13. data/benchmark/watch/benchmark-scan.rb +210 -0
  14. data/benchmark/watch/catalog.json +32 -0
  15. data/benchmark/watch/fluentd.conf +12 -0
  16. data/bin/grn2jsons +85 -0
  17. data/fluent-plugin-droonga.gemspec +41 -0
  18. data/lib/droonga/adapter.rb +156 -0
  19. data/lib/droonga/catalog.rb +153 -0
  20. data/lib/droonga/command_mapper.rb +45 -0
  21. data/lib/droonga/engine.rb +83 -0
  22. data/lib/droonga/executor.rb +289 -0
  23. data/lib/droonga/handler.rb +140 -0
  24. data/lib/droonga/handler_plugin.rb +35 -0
  25. data/lib/droonga/job_queue.rb +83 -0
  26. data/lib/droonga/job_queue_schema.rb +65 -0
  27. data/lib/droonga/logger.rb +34 -0
  28. data/lib/droonga/plugin.rb +41 -0
  29. data/lib/droonga/plugin/adapter/groonga/select.rb +88 -0
  30. data/lib/droonga/plugin/adapter_groonga.rb +40 -0
  31. data/lib/droonga/plugin/handler/groonga/column_create.rb +103 -0
  32. data/lib/droonga/plugin/handler/groonga/table_create.rb +100 -0
  33. data/lib/droonga/plugin/handler_add.rb +44 -0
  34. data/lib/droonga/plugin/handler_forward.rb +70 -0
  35. data/lib/droonga/plugin/handler_groonga.rb +52 -0
  36. data/lib/droonga/plugin/handler_proxy.rb +82 -0
  37. data/lib/droonga/plugin/handler_search.rb +33 -0
  38. data/lib/droonga/plugin/handler_watch.rb +102 -0
  39. data/lib/droonga/proxy.rb +371 -0
  40. data/lib/droonga/searcher.rb +415 -0
  41. data/lib/droonga/server.rb +112 -0
  42. data/lib/droonga/sweeper.rb +42 -0
  43. data/lib/droonga/watch_schema.rb +88 -0
  44. data/lib/droonga/watcher.rb +256 -0
  45. data/lib/droonga/worker.rb +51 -0
  46. data/lib/fluent/plugin/out_droonga.rb +56 -0
  47. data/lib/groonga_command_converter.rb +137 -0
  48. data/sample/cluster/catalog.json +43 -0
  49. data/sample/cluster/fluentd.conf +12 -0
  50. data/sample/fluentd.conf +8 -0
  51. data/test/fixtures/catalog.json +43 -0
  52. data/test/fixtures/document.grn +23 -0
  53. data/test/helper.rb +24 -0
  54. data/test/helper/fixture.rb +28 -0
  55. data/test/helper/sandbox.rb +73 -0
  56. data/test/helper/stub_worker.rb +27 -0
  57. data/test/helper/watch_helper.rb +35 -0
  58. data/test/plugin/adapter/groonga/test_select.rb +176 -0
  59. data/test/plugin/handler/groonga/test_column_create.rb +127 -0
  60. data/test/plugin/handler/groonga/test_table_create.rb +140 -0
  61. data/test/plugin/handler/test_handler_add.rb +135 -0
  62. data/test/plugin/handler/test_handler_groonga.rb +64 -0
  63. data/test/plugin/handler/test_handler_search.rb +512 -0
  64. data/test/plugin/handler/test_handler_watch.rb +168 -0
  65. data/test/run-test.rb +55 -0
  66. data/test/test_adapter.rb +48 -0
  67. data/test/test_catalog.rb +59 -0
  68. data/test/test_command_mapper.rb +44 -0
  69. data/test/test_groonga_command_converter.rb +242 -0
  70. data/test/test_handler.rb +53 -0
  71. data/test/test_job_queue_schema.rb +45 -0
  72. data/test/test_output.rb +99 -0
  73. data/test/test_sweeper.rb +95 -0
  74. data/test/test_watch_schema.rb +57 -0
  75. data/test/test_watcher.rb +336 -0
  76. data/test/test_worker.rb +144 -0
  77. metadata +299 -0
@@ -0,0 +1,135 @@
1
+ # Copyright (C) 2013 droonga project
2
+ #
3
+ # This library is free software; you can redistribute it and/or
4
+ # modify it under the terms of the GNU Lesser General Public
5
+ # License version 2.1 as published by the Free Software Foundation.
6
+ #
7
+ # This library is distributed in the hope that it will be useful,
8
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
9
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
10
+ # Lesser General Public License for more details.
11
+ #
12
+ # You should have received a copy of the GNU Lesser General Public
13
+ # License along with this library; if not, write to the Free Software
14
+ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
15
+
16
+ require "droonga/plugin/handler_add"
17
+
18
+ class AddHandlerTest < Test::Unit::TestCase
19
+ def setup
20
+ setup_database
21
+ setup_schema
22
+ setup_handler
23
+ end
24
+
25
+ def teardown
26
+ teardown_handler
27
+ teardown_database
28
+ end
29
+
30
+ private
31
+ def setup_database
32
+ FileUtils.rm_rf(@database_path.dirname.to_s)
33
+ FileUtils.mkdir_p(@database_path.dirname.to_s)
34
+ @database = Groonga::Database.create(:path => @database_path.to_s)
35
+ end
36
+
37
+ def setup_schema
38
+ end
39
+
40
+ def teardown_database
41
+ @database.close
42
+ @database = nil
43
+ FileUtils.rm_rf(@database_path.dirname.to_s)
44
+ end
45
+
46
+ def setup_handler
47
+ @worker = StubWorker.new
48
+ @handler = Droonga::AddHandler.new(@worker)
49
+ end
50
+
51
+ def teardown_handler
52
+ @handler = nil
53
+ end
54
+
55
+ public
56
+ class HasKeyTest < self
57
+ def setup_schema
58
+ Groonga::Schema.define do |schema|
59
+ schema.create_table("Users",
60
+ :type => :hash,
61
+ :key_type => :short_text) do |table|
62
+ table.short_text("country")
63
+ end
64
+ end
65
+ end
66
+
67
+ def test_empty_values
68
+ request = {
69
+ "table" => "Users",
70
+ "key" => "mori",
71
+ "values" => {},
72
+ }
73
+ mock(@handler).emit([true])
74
+ @handler.add(request)
75
+ table = @worker.context["Users"]
76
+ assert_equal(["mori"], table.collect(&:key))
77
+ end
78
+
79
+ def test_values
80
+ request = {
81
+ "table" => "Users",
82
+ "key" => "asami",
83
+ "values" => {"country" => "japan"},
84
+ }
85
+ mock(@handler).emit([true])
86
+ @handler.add(request)
87
+ table = @worker.context["Users"]
88
+ assert_equal(["japan"], table.collect(&:country))
89
+ end
90
+ end
91
+
92
+ class NoKeyTest < self
93
+ def setup_schema
94
+ Groonga::Schema.define do |schema|
95
+ schema.create_table("Books",
96
+ :type => :array) do |table|
97
+ table.short_text("title")
98
+ end
99
+ end
100
+ end
101
+
102
+ def test_empty_values
103
+ request = {
104
+ "table" => "Books",
105
+ "values" => {},
106
+ }
107
+ mock(@handler).emit([true])
108
+ @handler.add(request)
109
+ table = @worker.context["Books"]
110
+ assert_equal([nil], table.collect(&:title))
111
+ end
112
+
113
+ def test_with_values
114
+ request = {
115
+ "table" => "Books",
116
+ "values" => {"title" => "CSS"},
117
+ }
118
+ mock(@handler).emit([true])
119
+ @handler.add(request)
120
+ table = @worker.context["Books"]
121
+ assert_equal(["CSS"], table.collect(&:title))
122
+ end
123
+ end
124
+
125
+ class FailureTest < self
126
+ def test_nonexistent_table
127
+ request = {
128
+ "table" => "Nonexistent",
129
+ "values" => {},
130
+ }
131
+ mock(@handler).emit([false])
132
+ @handler.add(request)
133
+ end
134
+ end
135
+ end
@@ -0,0 +1,64 @@
1
+ # Copyright (C) 2013 droonga project
2
+ #
3
+ # This library is free software; you can redistribute it and/or
4
+ # modify it under the terms of the GNU Lesser General Public
5
+ # License version 2.1 as published by the Free Software Foundation.
6
+ #
7
+ # This library is distributed in the hope that it will be useful,
8
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
9
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
10
+ # Lesser General Public License for more details.
11
+ #
12
+ # You should have received a copy of the GNU Lesser General Public
13
+ # License along with this library; if not, write to the Free Software
14
+ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
15
+
16
+ require "droonga/plugin/handler_groonga"
17
+
18
+ class GroongaHandlerTest < Test::Unit::TestCase
19
+ def setup
20
+ setup_database
21
+ setup_handler
22
+ end
23
+
24
+ def teardown
25
+ teardown_handler
26
+ teardown_database
27
+ end
28
+
29
+ private
30
+ def setup_database
31
+ FileUtils.rm_rf(@database_path.dirname.to_s)
32
+ FileUtils.mkdir_p(@database_path.dirname.to_s)
33
+ @database = Groonga::Database.create(:path => @database_path.to_s)
34
+ end
35
+
36
+ def teardown_database
37
+ @database.close
38
+ @database = nil
39
+ FileUtils.rm_rf(@database_path.dirname.to_s)
40
+ end
41
+
42
+ def setup_handler
43
+ @worker = StubWorker.new
44
+ @handler = Droonga::GroongaHandler.new(@worker)
45
+ end
46
+
47
+ def teardown_handler
48
+ @handler = nil
49
+ end
50
+
51
+ private
52
+ def dump
53
+ database_dumper = Groonga::DatabaseDumper.new(:database => @database)
54
+ database_dumper.dump
55
+ end
56
+
57
+ NORMALIZED_START_TIME = Time.parse("2013-07-11T16:04:28+0900").to_i
58
+ NORMALIZED_ELAPSED_TIME = 1
59
+ def normalize_header(header)
60
+ start_time = NORMALIZED_START_TIME
61
+ elapsed_time = NORMALIZED_ELAPSED_TIME
62
+ [header[0], start_time, elapsed_time]
63
+ end
64
+ end
@@ -0,0 +1,512 @@
1
+ # Copyright (C) 2013 droonga project
2
+ #
3
+ # This library is free software; you can redistribute it and/or
4
+ # modify it under the terms of the GNU Lesser General Public
5
+ # License version 2.1 as published by the Free Software Foundation.
6
+ #
7
+ # This library is distributed in the hope that it will be useful,
8
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
9
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
10
+ # Lesser General Public License for more details.
11
+ #
12
+ # You should have received a copy of the GNU Lesser General Public
13
+ # License along with this library; if not, write to the Free Software
14
+ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
15
+
16
+ require "droonga/plugin/handler_search"
17
+
18
+ class SearchHandlerTest < Test::Unit::TestCase
19
+ def setup
20
+ setup_database
21
+ setup_handler
22
+ end
23
+
24
+ def teardown
25
+ teardown_handler
26
+ teardown_database
27
+ end
28
+
29
+ private
30
+ def setup_database
31
+ restore(fixture_data("document.grn"))
32
+ @database = Groonga::Database.open(@database_path.to_s)
33
+ end
34
+
35
+ def teardown_database
36
+ @database.close
37
+ @database = nil
38
+ end
39
+
40
+ def setup_handler
41
+ @worker = StubWorker.new
42
+ @handler = Droonga::SearchHandler.new(@worker)
43
+ end
44
+
45
+ def teardown_handler
46
+ @handler = nil
47
+ end
48
+
49
+ def search(request)
50
+ @handler.search(request)
51
+ normalize_result_set(@worker.body)
52
+ end
53
+
54
+ def normalize_result_set(result_set)
55
+ result_set.each do |name, result|
56
+ result["startTime"] = start_time if result["startTime"]
57
+ result["elapsedTime"] = elapsed_time if result["elapsedTime"]
58
+ end
59
+ result_set
60
+ end
61
+
62
+ def start_time
63
+ "2013-01-31T14:34:47+09:00"
64
+ end
65
+
66
+ def elapsed_time
67
+ 0.01
68
+ end
69
+
70
+ def assert_search(expected, request)
71
+ assert_equal(expected, search(request))
72
+ end
73
+
74
+ class NoParameterTest < self
75
+ def test_empty
76
+ assert_search({}, {})
77
+ end
78
+ end
79
+
80
+ class QueriesTest < self
81
+ def test_empty
82
+ assert_search({}, {"queries" => {}})
83
+ end
84
+ end
85
+
86
+ class HashQueryTest < self
87
+ def test_string_matchTo
88
+ request = base_request
89
+ request["queries"]["sections-result"]["condition"] = {
90
+ "query" => "Groonga",
91
+ "matchTo" => "title"
92
+ }
93
+ assert_search({
94
+ "sections-result" => {
95
+ "records" => [
96
+ { "title" => "Groonga overview" },
97
+ ],
98
+ },
99
+ },
100
+ request)
101
+ end
102
+
103
+ def test_array_matchTo
104
+ request = base_request
105
+ request["queries"]["sections-result"]["condition"] = {
106
+ "query" => "Groonga",
107
+ "matchTo" => ["title"]
108
+ }
109
+ assert_search({
110
+ "sections-result" => {
111
+ "records" => [
112
+ { "title" => "Groonga overview" },
113
+ ],
114
+ },
115
+ },
116
+ request)
117
+ end
118
+
119
+ def base_request
120
+ {
121
+ "queries" => {
122
+ "sections-result" => {
123
+ "source" => "Sections",
124
+ "output" => {
125
+ "elements" => [
126
+ "records",
127
+ ],
128
+ "format" => "complex",
129
+ "limit" => 1,
130
+ "attributes" => ["title"],
131
+ },
132
+ },
133
+ },
134
+ }
135
+ end
136
+ end
137
+
138
+ class SourceTest < self
139
+ def test_non_existent
140
+ assert_raise(Droonga::Searcher::UndefinedSourceError) do
141
+ search({
142
+ "queries" => {
143
+ "non-existent-result" => {
144
+ "source" => "non-existent",
145
+ },
146
+ },
147
+ })
148
+ end
149
+ end
150
+
151
+ def test_existent
152
+ assert_search({
153
+ "sections-result" => {},
154
+ },
155
+ {
156
+ "queries" => {
157
+ "sections-result" => {
158
+ "source" => "Sections",
159
+ "output" => {},
160
+ },
161
+ },
162
+ })
163
+ end
164
+ end
165
+
166
+ class OutputTest < self
167
+ def test_count
168
+ assert_search({
169
+ "sections-result" => {
170
+ "count" => 9,
171
+ },
172
+ },
173
+ {
174
+ "queries" => {
175
+ "sections-result" => {
176
+ "source" => "Sections",
177
+ "output" => {
178
+ "elements" => [
179
+ "count",
180
+ ],
181
+ },
182
+ },
183
+ },
184
+ })
185
+ end
186
+
187
+ def test_elapsed_time
188
+ assert_search({
189
+ "sections-result" => {
190
+ "startTime" => start_time,
191
+ "elapsedTime" => elapsed_time,
192
+ },
193
+ },
194
+ {
195
+ "queries" => {
196
+ "sections-result" => {
197
+ "source" => "Sections",
198
+ "output" => {
199
+ "elements" => [
200
+ "startTime",
201
+ "elapsedTime",
202
+ ],
203
+ },
204
+ },
205
+ },
206
+ })
207
+ end
208
+
209
+ class AttributesTest < self
210
+ def test_source_only
211
+ expected = {
212
+ "sections-result" => {
213
+ "records" => [
214
+ {
215
+ "_key" => "1.1",
216
+ "title" => "Groonga overview",
217
+ },
218
+ {
219
+ "_key" => "1.2",
220
+ "title" => "Full text search and Instant update",
221
+ },
222
+ {
223
+ "_key" => "1.3",
224
+ "title" => "Column store and aggregate query",
225
+ },
226
+ ],
227
+ },
228
+ }
229
+ request = {
230
+ "queries" => {
231
+ "sections-result" => {
232
+ "source" => "Sections",
233
+ "output" => {
234
+ "elements" => [
235
+ "records",
236
+ ],
237
+ "format" => "complex",
238
+ "limit" => 3,
239
+ "attributes" => ["_key", "title"],
240
+ },
241
+ },
242
+ },
243
+ }
244
+ assert_search(expected, request)
245
+ end
246
+
247
+ def test_label
248
+ expected = {
249
+ "sections-result" => {
250
+ "records" => [
251
+ {
252
+ "key" => "1.1",
253
+ "title" => "Groonga overview",
254
+ },
255
+ {
256
+ "key" => "1.2",
257
+ "title" => "Full text search and Instant update",
258
+ },
259
+ {
260
+ "key" => "1.3",
261
+ "title" => "Column store and aggregate query",
262
+ },
263
+ ],
264
+ },
265
+ }
266
+ request = {
267
+ "queries" => {
268
+ "sections-result" => {
269
+ "source" => "Sections",
270
+ "output" => {
271
+ "elements" => [
272
+ "records",
273
+ ],
274
+ "format" => "complex",
275
+ "limit" => 3,
276
+ "attributes" => [
277
+ {
278
+ "label" => "key",
279
+ "source" => "_key",
280
+ },
281
+ "title",
282
+ ],
283
+ },
284
+ },
285
+ },
286
+ }
287
+ assert_search(expected, request)
288
+ end
289
+
290
+ def test_static_value
291
+ expected = {
292
+ "sections-result" => {
293
+ "records" => [
294
+ {
295
+ "single_quote_string" => "string value",
296
+ "double_quote_string" => "string value",
297
+ "integer" => 29,
298
+ "complex_negative_number" => -29.29,
299
+ },
300
+ {
301
+ "single_quote_string" => "string value",
302
+ "double_quote_string" => "string value",
303
+ "integer" => 29,
304
+ "complex_negative_number" => -29.29,
305
+ },
306
+ ],
307
+ },
308
+ }
309
+ request = {
310
+ "queries" => {
311
+ "sections-result" => {
312
+ "source" => "Sections",
313
+ "output" => {
314
+ "elements" => [
315
+ "records",
316
+ ],
317
+ "format" => "complex",
318
+ "limit" => 2,
319
+ "attributes" => [
320
+ {
321
+ "label" => "single_quote_string",
322
+ "source" => "'string value'",
323
+ },
324
+ {
325
+ "label" => "double_quote_string",
326
+ "source" => '"string value"',
327
+ },
328
+ {
329
+ "label" => "integer",
330
+ "source" => "29",
331
+ },
332
+ {
333
+ "label" => "complex_negative_number",
334
+ "source" => "-29.29",
335
+ },
336
+ ],
337
+ },
338
+ },
339
+ },
340
+ }
341
+ assert_search(expected, request)
342
+ end
343
+
344
+ def test_expression
345
+ expected = {
346
+ "sections-result" => {
347
+ "records" => [
348
+ {
349
+ "formatted title" => "<Groonga overview>",
350
+ "title" => "Groonga overview",
351
+ },
352
+ ],
353
+ },
354
+ }
355
+ request = {
356
+ "queries" => {
357
+ "sections-result" => {
358
+ "source" => "Sections",
359
+ "output" => {
360
+ "elements" => [
361
+ "records",
362
+ ],
363
+ "format" => "complex",
364
+ "limit" => 1,
365
+ "attributes" => [
366
+ "title",
367
+ {
368
+ "label" => "formatted title",
369
+ "source" => "'<' + title + '>'",
370
+ },
371
+ ],
372
+ },
373
+ },
374
+ },
375
+ }
376
+ assert_search(expected, request)
377
+ end
378
+
379
+ def test_snippet_html
380
+ expected = {
381
+ "sections-result" => {
382
+ "records" => [
383
+ {
384
+ "title" => "Groonga overview",
385
+ "snippet" => [
386
+ "<span class=\"keyword\">Groonga</span> overview",
387
+ ],
388
+ },
389
+ ],
390
+ },
391
+ }
392
+ request = {
393
+ "queries" => {
394
+ "sections-result" => {
395
+ "source" => "Sections",
396
+ "condition" => {
397
+ "query" => "Groonga",
398
+ "matchTo" => ["title"],
399
+ },
400
+ "output" => {
401
+ "elements" => [
402
+ "records",
403
+ ],
404
+ "format" => "complex",
405
+ "limit" => 1,
406
+ "attributes" => [
407
+ "title",
408
+ {
409
+ "label" => "snippet",
410
+ "source" => "snippet_html(title)",
411
+ },
412
+ ],
413
+ },
414
+ },
415
+ },
416
+ }
417
+ assert_search(expected, request)
418
+ end
419
+ end
420
+
421
+ class FormatTest < self
422
+ def test_complex
423
+ request = {
424
+ "queries" => {
425
+ "sections-result" => {
426
+ "source" => "Sections",
427
+ "output" => {
428
+ "elements" => [
429
+ "records",
430
+ ],
431
+ "format" => "complex",
432
+ "limit" => 3,
433
+ "attributes" => ["_key", "title"],
434
+ },
435
+ },
436
+ },
437
+ }
438
+ assert_search(complex_result, request)
439
+ end
440
+
441
+ def test_simple
442
+ request = {
443
+ "queries" => {
444
+ "sections-result" => {
445
+ "source" => "Sections",
446
+ "output" => {
447
+ "elements" => [
448
+ "records",
449
+ ],
450
+ "format" => "simple",
451
+ "limit" => 3,
452
+ "attributes" => ["_key", "title"],
453
+ },
454
+ },
455
+ },
456
+ }
457
+ assert_search(simple_result, request)
458
+ end
459
+
460
+ def test_default
461
+ request = {
462
+ "queries" => {
463
+ "sections-result" => {
464
+ "source" => "Sections",
465
+ "output" => {
466
+ "elements" => [
467
+ "records",
468
+ ],
469
+ "limit" => 3,
470
+ "attributes" => ["_key", "title"],
471
+ },
472
+ },
473
+ },
474
+ }
475
+ assert_search(simple_result, request)
476
+ end
477
+
478
+ def complex_result
479
+ {
480
+ "sections-result" => {
481
+ "records" => [
482
+ {
483
+ "_key" => "1.1",
484
+ "title" => "Groonga overview",
485
+ },
486
+ {
487
+ "_key" => "1.2",
488
+ "title" => "Full text search and Instant update",
489
+ },
490
+ {
491
+ "_key" => "1.3",
492
+ "title" => "Column store and aggregate query",
493
+ },
494
+ ],
495
+ },
496
+ }
497
+ end
498
+
499
+ def simple_result
500
+ {
501
+ "sections-result" => {
502
+ "records" => [
503
+ ["1.1", "Groonga overview"],
504
+ ["1.2", "Full text search and Instant update"],
505
+ ["1.3", "Column store and aggregate query"],
506
+ ],
507
+ },
508
+ }
509
+ end
510
+ end
511
+ end
512
+ end