fluentd 0.14.17-x86-mingw32 → 1.3.1-x86-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (159) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +16 -5
  3. data/ADOPTERS.md +5 -0
  4. data/{ChangeLog → CHANGELOG.md} +495 -6
  5. data/CONTRIBUTING.md +5 -2
  6. data/GOVERNANCE.md +55 -0
  7. data/LICENSE +202 -0
  8. data/MAINTAINERS.md +7 -5
  9. data/README.md +17 -10
  10. data/bin/fluent-ca-generate +6 -0
  11. data/example/counter.conf +18 -0
  12. data/example/secondary_file.conf +3 -2
  13. data/fluentd.gemspec +3 -3
  14. data/lib/fluent/agent.rb +1 -1
  15. data/lib/fluent/command/binlog_reader.rb +11 -2
  16. data/lib/fluent/command/ca_generate.rb +181 -0
  17. data/lib/fluent/command/cat.rb +28 -15
  18. data/lib/fluent/command/debug.rb +4 -4
  19. data/lib/fluent/command/fluentd.rb +2 -2
  20. data/lib/fluent/command/plugin_config_formatter.rb +24 -2
  21. data/lib/fluent/command/plugin_generator.rb +26 -8
  22. data/lib/fluent/config/configure_proxy.rb +7 -1
  23. data/lib/fluent/config/dsl.rb +8 -5
  24. data/lib/fluent/config/element.rb +5 -0
  25. data/lib/fluent/config/literal_parser.rb +7 -1
  26. data/lib/fluent/config/types.rb +28 -2
  27. data/lib/fluent/config/v1_parser.rb +1 -2
  28. data/lib/fluent/configurable.rb +1 -0
  29. data/lib/fluent/counter.rb +23 -0
  30. data/lib/fluent/counter/base_socket.rb +46 -0
  31. data/lib/fluent/counter/client.rb +297 -0
  32. data/lib/fluent/counter/error.rb +86 -0
  33. data/lib/fluent/counter/mutex_hash.rb +163 -0
  34. data/lib/fluent/counter/server.rb +273 -0
  35. data/lib/fluent/counter/store.rb +205 -0
  36. data/lib/fluent/counter/validator.rb +145 -0
  37. data/lib/fluent/env.rb +1 -0
  38. data/lib/fluent/event_router.rb +1 -1
  39. data/lib/fluent/log.rb +119 -29
  40. data/lib/fluent/plugin/base.rb +12 -0
  41. data/lib/fluent/plugin/buf_file.rb +20 -16
  42. data/lib/fluent/plugin/buffer.rb +130 -32
  43. data/lib/fluent/plugin/buffer/file_chunk.rb +23 -4
  44. data/lib/fluent/plugin/compressable.rb +1 -1
  45. data/lib/fluent/plugin/filter_grep.rb +135 -21
  46. data/lib/fluent/plugin/filter_parser.rb +13 -2
  47. data/lib/fluent/plugin/filter_record_transformer.rb +16 -14
  48. data/lib/fluent/plugin/formatter_stdout.rb +3 -2
  49. data/lib/fluent/plugin/formatter_tsv.rb +5 -1
  50. data/lib/fluent/plugin/in_debug_agent.rb +8 -1
  51. data/lib/fluent/plugin/in_forward.rb +1 -1
  52. data/lib/fluent/plugin/in_http.rb +84 -3
  53. data/lib/fluent/plugin/in_monitor_agent.rb +7 -1
  54. data/lib/fluent/plugin/in_syslog.rb +31 -10
  55. data/lib/fluent/plugin/in_tail.rb +142 -53
  56. data/lib/fluent/plugin/in_tcp.rb +5 -6
  57. data/lib/fluent/plugin/in_udp.rb +6 -2
  58. data/lib/fluent/plugin/in_unix.rb +1 -1
  59. data/lib/fluent/plugin/multi_output.rb +1 -0
  60. data/lib/fluent/plugin/out_copy.rb +25 -2
  61. data/lib/fluent/plugin/out_file.rb +26 -7
  62. data/lib/fluent/plugin/out_forward.rb +81 -42
  63. data/lib/fluent/plugin/out_secondary_file.rb +2 -2
  64. data/lib/fluent/plugin/out_stdout.rb +0 -1
  65. data/lib/fluent/plugin/out_stream.rb +1 -1
  66. data/lib/fluent/plugin/output.rb +221 -57
  67. data/lib/fluent/plugin/parser_apache.rb +1 -1
  68. data/lib/fluent/plugin/parser_apache2.rb +5 -1
  69. data/lib/fluent/plugin/parser_apache_error.rb +1 -1
  70. data/lib/fluent/plugin/parser_json.rb +10 -3
  71. data/lib/fluent/plugin/parser_ltsv.rb +7 -0
  72. data/lib/fluent/plugin/parser_multiline.rb +2 -1
  73. data/lib/fluent/plugin/parser_nginx.rb +1 -1
  74. data/lib/fluent/plugin/parser_none.rb +1 -0
  75. data/lib/fluent/plugin/parser_regexp.rb +15 -14
  76. data/lib/fluent/plugin/parser_syslog.rb +9 -5
  77. data/lib/fluent/plugin_helper.rb +2 -0
  78. data/lib/fluent/plugin_helper/cert_option.rb +28 -9
  79. data/lib/fluent/plugin_helper/compat_parameters.rb +3 -1
  80. data/lib/fluent/plugin_helper/counter.rb +51 -0
  81. data/lib/fluent/plugin_helper/event_loop.rb +9 -0
  82. data/lib/fluent/plugin_helper/record_accessor.rb +210 -0
  83. data/lib/fluent/plugin_helper/retry_state.rb +15 -7
  84. data/lib/fluent/plugin_helper/server.rb +87 -25
  85. data/lib/fluent/plugin_helper/socket_option.rb +5 -2
  86. data/lib/fluent/plugin_helper/timer.rb +8 -7
  87. data/lib/fluent/root_agent.rb +18 -9
  88. data/lib/fluent/supervisor.rb +63 -23
  89. data/lib/fluent/system_config.rb +30 -2
  90. data/lib/fluent/test/helpers.rb +1 -1
  91. data/lib/fluent/time.rb +15 -7
  92. data/lib/fluent/timezone.rb +26 -2
  93. data/lib/fluent/version.rb +1 -1
  94. data/templates/new_gem/README.md.erb +2 -2
  95. data/templates/new_gem/lib/fluent/plugin/filter.rb.erb +1 -1
  96. data/templates/new_gem/lib/fluent/plugin/input.rb.erb +1 -1
  97. data/templates/new_gem/lib/fluent/plugin/output.rb.erb +1 -1
  98. data/templates/new_gem/lib/fluent/plugin/parser.rb.erb +4 -4
  99. data/test/command/test_ca_generate.rb +70 -0
  100. data/test/command/test_fluentd.rb +2 -2
  101. data/test/command/test_plugin_config_formatter.rb +8 -7
  102. data/test/command/test_plugin_generator.rb +65 -39
  103. data/test/config/test_config_parser.rb +7 -2
  104. data/test/config/test_configurable.rb +7 -2
  105. data/test/config/test_configure_proxy.rb +41 -3
  106. data/test/config/test_dsl.rb +10 -10
  107. data/test/config/test_element.rb +10 -0
  108. data/test/config/test_literal_parser.rb +8 -0
  109. data/test/config/test_plugin_configuration.rb +56 -0
  110. data/test/config/test_system_config.rb +19 -1
  111. data/test/config/test_types.rb +37 -0
  112. data/test/counter/test_client.rb +559 -0
  113. data/test/counter/test_error.rb +44 -0
  114. data/test/counter/test_mutex_hash.rb +179 -0
  115. data/test/counter/test_server.rb +589 -0
  116. data/test/counter/test_store.rb +258 -0
  117. data/test/counter/test_validator.rb +137 -0
  118. data/test/plugin/test_buf_file.rb +124 -0
  119. data/test/plugin/test_buffer.rb +3 -2
  120. data/test/plugin/test_filter_grep.rb +580 -2
  121. data/test/plugin/test_filter_parser.rb +33 -2
  122. data/test/plugin/test_filter_record_transformer.rb +22 -1
  123. data/test/plugin/test_formatter_ltsv.rb +3 -0
  124. data/test/plugin/test_formatter_tsv.rb +68 -0
  125. data/test/plugin/test_in_debug_agent.rb +21 -0
  126. data/test/plugin/test_in_exec.rb +3 -5
  127. data/test/plugin/test_in_http.rb +178 -0
  128. data/test/plugin/test_in_monitor_agent.rb +1 -1
  129. data/test/plugin/test_in_syslog.rb +64 -0
  130. data/test/plugin/test_in_tail.rb +116 -6
  131. data/test/plugin/test_in_tcp.rb +21 -0
  132. data/test/plugin/test_in_udp.rb +78 -0
  133. data/test/plugin/test_metadata.rb +89 -0
  134. data/test/plugin/test_out_copy.rb +31 -0
  135. data/test/plugin/test_out_file.rb +108 -2
  136. data/test/plugin/test_out_forward.rb +195 -2
  137. data/test/plugin/test_out_secondary_file.rb +14 -0
  138. data/test/plugin/test_output.rb +159 -45
  139. data/test/plugin/test_output_as_buffered.rb +19 -0
  140. data/test/plugin/test_output_as_buffered_backup.rb +307 -0
  141. data/test/plugin/test_output_as_buffered_retries.rb +70 -0
  142. data/test/plugin/test_output_as_buffered_secondary.rb +1 -1
  143. data/test/plugin/test_parser_apache2.rb +1 -0
  144. data/test/plugin/test_parser_labeled_tsv.rb +17 -0
  145. data/test/plugin/test_parser_nginx.rb +40 -0
  146. data/test/plugin/test_parser_regexp.rb +6 -7
  147. data/test/plugin/test_parser_syslog.rb +155 -5
  148. data/test/plugin_helper/test_child_process.rb +4 -4
  149. data/test/plugin_helper/test_compat_parameters.rb +22 -0
  150. data/test/plugin_helper/test_record_accessor.rb +197 -0
  151. data/test/plugin_helper/test_retry_state.rb +20 -0
  152. data/test/plugin_helper/test_server.rb +30 -2
  153. data/test/test_config.rb +3 -3
  154. data/test/test_configdsl.rb +2 -2
  155. data/test/test_log.rb +51 -1
  156. data/test/test_root_agent.rb +33 -0
  157. data/test/test_supervisor.rb +105 -0
  158. metadata +68 -8
  159. data/COPYING +0 -14
@@ -170,7 +170,7 @@ class BufferTest < Test::Unit::TestCase
170
170
 
171
171
  sub_test_case 'with default configuration and dummy implementation' do
172
172
  setup do
173
- @p = create_buffer({})
173
+ @p = create_buffer({'queued_chunks_limit_size' => 100})
174
174
  @dm0 = create_metadata(Time.parse('2016-04-11 16:00:00 +0000').to_i, nil, nil)
175
175
  @dm1 = create_metadata(Time.parse('2016-04-11 16:10:00 +0000').to_i, nil, nil)
176
176
  @dm2 = create_metadata(Time.parse('2016-04-11 16:20:00 +0000').to_i, nil, nil)
@@ -851,7 +851,8 @@ class BufferTest < Test::Unit::TestCase
851
851
  assert{ @p.stage[@dm1].size == 2 }
852
852
  assert !@p.stage[@dm1].rollbacked
853
853
 
854
- @p.stage[@dm1].failing = true
854
+ meta_list = [@dm0, @dm1, @dm2, @dm3].sort
855
+ @p.stage[meta_list.last].failing = true
855
856
 
856
857
  assert_raise(FluentPluginBufferTest::DummyMemoryChunkError) do
857
858
  @p.write({ @dm2 => [row], @dm3 => [row], @dm0 => [row, row, row], @dm1 => [row, row] })
@@ -23,12 +23,223 @@ class GrepFilterTest < Test::Unit::TestCase
23
23
 
24
24
  test "regexpN can contain a space" do
25
25
  d = create_driver(%[regexp1 message foo])
26
- assert_equal(Regexp.compile(/ foo/), d.instance.regexps['message'])
26
+ d.instance._regexp_and_conditions.each { |value|
27
+ assert_equal(Regexp.compile(/ foo/), value.pattern)
28
+ }
27
29
  end
28
30
 
29
31
  test "excludeN can contain a space" do
30
32
  d = create_driver(%[exclude1 message foo])
31
- assert_equal(Regexp.compile(/ foo/), d.instance.excludes['message'])
33
+ d.instance._exclude_or_conditions.each { |value|
34
+ assert_equal(Regexp.compile(/ foo/), value.pattern)
35
+ }
36
+ end
37
+
38
+ sub_test_case "duplicate key" do
39
+ test "flat" do
40
+ conf = %[
41
+ regexp1 message test
42
+ regexp2 message test2
43
+ ]
44
+ assert_raise(Fluent::ConfigError) do
45
+ create_driver(conf)
46
+ end
47
+ end
48
+ test "section" do
49
+ conf = %[
50
+ <regexp>
51
+ key message
52
+ pattern test
53
+ </regexp>
54
+ <regexp>
55
+ key message
56
+ pattern test2
57
+ </regexp>
58
+ ]
59
+ assert_raise(Fluent::ConfigError) do
60
+ create_driver(conf)
61
+ end
62
+ end
63
+ test "mix" do
64
+ conf = %[
65
+ regexp1 message test
66
+ <regexp>
67
+ key message
68
+ pattern test
69
+ </regexp>
70
+ ]
71
+ assert_raise(Fluent::ConfigError) do
72
+ create_driver(conf)
73
+ end
74
+ end
75
+
76
+ test "and/regexp" do
77
+ conf = %[
78
+ <and>
79
+ <regexp>
80
+ key message
81
+ pattern test
82
+ </regexp>
83
+ <regexp>
84
+ key message
85
+ pattern test
86
+ </regexp>
87
+ </and>
88
+ ]
89
+ assert_raise(Fluent::ConfigError) do
90
+ create_driver(conf)
91
+ end
92
+ end
93
+
94
+ test "and/regexp, and/regexp" do
95
+ conf = %[
96
+ <and>
97
+ <regexp>
98
+ key message
99
+ pattern test
100
+ </regexp>
101
+ </and>
102
+ <and>
103
+ <regexp>
104
+ key message
105
+ pattern test
106
+ </regexp>
107
+ </and>
108
+ ]
109
+ assert_raise(Fluent::ConfigError) do
110
+ create_driver(conf)
111
+ end
112
+ end
113
+
114
+ test "regexp, and/regexp" do
115
+ conf = %[
116
+ <regexp>
117
+ key message
118
+ pattern test
119
+ </regexp>
120
+ <and>
121
+ <regexp>
122
+ key message
123
+ pattern test
124
+ </regexp>
125
+ </and>
126
+ ]
127
+ assert_raise(Fluent::ConfigError) do
128
+ create_driver(conf)
129
+ end
130
+ end
131
+
132
+ test "and/exclude" do
133
+ conf = %[
134
+ <and>
135
+ <exclude>
136
+ key message
137
+ pattern test
138
+ </exclude>
139
+ <exclude>
140
+ key message
141
+ pattern test
142
+ </exclude>
143
+ </and>
144
+ ]
145
+ assert_raise(Fluent::ConfigError) do
146
+ create_driver(conf)
147
+ end
148
+ end
149
+
150
+ test "and/exclude, and/exclude" do
151
+ conf = %[
152
+ <and>
153
+ <exclude>
154
+ key message
155
+ pattern test
156
+ </exclude>
157
+ </and>
158
+ <and>
159
+ <exclude>
160
+ key message
161
+ pattern test
162
+ </exclude>
163
+ </and>
164
+ ]
165
+ assert_raise(Fluent::ConfigError) do
166
+ create_driver(conf)
167
+ end
168
+ end
169
+
170
+ test "exclude, or/exclude" do
171
+ conf = %[
172
+ <exclude>
173
+ key message
174
+ pattern test
175
+ </exclude>
176
+ <or>
177
+ <exclude>
178
+ key message
179
+ pattern test
180
+ </exclude>
181
+ </or>
182
+ ]
183
+ assert_raise(Fluent::ConfigError) do
184
+ create_driver(conf)
185
+ end
186
+ end
187
+ end
188
+
189
+ sub_test_case "pattern with slashes" do
190
+ test "start with character classes" do
191
+ conf = %[
192
+ <regexp>
193
+ key message
194
+ pattern /[a-z]test/
195
+ </regexp>
196
+ <exclude>
197
+ key message
198
+ pattern /[A-Z]test/
199
+ </exclude>
200
+ ]
201
+ d = create_driver(conf)
202
+ assert_equal(/[a-z]test/, d.instance.regexps.first.pattern)
203
+ assert_equal(/[A-Z]test/, d.instance.excludes.first.pattern)
204
+ end
205
+ end
206
+
207
+ sub_test_case "and/or section" do
208
+ test "<and> section cannot include both <regexp> and <exclude>" do
209
+ conf = %[
210
+ <and>
211
+ <regexp>
212
+ key message
213
+ pattern /test/
214
+ </regexp>
215
+ <exclude>
216
+ key level
217
+ pattern /debug/
218
+ </exclude>
219
+ </and>
220
+ ]
221
+ assert_raise(Fluent::ConfigError) do
222
+ create_driver(conf)
223
+ end
224
+ end
225
+
226
+ test "<or> section cannot include both <regexp> and <exclude>" do
227
+ conf = %[
228
+ <or>
229
+ <regexp>
230
+ key message
231
+ pattern /test/
232
+ </regexp>
233
+ <exclude>
234
+ key level
235
+ pattern /debug/
236
+ </exclude>
237
+ </or>
238
+ ]
239
+ assert_raise(Fluent::ConfigError) do
240
+ create_driver(conf)
241
+ end
242
+ end
32
243
  end
33
244
  end
34
245
 
@@ -77,6 +288,38 @@ class GrepFilterTest < Test::Unit::TestCase
77
288
  end
78
289
  end
79
290
 
291
+ test 'regexps' do
292
+ conf = %[
293
+ <regexp>
294
+ key message
295
+ pattern WARN
296
+ </regexp>
297
+ ]
298
+ filtered_records = filter(conf, messages)
299
+ assert_equal(3, filtered_records.size)
300
+ assert_block('only WARN logs') do
301
+ filtered_records.all? { |r|
302
+ !r['message'].include?('INFO')
303
+ }
304
+ end
305
+ end
306
+
307
+ test 'excludes' do
308
+ conf = %[
309
+ <exclude>
310
+ key message
311
+ pattern favicon
312
+ </exclude>
313
+ ]
314
+ filtered_records = filter(conf, messages)
315
+ assert_equal(3, filtered_records.size)
316
+ assert_block('remove favicon logs') do
317
+ filtered_records.all? { |r|
318
+ !r['message'].include?('favicon')
319
+ }
320
+ end
321
+ end
322
+
80
323
  sub_test_case 'with invalid sequence' do
81
324
  def messages
82
325
  [
@@ -90,6 +333,341 @@ class GrepFilterTest < Test::Unit::TestCase
90
333
  }
91
334
  end
92
335
  end
336
+
337
+ sub_test_case "and/or section" do
338
+ def records
339
+ [
340
+ { "time" => "2013/01/13T07:02:11.124202", "level" => "INFO", "method" => "GET", "path" => "/ping" },
341
+ { "time" => "2013/01/13T07:02:13.232645", "level" => "WARN", "method" => "POST", "path" => "/auth" },
342
+ { "time" => "2013/01/13T07:02:21.542145", "level" => "WARN", "method" => "GET", "path" => "/favicon.ico" },
343
+ { "time" => "2013/01/13T07:02:43.632145", "level" => "WARN", "method" => "POST", "path" => "/login" },
344
+ ]
345
+ end
346
+
347
+ def filter(conf, records)
348
+ d = create_driver(conf)
349
+ d.run do
350
+ records.each do |record|
351
+ d.feed("filter.test", @time, record)
352
+ end
353
+ end
354
+ d.filtered_records
355
+ end
356
+
357
+ test "basic and/regexp" do
358
+ conf = %[
359
+ <and>
360
+ <regexp>
361
+ key level
362
+ pattern ^INFO$
363
+ </regexp>
364
+ <regexp>
365
+ key method
366
+ pattern ^GET$
367
+ </regexp>
368
+ </and>
369
+ ]
370
+ filtered_records = filter(conf, records)
371
+ assert_equal(records.values_at(0), filtered_records)
372
+ end
373
+
374
+ test "basic or/exclude" do
375
+ conf = %[
376
+ <or>
377
+ <exclude>
378
+ key level
379
+ pattern ^INFO$
380
+ </exclude>
381
+ <exclude>
382
+ key method
383
+ pattern ^GET$
384
+ </exclude>
385
+ </or>
386
+ ]
387
+ filtered_records = filter(conf, records)
388
+ assert_equal(records.values_at(1, 3), filtered_records)
389
+ end
390
+
391
+ test "basic or/regexp" do
392
+ conf = %[
393
+ <or>
394
+ <regexp>
395
+ key level
396
+ pattern ^INFO$
397
+ </regexp>
398
+ <regexp>
399
+ key method
400
+ pattern ^GET$
401
+ </regexp>
402
+ </or>
403
+ ]
404
+ filtered_records = filter(conf, records)
405
+ assert_equal(records.values_at(0, 2), filtered_records)
406
+ end
407
+
408
+ test "basic and/exclude" do
409
+ conf = %[
410
+ <and>
411
+ <exclude>
412
+ key level
413
+ pattern ^INFO$
414
+ </exclude>
415
+ <exclude>
416
+ key method
417
+ pattern ^GET$
418
+ </exclude>
419
+ </and>
420
+ ]
421
+ filtered_records = filter(conf, records)
422
+ assert_equal(records.values_at(1, 2, 3), filtered_records)
423
+ end
424
+
425
+ sub_test_case "and/or combo" do
426
+ def records
427
+ [
428
+ { "time" => "2013/01/13T07:02:11.124202", "level" => "INFO", "method" => "GET", "path" => "/ping" },
429
+ { "time" => "2013/01/13T07:02:13.232645", "level" => "WARN", "method" => "POST", "path" => "/auth" },
430
+ { "time" => "2013/01/13T07:02:21.542145", "level" => "WARN", "method" => "GET", "path" => "/favicon.ico" },
431
+ { "time" => "2013/01/13T07:02:43.632145", "level" => "WARN", "method" => "POST", "path" => "/login" },
432
+ { "time" => "2013/01/13T07:02:44.959307", "level" => "ERROR", "method" => "POST", "path" => "/login" },
433
+ { "time" => "2013/01/13T07:02:45.444992", "level" => "ERROR", "method" => "GET", "path" => "/ping" },
434
+ { "time" => "2013/01/13T07:02:51.247941", "level" => "WARN", "method" => "GET", "path" => "/info" },
435
+ { "time" => "2013/01/13T07:02:53.108366", "level" => "WARN", "method" => "POST", "path" => "/ban" },
436
+ ]
437
+ end
438
+
439
+ test "and/regexp, or/exclude" do
440
+ conf = %[
441
+ <and>
442
+ <regexp>
443
+ key level
444
+ pattern ^ERROR|WARN$
445
+ </regexp>
446
+ <regexp>
447
+ key method
448
+ pattern ^GET|POST$
449
+ </regexp>
450
+ </and>
451
+ <or>
452
+ <exclude>
453
+ key level
454
+ pattern ^WARN$
455
+ </exclude>
456
+ <exclude>
457
+ key method
458
+ pattern ^GET$
459
+ </exclude>
460
+ </or>
461
+ ]
462
+ filtered_records = filter(conf, records)
463
+ assert_equal(records.values_at(4), filtered_records)
464
+ end
465
+
466
+ test "and/regexp, and/exclude" do
467
+ conf = %[
468
+ <and>
469
+ <regexp>
470
+ key level
471
+ pattern ^ERROR|WARN$
472
+ </regexp>
473
+ <regexp>
474
+ key method
475
+ pattern ^GET|POST$
476
+ </regexp>
477
+ </and>
478
+ <and>
479
+ <exclude>
480
+ key level
481
+ pattern ^WARN$
482
+ </exclude>
483
+ <exclude>
484
+ key method
485
+ pattern ^GET$
486
+ </exclude>
487
+ </and>
488
+ ]
489
+ filtered_records = filter(conf, records)
490
+ assert_equal(records.values_at(1, 3, 4, 5, 7), filtered_records)
491
+ end
492
+
493
+ test "or/regexp, and/exclude" do
494
+ conf = %[
495
+ <or>
496
+ <regexp>
497
+ key level
498
+ pattern ^ERROR|WARN$
499
+ </regexp>
500
+ <regexp>
501
+ key method
502
+ pattern ^GET|POST$
503
+ </regexp>
504
+ </or>
505
+ <and>
506
+ <exclude>
507
+ key level
508
+ pattern ^WARN$
509
+ </exclude>
510
+ <exclude>
511
+ key method
512
+ pattern ^GET$
513
+ </exclude>
514
+ </and>
515
+ ]
516
+ filtered_records = filter(conf, records)
517
+ assert_equal(records.values_at(0, 1, 3, 4, 5, 7), filtered_records)
518
+ end
519
+
520
+ test "or/regexp, or/exclude" do
521
+ conf = %[
522
+ <or>
523
+ <regexp>
524
+ key level
525
+ pattern ^ERROR|WARN$
526
+ </regexp>
527
+ <regexp>
528
+ key method
529
+ pattern ^GET|POST$
530
+ </regexp>
531
+ </or>
532
+ <or>
533
+ <exclude>
534
+ key level
535
+ pattern ^WARN$
536
+ </exclude>
537
+ <exclude>
538
+ key method
539
+ pattern ^GET$
540
+ </exclude>
541
+ </or>
542
+ ]
543
+ filtered_records = filter(conf, records)
544
+ assert_equal(records.values_at(4), filtered_records)
545
+ end
546
+
547
+ test "regexp, and/regexp" do
548
+ conf = %[
549
+ <and>
550
+ <regexp>
551
+ key level
552
+ pattern ^ERROR|WARN$
553
+ </regexp>
554
+ <regexp>
555
+ key method
556
+ pattern ^GET|POST$
557
+ </regexp>
558
+ </and>
559
+ <regexp>
560
+ key path
561
+ pattern ^/login$
562
+ </regexp>
563
+ ]
564
+ filtered_records = filter(conf, records)
565
+ assert_equal(records.values_at(3, 4), filtered_records)
566
+ end
567
+
568
+ test "regexp, or/exclude" do
569
+ conf = %[
570
+ <regexp>
571
+ key level
572
+ pattern ^ERROR|WARN$
573
+ </regexp>
574
+ <regexp>
575
+ key method
576
+ pattern ^GET|POST$
577
+ </regexp>
578
+ <or>
579
+ <exclude>
580
+ key level
581
+ pattern ^WARN$
582
+ </exclude>
583
+ <exclude>
584
+ key method
585
+ pattern ^GET$
586
+ </exclude>
587
+ </or>
588
+ ]
589
+ filtered_records = filter(conf, records)
590
+ assert_equal(records.values_at(4), filtered_records)
591
+ end
592
+
593
+ test "regexp, and/exclude" do
594
+ conf = %[
595
+ <regexp>
596
+ key level
597
+ pattern ^ERROR|WARN$
598
+ </regexp>
599
+ <regexp>
600
+ key method
601
+ pattern ^GET|POST$
602
+ </regexp>
603
+ <and>
604
+ <exclude>
605
+ key level
606
+ pattern ^WARN$
607
+ </exclude>
608
+ <exclude>
609
+ key method
610
+ pattern ^GET$
611
+ </exclude>
612
+ </and>
613
+ ]
614
+ filtered_records = filter(conf, records)
615
+ assert_equal(records.values_at(1, 3, 4, 5, 7), filtered_records)
616
+ end
617
+ end
618
+ end
619
+ end
620
+
621
+ sub_test_case 'nested keys' do
622
+ def messages
623
+ [
624
+ {"nest1" => {"nest2" => "INFO"}},
625
+ {"nest1" => {"nest2" => "WARN"}},
626
+ {"nest1" => {"nest2" => "WARN"}}
627
+ ]
628
+ end
629
+
630
+ def filter(config, msgs)
631
+ d = create_driver(config)
632
+ d.run {
633
+ msgs.each { |msg|
634
+ d.feed("filter.test", @time, {'foo' => 'bar', 'message' => msg})
635
+ }
636
+ }
637
+ d.filtered_records
638
+ end
639
+
640
+ test 'regexps' do
641
+ conf = %[
642
+ <regexp>
643
+ key $.message.nest1.nest2
644
+ pattern WARN
645
+ </regexp>
646
+ ]
647
+ filtered_records = filter(conf, messages)
648
+ assert_equal(2, filtered_records.size)
649
+ assert_block('only 2 nested logs') do
650
+ filtered_records.all? { |r|
651
+ r['message']['nest1']['nest2'] == 'WARN'
652
+ }
653
+ end
654
+ end
655
+
656
+ test 'excludes' do
657
+ conf = %[
658
+ <exclude>
659
+ key $.message.nest1.nest2
660
+ pattern WARN
661
+ </exclude>
662
+ ]
663
+ filtered_records = filter(conf, messages)
664
+ assert_equal(1, filtered_records.size)
665
+ assert_block('only 2 nested logs') do
666
+ filtered_records.all? { |r|
667
+ r['message']['nest1']['nest2'] == 'INFO'
668
+ }
669
+ end
670
+ end
93
671
  end
94
672
 
95
673
  sub_test_case 'grep non-string jsonable values' do