fluentd 1.16.5-x64-mingw-ucrt → 1.17.1-x64-mingw-ucrt

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (268) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +88 -0
  3. data/README.md +2 -1
  4. data/Rakefile +1 -1
  5. data/SECURITY.md +2 -2
  6. data/fluent.conf +14 -14
  7. data/lib/fluent/command/binlog_reader.rb +1 -1
  8. data/lib/fluent/command/cap_ctl.rb +4 -4
  9. data/lib/fluent/compat/call_super_mixin.rb +3 -3
  10. data/lib/fluent/compat/propagate_default.rb +4 -4
  11. data/lib/fluent/config/configure_proxy.rb +2 -2
  12. data/lib/fluent/config/types.rb +1 -1
  13. data/lib/fluent/config/yaml_parser/parser.rb +4 -0
  14. data/lib/fluent/configurable.rb +2 -2
  15. data/lib/fluent/counter/mutex_hash.rb +1 -1
  16. data/lib/fluent/fluent_log_event_router.rb +0 -2
  17. data/lib/fluent/log/console_adapter.rb +4 -2
  18. data/lib/fluent/plugin/buf_file.rb +1 -1
  19. data/lib/fluent/plugin/buffer/file_chunk.rb +1 -1
  20. data/lib/fluent/plugin/buffer/file_single_chunk.rb +2 -3
  21. data/lib/fluent/plugin/filter_parser.rb +26 -8
  22. data/lib/fluent/plugin/in_exec.rb +14 -2
  23. data/lib/fluent/plugin/in_http.rb +19 -54
  24. data/lib/fluent/plugin/in_sample.rb +13 -7
  25. data/lib/fluent/plugin/in_tail.rb +99 -25
  26. data/lib/fluent/plugin/out_copy.rb +1 -1
  27. data/lib/fluent/plugin/out_file.rb +8 -0
  28. data/lib/fluent/plugin/out_http.rb +137 -13
  29. data/lib/fluent/plugin/owned_by_mixin.rb +0 -1
  30. data/lib/fluent/plugin/parser_json.rb +26 -17
  31. data/lib/fluent/plugin/parser_msgpack.rb +24 -3
  32. data/lib/fluent/plugin_helper/http_server/server.rb +1 -1
  33. data/lib/fluent/plugin_helper/metrics.rb +2 -2
  34. data/lib/fluent/registry.rb +6 -6
  35. data/lib/fluent/test/output_test.rb +1 -1
  36. data/lib/fluent/unique_id.rb +1 -1
  37. data/lib/fluent/version.rb +1 -1
  38. data/templates/new_gem/fluent-plugin.gemspec.erb +6 -5
  39. metadata +109 -459
  40. data/.github/ISSUE_TEMPLATE/bug_report.yml +0 -71
  41. data/.github/ISSUE_TEMPLATE/config.yml +0 -5
  42. data/.github/ISSUE_TEMPLATE/feature_request.yml +0 -39
  43. data/.github/ISSUE_TEMPLATE.md +0 -17
  44. data/.github/PULL_REQUEST_TEMPLATE.md +0 -14
  45. data/.github/workflows/stale-actions.yml +0 -24
  46. data/.github/workflows/test.yml +0 -32
  47. data/.gitignore +0 -30
  48. data/Gemfile +0 -9
  49. data/fluentd.gemspec +0 -54
  50. data/test/command/test_binlog_reader.rb +0 -362
  51. data/test/command/test_ca_generate.rb +0 -70
  52. data/test/command/test_cap_ctl.rb +0 -100
  53. data/test/command/test_cat.rb +0 -128
  54. data/test/command/test_ctl.rb +0 -56
  55. data/test/command/test_fluentd.rb +0 -1291
  56. data/test/command/test_plugin_config_formatter.rb +0 -397
  57. data/test/command/test_plugin_generator.rb +0 -109
  58. data/test/compat/test_calls_super.rb +0 -166
  59. data/test/compat/test_parser.rb +0 -92
  60. data/test/config/assertions.rb +0 -42
  61. data/test/config/test_config_parser.rb +0 -551
  62. data/test/config/test_configurable.rb +0 -1784
  63. data/test/config/test_configure_proxy.rb +0 -604
  64. data/test/config/test_dsl.rb +0 -415
  65. data/test/config/test_element.rb +0 -518
  66. data/test/config/test_literal_parser.rb +0 -309
  67. data/test/config/test_plugin_configuration.rb +0 -56
  68. data/test/config/test_section.rb +0 -191
  69. data/test/config/test_system_config.rb +0 -195
  70. data/test/config/test_types.rb +0 -408
  71. data/test/counter/test_client.rb +0 -563
  72. data/test/counter/test_error.rb +0 -44
  73. data/test/counter/test_mutex_hash.rb +0 -179
  74. data/test/counter/test_server.rb +0 -589
  75. data/test/counter/test_store.rb +0 -258
  76. data/test/counter/test_validator.rb +0 -137
  77. data/test/helper.rb +0 -155
  78. data/test/helpers/fuzzy_assert.rb +0 -89
  79. data/test/helpers/process_extenstion.rb +0 -33
  80. data/test/log/test_console_adapter.rb +0 -110
  81. data/test/plugin/data/2010/01/20100102-030405.log +0 -0
  82. data/test/plugin/data/2010/01/20100102-030406.log +0 -0
  83. data/test/plugin/data/2010/01/20100102.log +0 -0
  84. data/test/plugin/data/log/bar +0 -0
  85. data/test/plugin/data/log/foo/bar.log +0 -0
  86. data/test/plugin/data/log/foo/bar2 +0 -0
  87. data/test/plugin/data/log/test.log +0 -0
  88. data/test/plugin/data/sd_file/config +0 -11
  89. data/test/plugin/data/sd_file/config.json +0 -17
  90. data/test/plugin/data/sd_file/config.yaml +0 -11
  91. data/test/plugin/data/sd_file/config.yml +0 -11
  92. data/test/plugin/data/sd_file/invalid_config.yml +0 -7
  93. data/test/plugin/in_tail/test_fifo.rb +0 -121
  94. data/test/plugin/in_tail/test_io_handler.rb +0 -150
  95. data/test/plugin/in_tail/test_position_file.rb +0 -346
  96. data/test/plugin/out_forward/test_ack_handler.rb +0 -140
  97. data/test/plugin/out_forward/test_connection_manager.rb +0 -145
  98. data/test/plugin/out_forward/test_handshake_protocol.rb +0 -112
  99. data/test/plugin/out_forward/test_load_balancer.rb +0 -106
  100. data/test/plugin/out_forward/test_socket_cache.rb +0 -174
  101. data/test/plugin/test_bare_output.rb +0 -131
  102. data/test/plugin/test_base.rb +0 -247
  103. data/test/plugin/test_buf_file.rb +0 -1314
  104. data/test/plugin/test_buf_file_single.rb +0 -898
  105. data/test/plugin/test_buf_memory.rb +0 -42
  106. data/test/plugin/test_buffer.rb +0 -1493
  107. data/test/plugin/test_buffer_chunk.rb +0 -209
  108. data/test/plugin/test_buffer_file_chunk.rb +0 -871
  109. data/test/plugin/test_buffer_file_single_chunk.rb +0 -611
  110. data/test/plugin/test_buffer_memory_chunk.rb +0 -339
  111. data/test/plugin/test_compressable.rb +0 -87
  112. data/test/plugin/test_file_util.rb +0 -96
  113. data/test/plugin/test_filter.rb +0 -368
  114. data/test/plugin/test_filter_grep.rb +0 -697
  115. data/test/plugin/test_filter_parser.rb +0 -731
  116. data/test/plugin/test_filter_record_transformer.rb +0 -577
  117. data/test/plugin/test_filter_stdout.rb +0 -207
  118. data/test/plugin/test_formatter_csv.rb +0 -136
  119. data/test/plugin/test_formatter_hash.rb +0 -38
  120. data/test/plugin/test_formatter_json.rb +0 -61
  121. data/test/plugin/test_formatter_ltsv.rb +0 -70
  122. data/test/plugin/test_formatter_msgpack.rb +0 -28
  123. data/test/plugin/test_formatter_out_file.rb +0 -116
  124. data/test/plugin/test_formatter_single_value.rb +0 -44
  125. data/test/plugin/test_formatter_tsv.rb +0 -76
  126. data/test/plugin/test_in_debug_agent.rb +0 -49
  127. data/test/plugin/test_in_exec.rb +0 -261
  128. data/test/plugin/test_in_forward.rb +0 -1178
  129. data/test/plugin/test_in_gc_stat.rb +0 -62
  130. data/test/plugin/test_in_http.rb +0 -1102
  131. data/test/plugin/test_in_monitor_agent.rb +0 -922
  132. data/test/plugin/test_in_object_space.rb +0 -66
  133. data/test/plugin/test_in_sample.rb +0 -190
  134. data/test/plugin/test_in_syslog.rb +0 -505
  135. data/test/plugin/test_in_tail.rb +0 -3288
  136. data/test/plugin/test_in_tcp.rb +0 -328
  137. data/test/plugin/test_in_udp.rb +0 -296
  138. data/test/plugin/test_in_unix.rb +0 -181
  139. data/test/plugin/test_input.rb +0 -137
  140. data/test/plugin/test_metadata.rb +0 -89
  141. data/test/plugin/test_metrics.rb +0 -294
  142. data/test/plugin/test_metrics_local.rb +0 -96
  143. data/test/plugin/test_multi_output.rb +0 -204
  144. data/test/plugin/test_out_copy.rb +0 -308
  145. data/test/plugin/test_out_exec.rb +0 -312
  146. data/test/plugin/test_out_exec_filter.rb +0 -606
  147. data/test/plugin/test_out_file.rb +0 -1038
  148. data/test/plugin/test_out_forward.rb +0 -1349
  149. data/test/plugin/test_out_http.rb +0 -429
  150. data/test/plugin/test_out_null.rb +0 -105
  151. data/test/plugin/test_out_relabel.rb +0 -28
  152. data/test/plugin/test_out_roundrobin.rb +0 -146
  153. data/test/plugin/test_out_secondary_file.rb +0 -458
  154. data/test/plugin/test_out_stdout.rb +0 -205
  155. data/test/plugin/test_out_stream.rb +0 -103
  156. data/test/plugin/test_output.rb +0 -1334
  157. data/test/plugin/test_output_as_buffered.rb +0 -2024
  158. data/test/plugin/test_output_as_buffered_backup.rb +0 -363
  159. data/test/plugin/test_output_as_buffered_compress.rb +0 -179
  160. data/test/plugin/test_output_as_buffered_overflow.rb +0 -250
  161. data/test/plugin/test_output_as_buffered_retries.rb +0 -966
  162. data/test/plugin/test_output_as_buffered_secondary.rb +0 -882
  163. data/test/plugin/test_output_as_standard.rb +0 -374
  164. data/test/plugin/test_owned_by.rb +0 -35
  165. data/test/plugin/test_parser.rb +0 -399
  166. data/test/plugin/test_parser_apache.rb +0 -42
  167. data/test/plugin/test_parser_apache2.rb +0 -47
  168. data/test/plugin/test_parser_apache_error.rb +0 -45
  169. data/test/plugin/test_parser_csv.rb +0 -200
  170. data/test/plugin/test_parser_json.rb +0 -138
  171. data/test/plugin/test_parser_labeled_tsv.rb +0 -160
  172. data/test/plugin/test_parser_multiline.rb +0 -111
  173. data/test/plugin/test_parser_nginx.rb +0 -88
  174. data/test/plugin/test_parser_none.rb +0 -52
  175. data/test/plugin/test_parser_regexp.rb +0 -284
  176. data/test/plugin/test_parser_syslog.rb +0 -650
  177. data/test/plugin/test_parser_tsv.rb +0 -122
  178. data/test/plugin/test_sd_file.rb +0 -228
  179. data/test/plugin/test_sd_srv.rb +0 -230
  180. data/test/plugin/test_storage.rb +0 -167
  181. data/test/plugin/test_storage_local.rb +0 -335
  182. data/test/plugin/test_string_util.rb +0 -26
  183. data/test/plugin_helper/data/cert/cert-key.pem +0 -27
  184. data/test/plugin_helper/data/cert/cert-with-CRLF.pem +0 -19
  185. data/test/plugin_helper/data/cert/cert-with-no-newline.pem +0 -19
  186. data/test/plugin_helper/data/cert/cert.pem +0 -19
  187. data/test/plugin_helper/data/cert/cert_chains/ca-cert-key.pem +0 -27
  188. data/test/plugin_helper/data/cert/cert_chains/ca-cert.pem +0 -20
  189. data/test/plugin_helper/data/cert/cert_chains/cert-key.pem +0 -27
  190. data/test/plugin_helper/data/cert/cert_chains/cert.pem +0 -40
  191. data/test/plugin_helper/data/cert/empty.pem +0 -0
  192. data/test/plugin_helper/data/cert/generate_cert.rb +0 -125
  193. data/test/plugin_helper/data/cert/with_ca/ca-cert-key-pass.pem +0 -30
  194. data/test/plugin_helper/data/cert/with_ca/ca-cert-key.pem +0 -27
  195. data/test/plugin_helper/data/cert/with_ca/ca-cert-pass.pem +0 -20
  196. data/test/plugin_helper/data/cert/with_ca/ca-cert.pem +0 -20
  197. data/test/plugin_helper/data/cert/with_ca/cert-key-pass.pem +0 -30
  198. data/test/plugin_helper/data/cert/with_ca/cert-key.pem +0 -27
  199. data/test/plugin_helper/data/cert/with_ca/cert-pass.pem +0 -21
  200. data/test/plugin_helper/data/cert/with_ca/cert.pem +0 -21
  201. data/test/plugin_helper/data/cert/without_ca/cert-key-pass.pem +0 -30
  202. data/test/plugin_helper/data/cert/without_ca/cert-key.pem +0 -27
  203. data/test/plugin_helper/data/cert/without_ca/cert-pass.pem +0 -20
  204. data/test/plugin_helper/data/cert/without_ca/cert.pem +0 -20
  205. data/test/plugin_helper/http_server/test_app.rb +0 -65
  206. data/test/plugin_helper/http_server/test_route.rb +0 -32
  207. data/test/plugin_helper/service_discovery/test_manager.rb +0 -93
  208. data/test/plugin_helper/service_discovery/test_round_robin_balancer.rb +0 -21
  209. data/test/plugin_helper/test_cert_option.rb +0 -25
  210. data/test/plugin_helper/test_child_process.rb +0 -862
  211. data/test/plugin_helper/test_compat_parameters.rb +0 -358
  212. data/test/plugin_helper/test_event_emitter.rb +0 -80
  213. data/test/plugin_helper/test_event_loop.rb +0 -52
  214. data/test/plugin_helper/test_extract.rb +0 -194
  215. data/test/plugin_helper/test_formatter.rb +0 -255
  216. data/test/plugin_helper/test_http_server_helper.rb +0 -372
  217. data/test/plugin_helper/test_inject.rb +0 -561
  218. data/test/plugin_helper/test_metrics.rb +0 -137
  219. data/test/plugin_helper/test_parser.rb +0 -264
  220. data/test/plugin_helper/test_record_accessor.rb +0 -238
  221. data/test/plugin_helper/test_retry_state.rb +0 -1006
  222. data/test/plugin_helper/test_server.rb +0 -1895
  223. data/test/plugin_helper/test_service_discovery.rb +0 -165
  224. data/test/plugin_helper/test_socket.rb +0 -146
  225. data/test/plugin_helper/test_storage.rb +0 -542
  226. data/test/plugin_helper/test_thread.rb +0 -164
  227. data/test/plugin_helper/test_timer.rb +0 -130
  228. data/test/scripts/exec_script.rb +0 -32
  229. data/test/scripts/fluent/plugin/formatter1/formatter_test1.rb +0 -7
  230. data/test/scripts/fluent/plugin/formatter2/formatter_test2.rb +0 -7
  231. data/test/scripts/fluent/plugin/formatter_known.rb +0 -8
  232. data/test/scripts/fluent/plugin/out_test.rb +0 -81
  233. data/test/scripts/fluent/plugin/out_test2.rb +0 -80
  234. data/test/scripts/fluent/plugin/parser_known.rb +0 -4
  235. data/test/test_capability.rb +0 -74
  236. data/test/test_clock.rb +0 -164
  237. data/test/test_config.rb +0 -369
  238. data/test/test_configdsl.rb +0 -148
  239. data/test/test_daemonizer.rb +0 -91
  240. data/test/test_engine.rb +0 -203
  241. data/test/test_event.rb +0 -531
  242. data/test/test_event_router.rb +0 -348
  243. data/test/test_event_time.rb +0 -199
  244. data/test/test_file_wrapper.rb +0 -53
  245. data/test/test_filter.rb +0 -121
  246. data/test/test_fluent_log_event_router.rb +0 -99
  247. data/test/test_formatter.rb +0 -369
  248. data/test/test_input.rb +0 -31
  249. data/test/test_log.rb +0 -1076
  250. data/test/test_match.rb +0 -148
  251. data/test/test_mixin.rb +0 -351
  252. data/test/test_msgpack_factory.rb +0 -50
  253. data/test/test_oj_options.rb +0 -55
  254. data/test/test_output.rb +0 -278
  255. data/test/test_plugin.rb +0 -251
  256. data/test/test_plugin_classes.rb +0 -370
  257. data/test/test_plugin_helper.rb +0 -81
  258. data/test/test_plugin_id.rb +0 -119
  259. data/test/test_process.rb +0 -14
  260. data/test/test_root_agent.rb +0 -951
  261. data/test/test_static_config_analysis.rb +0 -177
  262. data/test/test_supervisor.rb +0 -821
  263. data/test/test_test_drivers.rb +0 -136
  264. data/test/test_time_formatter.rb +0 -301
  265. data/test/test_time_parser.rb +0 -362
  266. data/test/test_tls.rb +0 -65
  267. data/test/test_unique_id.rb +0 -47
  268. data/test/test_variable_store.rb +0 -65
@@ -1,200 +0,0 @@
1
- require_relative '../helper'
2
- require 'fluent/test/driver/parser'
3
- require 'fluent/plugin/parser_csv'
4
-
5
- class CSVParserTest < ::Test::Unit::TestCase
6
- def setup
7
- Fluent::Test.setup
8
- end
9
-
10
- def create_driver(conf={})
11
- Fluent::Test::Driver::Parser.new(Fluent::Plugin::CSVParser).configure(conf)
12
- end
13
-
14
- data('array param' => '["time","c","d"]', 'string param' => 'time,c,d')
15
- def test_parse(param)
16
- d = create_driver('keys' => param, 'time_key' => 'time')
17
- d.instance.parse("2013/02/28 12:00:00,192.168.0.1,111") { |time, record|
18
- assert_equal(event_time('2013/02/28 12:00:00', format: '%Y/%m/%d %H:%M:%S'), time)
19
- assert_equal({
20
- 'c' => '192.168.0.1',
21
- 'd' => '111',
22
- }, record)
23
- }
24
- end
25
-
26
- data('array param' => '["c","d"]', 'string param' => 'c,d')
27
- def test_parse_without_time(param)
28
- time_at_start = Time.now.to_i
29
-
30
- d = create_driver('keys' => param)
31
- d.instance.parse("192.168.0.1,111") { |time, record|
32
- assert time && time >= time_at_start, "parser puts current time without time input"
33
- assert_equal({
34
- 'c' => '192.168.0.1',
35
- 'd' => '111',
36
- }, record)
37
- }
38
-
39
- d = Fluent::Test::Driver::Parser.new(Fluent::Plugin::CSVParser)
40
- d.configure('keys' => param, 'estimate_current_event' => 'no')
41
- d.instance.parse("192.168.0.1,111") { |time, record|
42
- assert_equal({
43
- 'c' => '192.168.0.1',
44
- 'd' => '111',
45
- }, record)
46
- assert_nil time, "parser returns nil w/o time and if configured so"
47
- }
48
- end
49
-
50
- def test_parse_with_keep_time_key
51
- d = create_driver(
52
- 'keys'=>'time',
53
- 'time_key'=>'time',
54
- 'time_format'=>"%d/%b/%Y:%H:%M:%S %z",
55
- 'keep_time_key'=>'true',
56
- )
57
- text = '28/Feb/2013:12:00:00 +0900'
58
- d.instance.parse(text) do |time, record|
59
- assert_equal text, record['time']
60
- end
61
- end
62
-
63
- data('array param' => '["a","b","c","d","e","f"]', 'string param' => 'a,b,c,d,e,f')
64
- def test_parse_with_null_value_pattern(param)
65
- d = create_driver(
66
- 'keys'=>param,
67
- 'null_value_pattern'=>'^(-|null|NULL)$'
68
- )
69
- d.instance.parse("-,null,NULL,,--,nuLL") do |time, record|
70
- assert_nil record['a']
71
- assert_nil record['b']
72
- assert_nil record['c']
73
- assert_nil record['d']
74
- assert_equal record['e'], '--'
75
- assert_equal record['f'], 'nuLL'
76
- end
77
- end
78
-
79
- data('array param' => '["a","b"]', 'string param' => 'a,b')
80
- def test_parse_with_null_empty_string(param)
81
- d = create_driver(
82
- 'keys'=>param,
83
- 'null_empty_string'=>true
84
- )
85
- d.instance.parse(", ") do |time, record|
86
- assert_nil record['a']
87
- assert_equal record['b'], ' '
88
- end
89
- end
90
-
91
- data('array param' => '["a","b","c"]', 'string param' => 'a,b,c')
92
- def test_parse_with_option_delimiter(param)
93
- d = create_driver(
94
- 'keys'=>param,
95
- 'delimiter'=>' ',
96
- )
97
- d.instance.parse("123 456 789") do |time, record|
98
- assert_equal record['a'], '123'
99
- assert_equal record['b'], '456'
100
- assert_equal record['c'], '789'
101
- end
102
- end
103
-
104
- sub_test_case 'parser' do
105
- data('normal' => 'normal',
106
- 'fast' => 'fast')
107
- def test_compatibility_between_normal_and_fast_parser(param)
108
- d = create_driver(
109
- 'keys' => 'time,key1,key2,key3,key4,key5',
110
- 'time_key' => 'time',
111
- 'time_format' => "%d/%b/%Y:%H:%M:%S %z",
112
- 'keep_time_key' => 'false',
113
- 'parser_type' => param
114
- )
115
-
116
- # non quoted
117
- text = '28/Feb/2013:12:00:00 +0900,value1,value2,value3,value4,value5'
118
- expected = {'key1' => 'value1', 'key2' => 'value2', 'key3' => "value3",
119
- 'key4' => 'value4', 'key5' => "value5"}
120
- d.instance.parse(text) do |time, record|
121
- assert_equal(event_time("28/Feb/2013:12:00:00 +0900", format: '%d/%b/%Y:%H:%M:%S %z'), time)
122
- assert_equal expected, record
123
- end
124
-
125
- # quoted
126
- text = '28/Feb/2013:12:00:00 +0900,"value1","val,ue2","va,lu,e3","val ue4",""'
127
- expected = {'key1' => 'value1', 'key2' => 'val,ue2', 'key3' => "va,lu,e3",
128
- 'key4' => 'val ue4', 'key5' => ""}
129
- d.instance.parse(text) do |time, record|
130
- assert_equal(event_time("28/Feb/2013:12:00:00 +0900", format: '%d/%b/%Y:%H:%M:%S %z'), time)
131
- assert_equal expected, record
132
- end
133
-
134
- # mixed
135
- text = '28/Feb/2013:12:00:00 +0900,message,"mes,sage","me,ssa,ge",mess age,""'
136
- expected = {'key1' => 'message', 'key2' => 'mes,sage', 'key3' => "me,ssa,ge",
137
- 'key4' => 'mess age', 'key5' => ""}
138
- d.instance.parse(text) do |time, record|
139
- assert_equal(event_time("28/Feb/2013:12:00:00 +0900", format: '%d/%b/%Y:%H:%M:%S %z'), time)
140
- assert_equal expected, record
141
- end
142
-
143
- # escaped
144
- text = '28/Feb/2013:12:00:00 +0900,"message","mes""sage","""message""",,""""""'
145
- expected = {'key1' => 'message', 'key2' => 'mes"sage', 'key3' => '"message"',
146
- 'key4' => nil, 'key5' => '""'}
147
- d.instance.parse(text) do |time, record|
148
- assert_equal(event_time("28/Feb/2013:12:00:00 +0900", format: '%d/%b/%Y:%H:%M:%S %z'), time)
149
- assert_equal expected, record
150
- end
151
- end
152
-
153
- def test_incompatibility_between_normal_and_fast_parser
154
- normal = create_driver(
155
- 'keys' => 'key1,key2',
156
- 'parser_type' => 'normal'
157
- )
158
- fast = create_driver(
159
- 'keys' => 'key1,key2',
160
- 'parser_type' => 'fast'
161
- )
162
-
163
- # unexpected quote position
164
- text = 'a"b,"a"""c"'
165
- assert_raise(CSV::MalformedCSVError) {
166
- normal.instance.parse(text) { |t, r| }
167
- }
168
- assert_nothing_raised {
169
- # generate broken record
170
- fast.instance.parse(text) { |t, r| }
171
- }
172
-
173
- # incorrect the number of column
174
- text = 'a,b,c'
175
- expected = {"key1" => 'a', "key2" => 'b'}
176
- normal.instance.parse(text) { |t, r|
177
- assert_equal expected, r
178
- }
179
- fast.instance.parse(text) { |t, r|
180
- assert_not_equal expected, r
181
- }
182
-
183
- # And more...
184
- end
185
- end
186
-
187
- # "parser_type" config shouldn't hide Fluent::Plugin::Parser#plugin_type
188
- # https://github.com/fluent/fluentd/issues/3296
189
- data('normal' => :normal, 'fast' => :fast)
190
- def test_parser_type_method(engine)
191
- d = create_driver('keys' => '["time"]','time_key' => 'time', 'parser_type' => engine.to_s)
192
- assert_equal(:text_per_line, d.instance.parser_type)
193
- end
194
-
195
- data('normal' => :normal, 'fast' => :fast)
196
- def test_parser_engine(engine)
197
- d = create_driver('keys' => '["time"]', 'time_key' => 'time', 'parser_engine' => engine.to_s)
198
- assert_equal(engine, d.instance.parser_engine)
199
- end
200
- end
@@ -1,138 +0,0 @@
1
- require_relative '../helper'
2
- require 'fluent/test/driver/parser'
3
- require 'fluent/plugin/parser'
4
-
5
- class JsonParserTest < ::Test::Unit::TestCase
6
- def setup
7
- Fluent::Test.setup
8
- @parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::JSONParser)
9
- end
10
-
11
- data('oj' => 'oj', 'yajl' => 'yajl')
12
- def test_parse(data)
13
- @parser.configure('json_parser' => data)
14
- @parser.instance.parse('{"time":1362020400,"host":"192.168.0.1","size":777,"method":"PUT"}') { |time, record|
15
- assert_equal(event_time('2013-02-28 12:00:00 +0900').to_i, time)
16
- assert_equal({
17
- 'host' => '192.168.0.1',
18
- 'size' => 777,
19
- 'method' => 'PUT',
20
- }, record)
21
- }
22
- end
23
-
24
- data('oj' => 'oj', 'yajl' => 'yajl')
25
- def test_parse_with_large_float(data)
26
- @parser.configure('json_parser' => data)
27
- @parser.instance.parse('{"num":999999999999999999999999999999.99999}') { |time, record|
28
- assert_equal(Float, record['num'].class)
29
- }
30
- end
31
-
32
- data('oj' => 'oj', 'yajl' => 'yajl')
33
- def test_parse_without_time(data)
34
- time_at_start = Time.now.to_i
35
-
36
- @parser.configure('json_parser' => data)
37
- @parser.instance.parse('{"host":"192.168.0.1","size":777,"method":"PUT"}') { |time, record|
38
- assert time && time >= time_at_start, "parser puts current time without time input"
39
- assert_equal({
40
- 'host' => '192.168.0.1',
41
- 'size' => 777,
42
- 'method' => 'PUT',
43
- }, record)
44
- }
45
-
46
- parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::JSONParser)
47
- parser.configure('json_parser' => data, 'estimate_current_event' => 'false')
48
- parser.instance.parse('{"host":"192.168.0.1","size":777,"method":"PUT"}') { |time, record|
49
- assert_equal({
50
- 'host' => '192.168.0.1',
51
- 'size' => 777,
52
- 'method' => 'PUT',
53
- }, record)
54
- assert_nil time, "parser return nil w/o time and if specified so"
55
- }
56
- end
57
-
58
- data('oj' => 'oj', 'yajl' => 'yajl')
59
- def test_parse_with_colon_string(data)
60
- @parser.configure('json_parser' => data)
61
- @parser.instance.parse('{"time":1362020400,"log":":message"}') { |time, record|
62
- assert_equal(record['log'], ':message')
63
- }
64
- end
65
-
66
- data('oj' => 'oj', 'yajl' => 'yajl')
67
- def test_parse_with_invalid_time(data)
68
- @parser.configure('json_parser' => data)
69
- assert_raise Fluent::ParserError do
70
- @parser.instance.parse('{"time":[],"k":"v"}') { |time, record| }
71
- end
72
- end
73
-
74
- data('oj' => 'oj', 'yajl' => 'yajl')
75
- def test_parse_float_time(data)
76
- parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::JSONParser)
77
- parser.configure('json_parser' => data)
78
- text = "100.1"
79
- parser.instance.parse("{\"time\":\"#{text}\"}") do |time, record|
80
- assert_equal 100, time.sec
81
- assert_equal 100_000_000, time.nsec
82
- end
83
- end
84
-
85
- data('oj' => 'oj', 'yajl' => 'yajl')
86
- def test_parse_with_keep_time_key(data)
87
- parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::JSONParser)
88
- format = "%d/%b/%Y:%H:%M:%S %z"
89
- parser.configure(
90
- 'time_format' => format,
91
- 'keep_time_key' => 'true',
92
- 'json_parser' => data
93
- )
94
- text = "28/Feb/2013:12:00:00 +0900"
95
- parser.instance.parse("{\"time\":\"#{text}\"}") do |time, record|
96
- assert_equal Time.strptime(text, format).to_i, time.sec
97
- assert_equal text, record['time']
98
- end
99
- end
100
-
101
- data('oj' => 'oj', 'yajl' => 'yajl')
102
- def test_parse_with_keep_time_key_without_time_format(data)
103
- parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::JSONParser)
104
- parser.configure(
105
- 'keep_time_key' => 'true',
106
- 'json_parser' => data
107
- )
108
- text = "100"
109
- parser.instance.parse("{\"time\":\"#{text}\"}") do |time, record|
110
- assert_equal text.to_i, time.sec
111
- assert_equal text, record['time']
112
- end
113
- end
114
-
115
- def test_yajl_parse_io_with_buffer_smaller_than_input
116
- parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::JSONParser)
117
- parser.configure(
118
- 'keep_time_key' => 'true',
119
- 'json_parser' => 'yajl',
120
- 'stream_buffer_size' => 1,
121
- )
122
- text = "100"
123
-
124
- waiting(5) do
125
- rd, wr = IO.pipe
126
- wr.write "{\"time\":\"#{text}\"}"
127
-
128
- parser.instance.parse_io(rd) do |time, record|
129
- assert_equal text.to_i, time.sec
130
- assert_equal text, record['time']
131
-
132
- # Once a record has been received the 'write' end of the pipe must be
133
- # closed, otherwise the test will block waiting for more input.
134
- wr.close
135
- end
136
- end
137
- end
138
- end
@@ -1,160 +0,0 @@
1
- require_relative '../helper'
2
- require 'fluent/test/driver/parser'
3
- require 'fluent/plugin/parser'
4
-
5
- class LabeledTSVParserTest < ::Test::Unit::TestCase
6
- def setup
7
- Fluent::Test.setup
8
- end
9
-
10
- def test_config_params
11
- parser = Fluent::Test::Driver::Parser.new(Fluent::TextParser::LabeledTSVParser)
12
-
13
- assert_equal "\t", parser.instance.delimiter
14
- assert_equal ":", parser.instance.label_delimiter
15
-
16
- parser.configure(
17
- 'delimiter' => ',',
18
- 'label_delimiter' => '=',
19
- )
20
-
21
- assert_equal ",", parser.instance.delimiter
22
- assert_equal "=", parser.instance.label_delimiter
23
- end
24
-
25
- def test_parse
26
- parser = Fluent::Test::Driver::Parser.new(Fluent::TextParser::LabeledTSVParser)
27
- parser.configure({})
28
- parser.instance.parse("time:2013/02/28 12:00:00\thost:192.168.0.1\treq_id:111") { |time, record|
29
- assert_equal(event_time('2013/02/28 12:00:00', format: '%Y/%m/%d %H:%M:%S'), time)
30
- assert_equal({
31
- 'host' => '192.168.0.1',
32
- 'req_id' => '111',
33
- }, record)
34
- }
35
- end
36
-
37
- def test_parse_with_customized_delimiter
38
- parser = Fluent::Test::Driver::Parser.new(Fluent::TextParser::LabeledTSVParser)
39
- parser.configure(
40
- 'delimiter' => ',',
41
- 'label_delimiter' => '=',
42
- )
43
- parser.instance.parse('time=2013/02/28 12:00:00,host=192.168.0.1,req_id=111') { |time, record|
44
- assert_equal(event_time('2013/02/28 12:00:00', format: '%Y/%m/%d %H:%M:%S'), time)
45
- assert_equal({
46
- 'host' => '192.168.0.1',
47
- 'req_id' => '111',
48
- }, record)
49
- }
50
- end
51
-
52
- def test_parse_with_customized_time_format
53
- parser = Fluent::Test::Driver::Parser.new(Fluent::TextParser::LabeledTSVParser)
54
- parser.configure(
55
- 'time_key' => 'mytime',
56
- 'time_format' => '%d/%b/%Y:%H:%M:%S %z',
57
- )
58
- parser.instance.parse("mytime:28/Feb/2013:12:00:00 +0900\thost:192.168.0.1\treq_id:111") { |time, record|
59
- assert_equal(event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'), time)
60
- assert_equal({
61
- 'host' => '192.168.0.1',
62
- 'req_id' => '111',
63
- }, record)
64
- }
65
- end
66
-
67
- def test_parse_without_time
68
- time_at_start = Time.now.to_i
69
-
70
- parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::LabeledTSVParser)
71
- parser.configure({})
72
- parser.instance.parse("host:192.168.0.1\treq_id:111") { |time, record|
73
- assert time && time >= time_at_start, "parser puts current time without time input"
74
- assert_equal({
75
- 'host' => '192.168.0.1',
76
- 'req_id' => '111',
77
- }, record)
78
- }
79
-
80
- parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::LabeledTSVParser)
81
- parser.configure({'estimate_current_event' => 'no'})
82
- parser.instance.parse("host:192.168.0.1\treq_id:111") { |time, record|
83
- assert_equal({
84
- 'host' => '192.168.0.1',
85
- 'req_id' => '111',
86
- }, record)
87
- assert_nil time, "parser returns nil w/o time and if configured so"
88
- }
89
- end
90
-
91
- def test_parse_with_keep_time_key
92
- parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::LabeledTSVParser)
93
- parser.configure(
94
- 'time_format'=>"%d/%b/%Y:%H:%M:%S %z",
95
- 'keep_time_key'=>'true',
96
- )
97
- text = '28/Feb/2013:12:00:00 +0900'
98
- parser.instance.parse("time:#{text}") do |time, record|
99
- assert_equal text, record['time']
100
- end
101
- end
102
-
103
- def test_parse_and_reject_invalid_kv_pairs
104
- parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::LabeledTSVParser)
105
- parser.configure(
106
- 'delimiter' => ' ',
107
- 'label_delimiter' => '=',
108
- )
109
- text = 'A leading portion that is not LTSV : foo=bar baz=derp and a trailing portion'
110
-
111
- expected = {'foo' => 'bar', 'baz' => 'derp'}
112
- parser.instance.parse(text) do |time, record|
113
- assert_equal expected, record
114
- end
115
-
116
- end
117
-
118
- def test_parse_with_null_value_pattern
119
- parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::LabeledTSVParser)
120
- parser.configure(
121
- 'null_value_pattern'=>'^(-|null|NULL)$'
122
- )
123
- parser.instance.parse("a:-\tb:null\tc:NULL\td:\te:--\tf:nuLL") do |time, record|
124
- assert_nil record['a']
125
- assert_nil record['b']
126
- assert_nil record['c']
127
- assert_equal record['d'], ''
128
- assert_equal record['e'], '--'
129
- assert_equal record['f'], 'nuLL'
130
- end
131
- end
132
-
133
- def test_parse_with_null_empty_string
134
- parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::LabeledTSVParser)
135
- parser.configure(
136
- 'null_empty_string'=>true
137
- )
138
- parser.instance.parse("a:\tb: ") do |time, record|
139
- assert_nil record['a']
140
- assert_equal record['b'], ' '
141
- end
142
- end
143
-
144
- data("single space" => ["k1=v1 k2=v2", { "k1" => "v1", "k2" => "v2" }],
145
- "multiple space" => ["k1=v1 k2=v2", { "k1" => "v1", "k2" => "v2" }],
146
- "reverse" => ["k2=v2 k1=v1", { "k1" => "v1", "k2" => "v2" }],
147
- "tab" => ["k2=v2\tk1=v1", { "k1" => "v1", "k2" => "v2" }],
148
- "tab and space" => ["k2=v2\t k1=v1", { "k1" => "v1", "k2" => "v2" }])
149
- def test_parse_with_delimiter_pattern(data)
150
- text, expected = data
151
- parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::LabeledTSVParser)
152
- parser.configure(
153
- 'delimiter_pattern' => '/\s+/',
154
- 'label_delimiter' => '='
155
- )
156
- parser.instance.parse(text) do |_time, record|
157
- assert_equal(expected, record)
158
- end
159
- end
160
- end
@@ -1,111 +0,0 @@
1
- require_relative '../helper'
2
- require 'fluent/test/driver/parser'
3
- require 'fluent/plugin/parser'
4
-
5
- class MultilineParserTest < ::Test::Unit::TestCase
6
- def setup
7
- Fluent::Test.setup
8
- end
9
-
10
- def create_parser(conf)
11
- parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::MultilineParser).configure(conf)
12
- parser
13
- end
14
-
15
- def test_configure_with_invalid_params
16
- [{'format100' => '/(?<msg>.*)/'}, {'format1' => '/(?<msg>.*)/', 'format3' => '/(?<msg>.*)/'}, 'format1' => '/(?<msg>.*)'].each { |config|
17
- assert_raise(Fluent::ConfigError) {
18
- create_parser(config)
19
- }
20
- }
21
- end
22
-
23
- def test_parse
24
- parser = create_parser('format1' => '/^(?<time>\d{4}-\d{1,2}-\d{1,2} \d{1,2}:\d{1,2}:\d{1,2}) \[(?<thread>.*)\] (?<level>[^\s]+)(?<message>.*)/')
25
- parser.instance.parse(<<EOS.chomp) { |time, record|
26
- 2013-3-03 14:27:33 [main] ERROR Main - Exception
27
- javax.management.RuntimeErrorException: null
28
- \tat Main.main(Main.java:16) ~[bin/:na]
29
- EOS
30
-
31
- assert_equal(event_time('2013-3-03 14:27:33').to_i, time)
32
- assert_equal({
33
- "thread" => "main",
34
- "level" => "ERROR",
35
- "message" => " Main - Exception\njavax.management.RuntimeErrorException: null\n\tat Main.main(Main.java:16) ~[bin/:na]"
36
- }, record)
37
- }
38
- end
39
-
40
- def test_parse_with_firstline
41
- parser = create_parser('format_firstline' => '/----/', 'format1' => '/time=(?<time>\d{4}-\d{1,2}-\d{1,2} \d{1,2}:\d{1,2}:\d{1,2}).*message=(?<message>.*)/')
42
- parser.instance.parse(<<EOS.chomp) { |time, record|
43
- ----
44
- time=2013-3-03 14:27:33
45
- message=test1
46
- EOS
47
-
48
- assert(parser.instance.firstline?('----'))
49
- assert_equal(event_time('2013-3-03 14:27:33').to_i, time)
50
- assert_equal({"message" => "test1"}, record)
51
- }
52
- end
53
-
54
- def test_parse_with_multiple_formats
55
- parser = create_parser('format_firstline' => '/^Started/',
56
- 'format1' => '/Started (?<method>[^ ]+) "(?<path>[^"]+)" for (?<host>[^ ]+) at (?<time>[^ ]+ [^ ]+ [^ ]+)\n/',
57
- 'format2' => '/Processing by (?<controller>[^\u0023]+)\u0023(?<controller_method>[^ ]+) as (?<format>[^ ]+?)\n/',
58
- 'format3' => '/( Parameters: (?<parameters>[^ ]+)\n)?/',
59
- 'format4' => '/ Rendered (?<template>[^ ]+) within (?<layout>.+) \([\d\.]+ms\)\n/',
60
- 'format5' => '/Completed (?<code>[^ ]+) [^ ]+ in (?<runtime>[\d\.]+)ms \(Views: (?<view_runtime>[\d\.]+)ms \| ActiveRecord: (?<ar_runtime>[\d\.]+)ms\)/'
61
- )
62
- parser.instance.parse(<<EOS.chomp) { |time, record|
63
- Started GET "/users/123/" for 127.0.0.1 at 2013-06-14 12:00:11 +0900
64
- Processing by UsersController#show as HTML
65
- Parameters: {"user_id"=>"123"}
66
- Rendered users/show.html.erb within layouts/application (0.3ms)
67
- Completed 200 OK in 4ms (Views: 3.2ms | ActiveRecord: 0.0ms)
68
- EOS
69
-
70
- assert(parser.instance.firstline?('Started GET "/users/123/" for 127.0.0.1...'))
71
- assert_equal(event_time('2013-06-14 12:00:11 +0900').to_i, time)
72
- assert_equal({
73
- "method" => "GET",
74
- "path" => "/users/123/",
75
- "host" => "127.0.0.1",
76
- "controller" => "UsersController",
77
- "controller_method" => "show",
78
- "format" => "HTML",
79
- "parameters" => "{\"user_id\"=>\"123\"}",
80
- "template" => "users/show.html.erb",
81
- "layout" => "layouts/application",
82
- "code" => "200",
83
- "runtime" => "4",
84
- "view_runtime" => "3.2",
85
- "ar_runtime" => "0.0"
86
- }, record)
87
- }
88
- end
89
-
90
- def test_parse_with_keep_time_key
91
- parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::MultilineParser).configure(
92
- 'format1' => '/^(?<time>\d{4}-\d{1,2}-\d{1,2} \d{1,2}:\d{1,2}:\d{1,2})/',
93
- 'keep_time_key' => 'true'
94
- )
95
- text = '2013-3-03 14:27:33'
96
- parser.instance.parse(text) { |time, record|
97
- assert_equal text, record['time']
98
- }
99
- end
100
-
101
- def test_parse_unmatched_lines
102
- parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::MultilineParser).configure(
103
- 'format1' => '/^message (?<message_id>\d)/',
104
- 'unmatched_lines' => true,
105
- )
106
- text = "message 1\nmessage a"
107
- r = []
108
- parser.instance.parse(text) { |_, record| r << record }
109
- assert_equal [{ 'message_id' => '1' }, { 'unmatched_line' => 'message a'}], r
110
- end
111
- end