fluentd 1.17.0-x86-mingw32 → 1.17.1-x86-mingw32

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (259) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +46 -0
  3. data/README.md +1 -0
  4. data/SECURITY.md +2 -2
  5. data/fluent.conf +14 -14
  6. data/lib/fluent/command/cap_ctl.rb +4 -4
  7. data/lib/fluent/compat/call_super_mixin.rb +3 -3
  8. data/lib/fluent/compat/propagate_default.rb +4 -4
  9. data/lib/fluent/config/yaml_parser/parser.rb +4 -0
  10. data/lib/fluent/log/console_adapter.rb +4 -2
  11. data/lib/fluent/plugin/in_exec.rb +14 -2
  12. data/lib/fluent/plugin/in_http.rb +1 -1
  13. data/lib/fluent/plugin/in_sample.rb +13 -7
  14. data/lib/fluent/plugin/in_tail.rb +65 -23
  15. data/lib/fluent/plugin/out_copy.rb +1 -1
  16. data/lib/fluent/plugin/out_file.rb +8 -0
  17. data/lib/fluent/plugin/out_http.rb +12 -0
  18. data/lib/fluent/plugin/parser_json.rb +4 -12
  19. data/lib/fluent/plugin_helper/http_server/server.rb +1 -1
  20. data/lib/fluent/version.rb +1 -1
  21. data/templates/new_gem/fluent-plugin.gemspec.erb +6 -5
  22. metadata +25 -472
  23. data/.github/DISCUSSION_TEMPLATE/q-a-japanese.yml +0 -50
  24. data/.github/DISCUSSION_TEMPLATE/q-a.yml +0 -47
  25. data/.github/ISSUE_TEMPLATE/bug_report.yml +0 -71
  26. data/.github/ISSUE_TEMPLATE/config.yml +0 -5
  27. data/.github/ISSUE_TEMPLATE/feature_request.yml +0 -39
  28. data/.github/ISSUE_TEMPLATE.md +0 -17
  29. data/.github/PULL_REQUEST_TEMPLATE.md +0 -14
  30. data/.github/workflows/stale-actions.yml +0 -24
  31. data/.github/workflows/test-ruby-head.yml +0 -31
  32. data/.github/workflows/test.yml +0 -32
  33. data/.gitignore +0 -30
  34. data/Gemfile +0 -9
  35. data/fluentd.gemspec +0 -62
  36. data/test/command/test_binlog_reader.rb +0 -362
  37. data/test/command/test_ca_generate.rb +0 -70
  38. data/test/command/test_cap_ctl.rb +0 -100
  39. data/test/command/test_cat.rb +0 -128
  40. data/test/command/test_ctl.rb +0 -56
  41. data/test/command/test_fluentd.rb +0 -1291
  42. data/test/command/test_plugin_config_formatter.rb +0 -397
  43. data/test/command/test_plugin_generator.rb +0 -109
  44. data/test/compat/test_calls_super.rb +0 -166
  45. data/test/compat/test_parser.rb +0 -92
  46. data/test/config/assertions.rb +0 -42
  47. data/test/config/test_config_parser.rb +0 -551
  48. data/test/config/test_configurable.rb +0 -1784
  49. data/test/config/test_configure_proxy.rb +0 -604
  50. data/test/config/test_dsl.rb +0 -415
  51. data/test/config/test_element.rb +0 -518
  52. data/test/config/test_literal_parser.rb +0 -309
  53. data/test/config/test_plugin_configuration.rb +0 -56
  54. data/test/config/test_section.rb +0 -191
  55. data/test/config/test_system_config.rb +0 -195
  56. data/test/config/test_types.rb +0 -408
  57. data/test/counter/test_client.rb +0 -563
  58. data/test/counter/test_error.rb +0 -44
  59. data/test/counter/test_mutex_hash.rb +0 -179
  60. data/test/counter/test_server.rb +0 -589
  61. data/test/counter/test_store.rb +0 -258
  62. data/test/counter/test_validator.rb +0 -137
  63. data/test/helper.rb +0 -155
  64. data/test/helpers/fuzzy_assert.rb +0 -89
  65. data/test/helpers/process_extenstion.rb +0 -33
  66. data/test/log/test_console_adapter.rb +0 -117
  67. data/test/plugin/data/2010/01/20100102-030405.log +0 -0
  68. data/test/plugin/data/2010/01/20100102-030406.log +0 -0
  69. data/test/plugin/data/2010/01/20100102.log +0 -0
  70. data/test/plugin/data/log/bar +0 -0
  71. data/test/plugin/data/log/foo/bar.log +0 -0
  72. data/test/plugin/data/log/foo/bar2 +0 -0
  73. data/test/plugin/data/log/test.log +0 -0
  74. data/test/plugin/data/log_numeric/01.log +0 -0
  75. data/test/plugin/data/log_numeric/02.log +0 -0
  76. data/test/plugin/data/log_numeric/12.log +0 -0
  77. data/test/plugin/data/log_numeric/14.log +0 -0
  78. data/test/plugin/data/sd_file/config +0 -11
  79. data/test/plugin/data/sd_file/config.json +0 -17
  80. data/test/plugin/data/sd_file/config.yaml +0 -11
  81. data/test/plugin/data/sd_file/config.yml +0 -11
  82. data/test/plugin/data/sd_file/invalid_config.yml +0 -7
  83. data/test/plugin/in_tail/test_fifo.rb +0 -121
  84. data/test/plugin/in_tail/test_io_handler.rb +0 -150
  85. data/test/plugin/in_tail/test_position_file.rb +0 -346
  86. data/test/plugin/out_forward/test_ack_handler.rb +0 -140
  87. data/test/plugin/out_forward/test_connection_manager.rb +0 -145
  88. data/test/plugin/out_forward/test_handshake_protocol.rb +0 -112
  89. data/test/plugin/out_forward/test_load_balancer.rb +0 -106
  90. data/test/plugin/out_forward/test_socket_cache.rb +0 -174
  91. data/test/plugin/test_bare_output.rb +0 -131
  92. data/test/plugin/test_base.rb +0 -247
  93. data/test/plugin/test_buf_file.rb +0 -1314
  94. data/test/plugin/test_buf_file_single.rb +0 -898
  95. data/test/plugin/test_buf_memory.rb +0 -42
  96. data/test/plugin/test_buffer.rb +0 -1493
  97. data/test/plugin/test_buffer_chunk.rb +0 -209
  98. data/test/plugin/test_buffer_file_chunk.rb +0 -871
  99. data/test/plugin/test_buffer_file_single_chunk.rb +0 -611
  100. data/test/plugin/test_buffer_memory_chunk.rb +0 -339
  101. data/test/plugin/test_compressable.rb +0 -87
  102. data/test/plugin/test_file_util.rb +0 -96
  103. data/test/plugin/test_filter.rb +0 -368
  104. data/test/plugin/test_filter_grep.rb +0 -697
  105. data/test/plugin/test_filter_parser.rb +0 -731
  106. data/test/plugin/test_filter_record_transformer.rb +0 -577
  107. data/test/plugin/test_filter_stdout.rb +0 -207
  108. data/test/plugin/test_formatter_csv.rb +0 -136
  109. data/test/plugin/test_formatter_hash.rb +0 -38
  110. data/test/plugin/test_formatter_json.rb +0 -61
  111. data/test/plugin/test_formatter_ltsv.rb +0 -70
  112. data/test/plugin/test_formatter_msgpack.rb +0 -28
  113. data/test/plugin/test_formatter_out_file.rb +0 -116
  114. data/test/plugin/test_formatter_single_value.rb +0 -44
  115. data/test/plugin/test_formatter_tsv.rb +0 -76
  116. data/test/plugin/test_in_debug_agent.rb +0 -49
  117. data/test/plugin/test_in_exec.rb +0 -261
  118. data/test/plugin/test_in_forward.rb +0 -1178
  119. data/test/plugin/test_in_gc_stat.rb +0 -62
  120. data/test/plugin/test_in_http.rb +0 -1124
  121. data/test/plugin/test_in_monitor_agent.rb +0 -922
  122. data/test/plugin/test_in_object_space.rb +0 -66
  123. data/test/plugin/test_in_sample.rb +0 -190
  124. data/test/plugin/test_in_syslog.rb +0 -505
  125. data/test/plugin/test_in_tail.rb +0 -3429
  126. data/test/plugin/test_in_tcp.rb +0 -328
  127. data/test/plugin/test_in_udp.rb +0 -296
  128. data/test/plugin/test_in_unix.rb +0 -181
  129. data/test/plugin/test_input.rb +0 -137
  130. data/test/plugin/test_metadata.rb +0 -89
  131. data/test/plugin/test_metrics.rb +0 -294
  132. data/test/plugin/test_metrics_local.rb +0 -96
  133. data/test/plugin/test_multi_output.rb +0 -204
  134. data/test/plugin/test_out_copy.rb +0 -308
  135. data/test/plugin/test_out_exec.rb +0 -312
  136. data/test/plugin/test_out_exec_filter.rb +0 -606
  137. data/test/plugin/test_out_file.rb +0 -1038
  138. data/test/plugin/test_out_forward.rb +0 -1349
  139. data/test/plugin/test_out_http.rb +0 -557
  140. data/test/plugin/test_out_null.rb +0 -105
  141. data/test/plugin/test_out_relabel.rb +0 -28
  142. data/test/plugin/test_out_roundrobin.rb +0 -146
  143. data/test/plugin/test_out_secondary_file.rb +0 -458
  144. data/test/plugin/test_out_stdout.rb +0 -205
  145. data/test/plugin/test_out_stream.rb +0 -103
  146. data/test/plugin/test_output.rb +0 -1334
  147. data/test/plugin/test_output_as_buffered.rb +0 -2024
  148. data/test/plugin/test_output_as_buffered_backup.rb +0 -363
  149. data/test/plugin/test_output_as_buffered_compress.rb +0 -179
  150. data/test/plugin/test_output_as_buffered_overflow.rb +0 -250
  151. data/test/plugin/test_output_as_buffered_retries.rb +0 -966
  152. data/test/plugin/test_output_as_buffered_secondary.rb +0 -882
  153. data/test/plugin/test_output_as_standard.rb +0 -374
  154. data/test/plugin/test_owned_by.rb +0 -34
  155. data/test/plugin/test_parser.rb +0 -399
  156. data/test/plugin/test_parser_apache.rb +0 -42
  157. data/test/plugin/test_parser_apache2.rb +0 -47
  158. data/test/plugin/test_parser_apache_error.rb +0 -45
  159. data/test/plugin/test_parser_csv.rb +0 -200
  160. data/test/plugin/test_parser_json.rb +0 -244
  161. data/test/plugin/test_parser_labeled_tsv.rb +0 -160
  162. data/test/plugin/test_parser_msgpack.rb +0 -127
  163. data/test/plugin/test_parser_multiline.rb +0 -111
  164. data/test/plugin/test_parser_nginx.rb +0 -88
  165. data/test/plugin/test_parser_none.rb +0 -52
  166. data/test/plugin/test_parser_regexp.rb +0 -284
  167. data/test/plugin/test_parser_syslog.rb +0 -650
  168. data/test/plugin/test_parser_tsv.rb +0 -122
  169. data/test/plugin/test_sd_file.rb +0 -228
  170. data/test/plugin/test_sd_srv.rb +0 -230
  171. data/test/plugin/test_storage.rb +0 -166
  172. data/test/plugin/test_storage_local.rb +0 -335
  173. data/test/plugin/test_string_util.rb +0 -26
  174. data/test/plugin_helper/data/cert/cert-key.pem +0 -27
  175. data/test/plugin_helper/data/cert/cert-with-CRLF.pem +0 -19
  176. data/test/plugin_helper/data/cert/cert-with-no-newline.pem +0 -19
  177. data/test/plugin_helper/data/cert/cert.pem +0 -19
  178. data/test/plugin_helper/data/cert/cert_chains/ca-cert-key.pem +0 -27
  179. data/test/plugin_helper/data/cert/cert_chains/ca-cert.pem +0 -20
  180. data/test/plugin_helper/data/cert/cert_chains/cert-key.pem +0 -27
  181. data/test/plugin_helper/data/cert/cert_chains/cert.pem +0 -40
  182. data/test/plugin_helper/data/cert/empty.pem +0 -0
  183. data/test/plugin_helper/data/cert/generate_cert.rb +0 -125
  184. data/test/plugin_helper/data/cert/with_ca/ca-cert-key-pass.pem +0 -30
  185. data/test/plugin_helper/data/cert/with_ca/ca-cert-key.pem +0 -27
  186. data/test/plugin_helper/data/cert/with_ca/ca-cert-pass.pem +0 -20
  187. data/test/plugin_helper/data/cert/with_ca/ca-cert.pem +0 -20
  188. data/test/plugin_helper/data/cert/with_ca/cert-key-pass.pem +0 -30
  189. data/test/plugin_helper/data/cert/with_ca/cert-key.pem +0 -27
  190. data/test/plugin_helper/data/cert/with_ca/cert-pass.pem +0 -21
  191. data/test/plugin_helper/data/cert/with_ca/cert.pem +0 -21
  192. data/test/plugin_helper/data/cert/without_ca/cert-key-pass.pem +0 -30
  193. data/test/plugin_helper/data/cert/without_ca/cert-key.pem +0 -27
  194. data/test/plugin_helper/data/cert/without_ca/cert-pass.pem +0 -20
  195. data/test/plugin_helper/data/cert/without_ca/cert.pem +0 -20
  196. data/test/plugin_helper/http_server/test_app.rb +0 -65
  197. data/test/plugin_helper/http_server/test_route.rb +0 -32
  198. data/test/plugin_helper/service_discovery/test_manager.rb +0 -93
  199. data/test/plugin_helper/service_discovery/test_round_robin_balancer.rb +0 -21
  200. data/test/plugin_helper/test_cert_option.rb +0 -25
  201. data/test/plugin_helper/test_child_process.rb +0 -862
  202. data/test/plugin_helper/test_compat_parameters.rb +0 -358
  203. data/test/plugin_helper/test_event_emitter.rb +0 -80
  204. data/test/plugin_helper/test_event_loop.rb +0 -52
  205. data/test/plugin_helper/test_extract.rb +0 -194
  206. data/test/plugin_helper/test_formatter.rb +0 -255
  207. data/test/plugin_helper/test_http_server_helper.rb +0 -372
  208. data/test/plugin_helper/test_inject.rb +0 -561
  209. data/test/plugin_helper/test_metrics.rb +0 -137
  210. data/test/plugin_helper/test_parser.rb +0 -264
  211. data/test/plugin_helper/test_record_accessor.rb +0 -238
  212. data/test/plugin_helper/test_retry_state.rb +0 -1006
  213. data/test/plugin_helper/test_server.rb +0 -1895
  214. data/test/plugin_helper/test_service_discovery.rb +0 -165
  215. data/test/plugin_helper/test_socket.rb +0 -146
  216. data/test/plugin_helper/test_storage.rb +0 -542
  217. data/test/plugin_helper/test_thread.rb +0 -164
  218. data/test/plugin_helper/test_timer.rb +0 -130
  219. data/test/scripts/exec_script.rb +0 -32
  220. data/test/scripts/fluent/plugin/formatter1/formatter_test1.rb +0 -7
  221. data/test/scripts/fluent/plugin/formatter2/formatter_test2.rb +0 -7
  222. data/test/scripts/fluent/plugin/formatter_known.rb +0 -8
  223. data/test/scripts/fluent/plugin/out_test.rb +0 -81
  224. data/test/scripts/fluent/plugin/out_test2.rb +0 -80
  225. data/test/scripts/fluent/plugin/parser_known.rb +0 -4
  226. data/test/test_capability.rb +0 -74
  227. data/test/test_clock.rb +0 -164
  228. data/test/test_config.rb +0 -369
  229. data/test/test_configdsl.rb +0 -148
  230. data/test/test_daemonizer.rb +0 -91
  231. data/test/test_engine.rb +0 -203
  232. data/test/test_event.rb +0 -531
  233. data/test/test_event_router.rb +0 -348
  234. data/test/test_event_time.rb +0 -199
  235. data/test/test_file_wrapper.rb +0 -53
  236. data/test/test_filter.rb +0 -121
  237. data/test/test_fluent_log_event_router.rb +0 -99
  238. data/test/test_formatter.rb +0 -369
  239. data/test/test_input.rb +0 -31
  240. data/test/test_log.rb +0 -1076
  241. data/test/test_match.rb +0 -148
  242. data/test/test_mixin.rb +0 -351
  243. data/test/test_msgpack_factory.rb +0 -50
  244. data/test/test_oj_options.rb +0 -55
  245. data/test/test_output.rb +0 -278
  246. data/test/test_plugin.rb +0 -251
  247. data/test/test_plugin_classes.rb +0 -370
  248. data/test/test_plugin_helper.rb +0 -81
  249. data/test/test_plugin_id.rb +0 -119
  250. data/test/test_process.rb +0 -14
  251. data/test/test_root_agent.rb +0 -951
  252. data/test/test_static_config_analysis.rb +0 -177
  253. data/test/test_supervisor.rb +0 -821
  254. data/test/test_test_drivers.rb +0 -136
  255. data/test/test_time_formatter.rb +0 -301
  256. data/test/test_time_parser.rb +0 -362
  257. data/test/test_tls.rb +0 -65
  258. data/test/test_unique_id.rb +0 -47
  259. data/test/test_variable_store.rb +0 -65
@@ -1,200 +0,0 @@
1
- require_relative '../helper'
2
- require 'fluent/test/driver/parser'
3
- require 'fluent/plugin/parser_csv'
4
-
5
- class CSVParserTest < ::Test::Unit::TestCase
6
- def setup
7
- Fluent::Test.setup
8
- end
9
-
10
- def create_driver(conf={})
11
- Fluent::Test::Driver::Parser.new(Fluent::Plugin::CSVParser).configure(conf)
12
- end
13
-
14
- data('array param' => '["time","c","d"]', 'string param' => 'time,c,d')
15
- def test_parse(param)
16
- d = create_driver('keys' => param, 'time_key' => 'time')
17
- d.instance.parse("2013/02/28 12:00:00,192.168.0.1,111") { |time, record|
18
- assert_equal(event_time('2013/02/28 12:00:00', format: '%Y/%m/%d %H:%M:%S'), time)
19
- assert_equal({
20
- 'c' => '192.168.0.1',
21
- 'd' => '111',
22
- }, record)
23
- }
24
- end
25
-
26
- data('array param' => '["c","d"]', 'string param' => 'c,d')
27
- def test_parse_without_time(param)
28
- time_at_start = Time.now.to_i
29
-
30
- d = create_driver('keys' => param)
31
- d.instance.parse("192.168.0.1,111") { |time, record|
32
- assert time && time >= time_at_start, "parser puts current time without time input"
33
- assert_equal({
34
- 'c' => '192.168.0.1',
35
- 'd' => '111',
36
- }, record)
37
- }
38
-
39
- d = Fluent::Test::Driver::Parser.new(Fluent::Plugin::CSVParser)
40
- d.configure('keys' => param, 'estimate_current_event' => 'no')
41
- d.instance.parse("192.168.0.1,111") { |time, record|
42
- assert_equal({
43
- 'c' => '192.168.0.1',
44
- 'd' => '111',
45
- }, record)
46
- assert_nil time, "parser returns nil w/o time and if configured so"
47
- }
48
- end
49
-
50
- def test_parse_with_keep_time_key
51
- d = create_driver(
52
- 'keys'=>'time',
53
- 'time_key'=>'time',
54
- 'time_format'=>"%d/%b/%Y:%H:%M:%S %z",
55
- 'keep_time_key'=>'true',
56
- )
57
- text = '28/Feb/2013:12:00:00 +0900'
58
- d.instance.parse(text) do |time, record|
59
- assert_equal text, record['time']
60
- end
61
- end
62
-
63
- data('array param' => '["a","b","c","d","e","f"]', 'string param' => 'a,b,c,d,e,f')
64
- def test_parse_with_null_value_pattern(param)
65
- d = create_driver(
66
- 'keys'=>param,
67
- 'null_value_pattern'=>'^(-|null|NULL)$'
68
- )
69
- d.instance.parse("-,null,NULL,,--,nuLL") do |time, record|
70
- assert_nil record['a']
71
- assert_nil record['b']
72
- assert_nil record['c']
73
- assert_nil record['d']
74
- assert_equal record['e'], '--'
75
- assert_equal record['f'], 'nuLL'
76
- end
77
- end
78
-
79
- data('array param' => '["a","b"]', 'string param' => 'a,b')
80
- def test_parse_with_null_empty_string(param)
81
- d = create_driver(
82
- 'keys'=>param,
83
- 'null_empty_string'=>true
84
- )
85
- d.instance.parse(", ") do |time, record|
86
- assert_nil record['a']
87
- assert_equal record['b'], ' '
88
- end
89
- end
90
-
91
- data('array param' => '["a","b","c"]', 'string param' => 'a,b,c')
92
- def test_parse_with_option_delimiter(param)
93
- d = create_driver(
94
- 'keys'=>param,
95
- 'delimiter'=>' ',
96
- )
97
- d.instance.parse("123 456 789") do |time, record|
98
- assert_equal record['a'], '123'
99
- assert_equal record['b'], '456'
100
- assert_equal record['c'], '789'
101
- end
102
- end
103
-
104
- sub_test_case 'parser' do
105
- data('normal' => 'normal',
106
- 'fast' => 'fast')
107
- def test_compatibility_between_normal_and_fast_parser(param)
108
- d = create_driver(
109
- 'keys' => 'time,key1,key2,key3,key4,key5',
110
- 'time_key' => 'time',
111
- 'time_format' => "%d/%b/%Y:%H:%M:%S %z",
112
- 'keep_time_key' => 'false',
113
- 'parser_type' => param
114
- )
115
-
116
- # non quoted
117
- text = '28/Feb/2013:12:00:00 +0900,value1,value2,value3,value4,value5'
118
- expected = {'key1' => 'value1', 'key2' => 'value2', 'key3' => "value3",
119
- 'key4' => 'value4', 'key5' => "value5"}
120
- d.instance.parse(text) do |time, record|
121
- assert_equal(event_time("28/Feb/2013:12:00:00 +0900", format: '%d/%b/%Y:%H:%M:%S %z'), time)
122
- assert_equal expected, record
123
- end
124
-
125
- # quoted
126
- text = '28/Feb/2013:12:00:00 +0900,"value1","val,ue2","va,lu,e3","val ue4",""'
127
- expected = {'key1' => 'value1', 'key2' => 'val,ue2', 'key3' => "va,lu,e3",
128
- 'key4' => 'val ue4', 'key5' => ""}
129
- d.instance.parse(text) do |time, record|
130
- assert_equal(event_time("28/Feb/2013:12:00:00 +0900", format: '%d/%b/%Y:%H:%M:%S %z'), time)
131
- assert_equal expected, record
132
- end
133
-
134
- # mixed
135
- text = '28/Feb/2013:12:00:00 +0900,message,"mes,sage","me,ssa,ge",mess age,""'
136
- expected = {'key1' => 'message', 'key2' => 'mes,sage', 'key3' => "me,ssa,ge",
137
- 'key4' => 'mess age', 'key5' => ""}
138
- d.instance.parse(text) do |time, record|
139
- assert_equal(event_time("28/Feb/2013:12:00:00 +0900", format: '%d/%b/%Y:%H:%M:%S %z'), time)
140
- assert_equal expected, record
141
- end
142
-
143
- # escaped
144
- text = '28/Feb/2013:12:00:00 +0900,"message","mes""sage","""message""",,""""""'
145
- expected = {'key1' => 'message', 'key2' => 'mes"sage', 'key3' => '"message"',
146
- 'key4' => nil, 'key5' => '""'}
147
- d.instance.parse(text) do |time, record|
148
- assert_equal(event_time("28/Feb/2013:12:00:00 +0900", format: '%d/%b/%Y:%H:%M:%S %z'), time)
149
- assert_equal expected, record
150
- end
151
- end
152
-
153
- def test_incompatibility_between_normal_and_fast_parser
154
- normal = create_driver(
155
- 'keys' => 'key1,key2',
156
- 'parser_type' => 'normal'
157
- )
158
- fast = create_driver(
159
- 'keys' => 'key1,key2',
160
- 'parser_type' => 'fast'
161
- )
162
-
163
- # unexpected quote position
164
- text = 'a"b,"a"""c"'
165
- assert_raise(CSV::MalformedCSVError) {
166
- normal.instance.parse(text) { |t, r| }
167
- }
168
- assert_nothing_raised {
169
- # generate broken record
170
- fast.instance.parse(text) { |t, r| }
171
- }
172
-
173
- # incorrect the number of column
174
- text = 'a,b,c'
175
- expected = {"key1" => 'a', "key2" => 'b'}
176
- normal.instance.parse(text) { |t, r|
177
- assert_equal expected, r
178
- }
179
- fast.instance.parse(text) { |t, r|
180
- assert_not_equal expected, r
181
- }
182
-
183
- # And more...
184
- end
185
- end
186
-
187
- # "parser_type" config shouldn't hide Fluent::Plugin::Parser#plugin_type
188
- # https://github.com/fluent/fluentd/issues/3296
189
- data('normal' => :normal, 'fast' => :fast)
190
- def test_parser_type_method(engine)
191
- d = create_driver('keys' => '["time"]','time_key' => 'time', 'parser_type' => engine.to_s)
192
- assert_equal(:text_per_line, d.instance.parser_type)
193
- end
194
-
195
- data('normal' => :normal, 'fast' => :fast)
196
- def test_parser_engine(engine)
197
- d = create_driver('keys' => '["time"]', 'time_key' => 'time', 'parser_engine' => engine.to_s)
198
- assert_equal(engine, d.instance.parser_engine)
199
- end
200
- end
@@ -1,244 +0,0 @@
1
- require_relative '../helper'
2
- require 'fluent/test/driver/parser'
3
- require 'fluent/plugin/parser'
4
-
5
- class JsonParserTest < ::Test::Unit::TestCase
6
- def setup
7
- Fluent::Test.setup
8
- @parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::JSONParser)
9
- end
10
-
11
- data('oj' => 'oj', 'yajl' => 'yajl')
12
- def test_parse(data)
13
- @parser.configure('json_parser' => data)
14
- @parser.instance.parse('{"time":1362020400,"host":"192.168.0.1","size":777,"method":"PUT"}') { |time, record|
15
- assert_equal(event_time('2013-02-28 12:00:00 +0900').to_i, time)
16
- assert_equal({
17
- 'host' => '192.168.0.1',
18
- 'size' => 777,
19
- 'method' => 'PUT',
20
- }, record)
21
- }
22
- end
23
-
24
- data('oj' => 'oj', 'yajl' => 'yajl')
25
- def test_parse_with_large_float(data)
26
- @parser.configure('json_parser' => data)
27
- @parser.instance.parse('{"num":999999999999999999999999999999.99999}') { |time, record|
28
- assert_equal(Float, record['num'].class)
29
- }
30
- end
31
-
32
- data('oj' => 'oj', 'yajl' => 'yajl')
33
- def test_parse_without_time(data)
34
- time_at_start = Time.now.to_i
35
-
36
- @parser.configure('json_parser' => data)
37
- @parser.instance.parse('{"host":"192.168.0.1","size":777,"method":"PUT"}') { |time, record|
38
- assert time && time >= time_at_start, "parser puts current time without time input"
39
- assert_equal({
40
- 'host' => '192.168.0.1',
41
- 'size' => 777,
42
- 'method' => 'PUT',
43
- }, record)
44
- }
45
-
46
- parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::JSONParser)
47
- parser.configure('json_parser' => data, 'estimate_current_event' => 'false')
48
- parser.instance.parse('{"host":"192.168.0.1","size":777,"method":"PUT"}') { |time, record|
49
- assert_equal({
50
- 'host' => '192.168.0.1',
51
- 'size' => 777,
52
- 'method' => 'PUT',
53
- }, record)
54
- assert_nil time, "parser return nil w/o time and if specified so"
55
- }
56
- end
57
-
58
- data('oj' => 'oj', 'yajl' => 'yajl')
59
- def test_parse_with_colon_string(data)
60
- @parser.configure('json_parser' => data)
61
- @parser.instance.parse('{"time":1362020400,"log":":message"}') { |time, record|
62
- assert_equal(record['log'], ':message')
63
- }
64
- end
65
-
66
- data('oj' => 'oj', 'yajl' => 'yajl')
67
- def test_parse_with_invalid_time(data)
68
- @parser.configure('json_parser' => data)
69
- assert_raise Fluent::ParserError do
70
- @parser.instance.parse('{"time":[],"k":"v"}') { |time, record| }
71
- end
72
- end
73
-
74
- data('oj' => 'oj', 'yajl' => 'yajl')
75
- def test_parse_float_time(data)
76
- parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::JSONParser)
77
- parser.configure('json_parser' => data)
78
- text = "100.1"
79
- parser.instance.parse("{\"time\":\"#{text}\"}") do |time, record|
80
- assert_equal 100, time.sec
81
- assert_equal 100_000_000, time.nsec
82
- end
83
- end
84
-
85
- data('oj' => 'oj', 'yajl' => 'yajl')
86
- def test_parse_with_keep_time_key(data)
87
- parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::JSONParser)
88
- format = "%d/%b/%Y:%H:%M:%S %z"
89
- parser.configure(
90
- 'time_format' => format,
91
- 'keep_time_key' => 'true',
92
- 'json_parser' => data
93
- )
94
- text = "28/Feb/2013:12:00:00 +0900"
95
- parser.instance.parse("{\"time\":\"#{text}\"}") do |time, record|
96
- assert_equal Time.strptime(text, format).to_i, time.sec
97
- assert_equal text, record['time']
98
- end
99
- end
100
-
101
- data('oj' => 'oj', 'yajl' => 'yajl')
102
- def test_parse_with_keep_time_key_without_time_format(data)
103
- parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::JSONParser)
104
- parser.configure(
105
- 'keep_time_key' => 'true',
106
- 'json_parser' => data
107
- )
108
- text = "100"
109
- parser.instance.parse("{\"time\":\"#{text}\"}") do |time, record|
110
- assert_equal text.to_i, time.sec
111
- assert_equal text, record['time']
112
- end
113
- end
114
-
115
- def test_yajl_parse_io_with_buffer_smaller_than_input
116
- parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::JSONParser)
117
- parser.configure(
118
- 'keep_time_key' => 'true',
119
- 'json_parser' => 'yajl',
120
- 'stream_buffer_size' => 1,
121
- )
122
- text = "100"
123
-
124
- waiting(5) do
125
- rd, wr = IO.pipe
126
- wr.write "{\"time\":\"#{text}\"}"
127
-
128
- parser.instance.parse_io(rd) do |time, record|
129
- assert_equal text.to_i, time.sec
130
- assert_equal text, record['time']
131
-
132
- # Once a record has been received the 'write' end of the pipe must be
133
- # closed, otherwise the test will block waiting for more input.
134
- wr.close
135
- end
136
- end
137
- end
138
-
139
- sub_test_case "various record pattern" do
140
- data("Only string", { record: '"message"', expected: [nil] }, keep: true)
141
- data("Only string without quotation", { record: "message", expected: [nil] }, keep: true)
142
- data("Only number", { record: "0", expected: [nil] }, keep: true)
143
- data(
144
- "Array of Hash",
145
- {
146
- record: '[{"k1": 1}, {"k2": 2}]',
147
- expected: [{"k1" => 1}, {"k2" => 2}]
148
- },
149
- keep: true,
150
- )
151
- data(
152
- "Array of both Hash and invalid",
153
- {
154
- record: '[{"k1": 1}, "string", {"k2": 2}, 0]',
155
- expected: [{"k1" => 1}, nil, {"k2" => 2}, nil]
156
- },
157
- keep: true,
158
- )
159
- data(
160
- "Array of all invalid",
161
- {
162
- record: '["string", 0, [{"k": 0}]]',
163
- expected: [nil, nil, nil]
164
- },
165
- keep: true,
166
- )
167
-
168
- def test_oj(data)
169
- parsed_records = []
170
- @parser.configure("json_parser" => "oj")
171
- @parser.instance.parse(data[:record]) { |time, record|
172
- parsed_records.append(record)
173
- }
174
- assert_equal(data[:expected], parsed_records)
175
- end
176
-
177
- def test_yajl(data)
178
- parsed_records = []
179
- @parser.configure("json_parser" => "yajl")
180
- @parser.instance.parse(data[:record]) { |time, record|
181
- parsed_records.append(record)
182
- }
183
- assert_equal(data[:expected], parsed_records)
184
- end
185
-
186
- def test_json(json)
187
- parsed_records = []
188
- @parser.configure("json_parser" => "json")
189
- @parser.instance.parse(data[:record]) { |time, record|
190
- parsed_records.append(record)
191
- }
192
- assert_equal(data[:expected], parsed_records)
193
- end
194
- end
195
-
196
- # This becomes NoMethodError if a non-Hash object is passed to convert_values.
197
- # https://github.com/fluent/fluentd/issues/4100
198
- sub_test_case "execute_convert_values with null_empty_string" do
199
- data("Only string", { record: '"message"', expected: [nil] }, keep: true)
200
- data(
201
- "Hash",
202
- {
203
- record: '{"k1": 1, "k2": ""}',
204
- expected: [{"k1" => 1, "k2" => nil}]
205
- },
206
- keep: true,
207
- )
208
- data(
209
- "Array of Hash",
210
- {
211
- record: '[{"k1": 1}, {"k2": ""}]',
212
- expected: [{"k1" => 1}, {"k2" => nil}]
213
- },
214
- keep: true,
215
- )
216
-
217
- def test_oj(data)
218
- parsed_records = []
219
- @parser.configure("json_parser" => "oj", "null_empty_string" => true)
220
- @parser.instance.parse(data[:record]) { |time, record|
221
- parsed_records.append(record)
222
- }
223
- assert_equal(data[:expected], parsed_records)
224
- end
225
-
226
- def test_yajl(data)
227
- parsed_records = []
228
- @parser.configure("json_parser" => "yajl", "null_empty_string" => true)
229
- @parser.instance.parse(data[:record]) { |time, record|
230
- parsed_records.append(record)
231
- }
232
- assert_equal(data[:expected], parsed_records)
233
- end
234
-
235
- def test_json(json)
236
- parsed_records = []
237
- @parser.configure("json_parser" => "json", "null_empty_string" => true)
238
- @parser.instance.parse(data[:record]) { |time, record|
239
- parsed_records.append(record)
240
- }
241
- assert_equal(data[:expected], parsed_records)
242
- end
243
- end
244
- end
@@ -1,160 +0,0 @@
1
- require_relative '../helper'
2
- require 'fluent/test/driver/parser'
3
- require 'fluent/plugin/parser'
4
-
5
- class LabeledTSVParserTest < ::Test::Unit::TestCase
6
- def setup
7
- Fluent::Test.setup
8
- end
9
-
10
- def test_config_params
11
- parser = Fluent::Test::Driver::Parser.new(Fluent::TextParser::LabeledTSVParser)
12
-
13
- assert_equal "\t", parser.instance.delimiter
14
- assert_equal ":", parser.instance.label_delimiter
15
-
16
- parser.configure(
17
- 'delimiter' => ',',
18
- 'label_delimiter' => '=',
19
- )
20
-
21
- assert_equal ",", parser.instance.delimiter
22
- assert_equal "=", parser.instance.label_delimiter
23
- end
24
-
25
- def test_parse
26
- parser = Fluent::Test::Driver::Parser.new(Fluent::TextParser::LabeledTSVParser)
27
- parser.configure({})
28
- parser.instance.parse("time:2013/02/28 12:00:00\thost:192.168.0.1\treq_id:111") { |time, record|
29
- assert_equal(event_time('2013/02/28 12:00:00', format: '%Y/%m/%d %H:%M:%S'), time)
30
- assert_equal({
31
- 'host' => '192.168.0.1',
32
- 'req_id' => '111',
33
- }, record)
34
- }
35
- end
36
-
37
- def test_parse_with_customized_delimiter
38
- parser = Fluent::Test::Driver::Parser.new(Fluent::TextParser::LabeledTSVParser)
39
- parser.configure(
40
- 'delimiter' => ',',
41
- 'label_delimiter' => '=',
42
- )
43
- parser.instance.parse('time=2013/02/28 12:00:00,host=192.168.0.1,req_id=111') { |time, record|
44
- assert_equal(event_time('2013/02/28 12:00:00', format: '%Y/%m/%d %H:%M:%S'), time)
45
- assert_equal({
46
- 'host' => '192.168.0.1',
47
- 'req_id' => '111',
48
- }, record)
49
- }
50
- end
51
-
52
- def test_parse_with_customized_time_format
53
- parser = Fluent::Test::Driver::Parser.new(Fluent::TextParser::LabeledTSVParser)
54
- parser.configure(
55
- 'time_key' => 'mytime',
56
- 'time_format' => '%d/%b/%Y:%H:%M:%S %z',
57
- )
58
- parser.instance.parse("mytime:28/Feb/2013:12:00:00 +0900\thost:192.168.0.1\treq_id:111") { |time, record|
59
- assert_equal(event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'), time)
60
- assert_equal({
61
- 'host' => '192.168.0.1',
62
- 'req_id' => '111',
63
- }, record)
64
- }
65
- end
66
-
67
- def test_parse_without_time
68
- time_at_start = Time.now.to_i
69
-
70
- parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::LabeledTSVParser)
71
- parser.configure({})
72
- parser.instance.parse("host:192.168.0.1\treq_id:111") { |time, record|
73
- assert time && time >= time_at_start, "parser puts current time without time input"
74
- assert_equal({
75
- 'host' => '192.168.0.1',
76
- 'req_id' => '111',
77
- }, record)
78
- }
79
-
80
- parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::LabeledTSVParser)
81
- parser.configure({'estimate_current_event' => 'no'})
82
- parser.instance.parse("host:192.168.0.1\treq_id:111") { |time, record|
83
- assert_equal({
84
- 'host' => '192.168.0.1',
85
- 'req_id' => '111',
86
- }, record)
87
- assert_nil time, "parser returns nil w/o time and if configured so"
88
- }
89
- end
90
-
91
- def test_parse_with_keep_time_key
92
- parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::LabeledTSVParser)
93
- parser.configure(
94
- 'time_format'=>"%d/%b/%Y:%H:%M:%S %z",
95
- 'keep_time_key'=>'true',
96
- )
97
- text = '28/Feb/2013:12:00:00 +0900'
98
- parser.instance.parse("time:#{text}") do |time, record|
99
- assert_equal text, record['time']
100
- end
101
- end
102
-
103
- def test_parse_and_reject_invalid_kv_pairs
104
- parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::LabeledTSVParser)
105
- parser.configure(
106
- 'delimiter' => ' ',
107
- 'label_delimiter' => '=',
108
- )
109
- text = 'A leading portion that is not LTSV : foo=bar baz=derp and a trailing portion'
110
-
111
- expected = {'foo' => 'bar', 'baz' => 'derp'}
112
- parser.instance.parse(text) do |time, record|
113
- assert_equal expected, record
114
- end
115
-
116
- end
117
-
118
- def test_parse_with_null_value_pattern
119
- parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::LabeledTSVParser)
120
- parser.configure(
121
- 'null_value_pattern'=>'^(-|null|NULL)$'
122
- )
123
- parser.instance.parse("a:-\tb:null\tc:NULL\td:\te:--\tf:nuLL") do |time, record|
124
- assert_nil record['a']
125
- assert_nil record['b']
126
- assert_nil record['c']
127
- assert_equal record['d'], ''
128
- assert_equal record['e'], '--'
129
- assert_equal record['f'], 'nuLL'
130
- end
131
- end
132
-
133
- def test_parse_with_null_empty_string
134
- parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::LabeledTSVParser)
135
- parser.configure(
136
- 'null_empty_string'=>true
137
- )
138
- parser.instance.parse("a:\tb: ") do |time, record|
139
- assert_nil record['a']
140
- assert_equal record['b'], ' '
141
- end
142
- end
143
-
144
- data("single space" => ["k1=v1 k2=v2", { "k1" => "v1", "k2" => "v2" }],
145
- "multiple space" => ["k1=v1 k2=v2", { "k1" => "v1", "k2" => "v2" }],
146
- "reverse" => ["k2=v2 k1=v1", { "k1" => "v1", "k2" => "v2" }],
147
- "tab" => ["k2=v2\tk1=v1", { "k1" => "v1", "k2" => "v2" }],
148
- "tab and space" => ["k2=v2\t k1=v1", { "k1" => "v1", "k2" => "v2" }])
149
- def test_parse_with_delimiter_pattern(data)
150
- text, expected = data
151
- parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::LabeledTSVParser)
152
- parser.configure(
153
- 'delimiter_pattern' => '/\s+/',
154
- 'label_delimiter' => '='
155
- )
156
- parser.instance.parse(text) do |_time, record|
157
- assert_equal(expected, record)
158
- end
159
- end
160
- end