fluentd 1.16.5-x64-mingw32 → 1.17.1-x64-mingw32

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (268) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +88 -0
  3. data/README.md +2 -1
  4. data/Rakefile +1 -1
  5. data/SECURITY.md +2 -2
  6. data/fluent.conf +14 -14
  7. data/lib/fluent/command/binlog_reader.rb +1 -1
  8. data/lib/fluent/command/cap_ctl.rb +4 -4
  9. data/lib/fluent/compat/call_super_mixin.rb +3 -3
  10. data/lib/fluent/compat/propagate_default.rb +4 -4
  11. data/lib/fluent/config/configure_proxy.rb +2 -2
  12. data/lib/fluent/config/types.rb +1 -1
  13. data/lib/fluent/config/yaml_parser/parser.rb +4 -0
  14. data/lib/fluent/configurable.rb +2 -2
  15. data/lib/fluent/counter/mutex_hash.rb +1 -1
  16. data/lib/fluent/fluent_log_event_router.rb +0 -2
  17. data/lib/fluent/log/console_adapter.rb +4 -2
  18. data/lib/fluent/plugin/buf_file.rb +1 -1
  19. data/lib/fluent/plugin/buffer/file_chunk.rb +1 -1
  20. data/lib/fluent/plugin/buffer/file_single_chunk.rb +2 -3
  21. data/lib/fluent/plugin/filter_parser.rb +26 -8
  22. data/lib/fluent/plugin/in_exec.rb +14 -2
  23. data/lib/fluent/plugin/in_http.rb +19 -54
  24. data/lib/fluent/plugin/in_sample.rb +13 -7
  25. data/lib/fluent/plugin/in_tail.rb +99 -25
  26. data/lib/fluent/plugin/out_copy.rb +1 -1
  27. data/lib/fluent/plugin/out_file.rb +8 -0
  28. data/lib/fluent/plugin/out_http.rb +137 -13
  29. data/lib/fluent/plugin/owned_by_mixin.rb +0 -1
  30. data/lib/fluent/plugin/parser_json.rb +26 -17
  31. data/lib/fluent/plugin/parser_msgpack.rb +24 -3
  32. data/lib/fluent/plugin_helper/http_server/server.rb +1 -1
  33. data/lib/fluent/plugin_helper/metrics.rb +2 -2
  34. data/lib/fluent/registry.rb +6 -6
  35. data/lib/fluent/test/output_test.rb +1 -1
  36. data/lib/fluent/unique_id.rb +1 -1
  37. data/lib/fluent/version.rb +1 -1
  38. data/templates/new_gem/fluent-plugin.gemspec.erb +6 -5
  39. metadata +109 -459
  40. data/.github/ISSUE_TEMPLATE/bug_report.yml +0 -71
  41. data/.github/ISSUE_TEMPLATE/config.yml +0 -5
  42. data/.github/ISSUE_TEMPLATE/feature_request.yml +0 -39
  43. data/.github/ISSUE_TEMPLATE.md +0 -17
  44. data/.github/PULL_REQUEST_TEMPLATE.md +0 -14
  45. data/.github/workflows/stale-actions.yml +0 -24
  46. data/.github/workflows/test.yml +0 -32
  47. data/.gitignore +0 -30
  48. data/Gemfile +0 -9
  49. data/fluentd.gemspec +0 -54
  50. data/test/command/test_binlog_reader.rb +0 -362
  51. data/test/command/test_ca_generate.rb +0 -70
  52. data/test/command/test_cap_ctl.rb +0 -100
  53. data/test/command/test_cat.rb +0 -128
  54. data/test/command/test_ctl.rb +0 -56
  55. data/test/command/test_fluentd.rb +0 -1291
  56. data/test/command/test_plugin_config_formatter.rb +0 -397
  57. data/test/command/test_plugin_generator.rb +0 -109
  58. data/test/compat/test_calls_super.rb +0 -166
  59. data/test/compat/test_parser.rb +0 -92
  60. data/test/config/assertions.rb +0 -42
  61. data/test/config/test_config_parser.rb +0 -551
  62. data/test/config/test_configurable.rb +0 -1784
  63. data/test/config/test_configure_proxy.rb +0 -604
  64. data/test/config/test_dsl.rb +0 -415
  65. data/test/config/test_element.rb +0 -518
  66. data/test/config/test_literal_parser.rb +0 -309
  67. data/test/config/test_plugin_configuration.rb +0 -56
  68. data/test/config/test_section.rb +0 -191
  69. data/test/config/test_system_config.rb +0 -195
  70. data/test/config/test_types.rb +0 -408
  71. data/test/counter/test_client.rb +0 -563
  72. data/test/counter/test_error.rb +0 -44
  73. data/test/counter/test_mutex_hash.rb +0 -179
  74. data/test/counter/test_server.rb +0 -589
  75. data/test/counter/test_store.rb +0 -258
  76. data/test/counter/test_validator.rb +0 -137
  77. data/test/helper.rb +0 -155
  78. data/test/helpers/fuzzy_assert.rb +0 -89
  79. data/test/helpers/process_extenstion.rb +0 -33
  80. data/test/log/test_console_adapter.rb +0 -110
  81. data/test/plugin/data/2010/01/20100102-030405.log +0 -0
  82. data/test/plugin/data/2010/01/20100102-030406.log +0 -0
  83. data/test/plugin/data/2010/01/20100102.log +0 -0
  84. data/test/plugin/data/log/bar +0 -0
  85. data/test/plugin/data/log/foo/bar.log +0 -0
  86. data/test/plugin/data/log/foo/bar2 +0 -0
  87. data/test/plugin/data/log/test.log +0 -0
  88. data/test/plugin/data/sd_file/config +0 -11
  89. data/test/plugin/data/sd_file/config.json +0 -17
  90. data/test/plugin/data/sd_file/config.yaml +0 -11
  91. data/test/plugin/data/sd_file/config.yml +0 -11
  92. data/test/plugin/data/sd_file/invalid_config.yml +0 -7
  93. data/test/plugin/in_tail/test_fifo.rb +0 -121
  94. data/test/plugin/in_tail/test_io_handler.rb +0 -150
  95. data/test/plugin/in_tail/test_position_file.rb +0 -346
  96. data/test/plugin/out_forward/test_ack_handler.rb +0 -140
  97. data/test/plugin/out_forward/test_connection_manager.rb +0 -145
  98. data/test/plugin/out_forward/test_handshake_protocol.rb +0 -112
  99. data/test/plugin/out_forward/test_load_balancer.rb +0 -106
  100. data/test/plugin/out_forward/test_socket_cache.rb +0 -174
  101. data/test/plugin/test_bare_output.rb +0 -131
  102. data/test/plugin/test_base.rb +0 -247
  103. data/test/plugin/test_buf_file.rb +0 -1314
  104. data/test/plugin/test_buf_file_single.rb +0 -898
  105. data/test/plugin/test_buf_memory.rb +0 -42
  106. data/test/plugin/test_buffer.rb +0 -1493
  107. data/test/plugin/test_buffer_chunk.rb +0 -209
  108. data/test/plugin/test_buffer_file_chunk.rb +0 -871
  109. data/test/plugin/test_buffer_file_single_chunk.rb +0 -611
  110. data/test/plugin/test_buffer_memory_chunk.rb +0 -339
  111. data/test/plugin/test_compressable.rb +0 -87
  112. data/test/plugin/test_file_util.rb +0 -96
  113. data/test/plugin/test_filter.rb +0 -368
  114. data/test/plugin/test_filter_grep.rb +0 -697
  115. data/test/plugin/test_filter_parser.rb +0 -731
  116. data/test/plugin/test_filter_record_transformer.rb +0 -577
  117. data/test/plugin/test_filter_stdout.rb +0 -207
  118. data/test/plugin/test_formatter_csv.rb +0 -136
  119. data/test/plugin/test_formatter_hash.rb +0 -38
  120. data/test/plugin/test_formatter_json.rb +0 -61
  121. data/test/plugin/test_formatter_ltsv.rb +0 -70
  122. data/test/plugin/test_formatter_msgpack.rb +0 -28
  123. data/test/plugin/test_formatter_out_file.rb +0 -116
  124. data/test/plugin/test_formatter_single_value.rb +0 -44
  125. data/test/plugin/test_formatter_tsv.rb +0 -76
  126. data/test/plugin/test_in_debug_agent.rb +0 -49
  127. data/test/plugin/test_in_exec.rb +0 -261
  128. data/test/plugin/test_in_forward.rb +0 -1178
  129. data/test/plugin/test_in_gc_stat.rb +0 -62
  130. data/test/plugin/test_in_http.rb +0 -1102
  131. data/test/plugin/test_in_monitor_agent.rb +0 -922
  132. data/test/plugin/test_in_object_space.rb +0 -66
  133. data/test/plugin/test_in_sample.rb +0 -190
  134. data/test/plugin/test_in_syslog.rb +0 -505
  135. data/test/plugin/test_in_tail.rb +0 -3288
  136. data/test/plugin/test_in_tcp.rb +0 -328
  137. data/test/plugin/test_in_udp.rb +0 -296
  138. data/test/plugin/test_in_unix.rb +0 -181
  139. data/test/plugin/test_input.rb +0 -137
  140. data/test/plugin/test_metadata.rb +0 -89
  141. data/test/plugin/test_metrics.rb +0 -294
  142. data/test/plugin/test_metrics_local.rb +0 -96
  143. data/test/plugin/test_multi_output.rb +0 -204
  144. data/test/plugin/test_out_copy.rb +0 -308
  145. data/test/plugin/test_out_exec.rb +0 -312
  146. data/test/plugin/test_out_exec_filter.rb +0 -606
  147. data/test/plugin/test_out_file.rb +0 -1038
  148. data/test/plugin/test_out_forward.rb +0 -1349
  149. data/test/plugin/test_out_http.rb +0 -429
  150. data/test/plugin/test_out_null.rb +0 -105
  151. data/test/plugin/test_out_relabel.rb +0 -28
  152. data/test/plugin/test_out_roundrobin.rb +0 -146
  153. data/test/plugin/test_out_secondary_file.rb +0 -458
  154. data/test/plugin/test_out_stdout.rb +0 -205
  155. data/test/plugin/test_out_stream.rb +0 -103
  156. data/test/plugin/test_output.rb +0 -1334
  157. data/test/plugin/test_output_as_buffered.rb +0 -2024
  158. data/test/plugin/test_output_as_buffered_backup.rb +0 -363
  159. data/test/plugin/test_output_as_buffered_compress.rb +0 -179
  160. data/test/plugin/test_output_as_buffered_overflow.rb +0 -250
  161. data/test/plugin/test_output_as_buffered_retries.rb +0 -966
  162. data/test/plugin/test_output_as_buffered_secondary.rb +0 -882
  163. data/test/plugin/test_output_as_standard.rb +0 -374
  164. data/test/plugin/test_owned_by.rb +0 -35
  165. data/test/plugin/test_parser.rb +0 -399
  166. data/test/plugin/test_parser_apache.rb +0 -42
  167. data/test/plugin/test_parser_apache2.rb +0 -47
  168. data/test/plugin/test_parser_apache_error.rb +0 -45
  169. data/test/plugin/test_parser_csv.rb +0 -200
  170. data/test/plugin/test_parser_json.rb +0 -138
  171. data/test/plugin/test_parser_labeled_tsv.rb +0 -160
  172. data/test/plugin/test_parser_multiline.rb +0 -111
  173. data/test/plugin/test_parser_nginx.rb +0 -88
  174. data/test/plugin/test_parser_none.rb +0 -52
  175. data/test/plugin/test_parser_regexp.rb +0 -284
  176. data/test/plugin/test_parser_syslog.rb +0 -650
  177. data/test/plugin/test_parser_tsv.rb +0 -122
  178. data/test/plugin/test_sd_file.rb +0 -228
  179. data/test/plugin/test_sd_srv.rb +0 -230
  180. data/test/plugin/test_storage.rb +0 -167
  181. data/test/plugin/test_storage_local.rb +0 -335
  182. data/test/plugin/test_string_util.rb +0 -26
  183. data/test/plugin_helper/data/cert/cert-key.pem +0 -27
  184. data/test/plugin_helper/data/cert/cert-with-CRLF.pem +0 -19
  185. data/test/plugin_helper/data/cert/cert-with-no-newline.pem +0 -19
  186. data/test/plugin_helper/data/cert/cert.pem +0 -19
  187. data/test/plugin_helper/data/cert/cert_chains/ca-cert-key.pem +0 -27
  188. data/test/plugin_helper/data/cert/cert_chains/ca-cert.pem +0 -20
  189. data/test/plugin_helper/data/cert/cert_chains/cert-key.pem +0 -27
  190. data/test/plugin_helper/data/cert/cert_chains/cert.pem +0 -40
  191. data/test/plugin_helper/data/cert/empty.pem +0 -0
  192. data/test/plugin_helper/data/cert/generate_cert.rb +0 -125
  193. data/test/plugin_helper/data/cert/with_ca/ca-cert-key-pass.pem +0 -30
  194. data/test/plugin_helper/data/cert/with_ca/ca-cert-key.pem +0 -27
  195. data/test/plugin_helper/data/cert/with_ca/ca-cert-pass.pem +0 -20
  196. data/test/plugin_helper/data/cert/with_ca/ca-cert.pem +0 -20
  197. data/test/plugin_helper/data/cert/with_ca/cert-key-pass.pem +0 -30
  198. data/test/plugin_helper/data/cert/with_ca/cert-key.pem +0 -27
  199. data/test/plugin_helper/data/cert/with_ca/cert-pass.pem +0 -21
  200. data/test/plugin_helper/data/cert/with_ca/cert.pem +0 -21
  201. data/test/plugin_helper/data/cert/without_ca/cert-key-pass.pem +0 -30
  202. data/test/plugin_helper/data/cert/without_ca/cert-key.pem +0 -27
  203. data/test/plugin_helper/data/cert/without_ca/cert-pass.pem +0 -20
  204. data/test/plugin_helper/data/cert/without_ca/cert.pem +0 -20
  205. data/test/plugin_helper/http_server/test_app.rb +0 -65
  206. data/test/plugin_helper/http_server/test_route.rb +0 -32
  207. data/test/plugin_helper/service_discovery/test_manager.rb +0 -93
  208. data/test/plugin_helper/service_discovery/test_round_robin_balancer.rb +0 -21
  209. data/test/plugin_helper/test_cert_option.rb +0 -25
  210. data/test/plugin_helper/test_child_process.rb +0 -862
  211. data/test/plugin_helper/test_compat_parameters.rb +0 -358
  212. data/test/plugin_helper/test_event_emitter.rb +0 -80
  213. data/test/plugin_helper/test_event_loop.rb +0 -52
  214. data/test/plugin_helper/test_extract.rb +0 -194
  215. data/test/plugin_helper/test_formatter.rb +0 -255
  216. data/test/plugin_helper/test_http_server_helper.rb +0 -372
  217. data/test/plugin_helper/test_inject.rb +0 -561
  218. data/test/plugin_helper/test_metrics.rb +0 -137
  219. data/test/plugin_helper/test_parser.rb +0 -264
  220. data/test/plugin_helper/test_record_accessor.rb +0 -238
  221. data/test/plugin_helper/test_retry_state.rb +0 -1006
  222. data/test/plugin_helper/test_server.rb +0 -1895
  223. data/test/plugin_helper/test_service_discovery.rb +0 -165
  224. data/test/plugin_helper/test_socket.rb +0 -146
  225. data/test/plugin_helper/test_storage.rb +0 -542
  226. data/test/plugin_helper/test_thread.rb +0 -164
  227. data/test/plugin_helper/test_timer.rb +0 -130
  228. data/test/scripts/exec_script.rb +0 -32
  229. data/test/scripts/fluent/plugin/formatter1/formatter_test1.rb +0 -7
  230. data/test/scripts/fluent/plugin/formatter2/formatter_test2.rb +0 -7
  231. data/test/scripts/fluent/plugin/formatter_known.rb +0 -8
  232. data/test/scripts/fluent/plugin/out_test.rb +0 -81
  233. data/test/scripts/fluent/plugin/out_test2.rb +0 -80
  234. data/test/scripts/fluent/plugin/parser_known.rb +0 -4
  235. data/test/test_capability.rb +0 -74
  236. data/test/test_clock.rb +0 -164
  237. data/test/test_config.rb +0 -369
  238. data/test/test_configdsl.rb +0 -148
  239. data/test/test_daemonizer.rb +0 -91
  240. data/test/test_engine.rb +0 -203
  241. data/test/test_event.rb +0 -531
  242. data/test/test_event_router.rb +0 -348
  243. data/test/test_event_time.rb +0 -199
  244. data/test/test_file_wrapper.rb +0 -53
  245. data/test/test_filter.rb +0 -121
  246. data/test/test_fluent_log_event_router.rb +0 -99
  247. data/test/test_formatter.rb +0 -369
  248. data/test/test_input.rb +0 -31
  249. data/test/test_log.rb +0 -1076
  250. data/test/test_match.rb +0 -148
  251. data/test/test_mixin.rb +0 -351
  252. data/test/test_msgpack_factory.rb +0 -50
  253. data/test/test_oj_options.rb +0 -55
  254. data/test/test_output.rb +0 -278
  255. data/test/test_plugin.rb +0 -251
  256. data/test/test_plugin_classes.rb +0 -370
  257. data/test/test_plugin_helper.rb +0 -81
  258. data/test/test_plugin_id.rb +0 -119
  259. data/test/test_process.rb +0 -14
  260. data/test/test_root_agent.rb +0 -951
  261. data/test/test_static_config_analysis.rb +0 -177
  262. data/test/test_supervisor.rb +0 -821
  263. data/test/test_test_drivers.rb +0 -136
  264. data/test/test_time_formatter.rb +0 -301
  265. data/test/test_time_parser.rb +0 -362
  266. data/test/test_tls.rb +0 -65
  267. data/test/test_unique_id.rb +0 -47
  268. data/test/test_variable_store.rb +0 -65
@@ -1,731 +0,0 @@
1
- require_relative '../helper'
2
- require 'timecop'
3
- require 'fluent/test/driver/filter'
4
- require 'fluent/plugin/filter_parser'
5
- require 'flexmock/test_unit'
6
-
7
- class ParserFilterTest < Test::Unit::TestCase
8
- include FlexMock::TestCase
9
-
10
- def setup
11
- Fluent::Test.setup
12
- @tag = 'test'
13
- @default_time = Time.parse('2010-05-04 03:02:01 UTC')
14
- Timecop.freeze(@default_time)
15
- end
16
-
17
- def teardown
18
- super
19
- Timecop.return
20
- end
21
-
22
- def assert_equal_parsed_time(expected, actual)
23
- if expected.is_a?(Integer)
24
- assert_equal(expected, actual.to_i)
25
- else
26
- assert_equal_event_time(expected, actual)
27
- end
28
- end
29
-
30
- ParserError = Fluent::Plugin::Parser::ParserError
31
- CONFIG = %[
32
- key_name message
33
- reserve_data true
34
- <parse>
35
- @type regexp
36
- expression /^(?<x>.)(?<y>.) (?<time>.+)$/
37
- time_format %Y%m%d%H%M%S
38
- </parse>
39
- ]
40
-
41
- def create_driver(conf=CONFIG)
42
- Fluent::Test::Driver::Filter.new(Fluent::Plugin::ParserFilter).configure(conf)
43
- end
44
-
45
- def test_configure
46
- assert_raise(Fluent::ConfigError) {
47
- create_driver('')
48
- }
49
- assert_raise(Fluent::NotFoundPluginError) {
50
- create_driver %[
51
- key_name foo
52
- <parse>
53
- @type unknown_format_that_will_never_be_implemented
54
- </parse>
55
- ]
56
- }
57
- assert_nothing_raised {
58
- create_driver %[
59
- key_name foo
60
- <parse>
61
- @type regexp
62
- expression /(?<x>.)/
63
- </parse>
64
- ]
65
- }
66
- assert_nothing_raised {
67
- create_driver %[
68
- key_name foo
69
- <parse>
70
- @type json
71
- </parse>
72
- ]
73
- }
74
- assert_nothing_raised {
75
- create_driver %[
76
- key_name foo
77
- format json
78
- ]
79
- }
80
- assert_nothing_raised {
81
- create_driver %[
82
- key_name foo
83
- <parse>
84
- @type ltsv
85
- </parse>
86
- ]
87
- }
88
- assert_nothing_raised {
89
- create_driver %[
90
- key_name message
91
- <parse>
92
- @type regexp
93
- expression /^col1=(?<col1>.+) col2=(?<col2>.+)$/
94
- </parse>
95
- ]
96
- }
97
- d = create_driver %[
98
- key_name foo
99
- <parse>
100
- @type regexp
101
- expression /(?<x>.)/
102
- </parse>
103
- ]
104
- assert_false d.instance.reserve_data
105
- end
106
-
107
- # CONFIG = %[
108
- # remove_prefix test
109
- # add_prefix parsed
110
- # key_name message
111
- # format /^(?<x>.)(?<y>.) (?<time>.+)$/
112
- # time_format %Y%m%d%H%M%S
113
- # reserve_data true
114
- # ]
115
- def test_filter
116
- d1 = create_driver(CONFIG)
117
- time = event_time("2012-01-02 13:14:15")
118
- d1.run(default_tag: @tag) do
119
- d1.feed(time, {'message' => '12 20120402182059'})
120
- d1.feed(time, {'message' => '34 20120402182100'})
121
- d1.feed(time, {'message' => '56 20120402182100'})
122
- d1.feed(time, {'message' => '78 20120402182101'})
123
- d1.feed(time, {'message' => '90 20120402182100'})
124
- end
125
- filtered = d1.filtered
126
- assert_equal 5, filtered.length
127
-
128
- first = filtered[0]
129
- assert_equal_event_time event_time("2012-04-02 18:20:59"), first[0]
130
- assert_equal '1', first[1]['x']
131
- assert_equal '2', first[1]['y']
132
- assert_equal '12 20120402182059', first[1]['message']
133
-
134
- second = filtered[1]
135
- assert_equal_event_time event_time("2012-04-02 18:21:00"), second[0]
136
- assert_equal '3', second[1]['x']
137
- assert_equal '4', second[1]['y']
138
-
139
- third = filtered[2]
140
- assert_equal_event_time event_time("2012-04-02 18:21:00"), third[0]
141
- assert_equal '5', third[1]['x']
142
- assert_equal '6', third[1]['y']
143
-
144
- fourth = filtered[3]
145
- assert_equal_event_time event_time("2012-04-02 18:21:01"), fourth[0]
146
- assert_equal '7', fourth[1]['x']
147
- assert_equal '8', fourth[1]['y']
148
-
149
- fifth = filtered[4]
150
- assert_equal_event_time event_time("2012-04-02 18:21:00"), fifth[0]
151
- assert_equal '9', fifth[1]['x']
152
- assert_equal '0', fifth[1]['y']
153
-
154
- d2 = create_driver(%[
155
- key_name data
156
- format /^(?<x>.)(?<y>.) (?<t>.+)$/
157
- ])
158
- time = Fluent::EventTime.from_time(@default_time) # EventTime emit test
159
- d2.run(default_tag: @tag) do
160
- d2.feed(time, {'data' => '12 20120402182059'})
161
- d2.feed(time, {'data' => '34 20120402182100'})
162
- end
163
- filtered = d2.filtered
164
- assert_equal 2, filtered.length
165
-
166
- first = filtered[0]
167
- assert_equal_event_time time, first[0]
168
- assert_nil first[1]['data']
169
- assert_equal '1', first[1]['x']
170
- assert_equal '2', first[1]['y']
171
- assert_equal '20120402182059', first[1]['t']
172
-
173
- second = filtered[1]
174
- assert_equal_event_time time, second[0]
175
- assert_nil second[1]['data']
176
- assert_equal '3', second[1]['x']
177
- assert_equal '4', second[1]['y']
178
- assert_equal '20120402182100', second[1]['t']
179
-
180
- d3 = create_driver(%[
181
- key_name data
182
- <parse>
183
- @type regexp
184
- expression /^(?<x>[0-9])(?<y>[0-9]) (?<t>.+)$/
185
- </parse>
186
- ])
187
- time = Time.parse("2012-04-02 18:20:59").to_i
188
- d3.run(default_tag: @tag) do
189
- d3.feed(time, {'data' => '12 20120402182059'})
190
- d3.feed(time, {'data' => '34 20120402182100'})
191
- d3.feed(time, {'data' => 'xy 20120402182101'})
192
- end
193
- filtered = d3.filtered
194
- assert_equal 2, filtered.length
195
-
196
- d4 = create_driver(%[
197
- key_name data
198
- <parse>
199
- @type json
200
- </parse>
201
- ])
202
- time = Time.parse("2012-04-02 18:20:59").to_i
203
- d4.run(default_tag: @tag) do
204
- d4.feed(time, {'data' => '{"xxx":"first","yyy":"second"}', 'xxx' => 'x', 'yyy' => 'y'})
205
- d4.feed(time, {'data' => 'foobar', 'xxx' => 'x', 'yyy' => 'y'})
206
- end
207
- filtered = d4.filtered
208
- assert_equal 1, filtered.length
209
-
210
- end
211
-
212
- data(:keep_key_name => false,
213
- :remove_key_name => true)
214
- def test_filter_with_reserved_data(remove_key_name)
215
- d1 = create_driver(%[
216
- key_name data
217
- reserve_data yes
218
- remove_key_name_field #{remove_key_name}
219
- <parse>
220
- @type regexp
221
- expression /^(?<x>\\d)(?<y>\\d) (?<t>.+)$/
222
- </parse>
223
- ])
224
- time = event_time("2012-04-02 18:20:59")
225
- d1.run(default_tag: @tag) do
226
- d1.feed(time, {'data' => '12 20120402182059'})
227
- d1.feed(time, {'data' => '34 20120402182100'})
228
- d1.feed(time, {'data' => 'xy 20120402182101'})
229
- end
230
- filtered = d1.filtered
231
- assert_equal 3, filtered.length
232
-
233
- d2 = create_driver(%[
234
- key_name data
235
- reserve_data yes
236
- remove_key_name_field #{remove_key_name}
237
- <parse>
238
- @type json
239
- </parse>
240
- ])
241
- time = Fluent::EventTime.from_time(@default_time)
242
- d2.run(default_tag: @tag) do
243
- d2.feed(time, {'data' => '{"xxx":"first","yyy":"second"}', 'xxx' => 'x', 'yyy' => 'y'})
244
- d2.feed(time, {'data' => 'foobar', 'xxx' => 'x', 'yyy' => 'y'})
245
- end
246
- filtered = d2.filtered
247
- assert_equal 2, filtered.length
248
-
249
- first = filtered[0]
250
- assert_equal_event_time time, first[0]
251
- if remove_key_name
252
- assert_not_include first[1], 'data'
253
- else
254
- assert_equal '{"xxx":"first","yyy":"second"}', first[1]['data']
255
- end
256
- assert_equal 'first', first[1]['xxx']
257
- assert_equal 'second', first[1]['yyy']
258
-
259
- second = filtered[1]
260
- assert_equal_event_time time, second[0]
261
- assert_equal 'foobar', second[1]['data']
262
- assert_equal 'x', second[1]['xxx']
263
- assert_equal 'y', second[1]['yyy']
264
- end
265
-
266
-
267
- CONFIG_LTSV = %[
268
- key_name data
269
- <parse>
270
- @type ltsv
271
- </parse>
272
- ]
273
- CONFIG_LTSV_WITH_TYPES = %[
274
- key_name data
275
- <parse>
276
- @type ltsv
277
- types i:integer,s:string,f:float,b:bool
278
- </parse>
279
- ]
280
- data(:event_time => lambda { |time| Fluent::EventTime.from_time(time) },
281
- :int_time => lambda { |time| time.to_i })
282
- def test_filter_ltsv(time_parse)
283
- d = create_driver(CONFIG_LTSV)
284
- time = time_parse.call(@default_time)
285
- d.run(default_tag: @tag) do
286
- d.feed(time, {'data' => "xxx:first\tyyy:second", 'xxx' => 'x', 'yyy' => 'y'})
287
- d.feed(time, {'data' => "xxx:first\tyyy:second2", 'xxx' => 'x', 'yyy' => 'y'})
288
- end
289
- filtered = d.filtered
290
- assert_equal 2, filtered.length
291
-
292
- first = filtered[0]
293
- assert_equal_parsed_time time, first[0]
294
- assert_nil first[1]['data']
295
- assert_equal 'first', first[1]['xxx']
296
- assert_equal 'second', first[1]['yyy']
297
-
298
- second = filtered[1]
299
- assert_equal_parsed_time time, second[0]
300
- assert_nil first[1]['data']
301
- assert_equal 'first', second[1]['xxx']
302
- assert_equal 'second2', second[1]['yyy']
303
-
304
- d = create_driver(CONFIG_LTSV + %[reserve_data yes])
305
- time = @default_time.to_i
306
- d.run(default_tag: @tag) do
307
- d.feed(time, {'data' => "xxx:first\tyyy:second", 'xxx' => 'x', 'yyy' => 'y'})
308
- d.feed(time, {'data' => "xxx:first\tyyy:second2", 'xxx' => 'x', 'yyy' => 'y'})
309
- end
310
- filtered = d.filtered
311
- assert_equal 2, filtered.length
312
-
313
- first = filtered[0]
314
- assert_equal_parsed_time time, first[0]
315
- assert_equal "xxx:first\tyyy:second", first[1]['data']
316
- assert_equal 'first', first[1]['xxx']
317
- assert_equal 'second', first[1]['yyy']
318
-
319
- second = filtered[1]
320
- assert_equal_parsed_time time, second[0]
321
- assert_equal "xxx:first\tyyy:second", first[1]['data']
322
- assert_equal 'first', second[1]['xxx']
323
- assert_equal 'second2', second[1]['yyy']
324
-
325
- # convert types
326
- #d = create_driver(CONFIG_LTSV + %[
327
- d = create_driver(CONFIG_LTSV_WITH_TYPES)
328
- time = @default_time.to_i
329
- d.run do
330
- d.feed(@tag, time, {'data' => "i:1\ts:2\tf:3\tb:true\tx:123"})
331
- end
332
- filtered = d.filtered
333
- assert_equal 1, filtered.length
334
-
335
- first = filtered[0]
336
- assert_equal_parsed_time time, first[0]
337
- assert_equal 1, first[1]['i']
338
- assert_equal '2', first[1]['s']
339
- assert_equal 3.0, first[1]['f']
340
- assert_equal true, first[1]['b']
341
- assert_equal '123', first[1]['x']
342
- end
343
-
344
- CONFIG_TSV = %[
345
- key_name data
346
- <parse>
347
- @type tsv
348
- keys key1,key2,key3
349
- </parse>
350
- ]
351
- data(:event_time => lambda { |time| Fluent::EventTime.from_time(time) },
352
- :int_time => lambda { |time| time.to_i })
353
- def test_filter_tsv(time_parse)
354
- d = create_driver(CONFIG_TSV)
355
- time = time_parse.call(@default_time)
356
- d.run do
357
- d.feed(@tag, time, {'data' => "value1\tvalue2\tvalueThree", 'xxx' => 'x', 'yyy' => 'y'})
358
- end
359
- filtered = d.filtered
360
- assert_equal 1, filtered.length
361
-
362
- first = filtered[0]
363
- assert_equal_parsed_time time, first[0]
364
- assert_nil first[1]['data']
365
- assert_equal 'value1', first[1]['key1']
366
- assert_equal 'value2', first[1]['key2']
367
- assert_equal 'valueThree', first[1]['key3']
368
- end
369
-
370
- CONFIG_CSV = %[
371
- key_name data
372
- <parse>
373
- @type csv
374
- keys key1,key2,key3
375
- </parse>
376
- ]
377
- CONFIG_CSV_COMPAT = %[
378
- key_name data
379
- format csv
380
- keys key1,key2,key3
381
- ]
382
- data(new_conf: CONFIG_CSV,
383
- compat_conf: CONFIG_CSV_COMPAT)
384
- def test_filter_csv(conf)
385
- d = create_driver(conf)
386
- time = @default_time.to_i
387
- d.run do
388
- d.feed(@tag, time, {'data' => 'value1,"value2","value""ThreeYes!"', 'xxx' => 'x', 'yyy' => 'y'})
389
- end
390
- filtered = d.filtered
391
- assert_equal 1, filtered.length
392
-
393
- first = filtered[0]
394
- assert_equal time, first[0]
395
- assert_nil first[1]['data']
396
- assert_equal 'value1', first[1]['key1']
397
- assert_equal 'value2', first[1]['key2']
398
- assert_equal 'value"ThreeYes!', first[1]['key3']
399
- end
400
-
401
- def test_filter_with_nested_record
402
- d = create_driver(%[
403
- key_name $.data.log
404
- <parse>
405
- @type csv
406
- keys key1,key2,key3
407
- </parse>
408
- ])
409
- time = @default_time.to_i
410
- d.run do
411
- d.feed(@tag, time, {'data' => {'log' => 'value1,"value2","value""ThreeYes!"'}, 'xxx' => 'x', 'yyy' => 'y'})
412
- end
413
- filtered = d.filtered
414
- assert_equal 1, filtered.length
415
-
416
- first = filtered[0]
417
- assert_equal time, first[0]
418
- assert_nil first[1]['data']
419
- assert_equal 'value1', first[1]['key1']
420
- assert_equal 'value2', first[1]['key2']
421
- assert_equal 'value"ThreeYes!', first[1]['key3']
422
- end
423
-
424
- CONFIG_HASH_VALUE_FIELD = %[
425
- key_name data
426
- hash_value_field parsed
427
- <parse>
428
- @type json
429
- </parse>
430
- ]
431
- CONFIG_HASH_VALUE_FIELD_RESERVE_DATA = %[
432
- key_name data
433
- reserve_data yes
434
- hash_value_field parsed
435
- <parse>
436
- @type json
437
- </parse>
438
- ]
439
- CONFIG_HASH_VALUE_FIELD_WITH_INJECT_KEY_PREFIX = %[
440
- key_name data
441
- hash_value_field parsed
442
- inject_key_prefix data.
443
- <parse>
444
- @type json
445
- </parse>
446
- ]
447
- data(:event_time => lambda { |time| Fluent::EventTime.from_time(time) },
448
- :int_time => lambda { |time| time.to_i })
449
- def test_filter_inject_hash_value_field(time_parse)
450
- original = {'data' => '{"xxx":"first","yyy":"second"}', 'xxx' => 'x', 'yyy' => 'y'}
451
-
452
- d = create_driver(CONFIG_HASH_VALUE_FIELD)
453
- time = time_parse.call(@default_time)
454
- d.run do
455
- d.feed(@tag, time, original)
456
- end
457
- filtered = d.filtered
458
- assert_equal 1, filtered.length
459
-
460
- first = filtered[0]
461
- assert_equal_parsed_time time, first[0]
462
-
463
- record = first[1]
464
- assert_equal 1, record.keys.size
465
- assert_equal({"xxx"=>"first","yyy"=>"second"}, record['parsed'])
466
-
467
- d = create_driver(CONFIG_HASH_VALUE_FIELD_RESERVE_DATA)
468
- time = @default_time.to_i
469
- d.run do
470
- d.feed(@tag, time, original)
471
- end
472
- filtered = d.filtered
473
- assert_equal 1, filtered.length
474
-
475
- first = filtered[0]
476
- assert_equal_parsed_time time, first[0]
477
-
478
- record = first[1]
479
- assert_equal 4, record.keys.size
480
- assert_equal original['data'], record['data']
481
- assert_equal original['xxx'], record['xxx']
482
- assert_equal original['yyy'], record['yyy']
483
- assert_equal({"xxx"=>"first","yyy"=>"second"}, record['parsed'])
484
-
485
- d = create_driver(CONFIG_HASH_VALUE_FIELD_WITH_INJECT_KEY_PREFIX)
486
- time = @default_time.to_i
487
- d.run do
488
- d.feed(@tag, time, original)
489
- end
490
- filtered = d.filtered
491
- assert_equal 1, filtered.length
492
-
493
- first = filtered[0]
494
- assert_equal_parsed_time time, first[0]
495
-
496
- record = first[1]
497
- assert_equal 1, record.keys.size
498
- assert_equal({"data.xxx"=>"first","data.yyy"=>"second"}, record['parsed'])
499
- end
500
-
501
- CONFIG_DONT_PARSE_TIME = %[
502
- key_name data
503
- reserve_time true
504
- <parse>
505
- @type json
506
- keep_time_key true
507
- </parse>
508
- ]
509
- CONFIG_DONT_PARSE_TIME_COMPAT = %[
510
- key_name data
511
- reserve_time true
512
- format json
513
- keep_time_key true
514
- ]
515
- data(new_conf: CONFIG_DONT_PARSE_TIME,
516
- compat_conf: CONFIG_DONT_PARSE_TIME_COMPAT)
517
- def test_time_should_be_reserved(conf)
518
- t = Time.now.to_i
519
- d = create_driver(conf)
520
- d.run(default_tag: @tag) do
521
- d.feed(t, {'data' => '{"time":1383190430, "f1":"v1"}'})
522
- d.feed(t, {'data' => '{"time":"1383190430", "f1":"v1"}'})
523
- d.feed(t, {'data' => '{"time":"2013-10-31 12:34:03 +0900", "f1":"v1"}'})
524
- end
525
- filtered = d.filtered
526
- assert_equal 3, filtered.length
527
-
528
- assert_equal 'v1', filtered[0][1]['f1']
529
- assert_equal 1383190430, filtered[0][1]['time']
530
- assert_equal t, filtered[0][0]
531
-
532
- assert_equal 'v1', filtered[1][1]['f1']
533
- assert_equal "1383190430", filtered[1][1]['time']
534
- assert_equal t, filtered[1][0]
535
-
536
- assert_equal 'v1', filtered[2][1]['f1']
537
- assert_equal '2013-10-31 12:34:03 +0900', filtered[2][1]['time']
538
- assert_equal t, filtered[2][0]
539
- end
540
-
541
- CONFIG_INVALID_TIME_VALUE = %[
542
- key_name data
543
- <parse>
544
- @type json
545
- </parse>
546
- ] # 'time' is implicit @time_key
547
- def test_filter_invalid_time_data
548
- # should not raise errors
549
- time = Time.now.to_i
550
- d = create_driver(CONFIG_INVALID_TIME_VALUE)
551
- assert_nothing_raised {
552
- d.run(default_tag: @tag) do
553
- d.feed(time, {'data' => '{"time":[], "f1":"v1"}'})
554
- d.feed(time, {'data' => '{"time":"thisisnottime", "f1":"v1"}'})
555
- end
556
- }
557
- filtered = d.filtered
558
- assert_equal 1, filtered.length
559
-
560
- assert_equal 0, filtered[0][0].to_i
561
- assert_equal 'v1', filtered[0][1]['f1']
562
- assert_equal nil, filtered[0][1]['time']
563
- end
564
-
565
- # REGEXP = /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/
566
-
567
- CONFIG_NOT_REPLACE = %[
568
- key_name data
569
- <parse>
570
- @type regexp
571
- expression /^(?<message>.*)$/
572
- </parse>
573
- ]
574
- CONFIG_INVALID_BYTE = CONFIG_NOT_REPLACE + %[
575
- replace_invalid_sequence true
576
- ]
577
- def test_filter_invalid_byte
578
- invalid_utf8 = "\xff".force_encoding('UTF-8')
579
-
580
- d = create_driver(CONFIG_NOT_REPLACE)
581
- d.run do
582
- d.feed(@tag, Fluent::EventTime.now.to_i, {'data' => invalid_utf8})
583
- end
584
- error_event = d.error_events.first
585
- assert_equal "test", error_event[0]
586
- assert_instance_of ArgumentError, error_event[3]
587
-
588
- d = create_driver(CONFIG_INVALID_BYTE)
589
- assert_nothing_raised {
590
- d.run do
591
- d.feed(@tag, Fluent::EventTime.now.to_i, {'data' => invalid_utf8})
592
- end
593
- }
594
- filtered = d.filtered
595
- assert_equal 1, filtered.length
596
- assert_nil filtered[0][1]['data']
597
- assert_equal '?'.force_encoding('UTF-8'), filtered[0][1]['message']
598
-
599
- d = create_driver(CONFIG_INVALID_BYTE + %[reserve_data yes])
600
- assert_nothing_raised {
601
- d.run do
602
- d.feed(@tag, Fluent::EventTime.now.to_i, {'data' => invalid_utf8})
603
- end
604
- }
605
- filtered = d.filtered
606
- assert_equal 1, filtered.length
607
- assert_equal invalid_utf8, filtered[0][1]['data']
608
- assert_equal '?'.force_encoding('UTF-8'), filtered[0][1]['message']
609
-
610
- invalid_ascii = "\xff".force_encoding('US-ASCII')
611
- d = create_driver(CONFIG_INVALID_BYTE)
612
- assert_nothing_raised {
613
- d.run do
614
- d.feed(@tag, Fluent::EventTime.now.to_i, {'data' => invalid_ascii})
615
- end
616
- }
617
- filtered = d.filtered
618
- assert_equal 1, filtered.length
619
- assert_nil filtered[0][1]['data']
620
- assert_equal '?'.force_encoding('US-ASCII'), filtered[0][1]['message']
621
- end
622
-
623
- CONFIG_NOT_IGNORE = %[
624
- key_name data
625
- hash_value_field parsed
626
- <parse>
627
- @type json
628
- </parse>
629
- ]
630
- CONFIG_PASS_SAME_RECORD = CONFIG_NOT_IGNORE + %[
631
- reserve_data true
632
- ]
633
- def test_filter_key_not_exist
634
- d = create_driver(CONFIG_NOT_IGNORE)
635
- flexmock(d.instance.router).should_receive(:emit_error_event).
636
- with(String, Integer, Hash, ArgumentError.new("data does not exist")).once
637
- assert_nothing_raised {
638
- d.run do
639
- d.feed(@tag, Fluent::EventTime.now.to_i, {'foo' => 'bar'})
640
- end
641
- }
642
-
643
- d = create_driver(CONFIG_PASS_SAME_RECORD)
644
- assert_nothing_raised {
645
- d.run do
646
- d.feed(@tag, Fluent::EventTime.now.to_i, {'foo' => 'bar'})
647
- end
648
- }
649
- filtered = d.filtered
650
- assert_equal 1, filtered.length
651
- assert_nil filtered[0][1]['data']
652
- assert_equal 'bar', filtered[0][1]['foo']
653
- end
654
-
655
- # emit_error_event test
656
- INVALID_MESSAGE = 'foo bar'
657
- VALID_MESSAGE = 'col1=foo col2=bar'
658
-
659
- def test_call_emit_error_event_when_parser_error
660
- d = create_driver(CONFIG_INVALID_TIME_VALUE)
661
- flexmock(d.instance.router).should_receive(:emit_error_event).
662
- with(String, Integer, Hash, ParserError).once
663
- d.run do
664
- d.feed(@tag, Fluent::EventTime.now.to_i, {'data' => '{"time":[], "f1":"v1"}'})
665
- end
666
- end
667
-
668
- CONFIG_UNMATCHED_PATTERN_LOG = %[
669
- key_name message
670
- <parse>
671
- @type regexp
672
- expression /^col1=(?<col1>.+) col2=(?<col2>.+)$/
673
- </parse>
674
- ]
675
-
676
- class UnmatchedPatternLogTest < self
677
- setup do
678
- @d = create_driver(CONFIG_UNMATCHED_PATTERN_LOG)
679
- end
680
-
681
- def test_call_emit_error_event_when_pattern_is_mismached
682
- flexmock(@d.instance.router).should_receive(:emit_error_event).
683
- with(String, Integer, Hash, ParserError.new("pattern not matched with data '#{INVALID_MESSAGE}'")).once
684
- @d.run do
685
- @d.feed(@tag, Fluent::EventTime.now.to_i, {'message' => INVALID_MESSAGE})
686
- end
687
- end
688
-
689
- def test_not_call_emit_error_event_when_pattern_is_mached
690
- flexmock(@d.instance.router).should_receive(:emit_error_event).never
691
- @d.run do
692
- @d.feed(@tag, Fluent::EventTime.now.to_i, {'message' => VALID_MESSAGE})
693
- end
694
- end
695
- end
696
-
697
- class EmitInvalidRecordToErrorTest < self
698
- def test_pattern_is_mismached_with_emit_invalid_record_to_error
699
- d = create_driver(CONFIG_UNMATCHED_PATTERN_LOG + "emit_invalid_record_to_error false")
700
- flexmock(d.instance.router).should_receive(:emit_error_event).never
701
- assert_nothing_raised {
702
- d.run do
703
- d.feed(@tag, Fluent::EventTime.now.to_i, {'message' => INVALID_MESSAGE})
704
- end
705
- }
706
- assert_equal 0, d.filtered.length
707
- end
708
-
709
- def test_parser_error_with_emit_invalid_record_to_error
710
- d = create_driver(CONFIG_INVALID_TIME_VALUE + "emit_invalid_record_to_error false")
711
- flexmock(d.instance.router).should_receive(:emit_error_event).never
712
- assert_nothing_raised {
713
- d.run do
714
- d.feed(@tag, Fluent::EventTime.now.to_i, {'data' => '{"time":[], "f1":"v1"}'})
715
- end
716
- }
717
- assert_equal 0, d.filtered.length
718
- end
719
-
720
- def test_key_not_exist_with_emit_invalid_record_to_error
721
- d = create_driver(CONFIG_NOT_IGNORE + "emit_invalid_record_to_error false")
722
- flexmock(d.instance.router).should_receive(:emit_error_event).never
723
- assert_nothing_raised {
724
- d.run do
725
- d.feed(@tag, Fluent::EventTime.now.to_i, {'foo' => 'bar'})
726
- end
727
- }
728
- assert_equal 0, d.filtered.length
729
- end
730
- end
731
- end