fluentd 1.17.0-x86-mingw32 → 1.17.1-x86-mingw32

Sign up to get free protection for your applications and to get access to all the features.
Files changed (259) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +46 -0
  3. data/README.md +1 -0
  4. data/SECURITY.md +2 -2
  5. data/fluent.conf +14 -14
  6. data/lib/fluent/command/cap_ctl.rb +4 -4
  7. data/lib/fluent/compat/call_super_mixin.rb +3 -3
  8. data/lib/fluent/compat/propagate_default.rb +4 -4
  9. data/lib/fluent/config/yaml_parser/parser.rb +4 -0
  10. data/lib/fluent/log/console_adapter.rb +4 -2
  11. data/lib/fluent/plugin/in_exec.rb +14 -2
  12. data/lib/fluent/plugin/in_http.rb +1 -1
  13. data/lib/fluent/plugin/in_sample.rb +13 -7
  14. data/lib/fluent/plugin/in_tail.rb +65 -23
  15. data/lib/fluent/plugin/out_copy.rb +1 -1
  16. data/lib/fluent/plugin/out_file.rb +8 -0
  17. data/lib/fluent/plugin/out_http.rb +12 -0
  18. data/lib/fluent/plugin/parser_json.rb +4 -12
  19. data/lib/fluent/plugin_helper/http_server/server.rb +1 -1
  20. data/lib/fluent/version.rb +1 -1
  21. data/templates/new_gem/fluent-plugin.gemspec.erb +6 -5
  22. metadata +25 -472
  23. data/.github/DISCUSSION_TEMPLATE/q-a-japanese.yml +0 -50
  24. data/.github/DISCUSSION_TEMPLATE/q-a.yml +0 -47
  25. data/.github/ISSUE_TEMPLATE/bug_report.yml +0 -71
  26. data/.github/ISSUE_TEMPLATE/config.yml +0 -5
  27. data/.github/ISSUE_TEMPLATE/feature_request.yml +0 -39
  28. data/.github/ISSUE_TEMPLATE.md +0 -17
  29. data/.github/PULL_REQUEST_TEMPLATE.md +0 -14
  30. data/.github/workflows/stale-actions.yml +0 -24
  31. data/.github/workflows/test-ruby-head.yml +0 -31
  32. data/.github/workflows/test.yml +0 -32
  33. data/.gitignore +0 -30
  34. data/Gemfile +0 -9
  35. data/fluentd.gemspec +0 -62
  36. data/test/command/test_binlog_reader.rb +0 -362
  37. data/test/command/test_ca_generate.rb +0 -70
  38. data/test/command/test_cap_ctl.rb +0 -100
  39. data/test/command/test_cat.rb +0 -128
  40. data/test/command/test_ctl.rb +0 -56
  41. data/test/command/test_fluentd.rb +0 -1291
  42. data/test/command/test_plugin_config_formatter.rb +0 -397
  43. data/test/command/test_plugin_generator.rb +0 -109
  44. data/test/compat/test_calls_super.rb +0 -166
  45. data/test/compat/test_parser.rb +0 -92
  46. data/test/config/assertions.rb +0 -42
  47. data/test/config/test_config_parser.rb +0 -551
  48. data/test/config/test_configurable.rb +0 -1784
  49. data/test/config/test_configure_proxy.rb +0 -604
  50. data/test/config/test_dsl.rb +0 -415
  51. data/test/config/test_element.rb +0 -518
  52. data/test/config/test_literal_parser.rb +0 -309
  53. data/test/config/test_plugin_configuration.rb +0 -56
  54. data/test/config/test_section.rb +0 -191
  55. data/test/config/test_system_config.rb +0 -195
  56. data/test/config/test_types.rb +0 -408
  57. data/test/counter/test_client.rb +0 -563
  58. data/test/counter/test_error.rb +0 -44
  59. data/test/counter/test_mutex_hash.rb +0 -179
  60. data/test/counter/test_server.rb +0 -589
  61. data/test/counter/test_store.rb +0 -258
  62. data/test/counter/test_validator.rb +0 -137
  63. data/test/helper.rb +0 -155
  64. data/test/helpers/fuzzy_assert.rb +0 -89
  65. data/test/helpers/process_extenstion.rb +0 -33
  66. data/test/log/test_console_adapter.rb +0 -117
  67. data/test/plugin/data/2010/01/20100102-030405.log +0 -0
  68. data/test/plugin/data/2010/01/20100102-030406.log +0 -0
  69. data/test/plugin/data/2010/01/20100102.log +0 -0
  70. data/test/plugin/data/log/bar +0 -0
  71. data/test/plugin/data/log/foo/bar.log +0 -0
  72. data/test/plugin/data/log/foo/bar2 +0 -0
  73. data/test/plugin/data/log/test.log +0 -0
  74. data/test/plugin/data/log_numeric/01.log +0 -0
  75. data/test/plugin/data/log_numeric/02.log +0 -0
  76. data/test/plugin/data/log_numeric/12.log +0 -0
  77. data/test/plugin/data/log_numeric/14.log +0 -0
  78. data/test/plugin/data/sd_file/config +0 -11
  79. data/test/plugin/data/sd_file/config.json +0 -17
  80. data/test/plugin/data/sd_file/config.yaml +0 -11
  81. data/test/plugin/data/sd_file/config.yml +0 -11
  82. data/test/plugin/data/sd_file/invalid_config.yml +0 -7
  83. data/test/plugin/in_tail/test_fifo.rb +0 -121
  84. data/test/plugin/in_tail/test_io_handler.rb +0 -150
  85. data/test/plugin/in_tail/test_position_file.rb +0 -346
  86. data/test/plugin/out_forward/test_ack_handler.rb +0 -140
  87. data/test/plugin/out_forward/test_connection_manager.rb +0 -145
  88. data/test/plugin/out_forward/test_handshake_protocol.rb +0 -112
  89. data/test/plugin/out_forward/test_load_balancer.rb +0 -106
  90. data/test/plugin/out_forward/test_socket_cache.rb +0 -174
  91. data/test/plugin/test_bare_output.rb +0 -131
  92. data/test/plugin/test_base.rb +0 -247
  93. data/test/plugin/test_buf_file.rb +0 -1314
  94. data/test/plugin/test_buf_file_single.rb +0 -898
  95. data/test/plugin/test_buf_memory.rb +0 -42
  96. data/test/plugin/test_buffer.rb +0 -1493
  97. data/test/plugin/test_buffer_chunk.rb +0 -209
  98. data/test/plugin/test_buffer_file_chunk.rb +0 -871
  99. data/test/plugin/test_buffer_file_single_chunk.rb +0 -611
  100. data/test/plugin/test_buffer_memory_chunk.rb +0 -339
  101. data/test/plugin/test_compressable.rb +0 -87
  102. data/test/plugin/test_file_util.rb +0 -96
  103. data/test/plugin/test_filter.rb +0 -368
  104. data/test/plugin/test_filter_grep.rb +0 -697
  105. data/test/plugin/test_filter_parser.rb +0 -731
  106. data/test/plugin/test_filter_record_transformer.rb +0 -577
  107. data/test/plugin/test_filter_stdout.rb +0 -207
  108. data/test/plugin/test_formatter_csv.rb +0 -136
  109. data/test/plugin/test_formatter_hash.rb +0 -38
  110. data/test/plugin/test_formatter_json.rb +0 -61
  111. data/test/plugin/test_formatter_ltsv.rb +0 -70
  112. data/test/plugin/test_formatter_msgpack.rb +0 -28
  113. data/test/plugin/test_formatter_out_file.rb +0 -116
  114. data/test/plugin/test_formatter_single_value.rb +0 -44
  115. data/test/plugin/test_formatter_tsv.rb +0 -76
  116. data/test/plugin/test_in_debug_agent.rb +0 -49
  117. data/test/plugin/test_in_exec.rb +0 -261
  118. data/test/plugin/test_in_forward.rb +0 -1178
  119. data/test/plugin/test_in_gc_stat.rb +0 -62
  120. data/test/plugin/test_in_http.rb +0 -1124
  121. data/test/plugin/test_in_monitor_agent.rb +0 -922
  122. data/test/plugin/test_in_object_space.rb +0 -66
  123. data/test/plugin/test_in_sample.rb +0 -190
  124. data/test/plugin/test_in_syslog.rb +0 -505
  125. data/test/plugin/test_in_tail.rb +0 -3429
  126. data/test/plugin/test_in_tcp.rb +0 -328
  127. data/test/plugin/test_in_udp.rb +0 -296
  128. data/test/plugin/test_in_unix.rb +0 -181
  129. data/test/plugin/test_input.rb +0 -137
  130. data/test/plugin/test_metadata.rb +0 -89
  131. data/test/plugin/test_metrics.rb +0 -294
  132. data/test/plugin/test_metrics_local.rb +0 -96
  133. data/test/plugin/test_multi_output.rb +0 -204
  134. data/test/plugin/test_out_copy.rb +0 -308
  135. data/test/plugin/test_out_exec.rb +0 -312
  136. data/test/plugin/test_out_exec_filter.rb +0 -606
  137. data/test/plugin/test_out_file.rb +0 -1038
  138. data/test/plugin/test_out_forward.rb +0 -1349
  139. data/test/plugin/test_out_http.rb +0 -557
  140. data/test/plugin/test_out_null.rb +0 -105
  141. data/test/plugin/test_out_relabel.rb +0 -28
  142. data/test/plugin/test_out_roundrobin.rb +0 -146
  143. data/test/plugin/test_out_secondary_file.rb +0 -458
  144. data/test/plugin/test_out_stdout.rb +0 -205
  145. data/test/plugin/test_out_stream.rb +0 -103
  146. data/test/plugin/test_output.rb +0 -1334
  147. data/test/plugin/test_output_as_buffered.rb +0 -2024
  148. data/test/plugin/test_output_as_buffered_backup.rb +0 -363
  149. data/test/plugin/test_output_as_buffered_compress.rb +0 -179
  150. data/test/plugin/test_output_as_buffered_overflow.rb +0 -250
  151. data/test/plugin/test_output_as_buffered_retries.rb +0 -966
  152. data/test/plugin/test_output_as_buffered_secondary.rb +0 -882
  153. data/test/plugin/test_output_as_standard.rb +0 -374
  154. data/test/plugin/test_owned_by.rb +0 -34
  155. data/test/plugin/test_parser.rb +0 -399
  156. data/test/plugin/test_parser_apache.rb +0 -42
  157. data/test/plugin/test_parser_apache2.rb +0 -47
  158. data/test/plugin/test_parser_apache_error.rb +0 -45
  159. data/test/plugin/test_parser_csv.rb +0 -200
  160. data/test/plugin/test_parser_json.rb +0 -244
  161. data/test/plugin/test_parser_labeled_tsv.rb +0 -160
  162. data/test/plugin/test_parser_msgpack.rb +0 -127
  163. data/test/plugin/test_parser_multiline.rb +0 -111
  164. data/test/plugin/test_parser_nginx.rb +0 -88
  165. data/test/plugin/test_parser_none.rb +0 -52
  166. data/test/plugin/test_parser_regexp.rb +0 -284
  167. data/test/plugin/test_parser_syslog.rb +0 -650
  168. data/test/plugin/test_parser_tsv.rb +0 -122
  169. data/test/plugin/test_sd_file.rb +0 -228
  170. data/test/plugin/test_sd_srv.rb +0 -230
  171. data/test/plugin/test_storage.rb +0 -166
  172. data/test/plugin/test_storage_local.rb +0 -335
  173. data/test/plugin/test_string_util.rb +0 -26
  174. data/test/plugin_helper/data/cert/cert-key.pem +0 -27
  175. data/test/plugin_helper/data/cert/cert-with-CRLF.pem +0 -19
  176. data/test/plugin_helper/data/cert/cert-with-no-newline.pem +0 -19
  177. data/test/plugin_helper/data/cert/cert.pem +0 -19
  178. data/test/plugin_helper/data/cert/cert_chains/ca-cert-key.pem +0 -27
  179. data/test/plugin_helper/data/cert/cert_chains/ca-cert.pem +0 -20
  180. data/test/plugin_helper/data/cert/cert_chains/cert-key.pem +0 -27
  181. data/test/plugin_helper/data/cert/cert_chains/cert.pem +0 -40
  182. data/test/plugin_helper/data/cert/empty.pem +0 -0
  183. data/test/plugin_helper/data/cert/generate_cert.rb +0 -125
  184. data/test/plugin_helper/data/cert/with_ca/ca-cert-key-pass.pem +0 -30
  185. data/test/plugin_helper/data/cert/with_ca/ca-cert-key.pem +0 -27
  186. data/test/plugin_helper/data/cert/with_ca/ca-cert-pass.pem +0 -20
  187. data/test/plugin_helper/data/cert/with_ca/ca-cert.pem +0 -20
  188. data/test/plugin_helper/data/cert/with_ca/cert-key-pass.pem +0 -30
  189. data/test/plugin_helper/data/cert/with_ca/cert-key.pem +0 -27
  190. data/test/plugin_helper/data/cert/with_ca/cert-pass.pem +0 -21
  191. data/test/plugin_helper/data/cert/with_ca/cert.pem +0 -21
  192. data/test/plugin_helper/data/cert/without_ca/cert-key-pass.pem +0 -30
  193. data/test/plugin_helper/data/cert/without_ca/cert-key.pem +0 -27
  194. data/test/plugin_helper/data/cert/without_ca/cert-pass.pem +0 -20
  195. data/test/plugin_helper/data/cert/without_ca/cert.pem +0 -20
  196. data/test/plugin_helper/http_server/test_app.rb +0 -65
  197. data/test/plugin_helper/http_server/test_route.rb +0 -32
  198. data/test/plugin_helper/service_discovery/test_manager.rb +0 -93
  199. data/test/plugin_helper/service_discovery/test_round_robin_balancer.rb +0 -21
  200. data/test/plugin_helper/test_cert_option.rb +0 -25
  201. data/test/plugin_helper/test_child_process.rb +0 -862
  202. data/test/plugin_helper/test_compat_parameters.rb +0 -358
  203. data/test/plugin_helper/test_event_emitter.rb +0 -80
  204. data/test/plugin_helper/test_event_loop.rb +0 -52
  205. data/test/plugin_helper/test_extract.rb +0 -194
  206. data/test/plugin_helper/test_formatter.rb +0 -255
  207. data/test/plugin_helper/test_http_server_helper.rb +0 -372
  208. data/test/plugin_helper/test_inject.rb +0 -561
  209. data/test/plugin_helper/test_metrics.rb +0 -137
  210. data/test/plugin_helper/test_parser.rb +0 -264
  211. data/test/plugin_helper/test_record_accessor.rb +0 -238
  212. data/test/plugin_helper/test_retry_state.rb +0 -1006
  213. data/test/plugin_helper/test_server.rb +0 -1895
  214. data/test/plugin_helper/test_service_discovery.rb +0 -165
  215. data/test/plugin_helper/test_socket.rb +0 -146
  216. data/test/plugin_helper/test_storage.rb +0 -542
  217. data/test/plugin_helper/test_thread.rb +0 -164
  218. data/test/plugin_helper/test_timer.rb +0 -130
  219. data/test/scripts/exec_script.rb +0 -32
  220. data/test/scripts/fluent/plugin/formatter1/formatter_test1.rb +0 -7
  221. data/test/scripts/fluent/plugin/formatter2/formatter_test2.rb +0 -7
  222. data/test/scripts/fluent/plugin/formatter_known.rb +0 -8
  223. data/test/scripts/fluent/plugin/out_test.rb +0 -81
  224. data/test/scripts/fluent/plugin/out_test2.rb +0 -80
  225. data/test/scripts/fluent/plugin/parser_known.rb +0 -4
  226. data/test/test_capability.rb +0 -74
  227. data/test/test_clock.rb +0 -164
  228. data/test/test_config.rb +0 -369
  229. data/test/test_configdsl.rb +0 -148
  230. data/test/test_daemonizer.rb +0 -91
  231. data/test/test_engine.rb +0 -203
  232. data/test/test_event.rb +0 -531
  233. data/test/test_event_router.rb +0 -348
  234. data/test/test_event_time.rb +0 -199
  235. data/test/test_file_wrapper.rb +0 -53
  236. data/test/test_filter.rb +0 -121
  237. data/test/test_fluent_log_event_router.rb +0 -99
  238. data/test/test_formatter.rb +0 -369
  239. data/test/test_input.rb +0 -31
  240. data/test/test_log.rb +0 -1076
  241. data/test/test_match.rb +0 -148
  242. data/test/test_mixin.rb +0 -351
  243. data/test/test_msgpack_factory.rb +0 -50
  244. data/test/test_oj_options.rb +0 -55
  245. data/test/test_output.rb +0 -278
  246. data/test/test_plugin.rb +0 -251
  247. data/test/test_plugin_classes.rb +0 -370
  248. data/test/test_plugin_helper.rb +0 -81
  249. data/test/test_plugin_id.rb +0 -119
  250. data/test/test_process.rb +0 -14
  251. data/test/test_root_agent.rb +0 -951
  252. data/test/test_static_config_analysis.rb +0 -177
  253. data/test/test_supervisor.rb +0 -821
  254. data/test/test_test_drivers.rb +0 -136
  255. data/test/test_time_formatter.rb +0 -301
  256. data/test/test_time_parser.rb +0 -362
  257. data/test/test_tls.rb +0 -65
  258. data/test/test_unique_id.rb +0 -47
  259. data/test/test_variable_store.rb +0 -65
@@ -1,606 +0,0 @@
1
- require_relative '../helper'
2
- require 'fluent/test/driver/output'
3
- require 'fluent/plugin/out_exec_filter'
4
- require 'fileutils'
5
-
6
- class ExecFilterOutputTest < Test::Unit::TestCase
7
- def setup
8
- Fluent::Test.setup
9
- end
10
-
11
- CONFIG = %[
12
- command cat
13
- num_children 3
14
- <inject>
15
- tag_key tag
16
- time_key time_in
17
- time_type string
18
- time_format %Y-%m-%d %H:%M:%S
19
- </inject>
20
- <format>
21
- keys ["time_in", "tag", "k1"]
22
- </format>
23
- <parse>
24
- keys ["time_out", "tag", "k2"]
25
- </parse>
26
- <extract>
27
- tag_key tag
28
- time_key time_out
29
- time_type string
30
- time_format %Y-%m-%d %H:%M:%S
31
- </extract>
32
- ]
33
-
34
- CONFIG_COMPAT = %[
35
- command cat
36
- in_keys time_in,tag,k1
37
- out_keys time_out,tag,k2
38
- tag_key tag
39
- in_time_key time_in
40
- out_time_key time_out
41
- time_format %Y-%m-%d %H:%M:%S
42
- localtime
43
- num_children 3
44
- ]
45
-
46
- def create_driver(conf)
47
- Fluent::Test::Driver::Output.new(Fluent::Plugin::ExecFilterOutput).configure(conf)
48
- end
49
-
50
- SED_SUPPORT_UNBUFFERED_OPTION = ->(){
51
- system("echo xxx | sed --unbuffered -l -e 's/x/y/g' >#{IO::NULL} 2>&1")
52
- $?.success?
53
- }.call
54
- SED_UNBUFFERED_OPTION = SED_SUPPORT_UNBUFFERED_OPTION ? '--unbuffered' : ''
55
-
56
- data(
57
- 'with sections' => CONFIG,
58
- 'traditional' => CONFIG_COMPAT,
59
- )
60
- test 'configure' do |conf|
61
- d = create_driver(conf)
62
-
63
- assert_false d.instance.parser.estimate_current_event
64
-
65
- assert_equal ["time_in","tag","k1"], d.instance.formatter.keys
66
- assert_equal ["time_out","tag","k2"], d.instance.parser.keys
67
- assert_equal "tag", d.instance.inject_config.tag_key
68
- assert_equal "tag", d.instance.extract_config.tag_key
69
- assert_equal "time_in", d.instance.inject_config.time_key
70
- assert_equal "time_out", d.instance.extract_config.time_key
71
- assert_equal "%Y-%m-%d %H:%M:%S", d.instance.inject_config.time_format
72
- assert_equal "%Y-%m-%d %H:%M:%S", d.instance.extract_config.time_format
73
- assert_equal true, d.instance.inject_config.localtime
74
- assert_equal 3, d.instance.num_children
75
-
76
- d = create_driver %[
77
- command sed -l -e s/foo/bar/
78
- in_keys time,k1
79
- out_keys time,k2
80
- tag xxx
81
- time_key time
82
- num_children 3
83
- ]
84
- assert_equal "sed -l -e s/foo/bar/", d.instance.command
85
-
86
- d = create_driver(conf + %[
87
- remove_prefix before
88
- add_prefix after
89
- ])
90
- assert_equal "before", d.instance.remove_prefix
91
- assert_equal "after" , d.instance.add_prefix
92
- end
93
-
94
- data(
95
- 'with sections' => CONFIG,
96
- 'traditional' => CONFIG_COMPAT,
97
- )
98
- test 'emit events with TSV format' do |conf|
99
- d = create_driver(conf)
100
- time = event_time("2011-01-02 13:14:15")
101
-
102
- d.run(default_tag: 'test', expect_emits: 2, timeout: 10) do
103
- # sleep 0.1 until d.instance.children && !d.instance.children.empty? && d.instance.children.all?{|c| c.finished == false }
104
- d.feed(time, {"k1"=>1})
105
- d.feed(time, {"k1"=>2})
106
- end
107
-
108
- assert_equal "2011-01-02 13:14:15\ttest\t1\n", d.formatted[0]
109
- assert_equal "2011-01-02 13:14:15\ttest\t2\n", d.formatted[1]
110
-
111
- events = d.events
112
- assert_equal 2, events.length
113
- assert_equal_event_time time, events[0][1]
114
- assert_equal ["test", time, {"k2"=>"1"}], events[0]
115
- assert_equal_event_time time, events[1][1]
116
- assert_equal ["test", time, {"k2"=>"2"}], events[1]
117
- end
118
-
119
- CONFIG_WITHOUT_TIME_FORMAT = %[
120
- command cat
121
- num_children 3
122
- tag xxx
123
- <inject>
124
- time_key time
125
- time_type unixtime
126
- </inject>
127
- <format>
128
- keys time,k1
129
- </format>
130
- <parse>
131
- keys time,k2
132
- time_key time
133
- time_type unixtime
134
- </parse>
135
- ]
136
- CONFIG_WITHOUT_TIME_FORMAT_COMPAT = %[
137
- command cat
138
- in_keys time,k1
139
- out_keys time,k2
140
- tag xxx
141
- time_key time
142
- num_children 3
143
- ]
144
-
145
- data(
146
- 'with sections' => CONFIG_WITHOUT_TIME_FORMAT,
147
- 'traditional' => CONFIG_WITHOUT_TIME_FORMAT_COMPAT,
148
- )
149
- test 'emit events without time format configuration' do |conf|
150
- d = create_driver(conf)
151
- time = event_time("2011-01-02 13:14:15 +0900")
152
-
153
- d.run(default_tag: 'test', expect_emits: 2, timeout: 10) do
154
- d.feed(time, {"k1"=>1})
155
- d.feed(time, {"k1"=>2})
156
- end
157
-
158
- assert_equal "1293941655\t1\n", d.formatted[0]
159
- assert_equal "1293941655\t2\n", d.formatted[1]
160
-
161
- events = d.events
162
- assert_equal 2, events.length
163
- assert_equal_event_time time, events[0][1]
164
- assert_equal ["xxx", time, {"k2"=>"1"}], events[0]
165
- assert_equal_event_time time, events[1][1]
166
- assert_equal ["xxx", time, {"k2"=>"2"}], events[1]
167
- end
168
-
169
- CONFIG_TO_DO_GREP = %[
170
- command grep --line-buffered -v poo
171
- num_children 3
172
- tag xxx
173
- <inject>
174
- time_key time
175
- time_type unixtime
176
- </inject>
177
- <format>
178
- keys time, val1
179
- </format>
180
- <parse>
181
- keys time, val2
182
- time_key time
183
- time_type unixtime
184
- </parse>
185
- ]
186
- CONFIG_TO_DO_GREP_COMPAT = %[
187
- command grep --line-buffered -v poo
188
- in_keys time,val1
189
- out_keys time,val2
190
- tag xxx
191
- time_key time
192
- num_children 3
193
- ]
194
-
195
- data(
196
- 'with sections' => CONFIG_TO_DO_GREP,
197
- 'traditional' => CONFIG_TO_DO_GREP_COMPAT,
198
- )
199
- test 'emit events through grep command' do |conf|
200
- d = create_driver(conf)
201
- time = event_time("2011-01-02 13:14:15 +0900")
202
-
203
- d.run(default_tag: 'test', expect_emits: 1, timeout: 10) do
204
- d.feed(time, {"val1"=>"sed-ed value poo"})
205
- d.feed(time, {"val1"=>"sed-ed value foo"})
206
- end
207
-
208
- assert_equal "1293941655\tsed-ed value poo\n", d.formatted[0]
209
- assert_equal "1293941655\tsed-ed value foo\n", d.formatted[1]
210
-
211
- events = d.events
212
- assert_equal 1, events.length
213
- assert_equal_event_time time, events[0][1]
214
- assert_equal ["xxx", time, {"val2"=>"sed-ed value foo"}], events[0]
215
- end
216
-
217
- CONFIG_TO_DO_SED = %[
218
- command sed #{SED_UNBUFFERED_OPTION} -l -e s/foo/bar/
219
- num_children 3
220
- tag xxx
221
- <inject>
222
- time_key time
223
- time_type unixtime
224
- </inject>
225
- <format>
226
- keys time, val1
227
- </format>
228
- <parse>
229
- keys time, val2
230
- time_key time
231
- time_type unixtime
232
- </parse>
233
- ]
234
- CONFIG_TO_DO_SED_COMPAT = %[
235
- command sed #{SED_UNBUFFERED_OPTION} -l -e s/foo/bar/
236
- in_keys time,val1
237
- out_keys time,val2
238
- tag xxx
239
- time_key time
240
- num_children 3
241
- ]
242
-
243
- data(
244
- 'with sections' => CONFIG_TO_DO_SED,
245
- 'traditional' => CONFIG_TO_DO_SED_COMPAT,
246
- )
247
- test 'emit events through sed command' do |conf|
248
- d = create_driver(conf)
249
- time = event_time("2011-01-02 13:14:15 +0900")
250
-
251
- d.run(default_tag: 'test', expect_emits: 1, timeout: 10) do
252
- d.feed(time, {"val1"=>"sed-ed value poo"})
253
- d.feed(time, {"val1"=>"sed-ed value foo"})
254
- end
255
-
256
- assert_equal "1293941655\tsed-ed value poo\n", d.formatted[0]
257
- assert_equal "1293941655\tsed-ed value foo\n", d.formatted[1]
258
-
259
- events = d.events
260
- assert_equal 2, events.length
261
- assert_equal_event_time time, events[0][1]
262
- assert_equal ["xxx", time, {"val2"=>"sed-ed value poo"}], events[0]
263
- assert_equal_event_time time, events[1][1]
264
- assert_equal ["xxx", time, {"val2"=>"sed-ed value bar"}], events[1]
265
- end
266
-
267
- CONFIG_TO_DO_SED_WITH_TAG_MODIFY = %[
268
- command sed #{SED_UNBUFFERED_OPTION} -l -e s/foo/bar/
269
- num_children 3
270
- remove_prefix input
271
- add_prefix output
272
- <inject>
273
- tag_key tag
274
- time_key time
275
- </inject>
276
- <format>
277
- keys tag, time, val1
278
- </format>
279
- <parse>
280
- keys tag, time, val2
281
- </parse>
282
- <extract>
283
- tag_key tag
284
- time_key time
285
- </extract>
286
- ]
287
- CONFIG_TO_DO_SED_WITH_TAG_MODIFY_COMPAT = %[
288
- command sed #{SED_UNBUFFERED_OPTION} -l -e s/foo/bar/
289
- in_keys tag,time,val1
290
- remove_prefix input
291
- out_keys tag,time,val2
292
- add_prefix output
293
- tag_key tag
294
- time_key time
295
- num_children 3
296
- ]
297
-
298
- data(
299
- 'with sections' => CONFIG_TO_DO_SED_WITH_TAG_MODIFY,
300
- 'traditional' => CONFIG_TO_DO_SED_WITH_TAG_MODIFY_COMPAT,
301
- )
302
- test 'emit events with add/remove tag prefix' do |conf|
303
- d = create_driver(conf)
304
-
305
- time = event_time("2011-01-02 13:14:15 +0900")
306
-
307
- d.run(default_tag: 'input.test', expect_emits: 2, timeout: 10) do
308
- d.feed(time, {"val1"=>"sed-ed value foo"})
309
- d.feed(time, {"val1"=>"sed-ed value poo"})
310
- end
311
-
312
- assert_equal "test\t1293941655\tsed-ed value foo\n", d.formatted[0]
313
- assert_equal "test\t1293941655\tsed-ed value poo\n", d.formatted[1]
314
-
315
- events = d.events
316
- assert_equal 2, events.length
317
- assert_equal_event_time time, events[0][1]
318
- assert_equal ["output.test", time, {"val2"=>"sed-ed value bar"}], events[0]
319
- assert_equal_event_time time, events[1][1]
320
- assert_equal ["output.test", time, {"val2"=>"sed-ed value poo"}], events[1]
321
- end
322
-
323
- CONFIG_JSON = %[
324
- command cat
325
- <format>
326
- @type tsv
327
- keys message
328
- </format>
329
- <parse>
330
- @type json
331
- stream_buffer_size 1
332
- </parse>
333
- <extract>
334
- tag_key tag
335
- time_key time
336
- </extract>
337
- ]
338
- CONFIG_JSON_COMPAT = %[
339
- command cat
340
- in_keys message
341
- out_format json
342
- out_stream_buffer_size 1
343
- time_key time
344
- tag_key tag
345
- ]
346
-
347
- data(
348
- 'with sections' => CONFIG_JSON,
349
- 'traditional' => CONFIG_JSON_COMPAT,
350
- )
351
- test 'using json format' do |conf|
352
- d = create_driver(conf)
353
- time = event_time("2011-01-02 13:14:15 +0900")
354
-
355
- d.run(default_tag: 'input.test', expect_emits: 1, timeout: 10) do
356
- i = d.instance
357
- assert{ i.router }
358
- d.feed(time, {"message"=>%[{"time":#{time},"tag":"t1","k1":"v1"}]})
359
- end
360
-
361
- assert_equal '{"time":1293941655,"tag":"t1","k1":"v1"}' + "\n", d.formatted[0]
362
-
363
- events = d.events
364
- assert_equal 1, events.length
365
- assert_equal_event_time time, events[0][1]
366
- assert_equal ["t1", time, {"k1"=>"v1"}], events[0]
367
- end
368
-
369
- CONFIG_JSON_WITH_FLOAT_TIME = %[
370
- command cat
371
- <format>
372
- @type tsv
373
- keys message
374
- </format>
375
- <parse>
376
- @type json
377
- stream_buffer_size 1
378
- </parse>
379
- <extract>
380
- tag_key tag
381
- time_key time
382
- </extract>
383
- ]
384
- CONFIG_JSON_WITH_FLOAT_TIME_COMPAT = %[
385
- command cat
386
- in_keys message
387
- out_format json
388
- out_stream_buffer_size 1
389
- time_key time
390
- tag_key tag
391
- ]
392
-
393
- data(
394
- 'with sections' => CONFIG_JSON_WITH_FLOAT_TIME,
395
- 'traditional' => CONFIG_JSON_WITH_FLOAT_TIME_COMPAT,
396
- )
397
- test 'using json format with float time' do |conf|
398
- d = create_driver(conf)
399
- time = event_time("2011-01-02 13:14:15.123 +0900")
400
-
401
- d.run(default_tag: 'input.test', expect_emits: 1, timeout: 10) do
402
- d.feed(time + 10, {"message"=>%[{"time":#{time.sec}.#{time.nsec},"tag":"t1","k1":"v1"}]})
403
- end
404
-
405
- assert_equal '{"time":1293941655.123000000,"tag":"t1","k1":"v1"}' + "\n", d.formatted[0]
406
-
407
- events = d.events
408
- assert_equal 1, events.length
409
- assert_equal_event_time time, events[0][1]
410
- assert_equal ["t1", time, {"k1"=>"v1"}], events[0]
411
- end
412
-
413
- CONFIG_JSON_WITH_TIME_FORMAT = %[
414
- command cat
415
- <format>
416
- @type tsv
417
- keys message
418
- </format>
419
- <parse>
420
- @type json
421
- stream_buffer_size 1
422
- </parse>
423
- <extract>
424
- tag_key tag
425
- time_key time
426
- time_type string
427
- time_format %d/%b/%Y %H:%M:%S.%N %z
428
- </extract>
429
- ]
430
- CONFIG_JSON_WITH_TIME_FORMAT_COMPAT = %[
431
- command cat
432
- in_keys message
433
- out_format json
434
- out_stream_buffer_size 1
435
- time_key time
436
- time_format %d/%b/%Y %H:%M:%S.%N %z
437
- tag_key tag
438
- ]
439
-
440
- data(
441
- 'with sections' => CONFIG_JSON_WITH_TIME_FORMAT,
442
- 'traditional' => CONFIG_JSON_WITH_TIME_FORMAT_COMPAT,
443
- )
444
- test 'using json format with custom time format' do |conf|
445
- d = create_driver(conf)
446
- time_str = "28/Feb/2013 12:00:00.123456789 +0900"
447
- time = event_time(time_str, format: "%d/%b/%Y %H:%M:%S.%N %z")
448
-
449
- d.run(default_tag: 'input.test', expect_emits: 1, timeout: 10) do
450
- d.feed(time + 10, {"message"=>%[{"time":"#{time_str}","tag":"t1","k1":"v1"}]})
451
- end
452
-
453
- assert_equal '{"time":"28/Feb/2013 12:00:00.123456789 +0900","tag":"t1","k1":"v1"}' + "\n", d.formatted[0]
454
-
455
- events = d.events
456
- assert_equal 1, events.length
457
- assert_equal_event_time time, events[0][1]
458
- assert_equal ["t1", time, {"k1"=>"v1"}], events[0]
459
- end
460
-
461
- CONFIG_ROUND_ROBIN = %[
462
- command ruby -e 'STDOUT.sync = true; STDIN.each_line{|line| puts line.chomp + "\t" + Process.pid.to_s }'
463
- num_children 2
464
- <inject>
465
- tag_key tag
466
- time_key time_in
467
- time_type string
468
- time_format %Y-%m-%d %H:%M:%S
469
- </inject>
470
- <format>
471
- keys ["time_in", "tag", "k1"]
472
- </format>
473
- <parse>
474
- keys ["time_out", "tag", "k2", "child_pid"]
475
- </parse>
476
- <extract>
477
- tag_key tag
478
- time_key time_out
479
- time_type string
480
- time_format %Y-%m-%d %H:%M:%S
481
- </extract>
482
- ]
483
- CONFIG_ROUND_ROBIN_COMPAT = %[
484
- command ruby -e 'STDOUT.sync = true; STDIN.each_line{|line| puts line.chomp + "\t" + Process.pid.to_s }'
485
- in_keys time_in,tag,k1
486
- out_keys time_out,tag,k2,child_pid
487
- tag_key tag
488
- in_time_key time_in
489
- out_time_key time_out
490
- time_format %Y-%m-%d %H:%M:%S
491
- localtime
492
- num_children 2
493
- ]
494
-
495
- data(
496
- 'with sections' => CONFIG_ROUND_ROBIN,
497
- 'traditional' => CONFIG_ROUND_ROBIN_COMPAT,
498
- )
499
- test 'using child processes by round robin' do |conf|
500
- d = create_driver(conf)
501
- time = event_time('2011-01-02 13:14:15')
502
-
503
- d.run(default_tag: 'test', expect_emits: 1, timeout: 10, start: true, shutdown: false){ d.feed(time, {"k1" => 0}) }
504
- d.run(default_tag: 'test', expect_emits: 1, timeout: 10, start: false, shutdown: false){ d.feed(time, {"k1" => 1}) }
505
- d.run(default_tag: 'test', expect_emits: 1, timeout: 10, start: false, shutdown: false){ d.feed(time, {"k1" => 2}) }
506
- d.run(default_tag: 'test', expect_emits: 1, timeout: 10, start: false, shutdown: false){ d.feed(time, {"k1" => 3}) }
507
-
508
- assert_equal "2011-01-02 13:14:15\ttest\t0\n", d.formatted[0]
509
- assert_equal "2011-01-02 13:14:15\ttest\t1\n", d.formatted[1]
510
- assert_equal "2011-01-02 13:14:15\ttest\t2\n", d.formatted[2]
511
- assert_equal "2011-01-02 13:14:15\ttest\t3\n", d.formatted[3]
512
-
513
- events = d.events
514
- assert_equal 4, events.length
515
-
516
- pid_list = []
517
- events.each do |event|
518
- pid = event[2]['child_pid']
519
- pid_list << pid unless pid_list.include?(pid)
520
- end
521
- assert_equal 2, pid_list.size, "the number of pids should be same with number of child processes: #{pid_list.inspect}"
522
-
523
- assert_equal pid_list[0], events[0][2]['child_pid']
524
- assert_equal pid_list[1], events[1][2]['child_pid']
525
- assert_equal pid_list[0], events[2][2]['child_pid']
526
- assert_equal pid_list[1], events[3][2]['child_pid']
527
-
528
- ensure
529
- d.run(start: false, shutdown: true)
530
- end
531
-
532
- # child process exits per 3 lines
533
- CONFIG_RESPAWN = %[
534
- command ruby -e 'STDOUT.sync = true; proc = ->(){line = STDIN.readline.chomp; puts line + "\t" + Process.pid.to_s}; proc.call; proc.call; proc.call'
535
- num_children 2
536
- child_respawn -1
537
- <inject>
538
- tag_key tag
539
- time_key time_in
540
- time_type unixtime
541
- </inject>
542
- <format>
543
- keys ["time_in", "tag", "k1"]
544
- </format>
545
- <parse>
546
- keys ["time_out", "tag", "k2", "child_pid"]
547
- </parse>
548
- <extract>
549
- tag_key tag
550
- time_key time_out
551
- time_type unixtime
552
- </extract>
553
- ]
554
-
555
- CONFIG_RESPAWN_COMPAT = %[
556
- command ruby -e 'STDOUT.sync = true; proc = ->(){line = STDIN.readline.chomp; puts line + "\t" + Process.pid.to_s}; proc.call; proc.call; proc.call'
557
- num_children 2
558
- child_respawn -1
559
- in_keys time_in,tag,k1
560
- out_keys time_out,tag,k2,child_pid
561
- tag_key tag
562
- in_time_key time_in
563
- out_time_key time_out
564
- # time_format %Y-%m-%d %H:%M:%S
565
- # localtime
566
- ]
567
-
568
- data(
569
- 'with sections' => CONFIG_RESPAWN,
570
- 'traditional' => CONFIG_RESPAWN_COMPAT,
571
- )
572
- test 'emit events via child processes which exits sometimes' do |conf|
573
- d = create_driver(conf)
574
- time = event_time("2011-01-02 13:14:15")
575
- countup = 0
576
-
577
- d.run(start: true, shutdown: false)
578
- assert_equal 2, d.instance.instance_eval{ @_child_process_processes.size }
579
-
580
- 2.times do
581
- d.run(default_tag: 'test', expect_emits: 3, timeout: 3, force_flush_retry: true, start: false, shutdown: false) do
582
- d.feed(time, { "k1" => countup }); countup += 1
583
- d.feed(time, { "k1" => countup }); countup += 1
584
- d.feed(time, { "k1" => countup }); countup += 1
585
- end
586
- end
587
-
588
- events = d.events
589
- assert_equal 6, events.length
590
-
591
- pid_list = []
592
- events.each do |event|
593
- pid = event[2]['child_pid']
594
- pid_list << pid unless pid_list.include?(pid)
595
- end
596
-
597
- # the number of pids should be same with number of child processes
598
- assert_equal 2, pid_list.size
599
- logs = d.instance.log.out.logs
600
- assert_equal 2, logs.count { |l| l.include?('child process exits with error code') }
601
- assert_equal 2, logs.count { |l| l.include?('respawning child process') }
602
-
603
- ensure
604
- d.run(start: false, shutdown: true)
605
- end
606
- end