fluentd 1.17.0-x86-mingw32 → 1.17.1-x86-mingw32

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (259) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +46 -0
  3. data/README.md +1 -0
  4. data/SECURITY.md +2 -2
  5. data/fluent.conf +14 -14
  6. data/lib/fluent/command/cap_ctl.rb +4 -4
  7. data/lib/fluent/compat/call_super_mixin.rb +3 -3
  8. data/lib/fluent/compat/propagate_default.rb +4 -4
  9. data/lib/fluent/config/yaml_parser/parser.rb +4 -0
  10. data/lib/fluent/log/console_adapter.rb +4 -2
  11. data/lib/fluent/plugin/in_exec.rb +14 -2
  12. data/lib/fluent/plugin/in_http.rb +1 -1
  13. data/lib/fluent/plugin/in_sample.rb +13 -7
  14. data/lib/fluent/plugin/in_tail.rb +65 -23
  15. data/lib/fluent/plugin/out_copy.rb +1 -1
  16. data/lib/fluent/plugin/out_file.rb +8 -0
  17. data/lib/fluent/plugin/out_http.rb +12 -0
  18. data/lib/fluent/plugin/parser_json.rb +4 -12
  19. data/lib/fluent/plugin_helper/http_server/server.rb +1 -1
  20. data/lib/fluent/version.rb +1 -1
  21. data/templates/new_gem/fluent-plugin.gemspec.erb +6 -5
  22. metadata +25 -472
  23. data/.github/DISCUSSION_TEMPLATE/q-a-japanese.yml +0 -50
  24. data/.github/DISCUSSION_TEMPLATE/q-a.yml +0 -47
  25. data/.github/ISSUE_TEMPLATE/bug_report.yml +0 -71
  26. data/.github/ISSUE_TEMPLATE/config.yml +0 -5
  27. data/.github/ISSUE_TEMPLATE/feature_request.yml +0 -39
  28. data/.github/ISSUE_TEMPLATE.md +0 -17
  29. data/.github/PULL_REQUEST_TEMPLATE.md +0 -14
  30. data/.github/workflows/stale-actions.yml +0 -24
  31. data/.github/workflows/test-ruby-head.yml +0 -31
  32. data/.github/workflows/test.yml +0 -32
  33. data/.gitignore +0 -30
  34. data/Gemfile +0 -9
  35. data/fluentd.gemspec +0 -62
  36. data/test/command/test_binlog_reader.rb +0 -362
  37. data/test/command/test_ca_generate.rb +0 -70
  38. data/test/command/test_cap_ctl.rb +0 -100
  39. data/test/command/test_cat.rb +0 -128
  40. data/test/command/test_ctl.rb +0 -56
  41. data/test/command/test_fluentd.rb +0 -1291
  42. data/test/command/test_plugin_config_formatter.rb +0 -397
  43. data/test/command/test_plugin_generator.rb +0 -109
  44. data/test/compat/test_calls_super.rb +0 -166
  45. data/test/compat/test_parser.rb +0 -92
  46. data/test/config/assertions.rb +0 -42
  47. data/test/config/test_config_parser.rb +0 -551
  48. data/test/config/test_configurable.rb +0 -1784
  49. data/test/config/test_configure_proxy.rb +0 -604
  50. data/test/config/test_dsl.rb +0 -415
  51. data/test/config/test_element.rb +0 -518
  52. data/test/config/test_literal_parser.rb +0 -309
  53. data/test/config/test_plugin_configuration.rb +0 -56
  54. data/test/config/test_section.rb +0 -191
  55. data/test/config/test_system_config.rb +0 -195
  56. data/test/config/test_types.rb +0 -408
  57. data/test/counter/test_client.rb +0 -563
  58. data/test/counter/test_error.rb +0 -44
  59. data/test/counter/test_mutex_hash.rb +0 -179
  60. data/test/counter/test_server.rb +0 -589
  61. data/test/counter/test_store.rb +0 -258
  62. data/test/counter/test_validator.rb +0 -137
  63. data/test/helper.rb +0 -155
  64. data/test/helpers/fuzzy_assert.rb +0 -89
  65. data/test/helpers/process_extenstion.rb +0 -33
  66. data/test/log/test_console_adapter.rb +0 -117
  67. data/test/plugin/data/2010/01/20100102-030405.log +0 -0
  68. data/test/plugin/data/2010/01/20100102-030406.log +0 -0
  69. data/test/plugin/data/2010/01/20100102.log +0 -0
  70. data/test/plugin/data/log/bar +0 -0
  71. data/test/plugin/data/log/foo/bar.log +0 -0
  72. data/test/plugin/data/log/foo/bar2 +0 -0
  73. data/test/plugin/data/log/test.log +0 -0
  74. data/test/plugin/data/log_numeric/01.log +0 -0
  75. data/test/plugin/data/log_numeric/02.log +0 -0
  76. data/test/plugin/data/log_numeric/12.log +0 -0
  77. data/test/plugin/data/log_numeric/14.log +0 -0
  78. data/test/plugin/data/sd_file/config +0 -11
  79. data/test/plugin/data/sd_file/config.json +0 -17
  80. data/test/plugin/data/sd_file/config.yaml +0 -11
  81. data/test/plugin/data/sd_file/config.yml +0 -11
  82. data/test/plugin/data/sd_file/invalid_config.yml +0 -7
  83. data/test/plugin/in_tail/test_fifo.rb +0 -121
  84. data/test/plugin/in_tail/test_io_handler.rb +0 -150
  85. data/test/plugin/in_tail/test_position_file.rb +0 -346
  86. data/test/plugin/out_forward/test_ack_handler.rb +0 -140
  87. data/test/plugin/out_forward/test_connection_manager.rb +0 -145
  88. data/test/plugin/out_forward/test_handshake_protocol.rb +0 -112
  89. data/test/plugin/out_forward/test_load_balancer.rb +0 -106
  90. data/test/plugin/out_forward/test_socket_cache.rb +0 -174
  91. data/test/plugin/test_bare_output.rb +0 -131
  92. data/test/plugin/test_base.rb +0 -247
  93. data/test/plugin/test_buf_file.rb +0 -1314
  94. data/test/plugin/test_buf_file_single.rb +0 -898
  95. data/test/plugin/test_buf_memory.rb +0 -42
  96. data/test/plugin/test_buffer.rb +0 -1493
  97. data/test/plugin/test_buffer_chunk.rb +0 -209
  98. data/test/plugin/test_buffer_file_chunk.rb +0 -871
  99. data/test/plugin/test_buffer_file_single_chunk.rb +0 -611
  100. data/test/plugin/test_buffer_memory_chunk.rb +0 -339
  101. data/test/plugin/test_compressable.rb +0 -87
  102. data/test/plugin/test_file_util.rb +0 -96
  103. data/test/plugin/test_filter.rb +0 -368
  104. data/test/plugin/test_filter_grep.rb +0 -697
  105. data/test/plugin/test_filter_parser.rb +0 -731
  106. data/test/plugin/test_filter_record_transformer.rb +0 -577
  107. data/test/plugin/test_filter_stdout.rb +0 -207
  108. data/test/plugin/test_formatter_csv.rb +0 -136
  109. data/test/plugin/test_formatter_hash.rb +0 -38
  110. data/test/plugin/test_formatter_json.rb +0 -61
  111. data/test/plugin/test_formatter_ltsv.rb +0 -70
  112. data/test/plugin/test_formatter_msgpack.rb +0 -28
  113. data/test/plugin/test_formatter_out_file.rb +0 -116
  114. data/test/plugin/test_formatter_single_value.rb +0 -44
  115. data/test/plugin/test_formatter_tsv.rb +0 -76
  116. data/test/plugin/test_in_debug_agent.rb +0 -49
  117. data/test/plugin/test_in_exec.rb +0 -261
  118. data/test/plugin/test_in_forward.rb +0 -1178
  119. data/test/plugin/test_in_gc_stat.rb +0 -62
  120. data/test/plugin/test_in_http.rb +0 -1124
  121. data/test/plugin/test_in_monitor_agent.rb +0 -922
  122. data/test/plugin/test_in_object_space.rb +0 -66
  123. data/test/plugin/test_in_sample.rb +0 -190
  124. data/test/plugin/test_in_syslog.rb +0 -505
  125. data/test/plugin/test_in_tail.rb +0 -3429
  126. data/test/plugin/test_in_tcp.rb +0 -328
  127. data/test/plugin/test_in_udp.rb +0 -296
  128. data/test/plugin/test_in_unix.rb +0 -181
  129. data/test/plugin/test_input.rb +0 -137
  130. data/test/plugin/test_metadata.rb +0 -89
  131. data/test/plugin/test_metrics.rb +0 -294
  132. data/test/plugin/test_metrics_local.rb +0 -96
  133. data/test/plugin/test_multi_output.rb +0 -204
  134. data/test/plugin/test_out_copy.rb +0 -308
  135. data/test/plugin/test_out_exec.rb +0 -312
  136. data/test/plugin/test_out_exec_filter.rb +0 -606
  137. data/test/plugin/test_out_file.rb +0 -1038
  138. data/test/plugin/test_out_forward.rb +0 -1349
  139. data/test/plugin/test_out_http.rb +0 -557
  140. data/test/plugin/test_out_null.rb +0 -105
  141. data/test/plugin/test_out_relabel.rb +0 -28
  142. data/test/plugin/test_out_roundrobin.rb +0 -146
  143. data/test/plugin/test_out_secondary_file.rb +0 -458
  144. data/test/plugin/test_out_stdout.rb +0 -205
  145. data/test/plugin/test_out_stream.rb +0 -103
  146. data/test/plugin/test_output.rb +0 -1334
  147. data/test/plugin/test_output_as_buffered.rb +0 -2024
  148. data/test/plugin/test_output_as_buffered_backup.rb +0 -363
  149. data/test/plugin/test_output_as_buffered_compress.rb +0 -179
  150. data/test/plugin/test_output_as_buffered_overflow.rb +0 -250
  151. data/test/plugin/test_output_as_buffered_retries.rb +0 -966
  152. data/test/plugin/test_output_as_buffered_secondary.rb +0 -882
  153. data/test/plugin/test_output_as_standard.rb +0 -374
  154. data/test/plugin/test_owned_by.rb +0 -34
  155. data/test/plugin/test_parser.rb +0 -399
  156. data/test/plugin/test_parser_apache.rb +0 -42
  157. data/test/plugin/test_parser_apache2.rb +0 -47
  158. data/test/plugin/test_parser_apache_error.rb +0 -45
  159. data/test/plugin/test_parser_csv.rb +0 -200
  160. data/test/plugin/test_parser_json.rb +0 -244
  161. data/test/plugin/test_parser_labeled_tsv.rb +0 -160
  162. data/test/plugin/test_parser_msgpack.rb +0 -127
  163. data/test/plugin/test_parser_multiline.rb +0 -111
  164. data/test/plugin/test_parser_nginx.rb +0 -88
  165. data/test/plugin/test_parser_none.rb +0 -52
  166. data/test/plugin/test_parser_regexp.rb +0 -284
  167. data/test/plugin/test_parser_syslog.rb +0 -650
  168. data/test/plugin/test_parser_tsv.rb +0 -122
  169. data/test/plugin/test_sd_file.rb +0 -228
  170. data/test/plugin/test_sd_srv.rb +0 -230
  171. data/test/plugin/test_storage.rb +0 -166
  172. data/test/plugin/test_storage_local.rb +0 -335
  173. data/test/plugin/test_string_util.rb +0 -26
  174. data/test/plugin_helper/data/cert/cert-key.pem +0 -27
  175. data/test/plugin_helper/data/cert/cert-with-CRLF.pem +0 -19
  176. data/test/plugin_helper/data/cert/cert-with-no-newline.pem +0 -19
  177. data/test/plugin_helper/data/cert/cert.pem +0 -19
  178. data/test/plugin_helper/data/cert/cert_chains/ca-cert-key.pem +0 -27
  179. data/test/plugin_helper/data/cert/cert_chains/ca-cert.pem +0 -20
  180. data/test/plugin_helper/data/cert/cert_chains/cert-key.pem +0 -27
  181. data/test/plugin_helper/data/cert/cert_chains/cert.pem +0 -40
  182. data/test/plugin_helper/data/cert/empty.pem +0 -0
  183. data/test/plugin_helper/data/cert/generate_cert.rb +0 -125
  184. data/test/plugin_helper/data/cert/with_ca/ca-cert-key-pass.pem +0 -30
  185. data/test/plugin_helper/data/cert/with_ca/ca-cert-key.pem +0 -27
  186. data/test/plugin_helper/data/cert/with_ca/ca-cert-pass.pem +0 -20
  187. data/test/plugin_helper/data/cert/with_ca/ca-cert.pem +0 -20
  188. data/test/plugin_helper/data/cert/with_ca/cert-key-pass.pem +0 -30
  189. data/test/plugin_helper/data/cert/with_ca/cert-key.pem +0 -27
  190. data/test/plugin_helper/data/cert/with_ca/cert-pass.pem +0 -21
  191. data/test/plugin_helper/data/cert/with_ca/cert.pem +0 -21
  192. data/test/plugin_helper/data/cert/without_ca/cert-key-pass.pem +0 -30
  193. data/test/plugin_helper/data/cert/without_ca/cert-key.pem +0 -27
  194. data/test/plugin_helper/data/cert/without_ca/cert-pass.pem +0 -20
  195. data/test/plugin_helper/data/cert/without_ca/cert.pem +0 -20
  196. data/test/plugin_helper/http_server/test_app.rb +0 -65
  197. data/test/plugin_helper/http_server/test_route.rb +0 -32
  198. data/test/plugin_helper/service_discovery/test_manager.rb +0 -93
  199. data/test/plugin_helper/service_discovery/test_round_robin_balancer.rb +0 -21
  200. data/test/plugin_helper/test_cert_option.rb +0 -25
  201. data/test/plugin_helper/test_child_process.rb +0 -862
  202. data/test/plugin_helper/test_compat_parameters.rb +0 -358
  203. data/test/plugin_helper/test_event_emitter.rb +0 -80
  204. data/test/plugin_helper/test_event_loop.rb +0 -52
  205. data/test/plugin_helper/test_extract.rb +0 -194
  206. data/test/plugin_helper/test_formatter.rb +0 -255
  207. data/test/plugin_helper/test_http_server_helper.rb +0 -372
  208. data/test/plugin_helper/test_inject.rb +0 -561
  209. data/test/plugin_helper/test_metrics.rb +0 -137
  210. data/test/plugin_helper/test_parser.rb +0 -264
  211. data/test/plugin_helper/test_record_accessor.rb +0 -238
  212. data/test/plugin_helper/test_retry_state.rb +0 -1006
  213. data/test/plugin_helper/test_server.rb +0 -1895
  214. data/test/plugin_helper/test_service_discovery.rb +0 -165
  215. data/test/plugin_helper/test_socket.rb +0 -146
  216. data/test/plugin_helper/test_storage.rb +0 -542
  217. data/test/plugin_helper/test_thread.rb +0 -164
  218. data/test/plugin_helper/test_timer.rb +0 -130
  219. data/test/scripts/exec_script.rb +0 -32
  220. data/test/scripts/fluent/plugin/formatter1/formatter_test1.rb +0 -7
  221. data/test/scripts/fluent/plugin/formatter2/formatter_test2.rb +0 -7
  222. data/test/scripts/fluent/plugin/formatter_known.rb +0 -8
  223. data/test/scripts/fluent/plugin/out_test.rb +0 -81
  224. data/test/scripts/fluent/plugin/out_test2.rb +0 -80
  225. data/test/scripts/fluent/plugin/parser_known.rb +0 -4
  226. data/test/test_capability.rb +0 -74
  227. data/test/test_clock.rb +0 -164
  228. data/test/test_config.rb +0 -369
  229. data/test/test_configdsl.rb +0 -148
  230. data/test/test_daemonizer.rb +0 -91
  231. data/test/test_engine.rb +0 -203
  232. data/test/test_event.rb +0 -531
  233. data/test/test_event_router.rb +0 -348
  234. data/test/test_event_time.rb +0 -199
  235. data/test/test_file_wrapper.rb +0 -53
  236. data/test/test_filter.rb +0 -121
  237. data/test/test_fluent_log_event_router.rb +0 -99
  238. data/test/test_formatter.rb +0 -369
  239. data/test/test_input.rb +0 -31
  240. data/test/test_log.rb +0 -1076
  241. data/test/test_match.rb +0 -148
  242. data/test/test_mixin.rb +0 -351
  243. data/test/test_msgpack_factory.rb +0 -50
  244. data/test/test_oj_options.rb +0 -55
  245. data/test/test_output.rb +0 -278
  246. data/test/test_plugin.rb +0 -251
  247. data/test/test_plugin_classes.rb +0 -370
  248. data/test/test_plugin_helper.rb +0 -81
  249. data/test/test_plugin_id.rb +0 -119
  250. data/test/test_process.rb +0 -14
  251. data/test/test_root_agent.rb +0 -951
  252. data/test/test_static_config_analysis.rb +0 -177
  253. data/test/test_supervisor.rb +0 -821
  254. data/test/test_test_drivers.rb +0 -136
  255. data/test/test_time_formatter.rb +0 -301
  256. data/test/test_time_parser.rb +0 -362
  257. data/test/test_tls.rb +0 -65
  258. data/test/test_unique_id.rb +0 -47
  259. data/test/test_variable_store.rb +0 -65
@@ -1,1178 +0,0 @@
1
- require_relative '../helper'
2
-
3
- require 'fluent/test/driver/input'
4
- require 'fluent/test/startup_shutdown'
5
- require 'base64'
6
-
7
- require 'fluent/env'
8
- require 'fluent/event'
9
- require 'fluent/plugin/in_forward'
10
- require 'fluent/plugin/compressable'
11
-
12
- require 'timecop'
13
-
14
- class ForwardInputTest < Test::Unit::TestCase
15
- include Fluent::Plugin::Compressable
16
-
17
- def setup
18
- Fluent::Test.setup
19
- @responses = [] # for testing responses after sending data
20
- @d = nil
21
- @port = unused_port
22
- end
23
-
24
- def teardown
25
- @d.instance_shutdown if @d
26
- @port = nil
27
- end
28
-
29
- SHARED_KEY = 'foobar1'
30
- USER_NAME = 'tagomoris'
31
- USER_PASSWORD = 'fluentd'
32
-
33
- def base_config
34
- %[
35
- port #{@port}
36
- bind 127.0.0.1
37
- ]
38
- end
39
- LOCALHOST_HOSTNAME_GETTER = ->(){sock = UDPSocket.new(::Socket::AF_INET); sock.do_not_reverse_lookup = false; sock.connect("127.0.0.1", 2048); sock.peeraddr[2] }
40
- LOCALHOST_HOSTNAME = LOCALHOST_HOSTNAME_GETTER.call
41
- DUMMY_SOCK = Struct.new(:remote_host, :remote_addr, :remote_port).new(LOCALHOST_HOSTNAME, "127.0.0.1", 0)
42
-
43
- def config_auth
44
- base_config + %[
45
- <security>
46
- self_hostname localhost
47
- shared_key foobar1
48
- user_auth true
49
- <user>
50
- username #{USER_NAME}
51
- password #{USER_PASSWORD}
52
- </user>
53
- <client>
54
- network 127.0.0.0/8
55
- shared_key #{SHARED_KEY}
56
- users ["#{USER_NAME}"]
57
- </client>
58
- </security>
59
- ]
60
- end
61
-
62
- def create_driver(conf=base_config)
63
- Fluent::Test::Driver::Input.new(Fluent::Plugin::ForwardInput).configure(conf)
64
- end
65
-
66
- sub_test_case '#configure' do
67
- test 'simple' do
68
- @d = d = create_driver
69
- assert_equal @port, d.instance.port
70
- assert_equal '127.0.0.1', d.instance.bind
71
- assert_equal 0.5, d.instance.blocking_timeout
72
- assert !d.instance.backlog
73
- end
74
-
75
- test 'auth' do
76
- @d = d = create_driver(config_auth)
77
- assert_equal @port, d.instance.port
78
- assert_equal '127.0.0.1', d.instance.bind
79
- assert !d.instance.backlog
80
-
81
- assert d.instance.security
82
- assert_equal 1, d.instance.security.users.size
83
- assert_equal 1, d.instance.security.clients.size
84
- end
85
-
86
- data(tag: "tag",
87
- add_tag_prefix: "add_tag_prefix")
88
- test 'tag parameters' do |data|
89
- assert_raise(Fluent::ConfigError.new("'#{data}' parameter must not be empty")) {
90
- create_driver(base_config + "#{data} ''")
91
- }
92
- end
93
-
94
- test 'send_keepalive_packet is disabled by default' do
95
- @d = d = create_driver(config_auth)
96
- assert_false d.instance.send_keepalive_packet
97
- end
98
-
99
- test 'send_keepalive_packet can be enabled' do
100
- @d = d = create_driver(config_auth + %[
101
- send_keepalive_packet true
102
- ])
103
- assert_true d.instance.send_keepalive_packet
104
- end
105
-
106
- test 'both send_keepalive_packet and deny_keepalive cannot be enabled' do
107
- assert_raise(Fluent::ConfigError.new("both 'send_keepalive_packet' and 'deny_keepalive' cannot be set to true")) do
108
- create_driver(config_auth + %[
109
- send_keepalive_packet true
110
- deny_keepalive true
111
- ])
112
- end
113
- end
114
- end
115
-
116
- sub_test_case 'message' do
117
- test 'time' do
118
- time = event_time("2011-01-02 13:14:15 UTC")
119
- begin
120
- Timecop.freeze(Time.at(time))
121
- @d = d = create_driver
122
-
123
- records = [
124
- ["tag1", time, {"a"=>1}],
125
- ["tag2", time, {"a"=>2}],
126
- ]
127
-
128
- d.run(expect_records: records.length, timeout: 5) do
129
- records.each {|tag, _time, record|
130
- send_data packer.write([tag, 0, record]).to_s
131
- }
132
- end
133
- assert_equal(records, d.events.sort_by {|a| a[0] })
134
- ensure
135
- Timecop.return
136
- end
137
- end
138
-
139
- test 'plain' do
140
- @d = d = create_driver
141
-
142
- time = event_time("2011-01-02 13:14:15 UTC")
143
-
144
- records = [
145
- ["tag1", time, {"a"=>1}],
146
- ["tag2", time, {"a"=>2}],
147
- ]
148
-
149
- d.run(expect_records: records.length, timeout: 5) do
150
- records.each {|tag, _time, record|
151
- send_data packer.write([tag, _time, record]).to_s
152
- }
153
- end
154
- assert_equal(records, d.events)
155
- end
156
-
157
- test 'time_as_integer' do
158
- @d = d = create_driver
159
-
160
- time_i = event_time("2011-01-02 13:14:15 UTC").to_i
161
-
162
- records = [
163
- ["tag1", time_i, {"a"=>1}],
164
- ["tag2", time_i, {"a"=>2}],
165
- ]
166
-
167
- d.run(expect_records: records.length, timeout: 5) do
168
- records.each {|tag, _time, record|
169
- send_data packer.write([tag, _time, record]).to_s
170
- }
171
- end
172
-
173
- assert_equal(records, d.events)
174
- end
175
-
176
- test 'skip_invalid_event' do
177
- @d = d = create_driver(base_config + "skip_invalid_event true")
178
-
179
- time = event_time("2011-01-02 13:14:15 UTC")
180
-
181
- records = [
182
- ["tag1", time, {"a" => 1}],
183
- ["tag2", time, {"a" => 2}],
184
- ]
185
-
186
- d.run(shutdown: false, expect_records: 2, timeout: 10) do
187
- entries = []
188
- # These entries are skipped
189
- entries << ['tag1', true, {'a' => 3}] << ['tag2', time, 'invalid record']
190
- entries += records.map { |tag, _time, record| [tag, _time, record] }
191
-
192
- entries.each {|tag, _time, record|
193
- # Without ack, logs are sometimes not saved to logs during test.
194
- send_data packer.write([tag, _time, record]).to_s #, try_to_receive_response: true
195
- }
196
- end
197
-
198
- logs = d.instance.log.logs
199
- assert_equal 2, logs.count { |line| line =~ /got invalid event and drop it/ }
200
- assert_equal records[0], d.events[0]
201
- assert_equal records[1], d.events[1]
202
-
203
- d.instance_shutdown
204
- end
205
-
206
- test 'json_using_integer_time' do
207
- @d = d = create_driver
208
-
209
- time_i = event_time("2011-01-02 13:14:15 UTC").to_i
210
-
211
- records = [
212
- ["tag1", time_i, {"a"=>1}],
213
- ["tag2", time_i, {"a"=>2}],
214
- ]
215
-
216
- d.run(expect_records: records.length, timeout: 20) do
217
- records.each {|tag, _time, record|
218
- send_data [tag, _time, record].to_json
219
- }
220
- end
221
-
222
- assert_equal(records, d.events.sort_by {|a| a[0] })
223
- end
224
-
225
- test 'json_with_newline' do
226
- @d = d = create_driver
227
-
228
- time_i = event_time("2011-01-02 13:14:15 UTC").to_i
229
-
230
- records = [
231
- ["tag1", time_i, {"a"=>1}],
232
- ["tag2", time_i, {"a"=>2}],
233
- ]
234
-
235
- d.run(expect_records: records.length, timeout: 20, shutdown: true) do
236
- records.each {|tag, _time, record|
237
- send_data [tag, _time, record].to_json + "\n"
238
- }
239
- end
240
-
241
- assert_equal(records, d.events.sort_by {|a| a[0] })
242
- end
243
- end
244
-
245
- sub_test_case 'forward' do
246
- data(tcp: {
247
- options: {
248
- auth: false
249
- }
250
- },
251
- auth: {
252
- options: {
253
- auth: true
254
- }
255
- })
256
- test 'plain' do |data|
257
- options = data[:options]
258
- config = options[:auth] ? config_auth : base_config
259
- @d = d = create_driver(config)
260
-
261
- time = event_time("2011-01-02 13:14:15 UTC")
262
-
263
- records = [
264
- ["tag1", time, {"a"=>1}],
265
- ["tag1", time, {"a"=>2}]
266
- ]
267
-
268
- d.run(expect_records: records.length, timeout: 20) do
269
- entries = []
270
- records.each {|tag, _time, record|
271
- entries << [_time, record]
272
- }
273
- send_data packer.write(["tag1", entries]).to_s, **options
274
- end
275
- assert_equal(records, d.events)
276
- end
277
-
278
- data(tag: {
279
- param: "tag new_tag",
280
- result: "new_tag"
281
- },
282
- add_tag_prefix: {
283
- param: "add_tag_prefix new_prefix",
284
- result: "new_prefix.tag1"
285
- })
286
- test 'tag parameters' do |data|
287
- @d = create_driver(base_config + data[:param])
288
- time = event_time("2011-01-02 13:14:15 UTC")
289
- options = {auth: false}
290
-
291
- records = [
292
- ["tag1", time, {"a"=>1}],
293
- ["tag1", time, {"a"=>2}],
294
- ]
295
-
296
- @d.run(expect_records: records.length, timeout: 20) do
297
- entries = []
298
- records.each {|tag, _time, record|
299
- entries << [_time, record]
300
- }
301
- send_data packer.write(["tag1", entries]).to_s, **options
302
- end
303
- assert_equal(data[:result], @d.events[0][0])
304
- end
305
-
306
- data(tcp: {
307
- options: {
308
- auth: false
309
- }
310
- },
311
- auth: {
312
- options: {
313
- auth: true
314
- }
315
- })
316
- test 'time_as_integer' do |data|
317
- options = data[:options]
318
- config = options[:auth] ? config_auth : base_config
319
- @d = d = create_driver(config)
320
-
321
- time_i = event_time("2011-01-02 13:14:15 UTC")
322
-
323
- records = [
324
- ["tag1", time_i, {"a"=>1}],
325
- ["tag1", time_i, {"a"=>2}]
326
- ]
327
-
328
- d.run(expect_records: records.length, timeout: 20) do
329
- entries = []
330
- records.each {|tag, _time, record|
331
- entries << [_time, record]
332
- }
333
- send_data packer.write(["tag1", entries]).to_s, **options
334
- end
335
-
336
- assert_equal(records, d.events)
337
- end
338
-
339
- data(tcp: {
340
- options: {
341
- auth: false
342
- }
343
- },
344
- auth: {
345
- options: {
346
- auth: true
347
- }
348
- })
349
- test 'skip_invalid_event' do |data|
350
- options = data[:options]
351
- config = options[:auth] ? config_auth : base_config
352
- @d = d = create_driver(config + "skip_invalid_event true")
353
-
354
- time = event_time("2011-01-02 13:14:15 UTC")
355
-
356
- records = [
357
- ["tag1", time, {"a" => 1}],
358
- ["tag1", time, {"a" => 2}],
359
- ]
360
-
361
- d.run(shutdown: false, expect_records: records.length, timeout: 20) do
362
- entries = records.map { |tag, _time, record| [_time, record] }
363
- # These entries are skipped
364
- entries << ['invalid time', {'a' => 3}] << [time, 'invalid record']
365
-
366
- send_data packer.write(["tag1", entries]).to_s, **options
367
- end
368
-
369
- logs = d.instance.log.out.logs
370
- assert{ logs.count{|line| line =~ /skip invalid event/ } == 2 }
371
-
372
- d.instance_shutdown
373
- end
374
- end
375
-
376
- sub_test_case 'packed forward' do
377
- data(tcp: {
378
- options: {
379
- auth: false
380
- }
381
- },
382
- auth: {
383
- options: {
384
- auth: true
385
- }
386
- })
387
- test 'plain' do |data|
388
- options = data[:options]
389
- config = options[:auth] ? config_auth : base_config
390
- @d = d = create_driver(config)
391
-
392
- time = event_time("2011-01-02 13:14:15 UTC")
393
-
394
- records = [
395
- ["tag1", time, {"a"=>1}],
396
- ["tag1", time, {"a"=>2}],
397
- ]
398
-
399
- d.run(expect_records: records.length, timeout: 20) do
400
- entries = ''
401
- records.each {|_tag, _time, record|
402
- packer(entries).write([_time, record]).flush
403
- }
404
- send_data packer.write(["tag1", entries]).to_s, **options
405
- end
406
- assert_equal(records, d.events)
407
- end
408
-
409
- data(tag: {
410
- param: "tag new_tag",
411
- result: "new_tag"
412
- },
413
- add_tag_prefix: {
414
- param: "add_tag_prefix new_prefix",
415
- result: "new_prefix.tag1"
416
- })
417
- test 'tag parameters' do |data|
418
- @d = create_driver(base_config + data[:param])
419
- time = event_time("2011-01-02 13:14:15 UTC")
420
- options = {auth: false}
421
-
422
- records = [
423
- ["tag1", time, {"a"=>1}],
424
- ["tag1", time, {"a"=>2}],
425
- ]
426
-
427
- @d.run(expect_records: records.length, timeout: 20) do
428
- entries = ''
429
- records.each {|_tag, _time, record|
430
- packer(entries).write([_time, record]).flush
431
- }
432
- send_data packer.write(["tag1", entries]).to_s, **options
433
- end
434
- assert_equal(data[:result], @d.events[0][0])
435
- end
436
-
437
- data(tcp: {
438
- options: {
439
- auth: false
440
- }
441
- },
442
- auth: {
443
- options: {
444
- auth: true
445
- }
446
- })
447
- test 'time_as_integer' do |data|
448
- options = data[:options]
449
- config = options[:auth] ? config_auth : base_config
450
- @d = d = create_driver(config)
451
-
452
- time_i = event_time("2011-01-02 13:14:15 UTC").to_i
453
-
454
- records = [
455
- ["tag1", time_i, {"a"=>1}],
456
- ["tag1", time_i, {"a"=>2}],
457
- ]
458
-
459
- d.run(expect_records: records.length, timeout: 20) do
460
- entries = ''
461
- records.each {|tag, _time, record|
462
- packer(entries).write([_time, record]).flush
463
- }
464
- send_data packer.write(["tag1", entries]).to_s, **options
465
- end
466
- assert_equal(records, d.events)
467
- end
468
-
469
- data(tcp: {
470
- options: {
471
- auth: false
472
- }
473
- },
474
- auth: {
475
- options: {
476
- auth: true
477
- }
478
- })
479
- test 'skip_invalid_event' do |data|
480
- options = data[:options]
481
- config = options[:auth] ? config_auth : base_config
482
- @d = d = create_driver(config + "skip_invalid_event true")
483
-
484
- time = event_time("2011-01-02 13:14:15 UTC")
485
-
486
- records = [
487
- ["tag1", time, {"a" => 1}],
488
- ["tag1", time, {"a" => 2}],
489
- ]
490
-
491
- d.run(shutdown: false, expect_records: records.length, timeout: 20) do
492
- entries = records.map { |tag, _time, record| [_time, record] }
493
- # These entries are skipped
494
- entries << ['invalid time', {'a' => 3}] << [time, 'invalid record']
495
-
496
- packed_entries = ''
497
- entries.each { |_time, record|
498
- packer(packed_entries).write([_time, record]).flush
499
- }
500
- send_data packer.write(["tag1", packed_entries]).to_s, **options
501
- end
502
-
503
- logs = d.instance.log.logs
504
- assert_equal 2, logs.count { |line| line =~ /skip invalid event/ }
505
-
506
- d.instance_shutdown
507
- end
508
- end
509
-
510
- sub_test_case 'compressed packed forward' do
511
- test 'set_compress_to_option' do
512
- @d = d = create_driver
513
-
514
- time_i = event_time("2011-01-02 13:14:15 UTC").to_i
515
- events = [
516
- ["tag1", time_i, {"a"=>1}],
517
- ["tag1", time_i, {"a"=>2}]
518
- ]
519
-
520
- # create compressed entries
521
- entries = ''
522
- events.each do |_tag, _time, record|
523
- v = [_time, record].to_msgpack
524
- entries << compress(v)
525
- end
526
- chunk = ["tag1", entries, { 'compressed' => 'gzip' }].to_msgpack
527
-
528
- d.run do
529
- Fluent::MessagePackFactory.msgpack_unpacker.feed_each(chunk) do |obj|
530
- option = d.instance.send(:on_message, obj, chunk.size, DUMMY_SOCK)
531
- assert_equal 'gzip', option['compressed']
532
- end
533
- end
534
-
535
- assert_equal events, d.events
536
- end
537
-
538
- test 'create_CompressedMessagePackEventStream_with_gzip_compress_option' do
539
- @d = d = create_driver
540
-
541
- time_i = event_time("2011-01-02 13:14:15 UTC").to_i
542
- events = [
543
- ["tag1", time_i, {"a"=>1}],
544
- ["tag1", time_i, {"a"=>2}]
545
- ]
546
-
547
- # create compressed entries
548
- entries = ''
549
- events.each do |_tag, _time, record|
550
- v = [_time, record].to_msgpack
551
- entries << compress(v)
552
- end
553
- chunk = ["tag1", entries, { 'compressed' => 'gzip' }].to_msgpack
554
-
555
- # check CompressedMessagePackEventStream is created
556
- mock(Fluent::CompressedMessagePackEventStream).new(entries, nil, 0)
557
-
558
- d.run do
559
- Fluent::MessagePackFactory.msgpack_unpacker.feed_each(chunk) do |obj|
560
- option = d.instance.send(:on_message, obj, chunk.size, DUMMY_SOCK)
561
- assert_equal 'gzip', option['compressed']
562
- end
563
- end
564
- end
565
- end
566
-
567
- sub_test_case 'warning' do
568
- test 'send_large_chunk_warning' do
569
- @d = d = create_driver(base_config + %[
570
- chunk_size_warn_limit 16M
571
- chunk_size_limit 32M
572
- ])
573
-
574
- time = event_time("2014-04-25 13:14:15 UTC")
575
-
576
- # generate over 16M chunk
577
- str = "X" * 1024 * 1024
578
- chunk = [ "test.tag", (0...16).map{|i| [time + i, {"data" => str}] } ].to_msgpack
579
- assert chunk.size > (16 * 1024 * 1024)
580
- assert chunk.size < (32 * 1024 * 1024)
581
-
582
- d.run(shutdown: false) do
583
- Fluent::MessagePackFactory.msgpack_unpacker.feed_each(chunk) do |obj|
584
- d.instance.send(:on_message, obj, chunk.size, DUMMY_SOCK)
585
- end
586
- end
587
-
588
- # check emitted data
589
- emits = d.events
590
- assert_equal 16, emits.size
591
- assert emits.map(&:first).all?{|t| t == "test.tag" }
592
- assert_equal (0...16).to_a, emits.map{|_tag, t, _record| t - time }
593
-
594
- # check log
595
- logs = d.instance.log.logs
596
- assert_equal 1, logs.count{|line|
597
- line =~ / \[warn\]: Input chunk size is larger than 'chunk_size_warn_limit':/ &&
598
- line =~ / tag="test.tag" host="#{LOCALHOST_HOSTNAME}" limit=16777216 size=16777501/
599
- }, "large chunk warning is not logged"
600
-
601
- d.instance_shutdown
602
- end
603
-
604
- test 'send_large_chunk_only_warning' do
605
- @d = d = create_driver(base_config + %[
606
- chunk_size_warn_limit 16M
607
- ])
608
- time = event_time("2014-04-25 13:14:15 UTC")
609
-
610
- # generate over 16M chunk
611
- str = "X" * 1024 * 1024
612
- chunk = [ "test.tag", (0...16).map{|i| [time + i, {"data" => str}] } ].to_msgpack
613
-
614
- d.run(shutdown: false) do
615
- Fluent::MessagePackFactory.msgpack_unpacker.feed_each(chunk) do |obj|
616
- d.instance.send(:on_message, obj, chunk.size, DUMMY_SOCK)
617
- end
618
- end
619
-
620
- # check log
621
- logs = d.instance.log.logs
622
- assert_equal 1, logs.count{ |line|
623
- line =~ / \[warn\]: Input chunk size is larger than 'chunk_size_warn_limit':/ &&
624
- line =~ / tag="test.tag" host="#{LOCALHOST_HOSTNAME}" limit=16777216 size=16777501/
625
- }, "large chunk warning is not logged"
626
-
627
- d.instance_shutdown
628
- end
629
-
630
- test 'send_large_chunk_limit' do
631
- @d = d = create_driver(base_config + %[
632
- chunk_size_warn_limit 16M
633
- chunk_size_limit 32M
634
- ])
635
-
636
- time = event_time("2014-04-25 13:14:15 UTC")
637
-
638
- # generate over 32M chunk
639
- str = "X" * 1024 * 1024
640
- chunk = [ "test.tag", (0...32).map{|i| [time + i, {"data" => str}] } ].to_msgpack
641
- assert chunk.size > (32 * 1024 * 1024)
642
-
643
- # d.run => send_data
644
- d.run(shutdown: false) do
645
- Fluent::MessagePackFactory.msgpack_unpacker.feed_each(chunk) do |obj|
646
- d.instance.send(:on_message, obj, chunk.size, DUMMY_SOCK)
647
- end
648
- end
649
-
650
- # check emitted data
651
- emits = d.events
652
- assert_equal 0, emits.size
653
-
654
- # check log
655
- logs = d.instance.log.logs
656
- assert_equal 1, logs.count{|line|
657
- line =~ / \[warn\]: Input chunk size is larger than 'chunk_size_limit', dropped:/ &&
658
- line =~ / tag="test.tag" host="#{LOCALHOST_HOSTNAME}" limit=33554432 size=33554989/
659
- }, "large chunk warning is not logged"
660
-
661
- d.instance_shutdown
662
- end
663
-
664
- data('string chunk' => 'broken string',
665
- 'integer chunk' => 10)
666
- test 'send_broken_chunk' do |data|
667
- @d = d = create_driver
668
-
669
- # d.run => send_data
670
- d.run(shutdown: false) do
671
- d.instance.send(:on_message, data, 1000000000, DUMMY_SOCK)
672
- end
673
-
674
- # check emitted data
675
- assert_equal 0, d.events.size
676
-
677
- # check log
678
- logs = d.instance.log.logs
679
- assert_equal 1, logs.count{|line|
680
- line =~ / \[warn\]: incoming chunk is broken: host="#{LOCALHOST_HOSTNAME}" msg=#{data.inspect}/
681
- }, "should not accept broken chunk"
682
-
683
- d.instance_shutdown
684
- end
685
- end
686
-
687
- sub_test_case 'respond to required ack' do
688
- data(tcp: {
689
- options: {
690
- auth: false
691
- }
692
- },
693
- auth: {
694
- options: {
695
- auth: true
696
- }
697
- })
698
- test 'message' do |data|
699
- options = data[:options]
700
- config = options[:auth] ? config_auth : base_config
701
- @d = d = create_driver(config)
702
-
703
- time = event_time("2011-01-02 13:14:15 UTC")
704
-
705
- events = [
706
- ["tag1", time, {"a"=>1}],
707
- ["tag2", time, {"a"=>2}]
708
- ]
709
-
710
- expected_acks = []
711
-
712
- d.run(expect_records: events.size) do
713
- events.each {|tag, _time, record|
714
- op = { 'chunk' => Base64.encode64(record.object_id.to_s) }
715
- expected_acks << op['chunk']
716
- send_data([tag, _time, record, op].to_msgpack, try_to_receive_response: true, response_timeout: 1, **options)
717
- }
718
- end
719
-
720
- assert_equal events, d.events
721
- assert_equal expected_acks, @responses.map { |res| MessagePack.unpack(res)['ack'] }
722
- end
723
-
724
- # FIX: response is not pushed into @responses because IO.select has been blocked until InputForward shutdowns
725
- data(tcp: {
726
- options: {
727
- auth: false
728
- }
729
- },
730
- auth: {
731
- options: {
732
- auth: true
733
- }
734
- })
735
- test 'forward' do |data|
736
- options = data[:options]
737
- config = options[:auth] ? config_auth : base_config
738
- @d = d = create_driver(config)
739
-
740
- time = event_time("2011-01-02 13:14:15 UTC")
741
-
742
- events = [
743
- ["tag1", time, {"a"=>1}],
744
- ["tag1", time, {"a"=>2}]
745
- ]
746
-
747
- expected_acks = []
748
-
749
- d.run(expect_records: events.size) do
750
- entries = []
751
- events.each {|_tag, _time, record|
752
- entries << [_time, record]
753
- }
754
- op = { 'chunk' => Base64.encode64(entries.object_id.to_s) }
755
- expected_acks << op['chunk']
756
- send_data(["tag1", entries, op].to_msgpack, try_to_receive_response: true, response_timeout: 1, **options)
757
- end
758
-
759
- assert_equal events, d.events
760
- assert_equal expected_acks, @responses.map { |res| MessagePack.unpack(res)['ack'] }
761
- end
762
-
763
- data(tcp: {
764
- options: {
765
- auth: false
766
- }
767
- },
768
- auth: {
769
- options: {
770
- auth: true
771
- }
772
- })
773
- test 'packed_forward' do |data|
774
- options = data[:options]
775
- config = options[:auth] ? config_auth : base_config
776
- @d = d = create_driver(config)
777
-
778
- time = event_time("2011-01-02 13:14:15 UTC")
779
-
780
- events = [
781
- ["tag1", time, {"a"=>1}],
782
- ["tag1", time, {"a"=>2}]
783
- ]
784
-
785
- expected_acks = []
786
-
787
- d.run(expect_records: events.size) do
788
- entries = ''
789
- events.each {|_tag, _time,record|
790
- [time, record].to_msgpack(entries)
791
- }
792
- op = { 'chunk' => Base64.encode64(entries.object_id.to_s) }
793
- expected_acks << op['chunk']
794
- send_data(["tag1", entries, op].to_msgpack, try_to_receive_response: true, response_timeout: 1, **options)
795
- end
796
-
797
- assert_equal events, d.events
798
- assert_equal expected_acks, @responses.map { |res| MessagePack.unpack(res)['ack'] }
799
- end
800
-
801
- data(
802
- tcp: {
803
- options: {
804
- auth: false
805
- }
806
- },
807
- ### Auth is not supported with json
808
- # auth: {
809
- # options: {
810
- # auth: true
811
- # }
812
- # },
813
- )
814
- test 'message_json' do |data|
815
- options = data[:options]
816
- config = options[:auth] ? config_auth : base_config
817
- @d = d = create_driver(config)
818
-
819
- time_i = event_time("2011-01-02 13:14:15 UTC")
820
-
821
- events = [
822
- ["tag1", time_i, {"a"=>1}],
823
- ["tag2", time_i, {"a"=>2}]
824
- ]
825
-
826
- expected_acks = []
827
-
828
- d.run(expect_records: events.size, timeout: 20) do
829
- events.each {|tag, _time, record|
830
- op = { 'chunk' => Base64.encode64(record.object_id.to_s) }
831
- expected_acks << op['chunk']
832
- send_data([tag, _time, record, op].to_json, try_to_receive_response: true, response_timeout: 1, **options)
833
- }
834
- end
835
-
836
- assert_equal events, d.events
837
- assert_equal expected_acks, @responses.map { |res| JSON.parse(res)['ack'] }
838
- end
839
- end
840
-
841
- sub_test_case 'not respond without required ack' do
842
- data(tcp: {
843
- options: {
844
- auth: false
845
- }
846
- },
847
- auth: {
848
- options: {
849
- auth: true
850
- }
851
- })
852
- test 'message' do |data|
853
- options = data[:options]
854
- config = options[:auth] ? config_auth : base_config
855
- @d = d = create_driver(config)
856
-
857
- time = event_time("2011-01-02 13:14:15 UTC")
858
-
859
- events = [
860
- ["tag1", time, {"a"=>1}],
861
- ["tag2", time, {"a"=>2}]
862
- ]
863
-
864
- d.run(expect_records: events.size, timeout: 20) do
865
- events.each {|tag, _time, record|
866
- send_data([tag, _time, record].to_msgpack, try_to_receive_response: true, response_timeout: 1, **options)
867
- }
868
- end
869
-
870
- assert_equal events, d.events
871
- assert_equal [nil, nil], @responses
872
- end
873
-
874
- data(tcp: {
875
- options: {
876
- auth: false
877
- }
878
- },
879
- auth: {
880
- options: {
881
- auth: true
882
- }
883
- })
884
- test 'forward' do |data|
885
- options = data[:options]
886
- config = options[:auth] ? config_auth : base_config
887
- @d = d = create_driver(config)
888
-
889
- time = event_time("2011-01-02 13:14:15 UTC")
890
-
891
- events = [
892
- ["tag1", time, {"a"=>1}],
893
- ["tag1", time, {"a"=>2}]
894
- ]
895
-
896
- d.run(expect_records: events.size, timeout: 20) do
897
- entries = []
898
- events.each {|tag, _time, record|
899
- entries << [_time, record]
900
- }
901
- send_data(["tag1", entries].to_msgpack, try_to_receive_response: true, response_timeout: 1, **options)
902
- end
903
-
904
- assert_equal events, d.events
905
- assert_equal [nil], @responses
906
- end
907
-
908
- data(tcp: {
909
- options: {
910
- auth: false
911
- }
912
- },
913
- auth: {
914
- options: {
915
- auth: true
916
- }
917
- })
918
- test 'packed_forward' do |data|
919
- options = data[:options]
920
- config = options[:auth] ? config_auth : base_config
921
- @d = d = create_driver(config)
922
-
923
- time = event_time("2011-01-02 13:14:15 UTC")
924
-
925
- events = [
926
- ["tag1", time, {"a"=>1}],
927
- ["tag1", time, {"a"=>2}]
928
- ]
929
-
930
- d.run(expect_records: events.size, timeout: 20) do
931
- entries = ''
932
- events.each {|tag, _time, record|
933
- [_time, record].to_msgpack(entries)
934
- }
935
- send_data(["tag1", entries].to_msgpack, try_to_receive_response: true, response_timeout: 1, **options)
936
- end
937
-
938
- assert_equal events, d.events
939
- assert_equal [nil], @responses
940
- end
941
-
942
- data(
943
- tcp: {
944
- options: {
945
- auth: false
946
- }
947
- },
948
- ### Auth is not supported with json
949
- # auth: {
950
- # config: config_auth,
951
- # options: {
952
- # auth: true
953
- # }
954
- # },
955
- )
956
- test 'message_json' do |data|
957
- options = data[:options]
958
- config = options[:auth] ? config_auth : base_config
959
- @d = d = create_driver(config)
960
-
961
- time_i = event_time("2011-01-02 13:14:15 UTC").to_i
962
-
963
- events = [
964
- ["tag1", time_i, {"a"=>1}],
965
- ["tag2", time_i, {"a"=>2}]
966
- ]
967
-
968
- d.run(expect_records: events.size, timeout: 20) do
969
- events.each {|tag, _time, record|
970
- send_data([tag, _time, record].to_json, try_to_receive_response: true, response_timeout: 1, **options)
971
- }
972
- end
973
-
974
- assert_equal events, d.events
975
- assert_equal [nil, nil], @responses
976
- end
977
- end
978
-
979
- def packer(*args)
980
- Fluent::MessagePackFactory.msgpack_packer(*args)
981
- end
982
-
983
- def unpacker
984
- Fluent::MessagePackFactory.msgpack_unpacker
985
- end
986
-
987
- # res
988
- # '' : socket is disconnected without any data
989
- # nil: socket read timeout
990
- def read_data(io, timeout, &block)
991
- res = ''
992
- select_timeout = 0.5
993
- clock_id = Process::CLOCK_MONOTONIC_RAW rescue Process::CLOCK_MONOTONIC
994
- timeout_at = Process.clock_gettime(clock_id) + timeout
995
- begin
996
- buf = ''
997
- io_activated = false
998
- while Process.clock_gettime(clock_id) < timeout_at
999
- if IO.select([io], nil, nil, select_timeout)
1000
- io_activated = true
1001
- buf = io.readpartial(2048)
1002
- res ||= ''
1003
- res << buf
1004
-
1005
- break if block.call(res)
1006
- end
1007
- end
1008
- res = nil unless io_activated # timeout without no data arrival
1009
- rescue Errno::EAGAIN
1010
- sleep 0.01
1011
- retry if res == ''
1012
- # if res is not empty, all data in socket buffer are read, so do not retry
1013
- rescue IOError, EOFError, Errno::ECONNRESET
1014
- # socket disconnected
1015
- end
1016
- res
1017
- end
1018
-
1019
- def simulate_auth_sequence(io, shared_key=SHARED_KEY, username=USER_NAME, password=USER_PASSWORD)
1020
- auth_response_timeout = 30
1021
- shared_key_salt = 'salt'
1022
-
1023
- # reading helo
1024
- helo_data = read_data(io, auth_response_timeout){|data| MessagePack.unpack(data) rescue nil }
1025
- raise "Authentication packet timeout" unless helo_data
1026
- raise "Authentication connection closed" if helo_data == ''
1027
- # ['HELO', options(hash)]
1028
- helo = MessagePack.unpack(helo_data)
1029
- raise "Invalid HELO header" unless helo[0] == 'HELO'
1030
- raise "Invalid HELO option object" unless helo[1].is_a?(Hash)
1031
- @options = helo[1]
1032
-
1033
- # sending ping
1034
- ping = [
1035
- 'PING',
1036
- 'selfhostname',
1037
- shared_key_salt,
1038
- Digest::SHA512.new
1039
- .update(shared_key_salt)
1040
- .update('selfhostname')
1041
- .update(@options['nonce'])
1042
- .update(shared_key).hexdigest,
1043
- ]
1044
- if @options['auth'] # auth enabled -> value is auth salt
1045
- pass_digest = Digest::SHA512.new.update(@options['auth']).update(username).update(password).hexdigest
1046
- ping.push(username, pass_digest)
1047
- else
1048
- ping.push('', '')
1049
- end
1050
- io.write ping.to_msgpack
1051
- io.flush
1052
-
1053
- # reading pong
1054
- pong_data = read_data(io, auth_response_timeout){|data| MessagePack.unpack(data) rescue nil }
1055
- raise "PONG packet timeout" unless pong_data
1056
- raise "PONG connection closed" if pong_data == ''
1057
- # ['PING', bool(auth_result), string(reason_if_failed), self_hostname, shared_key_digest]
1058
- pong = MessagePack.unpack(pong_data)
1059
- raise "Invalid PONG header" unless pong[0] == 'PONG'
1060
- raise "Authentication Failure: #{pong[2]}" unless pong[1]
1061
- clientside_calculated = Digest::SHA512.new
1062
- .update(shared_key_salt)
1063
- .update(pong[3])
1064
- .update(@options['nonce'])
1065
- .update(shared_key).hexdigest
1066
- raise "Shared key digest mismatch" unless clientside_calculated == pong[4]
1067
-
1068
- # authentication success
1069
- true
1070
- end
1071
-
1072
- def connect
1073
- TCPSocket.new('127.0.0.1', @port)
1074
- end
1075
-
1076
- # Data ordering is not assured:
1077
- # Records in different sockets are processed on different thread, so its scheduling make effect
1078
- # on order of emitted records.
1079
- # So, we MUST sort emitted records in different `send_data` before assertion.
1080
- def send_data(data, try_to_receive_response: false, response_timeout: 5, auth: false)
1081
- io = connect
1082
-
1083
- if auth
1084
- simulate_auth_sequence(io)
1085
- end
1086
-
1087
- io.write data
1088
- io.flush
1089
- if try_to_receive_response
1090
- @responses << read_data(io, response_timeout){|d| MessagePack.unpack(d) rescue nil }
1091
- end
1092
- ensure
1093
- io.close rescue nil # SSL socket requires any writes to close sockets
1094
- end
1095
-
1096
- sub_test_case 'source_hostname_key and source_address_key features' do
1097
- data(
1098
- both: [:hostname, :address],
1099
- hostname: [:hostname],
1100
- address: [:address],
1101
- )
1102
- test 'message protocol' do |keys|
1103
- execute_test_with_source_hostname_key(*keys) { |events|
1104
- events.each { |tag, time, record|
1105
- send_data [tag, time, record].to_msgpack
1106
- }
1107
- }
1108
- end
1109
-
1110
- data(
1111
- both: [:hostname, :address],
1112
- hostname: [:hostname],
1113
- address: [:address],
1114
- )
1115
- test 'forward protocol' do |keys|
1116
- execute_test_with_source_hostname_key(*keys) { |events|
1117
- entries = []
1118
- events.each {|tag,time,record|
1119
- entries << [time, record]
1120
- }
1121
- send_data ['tag1', entries].to_msgpack
1122
- }
1123
- end
1124
-
1125
- data(
1126
- both: [:hostname, :address],
1127
- hostname: [:hostname],
1128
- address: [:address],
1129
- )
1130
- test 'packed forward protocol' do |keys|
1131
- execute_test_with_source_hostname_key(*keys) { |events|
1132
- entries = ''
1133
- events.each { |tag, time, record|
1134
- Fluent::MessagePackFactory.msgpack_packer(entries).write([time, record]).flush
1135
- }
1136
- send_data Fluent::MessagePackFactory.msgpack_packer.write(["tag1", entries]).to_s
1137
- }
1138
- end
1139
- end
1140
-
1141
- def execute_test_with_source_hostname_key(*keys, &block)
1142
- conf = base_config.dup
1143
- if keys.include?(:hostname)
1144
- conf << <<EOL
1145
- source_hostname_key source_hostname
1146
- EOL
1147
- end
1148
- if keys.include?(:address)
1149
- conf << <<EOL
1150
- source_address_key source_address
1151
- EOL
1152
- end
1153
- @d = d = create_driver(conf)
1154
-
1155
- time = event_time("2011-01-02 13:14:15 UTC")
1156
- events = [
1157
- ["tag1", time, {"a"=>1}],
1158
- ["tag1", time, {"a"=>2}]
1159
- ]
1160
-
1161
- d.run(expect_records: events.size) do
1162
- block.call(events)
1163
- end
1164
-
1165
- d.events.each { |tag, _time, record|
1166
- if keys.include?(:hostname)
1167
- assert_true record.has_key?('source_hostname')
1168
- assert_equal DUMMY_SOCK.remote_host, record['source_hostname']
1169
- end
1170
- if keys.include?(:address)
1171
- assert_true record.has_key?('source_address')
1172
- assert_equal DUMMY_SOCK.remote_addr, record['source_address']
1173
- end
1174
- }
1175
- end
1176
-
1177
- # TODO heartbeat
1178
- end