fluentd 1.17.0-x86-mingw32 → 1.17.1-x86-mingw32

Sign up to get free protection for your applications and to get access to all the features.
Files changed (259) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +46 -0
  3. data/README.md +1 -0
  4. data/SECURITY.md +2 -2
  5. data/fluent.conf +14 -14
  6. data/lib/fluent/command/cap_ctl.rb +4 -4
  7. data/lib/fluent/compat/call_super_mixin.rb +3 -3
  8. data/lib/fluent/compat/propagate_default.rb +4 -4
  9. data/lib/fluent/config/yaml_parser/parser.rb +4 -0
  10. data/lib/fluent/log/console_adapter.rb +4 -2
  11. data/lib/fluent/plugin/in_exec.rb +14 -2
  12. data/lib/fluent/plugin/in_http.rb +1 -1
  13. data/lib/fluent/plugin/in_sample.rb +13 -7
  14. data/lib/fluent/plugin/in_tail.rb +65 -23
  15. data/lib/fluent/plugin/out_copy.rb +1 -1
  16. data/lib/fluent/plugin/out_file.rb +8 -0
  17. data/lib/fluent/plugin/out_http.rb +12 -0
  18. data/lib/fluent/plugin/parser_json.rb +4 -12
  19. data/lib/fluent/plugin_helper/http_server/server.rb +1 -1
  20. data/lib/fluent/version.rb +1 -1
  21. data/templates/new_gem/fluent-plugin.gemspec.erb +6 -5
  22. metadata +25 -472
  23. data/.github/DISCUSSION_TEMPLATE/q-a-japanese.yml +0 -50
  24. data/.github/DISCUSSION_TEMPLATE/q-a.yml +0 -47
  25. data/.github/ISSUE_TEMPLATE/bug_report.yml +0 -71
  26. data/.github/ISSUE_TEMPLATE/config.yml +0 -5
  27. data/.github/ISSUE_TEMPLATE/feature_request.yml +0 -39
  28. data/.github/ISSUE_TEMPLATE.md +0 -17
  29. data/.github/PULL_REQUEST_TEMPLATE.md +0 -14
  30. data/.github/workflows/stale-actions.yml +0 -24
  31. data/.github/workflows/test-ruby-head.yml +0 -31
  32. data/.github/workflows/test.yml +0 -32
  33. data/.gitignore +0 -30
  34. data/Gemfile +0 -9
  35. data/fluentd.gemspec +0 -62
  36. data/test/command/test_binlog_reader.rb +0 -362
  37. data/test/command/test_ca_generate.rb +0 -70
  38. data/test/command/test_cap_ctl.rb +0 -100
  39. data/test/command/test_cat.rb +0 -128
  40. data/test/command/test_ctl.rb +0 -56
  41. data/test/command/test_fluentd.rb +0 -1291
  42. data/test/command/test_plugin_config_formatter.rb +0 -397
  43. data/test/command/test_plugin_generator.rb +0 -109
  44. data/test/compat/test_calls_super.rb +0 -166
  45. data/test/compat/test_parser.rb +0 -92
  46. data/test/config/assertions.rb +0 -42
  47. data/test/config/test_config_parser.rb +0 -551
  48. data/test/config/test_configurable.rb +0 -1784
  49. data/test/config/test_configure_proxy.rb +0 -604
  50. data/test/config/test_dsl.rb +0 -415
  51. data/test/config/test_element.rb +0 -518
  52. data/test/config/test_literal_parser.rb +0 -309
  53. data/test/config/test_plugin_configuration.rb +0 -56
  54. data/test/config/test_section.rb +0 -191
  55. data/test/config/test_system_config.rb +0 -195
  56. data/test/config/test_types.rb +0 -408
  57. data/test/counter/test_client.rb +0 -563
  58. data/test/counter/test_error.rb +0 -44
  59. data/test/counter/test_mutex_hash.rb +0 -179
  60. data/test/counter/test_server.rb +0 -589
  61. data/test/counter/test_store.rb +0 -258
  62. data/test/counter/test_validator.rb +0 -137
  63. data/test/helper.rb +0 -155
  64. data/test/helpers/fuzzy_assert.rb +0 -89
  65. data/test/helpers/process_extenstion.rb +0 -33
  66. data/test/log/test_console_adapter.rb +0 -117
  67. data/test/plugin/data/2010/01/20100102-030405.log +0 -0
  68. data/test/plugin/data/2010/01/20100102-030406.log +0 -0
  69. data/test/plugin/data/2010/01/20100102.log +0 -0
  70. data/test/plugin/data/log/bar +0 -0
  71. data/test/plugin/data/log/foo/bar.log +0 -0
  72. data/test/plugin/data/log/foo/bar2 +0 -0
  73. data/test/plugin/data/log/test.log +0 -0
  74. data/test/plugin/data/log_numeric/01.log +0 -0
  75. data/test/plugin/data/log_numeric/02.log +0 -0
  76. data/test/plugin/data/log_numeric/12.log +0 -0
  77. data/test/plugin/data/log_numeric/14.log +0 -0
  78. data/test/plugin/data/sd_file/config +0 -11
  79. data/test/plugin/data/sd_file/config.json +0 -17
  80. data/test/plugin/data/sd_file/config.yaml +0 -11
  81. data/test/plugin/data/sd_file/config.yml +0 -11
  82. data/test/plugin/data/sd_file/invalid_config.yml +0 -7
  83. data/test/plugin/in_tail/test_fifo.rb +0 -121
  84. data/test/plugin/in_tail/test_io_handler.rb +0 -150
  85. data/test/plugin/in_tail/test_position_file.rb +0 -346
  86. data/test/plugin/out_forward/test_ack_handler.rb +0 -140
  87. data/test/plugin/out_forward/test_connection_manager.rb +0 -145
  88. data/test/plugin/out_forward/test_handshake_protocol.rb +0 -112
  89. data/test/plugin/out_forward/test_load_balancer.rb +0 -106
  90. data/test/plugin/out_forward/test_socket_cache.rb +0 -174
  91. data/test/plugin/test_bare_output.rb +0 -131
  92. data/test/plugin/test_base.rb +0 -247
  93. data/test/plugin/test_buf_file.rb +0 -1314
  94. data/test/plugin/test_buf_file_single.rb +0 -898
  95. data/test/plugin/test_buf_memory.rb +0 -42
  96. data/test/plugin/test_buffer.rb +0 -1493
  97. data/test/plugin/test_buffer_chunk.rb +0 -209
  98. data/test/plugin/test_buffer_file_chunk.rb +0 -871
  99. data/test/plugin/test_buffer_file_single_chunk.rb +0 -611
  100. data/test/plugin/test_buffer_memory_chunk.rb +0 -339
  101. data/test/plugin/test_compressable.rb +0 -87
  102. data/test/plugin/test_file_util.rb +0 -96
  103. data/test/plugin/test_filter.rb +0 -368
  104. data/test/plugin/test_filter_grep.rb +0 -697
  105. data/test/plugin/test_filter_parser.rb +0 -731
  106. data/test/plugin/test_filter_record_transformer.rb +0 -577
  107. data/test/plugin/test_filter_stdout.rb +0 -207
  108. data/test/plugin/test_formatter_csv.rb +0 -136
  109. data/test/plugin/test_formatter_hash.rb +0 -38
  110. data/test/plugin/test_formatter_json.rb +0 -61
  111. data/test/plugin/test_formatter_ltsv.rb +0 -70
  112. data/test/plugin/test_formatter_msgpack.rb +0 -28
  113. data/test/plugin/test_formatter_out_file.rb +0 -116
  114. data/test/plugin/test_formatter_single_value.rb +0 -44
  115. data/test/plugin/test_formatter_tsv.rb +0 -76
  116. data/test/plugin/test_in_debug_agent.rb +0 -49
  117. data/test/plugin/test_in_exec.rb +0 -261
  118. data/test/plugin/test_in_forward.rb +0 -1178
  119. data/test/plugin/test_in_gc_stat.rb +0 -62
  120. data/test/plugin/test_in_http.rb +0 -1124
  121. data/test/plugin/test_in_monitor_agent.rb +0 -922
  122. data/test/plugin/test_in_object_space.rb +0 -66
  123. data/test/plugin/test_in_sample.rb +0 -190
  124. data/test/plugin/test_in_syslog.rb +0 -505
  125. data/test/plugin/test_in_tail.rb +0 -3429
  126. data/test/plugin/test_in_tcp.rb +0 -328
  127. data/test/plugin/test_in_udp.rb +0 -296
  128. data/test/plugin/test_in_unix.rb +0 -181
  129. data/test/plugin/test_input.rb +0 -137
  130. data/test/plugin/test_metadata.rb +0 -89
  131. data/test/plugin/test_metrics.rb +0 -294
  132. data/test/plugin/test_metrics_local.rb +0 -96
  133. data/test/plugin/test_multi_output.rb +0 -204
  134. data/test/plugin/test_out_copy.rb +0 -308
  135. data/test/plugin/test_out_exec.rb +0 -312
  136. data/test/plugin/test_out_exec_filter.rb +0 -606
  137. data/test/plugin/test_out_file.rb +0 -1038
  138. data/test/plugin/test_out_forward.rb +0 -1349
  139. data/test/plugin/test_out_http.rb +0 -557
  140. data/test/plugin/test_out_null.rb +0 -105
  141. data/test/plugin/test_out_relabel.rb +0 -28
  142. data/test/plugin/test_out_roundrobin.rb +0 -146
  143. data/test/plugin/test_out_secondary_file.rb +0 -458
  144. data/test/plugin/test_out_stdout.rb +0 -205
  145. data/test/plugin/test_out_stream.rb +0 -103
  146. data/test/plugin/test_output.rb +0 -1334
  147. data/test/plugin/test_output_as_buffered.rb +0 -2024
  148. data/test/plugin/test_output_as_buffered_backup.rb +0 -363
  149. data/test/plugin/test_output_as_buffered_compress.rb +0 -179
  150. data/test/plugin/test_output_as_buffered_overflow.rb +0 -250
  151. data/test/plugin/test_output_as_buffered_retries.rb +0 -966
  152. data/test/plugin/test_output_as_buffered_secondary.rb +0 -882
  153. data/test/plugin/test_output_as_standard.rb +0 -374
  154. data/test/plugin/test_owned_by.rb +0 -34
  155. data/test/plugin/test_parser.rb +0 -399
  156. data/test/plugin/test_parser_apache.rb +0 -42
  157. data/test/plugin/test_parser_apache2.rb +0 -47
  158. data/test/plugin/test_parser_apache_error.rb +0 -45
  159. data/test/plugin/test_parser_csv.rb +0 -200
  160. data/test/plugin/test_parser_json.rb +0 -244
  161. data/test/plugin/test_parser_labeled_tsv.rb +0 -160
  162. data/test/plugin/test_parser_msgpack.rb +0 -127
  163. data/test/plugin/test_parser_multiline.rb +0 -111
  164. data/test/plugin/test_parser_nginx.rb +0 -88
  165. data/test/plugin/test_parser_none.rb +0 -52
  166. data/test/plugin/test_parser_regexp.rb +0 -284
  167. data/test/plugin/test_parser_syslog.rb +0 -650
  168. data/test/plugin/test_parser_tsv.rb +0 -122
  169. data/test/plugin/test_sd_file.rb +0 -228
  170. data/test/plugin/test_sd_srv.rb +0 -230
  171. data/test/plugin/test_storage.rb +0 -166
  172. data/test/plugin/test_storage_local.rb +0 -335
  173. data/test/plugin/test_string_util.rb +0 -26
  174. data/test/plugin_helper/data/cert/cert-key.pem +0 -27
  175. data/test/plugin_helper/data/cert/cert-with-CRLF.pem +0 -19
  176. data/test/plugin_helper/data/cert/cert-with-no-newline.pem +0 -19
  177. data/test/plugin_helper/data/cert/cert.pem +0 -19
  178. data/test/plugin_helper/data/cert/cert_chains/ca-cert-key.pem +0 -27
  179. data/test/plugin_helper/data/cert/cert_chains/ca-cert.pem +0 -20
  180. data/test/plugin_helper/data/cert/cert_chains/cert-key.pem +0 -27
  181. data/test/plugin_helper/data/cert/cert_chains/cert.pem +0 -40
  182. data/test/plugin_helper/data/cert/empty.pem +0 -0
  183. data/test/plugin_helper/data/cert/generate_cert.rb +0 -125
  184. data/test/plugin_helper/data/cert/with_ca/ca-cert-key-pass.pem +0 -30
  185. data/test/plugin_helper/data/cert/with_ca/ca-cert-key.pem +0 -27
  186. data/test/plugin_helper/data/cert/with_ca/ca-cert-pass.pem +0 -20
  187. data/test/plugin_helper/data/cert/with_ca/ca-cert.pem +0 -20
  188. data/test/plugin_helper/data/cert/with_ca/cert-key-pass.pem +0 -30
  189. data/test/plugin_helper/data/cert/with_ca/cert-key.pem +0 -27
  190. data/test/plugin_helper/data/cert/with_ca/cert-pass.pem +0 -21
  191. data/test/plugin_helper/data/cert/with_ca/cert.pem +0 -21
  192. data/test/plugin_helper/data/cert/without_ca/cert-key-pass.pem +0 -30
  193. data/test/plugin_helper/data/cert/without_ca/cert-key.pem +0 -27
  194. data/test/plugin_helper/data/cert/without_ca/cert-pass.pem +0 -20
  195. data/test/plugin_helper/data/cert/without_ca/cert.pem +0 -20
  196. data/test/plugin_helper/http_server/test_app.rb +0 -65
  197. data/test/plugin_helper/http_server/test_route.rb +0 -32
  198. data/test/plugin_helper/service_discovery/test_manager.rb +0 -93
  199. data/test/plugin_helper/service_discovery/test_round_robin_balancer.rb +0 -21
  200. data/test/plugin_helper/test_cert_option.rb +0 -25
  201. data/test/plugin_helper/test_child_process.rb +0 -862
  202. data/test/plugin_helper/test_compat_parameters.rb +0 -358
  203. data/test/plugin_helper/test_event_emitter.rb +0 -80
  204. data/test/plugin_helper/test_event_loop.rb +0 -52
  205. data/test/plugin_helper/test_extract.rb +0 -194
  206. data/test/plugin_helper/test_formatter.rb +0 -255
  207. data/test/plugin_helper/test_http_server_helper.rb +0 -372
  208. data/test/plugin_helper/test_inject.rb +0 -561
  209. data/test/plugin_helper/test_metrics.rb +0 -137
  210. data/test/plugin_helper/test_parser.rb +0 -264
  211. data/test/plugin_helper/test_record_accessor.rb +0 -238
  212. data/test/plugin_helper/test_retry_state.rb +0 -1006
  213. data/test/plugin_helper/test_server.rb +0 -1895
  214. data/test/plugin_helper/test_service_discovery.rb +0 -165
  215. data/test/plugin_helper/test_socket.rb +0 -146
  216. data/test/plugin_helper/test_storage.rb +0 -542
  217. data/test/plugin_helper/test_thread.rb +0 -164
  218. data/test/plugin_helper/test_timer.rb +0 -130
  219. data/test/scripts/exec_script.rb +0 -32
  220. data/test/scripts/fluent/plugin/formatter1/formatter_test1.rb +0 -7
  221. data/test/scripts/fluent/plugin/formatter2/formatter_test2.rb +0 -7
  222. data/test/scripts/fluent/plugin/formatter_known.rb +0 -8
  223. data/test/scripts/fluent/plugin/out_test.rb +0 -81
  224. data/test/scripts/fluent/plugin/out_test2.rb +0 -80
  225. data/test/scripts/fluent/plugin/parser_known.rb +0 -4
  226. data/test/test_capability.rb +0 -74
  227. data/test/test_clock.rb +0 -164
  228. data/test/test_config.rb +0 -369
  229. data/test/test_configdsl.rb +0 -148
  230. data/test/test_daemonizer.rb +0 -91
  231. data/test/test_engine.rb +0 -203
  232. data/test/test_event.rb +0 -531
  233. data/test/test_event_router.rb +0 -348
  234. data/test/test_event_time.rb +0 -199
  235. data/test/test_file_wrapper.rb +0 -53
  236. data/test/test_filter.rb +0 -121
  237. data/test/test_fluent_log_event_router.rb +0 -99
  238. data/test/test_formatter.rb +0 -369
  239. data/test/test_input.rb +0 -31
  240. data/test/test_log.rb +0 -1076
  241. data/test/test_match.rb +0 -148
  242. data/test/test_mixin.rb +0 -351
  243. data/test/test_msgpack_factory.rb +0 -50
  244. data/test/test_oj_options.rb +0 -55
  245. data/test/test_output.rb +0 -278
  246. data/test/test_plugin.rb +0 -251
  247. data/test/test_plugin_classes.rb +0 -370
  248. data/test/test_plugin_helper.rb +0 -81
  249. data/test/test_plugin_id.rb +0 -119
  250. data/test/test_process.rb +0 -14
  251. data/test/test_root_agent.rb +0 -951
  252. data/test/test_static_config_analysis.rb +0 -177
  253. data/test/test_supervisor.rb +0 -821
  254. data/test/test_test_drivers.rb +0 -136
  255. data/test/test_time_formatter.rb +0 -301
  256. data/test/test_time_parser.rb +0 -362
  257. data/test/test_tls.rb +0 -65
  258. data/test/test_unique_id.rb +0 -47
  259. data/test/test_variable_store.rb +0 -65
@@ -1,1178 +0,0 @@
1
- require_relative '../helper'
2
-
3
- require 'fluent/test/driver/input'
4
- require 'fluent/test/startup_shutdown'
5
- require 'base64'
6
-
7
- require 'fluent/env'
8
- require 'fluent/event'
9
- require 'fluent/plugin/in_forward'
10
- require 'fluent/plugin/compressable'
11
-
12
- require 'timecop'
13
-
14
- class ForwardInputTest < Test::Unit::TestCase
15
- include Fluent::Plugin::Compressable
16
-
17
- def setup
18
- Fluent::Test.setup
19
- @responses = [] # for testing responses after sending data
20
- @d = nil
21
- @port = unused_port
22
- end
23
-
24
- def teardown
25
- @d.instance_shutdown if @d
26
- @port = nil
27
- end
28
-
29
- SHARED_KEY = 'foobar1'
30
- USER_NAME = 'tagomoris'
31
- USER_PASSWORD = 'fluentd'
32
-
33
- def base_config
34
- %[
35
- port #{@port}
36
- bind 127.0.0.1
37
- ]
38
- end
39
- LOCALHOST_HOSTNAME_GETTER = ->(){sock = UDPSocket.new(::Socket::AF_INET); sock.do_not_reverse_lookup = false; sock.connect("127.0.0.1", 2048); sock.peeraddr[2] }
40
- LOCALHOST_HOSTNAME = LOCALHOST_HOSTNAME_GETTER.call
41
- DUMMY_SOCK = Struct.new(:remote_host, :remote_addr, :remote_port).new(LOCALHOST_HOSTNAME, "127.0.0.1", 0)
42
-
43
- def config_auth
44
- base_config + %[
45
- <security>
46
- self_hostname localhost
47
- shared_key foobar1
48
- user_auth true
49
- <user>
50
- username #{USER_NAME}
51
- password #{USER_PASSWORD}
52
- </user>
53
- <client>
54
- network 127.0.0.0/8
55
- shared_key #{SHARED_KEY}
56
- users ["#{USER_NAME}"]
57
- </client>
58
- </security>
59
- ]
60
- end
61
-
62
- def create_driver(conf=base_config)
63
- Fluent::Test::Driver::Input.new(Fluent::Plugin::ForwardInput).configure(conf)
64
- end
65
-
66
- sub_test_case '#configure' do
67
- test 'simple' do
68
- @d = d = create_driver
69
- assert_equal @port, d.instance.port
70
- assert_equal '127.0.0.1', d.instance.bind
71
- assert_equal 0.5, d.instance.blocking_timeout
72
- assert !d.instance.backlog
73
- end
74
-
75
- test 'auth' do
76
- @d = d = create_driver(config_auth)
77
- assert_equal @port, d.instance.port
78
- assert_equal '127.0.0.1', d.instance.bind
79
- assert !d.instance.backlog
80
-
81
- assert d.instance.security
82
- assert_equal 1, d.instance.security.users.size
83
- assert_equal 1, d.instance.security.clients.size
84
- end
85
-
86
- data(tag: "tag",
87
- add_tag_prefix: "add_tag_prefix")
88
- test 'tag parameters' do |data|
89
- assert_raise(Fluent::ConfigError.new("'#{data}' parameter must not be empty")) {
90
- create_driver(base_config + "#{data} ''")
91
- }
92
- end
93
-
94
- test 'send_keepalive_packet is disabled by default' do
95
- @d = d = create_driver(config_auth)
96
- assert_false d.instance.send_keepalive_packet
97
- end
98
-
99
- test 'send_keepalive_packet can be enabled' do
100
- @d = d = create_driver(config_auth + %[
101
- send_keepalive_packet true
102
- ])
103
- assert_true d.instance.send_keepalive_packet
104
- end
105
-
106
- test 'both send_keepalive_packet and deny_keepalive cannot be enabled' do
107
- assert_raise(Fluent::ConfigError.new("both 'send_keepalive_packet' and 'deny_keepalive' cannot be set to true")) do
108
- create_driver(config_auth + %[
109
- send_keepalive_packet true
110
- deny_keepalive true
111
- ])
112
- end
113
- end
114
- end
115
-
116
- sub_test_case 'message' do
117
- test 'time' do
118
- time = event_time("2011-01-02 13:14:15 UTC")
119
- begin
120
- Timecop.freeze(Time.at(time))
121
- @d = d = create_driver
122
-
123
- records = [
124
- ["tag1", time, {"a"=>1}],
125
- ["tag2", time, {"a"=>2}],
126
- ]
127
-
128
- d.run(expect_records: records.length, timeout: 5) do
129
- records.each {|tag, _time, record|
130
- send_data packer.write([tag, 0, record]).to_s
131
- }
132
- end
133
- assert_equal(records, d.events.sort_by {|a| a[0] })
134
- ensure
135
- Timecop.return
136
- end
137
- end
138
-
139
- test 'plain' do
140
- @d = d = create_driver
141
-
142
- time = event_time("2011-01-02 13:14:15 UTC")
143
-
144
- records = [
145
- ["tag1", time, {"a"=>1}],
146
- ["tag2", time, {"a"=>2}],
147
- ]
148
-
149
- d.run(expect_records: records.length, timeout: 5) do
150
- records.each {|tag, _time, record|
151
- send_data packer.write([tag, _time, record]).to_s
152
- }
153
- end
154
- assert_equal(records, d.events)
155
- end
156
-
157
- test 'time_as_integer' do
158
- @d = d = create_driver
159
-
160
- time_i = event_time("2011-01-02 13:14:15 UTC").to_i
161
-
162
- records = [
163
- ["tag1", time_i, {"a"=>1}],
164
- ["tag2", time_i, {"a"=>2}],
165
- ]
166
-
167
- d.run(expect_records: records.length, timeout: 5) do
168
- records.each {|tag, _time, record|
169
- send_data packer.write([tag, _time, record]).to_s
170
- }
171
- end
172
-
173
- assert_equal(records, d.events)
174
- end
175
-
176
- test 'skip_invalid_event' do
177
- @d = d = create_driver(base_config + "skip_invalid_event true")
178
-
179
- time = event_time("2011-01-02 13:14:15 UTC")
180
-
181
- records = [
182
- ["tag1", time, {"a" => 1}],
183
- ["tag2", time, {"a" => 2}],
184
- ]
185
-
186
- d.run(shutdown: false, expect_records: 2, timeout: 10) do
187
- entries = []
188
- # These entries are skipped
189
- entries << ['tag1', true, {'a' => 3}] << ['tag2', time, 'invalid record']
190
- entries += records.map { |tag, _time, record| [tag, _time, record] }
191
-
192
- entries.each {|tag, _time, record|
193
- # Without ack, logs are sometimes not saved to logs during test.
194
- send_data packer.write([tag, _time, record]).to_s #, try_to_receive_response: true
195
- }
196
- end
197
-
198
- logs = d.instance.log.logs
199
- assert_equal 2, logs.count { |line| line =~ /got invalid event and drop it/ }
200
- assert_equal records[0], d.events[0]
201
- assert_equal records[1], d.events[1]
202
-
203
- d.instance_shutdown
204
- end
205
-
206
- test 'json_using_integer_time' do
207
- @d = d = create_driver
208
-
209
- time_i = event_time("2011-01-02 13:14:15 UTC").to_i
210
-
211
- records = [
212
- ["tag1", time_i, {"a"=>1}],
213
- ["tag2", time_i, {"a"=>2}],
214
- ]
215
-
216
- d.run(expect_records: records.length, timeout: 20) do
217
- records.each {|tag, _time, record|
218
- send_data [tag, _time, record].to_json
219
- }
220
- end
221
-
222
- assert_equal(records, d.events.sort_by {|a| a[0] })
223
- end
224
-
225
- test 'json_with_newline' do
226
- @d = d = create_driver
227
-
228
- time_i = event_time("2011-01-02 13:14:15 UTC").to_i
229
-
230
- records = [
231
- ["tag1", time_i, {"a"=>1}],
232
- ["tag2", time_i, {"a"=>2}],
233
- ]
234
-
235
- d.run(expect_records: records.length, timeout: 20, shutdown: true) do
236
- records.each {|tag, _time, record|
237
- send_data [tag, _time, record].to_json + "\n"
238
- }
239
- end
240
-
241
- assert_equal(records, d.events.sort_by {|a| a[0] })
242
- end
243
- end
244
-
245
- sub_test_case 'forward' do
246
- data(tcp: {
247
- options: {
248
- auth: false
249
- }
250
- },
251
- auth: {
252
- options: {
253
- auth: true
254
- }
255
- })
256
- test 'plain' do |data|
257
- options = data[:options]
258
- config = options[:auth] ? config_auth : base_config
259
- @d = d = create_driver(config)
260
-
261
- time = event_time("2011-01-02 13:14:15 UTC")
262
-
263
- records = [
264
- ["tag1", time, {"a"=>1}],
265
- ["tag1", time, {"a"=>2}]
266
- ]
267
-
268
- d.run(expect_records: records.length, timeout: 20) do
269
- entries = []
270
- records.each {|tag, _time, record|
271
- entries << [_time, record]
272
- }
273
- send_data packer.write(["tag1", entries]).to_s, **options
274
- end
275
- assert_equal(records, d.events)
276
- end
277
-
278
- data(tag: {
279
- param: "tag new_tag",
280
- result: "new_tag"
281
- },
282
- add_tag_prefix: {
283
- param: "add_tag_prefix new_prefix",
284
- result: "new_prefix.tag1"
285
- })
286
- test 'tag parameters' do |data|
287
- @d = create_driver(base_config + data[:param])
288
- time = event_time("2011-01-02 13:14:15 UTC")
289
- options = {auth: false}
290
-
291
- records = [
292
- ["tag1", time, {"a"=>1}],
293
- ["tag1", time, {"a"=>2}],
294
- ]
295
-
296
- @d.run(expect_records: records.length, timeout: 20) do
297
- entries = []
298
- records.each {|tag, _time, record|
299
- entries << [_time, record]
300
- }
301
- send_data packer.write(["tag1", entries]).to_s, **options
302
- end
303
- assert_equal(data[:result], @d.events[0][0])
304
- end
305
-
306
- data(tcp: {
307
- options: {
308
- auth: false
309
- }
310
- },
311
- auth: {
312
- options: {
313
- auth: true
314
- }
315
- })
316
- test 'time_as_integer' do |data|
317
- options = data[:options]
318
- config = options[:auth] ? config_auth : base_config
319
- @d = d = create_driver(config)
320
-
321
- time_i = event_time("2011-01-02 13:14:15 UTC")
322
-
323
- records = [
324
- ["tag1", time_i, {"a"=>1}],
325
- ["tag1", time_i, {"a"=>2}]
326
- ]
327
-
328
- d.run(expect_records: records.length, timeout: 20) do
329
- entries = []
330
- records.each {|tag, _time, record|
331
- entries << [_time, record]
332
- }
333
- send_data packer.write(["tag1", entries]).to_s, **options
334
- end
335
-
336
- assert_equal(records, d.events)
337
- end
338
-
339
- data(tcp: {
340
- options: {
341
- auth: false
342
- }
343
- },
344
- auth: {
345
- options: {
346
- auth: true
347
- }
348
- })
349
- test 'skip_invalid_event' do |data|
350
- options = data[:options]
351
- config = options[:auth] ? config_auth : base_config
352
- @d = d = create_driver(config + "skip_invalid_event true")
353
-
354
- time = event_time("2011-01-02 13:14:15 UTC")
355
-
356
- records = [
357
- ["tag1", time, {"a" => 1}],
358
- ["tag1", time, {"a" => 2}],
359
- ]
360
-
361
- d.run(shutdown: false, expect_records: records.length, timeout: 20) do
362
- entries = records.map { |tag, _time, record| [_time, record] }
363
- # These entries are skipped
364
- entries << ['invalid time', {'a' => 3}] << [time, 'invalid record']
365
-
366
- send_data packer.write(["tag1", entries]).to_s, **options
367
- end
368
-
369
- logs = d.instance.log.out.logs
370
- assert{ logs.count{|line| line =~ /skip invalid event/ } == 2 }
371
-
372
- d.instance_shutdown
373
- end
374
- end
375
-
376
- sub_test_case 'packed forward' do
377
- data(tcp: {
378
- options: {
379
- auth: false
380
- }
381
- },
382
- auth: {
383
- options: {
384
- auth: true
385
- }
386
- })
387
- test 'plain' do |data|
388
- options = data[:options]
389
- config = options[:auth] ? config_auth : base_config
390
- @d = d = create_driver(config)
391
-
392
- time = event_time("2011-01-02 13:14:15 UTC")
393
-
394
- records = [
395
- ["tag1", time, {"a"=>1}],
396
- ["tag1", time, {"a"=>2}],
397
- ]
398
-
399
- d.run(expect_records: records.length, timeout: 20) do
400
- entries = ''
401
- records.each {|_tag, _time, record|
402
- packer(entries).write([_time, record]).flush
403
- }
404
- send_data packer.write(["tag1", entries]).to_s, **options
405
- end
406
- assert_equal(records, d.events)
407
- end
408
-
409
- data(tag: {
410
- param: "tag new_tag",
411
- result: "new_tag"
412
- },
413
- add_tag_prefix: {
414
- param: "add_tag_prefix new_prefix",
415
- result: "new_prefix.tag1"
416
- })
417
- test 'tag parameters' do |data|
418
- @d = create_driver(base_config + data[:param])
419
- time = event_time("2011-01-02 13:14:15 UTC")
420
- options = {auth: false}
421
-
422
- records = [
423
- ["tag1", time, {"a"=>1}],
424
- ["tag1", time, {"a"=>2}],
425
- ]
426
-
427
- @d.run(expect_records: records.length, timeout: 20) do
428
- entries = ''
429
- records.each {|_tag, _time, record|
430
- packer(entries).write([_time, record]).flush
431
- }
432
- send_data packer.write(["tag1", entries]).to_s, **options
433
- end
434
- assert_equal(data[:result], @d.events[0][0])
435
- end
436
-
437
- data(tcp: {
438
- options: {
439
- auth: false
440
- }
441
- },
442
- auth: {
443
- options: {
444
- auth: true
445
- }
446
- })
447
- test 'time_as_integer' do |data|
448
- options = data[:options]
449
- config = options[:auth] ? config_auth : base_config
450
- @d = d = create_driver(config)
451
-
452
- time_i = event_time("2011-01-02 13:14:15 UTC").to_i
453
-
454
- records = [
455
- ["tag1", time_i, {"a"=>1}],
456
- ["tag1", time_i, {"a"=>2}],
457
- ]
458
-
459
- d.run(expect_records: records.length, timeout: 20) do
460
- entries = ''
461
- records.each {|tag, _time, record|
462
- packer(entries).write([_time, record]).flush
463
- }
464
- send_data packer.write(["tag1", entries]).to_s, **options
465
- end
466
- assert_equal(records, d.events)
467
- end
468
-
469
- data(tcp: {
470
- options: {
471
- auth: false
472
- }
473
- },
474
- auth: {
475
- options: {
476
- auth: true
477
- }
478
- })
479
- test 'skip_invalid_event' do |data|
480
- options = data[:options]
481
- config = options[:auth] ? config_auth : base_config
482
- @d = d = create_driver(config + "skip_invalid_event true")
483
-
484
- time = event_time("2011-01-02 13:14:15 UTC")
485
-
486
- records = [
487
- ["tag1", time, {"a" => 1}],
488
- ["tag1", time, {"a" => 2}],
489
- ]
490
-
491
- d.run(shutdown: false, expect_records: records.length, timeout: 20) do
492
- entries = records.map { |tag, _time, record| [_time, record] }
493
- # These entries are skipped
494
- entries << ['invalid time', {'a' => 3}] << [time, 'invalid record']
495
-
496
- packed_entries = ''
497
- entries.each { |_time, record|
498
- packer(packed_entries).write([_time, record]).flush
499
- }
500
- send_data packer.write(["tag1", packed_entries]).to_s, **options
501
- end
502
-
503
- logs = d.instance.log.logs
504
- assert_equal 2, logs.count { |line| line =~ /skip invalid event/ }
505
-
506
- d.instance_shutdown
507
- end
508
- end
509
-
510
- sub_test_case 'compressed packed forward' do
511
- test 'set_compress_to_option' do
512
- @d = d = create_driver
513
-
514
- time_i = event_time("2011-01-02 13:14:15 UTC").to_i
515
- events = [
516
- ["tag1", time_i, {"a"=>1}],
517
- ["tag1", time_i, {"a"=>2}]
518
- ]
519
-
520
- # create compressed entries
521
- entries = ''
522
- events.each do |_tag, _time, record|
523
- v = [_time, record].to_msgpack
524
- entries << compress(v)
525
- end
526
- chunk = ["tag1", entries, { 'compressed' => 'gzip' }].to_msgpack
527
-
528
- d.run do
529
- Fluent::MessagePackFactory.msgpack_unpacker.feed_each(chunk) do |obj|
530
- option = d.instance.send(:on_message, obj, chunk.size, DUMMY_SOCK)
531
- assert_equal 'gzip', option['compressed']
532
- end
533
- end
534
-
535
- assert_equal events, d.events
536
- end
537
-
538
- test 'create_CompressedMessagePackEventStream_with_gzip_compress_option' do
539
- @d = d = create_driver
540
-
541
- time_i = event_time("2011-01-02 13:14:15 UTC").to_i
542
- events = [
543
- ["tag1", time_i, {"a"=>1}],
544
- ["tag1", time_i, {"a"=>2}]
545
- ]
546
-
547
- # create compressed entries
548
- entries = ''
549
- events.each do |_tag, _time, record|
550
- v = [_time, record].to_msgpack
551
- entries << compress(v)
552
- end
553
- chunk = ["tag1", entries, { 'compressed' => 'gzip' }].to_msgpack
554
-
555
- # check CompressedMessagePackEventStream is created
556
- mock(Fluent::CompressedMessagePackEventStream).new(entries, nil, 0)
557
-
558
- d.run do
559
- Fluent::MessagePackFactory.msgpack_unpacker.feed_each(chunk) do |obj|
560
- option = d.instance.send(:on_message, obj, chunk.size, DUMMY_SOCK)
561
- assert_equal 'gzip', option['compressed']
562
- end
563
- end
564
- end
565
- end
566
-
567
- sub_test_case 'warning' do
568
- test 'send_large_chunk_warning' do
569
- @d = d = create_driver(base_config + %[
570
- chunk_size_warn_limit 16M
571
- chunk_size_limit 32M
572
- ])
573
-
574
- time = event_time("2014-04-25 13:14:15 UTC")
575
-
576
- # generate over 16M chunk
577
- str = "X" * 1024 * 1024
578
- chunk = [ "test.tag", (0...16).map{|i| [time + i, {"data" => str}] } ].to_msgpack
579
- assert chunk.size > (16 * 1024 * 1024)
580
- assert chunk.size < (32 * 1024 * 1024)
581
-
582
- d.run(shutdown: false) do
583
- Fluent::MessagePackFactory.msgpack_unpacker.feed_each(chunk) do |obj|
584
- d.instance.send(:on_message, obj, chunk.size, DUMMY_SOCK)
585
- end
586
- end
587
-
588
- # check emitted data
589
- emits = d.events
590
- assert_equal 16, emits.size
591
- assert emits.map(&:first).all?{|t| t == "test.tag" }
592
- assert_equal (0...16).to_a, emits.map{|_tag, t, _record| t - time }
593
-
594
- # check log
595
- logs = d.instance.log.logs
596
- assert_equal 1, logs.count{|line|
597
- line =~ / \[warn\]: Input chunk size is larger than 'chunk_size_warn_limit':/ &&
598
- line =~ / tag="test.tag" host="#{LOCALHOST_HOSTNAME}" limit=16777216 size=16777501/
599
- }, "large chunk warning is not logged"
600
-
601
- d.instance_shutdown
602
- end
603
-
604
- test 'send_large_chunk_only_warning' do
605
- @d = d = create_driver(base_config + %[
606
- chunk_size_warn_limit 16M
607
- ])
608
- time = event_time("2014-04-25 13:14:15 UTC")
609
-
610
- # generate over 16M chunk
611
- str = "X" * 1024 * 1024
612
- chunk = [ "test.tag", (0...16).map{|i| [time + i, {"data" => str}] } ].to_msgpack
613
-
614
- d.run(shutdown: false) do
615
- Fluent::MessagePackFactory.msgpack_unpacker.feed_each(chunk) do |obj|
616
- d.instance.send(:on_message, obj, chunk.size, DUMMY_SOCK)
617
- end
618
- end
619
-
620
- # check log
621
- logs = d.instance.log.logs
622
- assert_equal 1, logs.count{ |line|
623
- line =~ / \[warn\]: Input chunk size is larger than 'chunk_size_warn_limit':/ &&
624
- line =~ / tag="test.tag" host="#{LOCALHOST_HOSTNAME}" limit=16777216 size=16777501/
625
- }, "large chunk warning is not logged"
626
-
627
- d.instance_shutdown
628
- end
629
-
630
- test 'send_large_chunk_limit' do
631
- @d = d = create_driver(base_config + %[
632
- chunk_size_warn_limit 16M
633
- chunk_size_limit 32M
634
- ])
635
-
636
- time = event_time("2014-04-25 13:14:15 UTC")
637
-
638
- # generate over 32M chunk
639
- str = "X" * 1024 * 1024
640
- chunk = [ "test.tag", (0...32).map{|i| [time + i, {"data" => str}] } ].to_msgpack
641
- assert chunk.size > (32 * 1024 * 1024)
642
-
643
- # d.run => send_data
644
- d.run(shutdown: false) do
645
- Fluent::MessagePackFactory.msgpack_unpacker.feed_each(chunk) do |obj|
646
- d.instance.send(:on_message, obj, chunk.size, DUMMY_SOCK)
647
- end
648
- end
649
-
650
- # check emitted data
651
- emits = d.events
652
- assert_equal 0, emits.size
653
-
654
- # check log
655
- logs = d.instance.log.logs
656
- assert_equal 1, logs.count{|line|
657
- line =~ / \[warn\]: Input chunk size is larger than 'chunk_size_limit', dropped:/ &&
658
- line =~ / tag="test.tag" host="#{LOCALHOST_HOSTNAME}" limit=33554432 size=33554989/
659
- }, "large chunk warning is not logged"
660
-
661
- d.instance_shutdown
662
- end
663
-
664
- data('string chunk' => 'broken string',
665
- 'integer chunk' => 10)
666
- test 'send_broken_chunk' do |data|
667
- @d = d = create_driver
668
-
669
- # d.run => send_data
670
- d.run(shutdown: false) do
671
- d.instance.send(:on_message, data, 1000000000, DUMMY_SOCK)
672
- end
673
-
674
- # check emitted data
675
- assert_equal 0, d.events.size
676
-
677
- # check log
678
- logs = d.instance.log.logs
679
- assert_equal 1, logs.count{|line|
680
- line =~ / \[warn\]: incoming chunk is broken: host="#{LOCALHOST_HOSTNAME}" msg=#{data.inspect}/
681
- }, "should not accept broken chunk"
682
-
683
- d.instance_shutdown
684
- end
685
- end
686
-
687
- sub_test_case 'respond to required ack' do
688
- data(tcp: {
689
- options: {
690
- auth: false
691
- }
692
- },
693
- auth: {
694
- options: {
695
- auth: true
696
- }
697
- })
698
- test 'message' do |data|
699
- options = data[:options]
700
- config = options[:auth] ? config_auth : base_config
701
- @d = d = create_driver(config)
702
-
703
- time = event_time("2011-01-02 13:14:15 UTC")
704
-
705
- events = [
706
- ["tag1", time, {"a"=>1}],
707
- ["tag2", time, {"a"=>2}]
708
- ]
709
-
710
- expected_acks = []
711
-
712
- d.run(expect_records: events.size) do
713
- events.each {|tag, _time, record|
714
- op = { 'chunk' => Base64.encode64(record.object_id.to_s) }
715
- expected_acks << op['chunk']
716
- send_data([tag, _time, record, op].to_msgpack, try_to_receive_response: true, response_timeout: 1, **options)
717
- }
718
- end
719
-
720
- assert_equal events, d.events
721
- assert_equal expected_acks, @responses.map { |res| MessagePack.unpack(res)['ack'] }
722
- end
723
-
724
- # FIX: response is not pushed into @responses because IO.select has been blocked until InputForward shutdowns
725
- data(tcp: {
726
- options: {
727
- auth: false
728
- }
729
- },
730
- auth: {
731
- options: {
732
- auth: true
733
- }
734
- })
735
- test 'forward' do |data|
736
- options = data[:options]
737
- config = options[:auth] ? config_auth : base_config
738
- @d = d = create_driver(config)
739
-
740
- time = event_time("2011-01-02 13:14:15 UTC")
741
-
742
- events = [
743
- ["tag1", time, {"a"=>1}],
744
- ["tag1", time, {"a"=>2}]
745
- ]
746
-
747
- expected_acks = []
748
-
749
- d.run(expect_records: events.size) do
750
- entries = []
751
- events.each {|_tag, _time, record|
752
- entries << [_time, record]
753
- }
754
- op = { 'chunk' => Base64.encode64(entries.object_id.to_s) }
755
- expected_acks << op['chunk']
756
- send_data(["tag1", entries, op].to_msgpack, try_to_receive_response: true, response_timeout: 1, **options)
757
- end
758
-
759
- assert_equal events, d.events
760
- assert_equal expected_acks, @responses.map { |res| MessagePack.unpack(res)['ack'] }
761
- end
762
-
763
- data(tcp: {
764
- options: {
765
- auth: false
766
- }
767
- },
768
- auth: {
769
- options: {
770
- auth: true
771
- }
772
- })
773
- test 'packed_forward' do |data|
774
- options = data[:options]
775
- config = options[:auth] ? config_auth : base_config
776
- @d = d = create_driver(config)
777
-
778
- time = event_time("2011-01-02 13:14:15 UTC")
779
-
780
- events = [
781
- ["tag1", time, {"a"=>1}],
782
- ["tag1", time, {"a"=>2}]
783
- ]
784
-
785
- expected_acks = []
786
-
787
- d.run(expect_records: events.size) do
788
- entries = ''
789
- events.each {|_tag, _time,record|
790
- [time, record].to_msgpack(entries)
791
- }
792
- op = { 'chunk' => Base64.encode64(entries.object_id.to_s) }
793
- expected_acks << op['chunk']
794
- send_data(["tag1", entries, op].to_msgpack, try_to_receive_response: true, response_timeout: 1, **options)
795
- end
796
-
797
- assert_equal events, d.events
798
- assert_equal expected_acks, @responses.map { |res| MessagePack.unpack(res)['ack'] }
799
- end
800
-
801
- data(
802
- tcp: {
803
- options: {
804
- auth: false
805
- }
806
- },
807
- ### Auth is not supported with json
808
- # auth: {
809
- # options: {
810
- # auth: true
811
- # }
812
- # },
813
- )
814
- test 'message_json' do |data|
815
- options = data[:options]
816
- config = options[:auth] ? config_auth : base_config
817
- @d = d = create_driver(config)
818
-
819
- time_i = event_time("2011-01-02 13:14:15 UTC")
820
-
821
- events = [
822
- ["tag1", time_i, {"a"=>1}],
823
- ["tag2", time_i, {"a"=>2}]
824
- ]
825
-
826
- expected_acks = []
827
-
828
- d.run(expect_records: events.size, timeout: 20) do
829
- events.each {|tag, _time, record|
830
- op = { 'chunk' => Base64.encode64(record.object_id.to_s) }
831
- expected_acks << op['chunk']
832
- send_data([tag, _time, record, op].to_json, try_to_receive_response: true, response_timeout: 1, **options)
833
- }
834
- end
835
-
836
- assert_equal events, d.events
837
- assert_equal expected_acks, @responses.map { |res| JSON.parse(res)['ack'] }
838
- end
839
- end
840
-
841
- sub_test_case 'not respond without required ack' do
842
- data(tcp: {
843
- options: {
844
- auth: false
845
- }
846
- },
847
- auth: {
848
- options: {
849
- auth: true
850
- }
851
- })
852
- test 'message' do |data|
853
- options = data[:options]
854
- config = options[:auth] ? config_auth : base_config
855
- @d = d = create_driver(config)
856
-
857
- time = event_time("2011-01-02 13:14:15 UTC")
858
-
859
- events = [
860
- ["tag1", time, {"a"=>1}],
861
- ["tag2", time, {"a"=>2}]
862
- ]
863
-
864
- d.run(expect_records: events.size, timeout: 20) do
865
- events.each {|tag, _time, record|
866
- send_data([tag, _time, record].to_msgpack, try_to_receive_response: true, response_timeout: 1, **options)
867
- }
868
- end
869
-
870
- assert_equal events, d.events
871
- assert_equal [nil, nil], @responses
872
- end
873
-
874
- data(tcp: {
875
- options: {
876
- auth: false
877
- }
878
- },
879
- auth: {
880
- options: {
881
- auth: true
882
- }
883
- })
884
- test 'forward' do |data|
885
- options = data[:options]
886
- config = options[:auth] ? config_auth : base_config
887
- @d = d = create_driver(config)
888
-
889
- time = event_time("2011-01-02 13:14:15 UTC")
890
-
891
- events = [
892
- ["tag1", time, {"a"=>1}],
893
- ["tag1", time, {"a"=>2}]
894
- ]
895
-
896
- d.run(expect_records: events.size, timeout: 20) do
897
- entries = []
898
- events.each {|tag, _time, record|
899
- entries << [_time, record]
900
- }
901
- send_data(["tag1", entries].to_msgpack, try_to_receive_response: true, response_timeout: 1, **options)
902
- end
903
-
904
- assert_equal events, d.events
905
- assert_equal [nil], @responses
906
- end
907
-
908
- data(tcp: {
909
- options: {
910
- auth: false
911
- }
912
- },
913
- auth: {
914
- options: {
915
- auth: true
916
- }
917
- })
918
- test 'packed_forward' do |data|
919
- options = data[:options]
920
- config = options[:auth] ? config_auth : base_config
921
- @d = d = create_driver(config)
922
-
923
- time = event_time("2011-01-02 13:14:15 UTC")
924
-
925
- events = [
926
- ["tag1", time, {"a"=>1}],
927
- ["tag1", time, {"a"=>2}]
928
- ]
929
-
930
- d.run(expect_records: events.size, timeout: 20) do
931
- entries = ''
932
- events.each {|tag, _time, record|
933
- [_time, record].to_msgpack(entries)
934
- }
935
- send_data(["tag1", entries].to_msgpack, try_to_receive_response: true, response_timeout: 1, **options)
936
- end
937
-
938
- assert_equal events, d.events
939
- assert_equal [nil], @responses
940
- end
941
-
942
- data(
943
- tcp: {
944
- options: {
945
- auth: false
946
- }
947
- },
948
- ### Auth is not supported with json
949
- # auth: {
950
- # config: config_auth,
951
- # options: {
952
- # auth: true
953
- # }
954
- # },
955
- )
956
- test 'message_json' do |data|
957
- options = data[:options]
958
- config = options[:auth] ? config_auth : base_config
959
- @d = d = create_driver(config)
960
-
961
- time_i = event_time("2011-01-02 13:14:15 UTC").to_i
962
-
963
- events = [
964
- ["tag1", time_i, {"a"=>1}],
965
- ["tag2", time_i, {"a"=>2}]
966
- ]
967
-
968
- d.run(expect_records: events.size, timeout: 20) do
969
- events.each {|tag, _time, record|
970
- send_data([tag, _time, record].to_json, try_to_receive_response: true, response_timeout: 1, **options)
971
- }
972
- end
973
-
974
- assert_equal events, d.events
975
- assert_equal [nil, nil], @responses
976
- end
977
- end
978
-
979
- def packer(*args)
980
- Fluent::MessagePackFactory.msgpack_packer(*args)
981
- end
982
-
983
- def unpacker
984
- Fluent::MessagePackFactory.msgpack_unpacker
985
- end
986
-
987
- # res
988
- # '' : socket is disconnected without any data
989
- # nil: socket read timeout
990
- def read_data(io, timeout, &block)
991
- res = ''
992
- select_timeout = 0.5
993
- clock_id = Process::CLOCK_MONOTONIC_RAW rescue Process::CLOCK_MONOTONIC
994
- timeout_at = Process.clock_gettime(clock_id) + timeout
995
- begin
996
- buf = ''
997
- io_activated = false
998
- while Process.clock_gettime(clock_id) < timeout_at
999
- if IO.select([io], nil, nil, select_timeout)
1000
- io_activated = true
1001
- buf = io.readpartial(2048)
1002
- res ||= ''
1003
- res << buf
1004
-
1005
- break if block.call(res)
1006
- end
1007
- end
1008
- res = nil unless io_activated # timeout without no data arrival
1009
- rescue Errno::EAGAIN
1010
- sleep 0.01
1011
- retry if res == ''
1012
- # if res is not empty, all data in socket buffer are read, so do not retry
1013
- rescue IOError, EOFError, Errno::ECONNRESET
1014
- # socket disconnected
1015
- end
1016
- res
1017
- end
1018
-
1019
- def simulate_auth_sequence(io, shared_key=SHARED_KEY, username=USER_NAME, password=USER_PASSWORD)
1020
- auth_response_timeout = 30
1021
- shared_key_salt = 'salt'
1022
-
1023
- # reading helo
1024
- helo_data = read_data(io, auth_response_timeout){|data| MessagePack.unpack(data) rescue nil }
1025
- raise "Authentication packet timeout" unless helo_data
1026
- raise "Authentication connection closed" if helo_data == ''
1027
- # ['HELO', options(hash)]
1028
- helo = MessagePack.unpack(helo_data)
1029
- raise "Invalid HELO header" unless helo[0] == 'HELO'
1030
- raise "Invalid HELO option object" unless helo[1].is_a?(Hash)
1031
- @options = helo[1]
1032
-
1033
- # sending ping
1034
- ping = [
1035
- 'PING',
1036
- 'selfhostname',
1037
- shared_key_salt,
1038
- Digest::SHA512.new
1039
- .update(shared_key_salt)
1040
- .update('selfhostname')
1041
- .update(@options['nonce'])
1042
- .update(shared_key).hexdigest,
1043
- ]
1044
- if @options['auth'] # auth enabled -> value is auth salt
1045
- pass_digest = Digest::SHA512.new.update(@options['auth']).update(username).update(password).hexdigest
1046
- ping.push(username, pass_digest)
1047
- else
1048
- ping.push('', '')
1049
- end
1050
- io.write ping.to_msgpack
1051
- io.flush
1052
-
1053
- # reading pong
1054
- pong_data = read_data(io, auth_response_timeout){|data| MessagePack.unpack(data) rescue nil }
1055
- raise "PONG packet timeout" unless pong_data
1056
- raise "PONG connection closed" if pong_data == ''
1057
- # ['PING', bool(auth_result), string(reason_if_failed), self_hostname, shared_key_digest]
1058
- pong = MessagePack.unpack(pong_data)
1059
- raise "Invalid PONG header" unless pong[0] == 'PONG'
1060
- raise "Authentication Failure: #{pong[2]}" unless pong[1]
1061
- clientside_calculated = Digest::SHA512.new
1062
- .update(shared_key_salt)
1063
- .update(pong[3])
1064
- .update(@options['nonce'])
1065
- .update(shared_key).hexdigest
1066
- raise "Shared key digest mismatch" unless clientside_calculated == pong[4]
1067
-
1068
- # authentication success
1069
- true
1070
- end
1071
-
1072
- def connect
1073
- TCPSocket.new('127.0.0.1', @port)
1074
- end
1075
-
1076
- # Data ordering is not assured:
1077
- # Records in different sockets are processed on different thread, so its scheduling make effect
1078
- # on order of emitted records.
1079
- # So, we MUST sort emitted records in different `send_data` before assertion.
1080
- def send_data(data, try_to_receive_response: false, response_timeout: 5, auth: false)
1081
- io = connect
1082
-
1083
- if auth
1084
- simulate_auth_sequence(io)
1085
- end
1086
-
1087
- io.write data
1088
- io.flush
1089
- if try_to_receive_response
1090
- @responses << read_data(io, response_timeout){|d| MessagePack.unpack(d) rescue nil }
1091
- end
1092
- ensure
1093
- io.close rescue nil # SSL socket requires any writes to close sockets
1094
- end
1095
-
1096
- sub_test_case 'source_hostname_key and source_address_key features' do
1097
- data(
1098
- both: [:hostname, :address],
1099
- hostname: [:hostname],
1100
- address: [:address],
1101
- )
1102
- test 'message protocol' do |keys|
1103
- execute_test_with_source_hostname_key(*keys) { |events|
1104
- events.each { |tag, time, record|
1105
- send_data [tag, time, record].to_msgpack
1106
- }
1107
- }
1108
- end
1109
-
1110
- data(
1111
- both: [:hostname, :address],
1112
- hostname: [:hostname],
1113
- address: [:address],
1114
- )
1115
- test 'forward protocol' do |keys|
1116
- execute_test_with_source_hostname_key(*keys) { |events|
1117
- entries = []
1118
- events.each {|tag,time,record|
1119
- entries << [time, record]
1120
- }
1121
- send_data ['tag1', entries].to_msgpack
1122
- }
1123
- end
1124
-
1125
- data(
1126
- both: [:hostname, :address],
1127
- hostname: [:hostname],
1128
- address: [:address],
1129
- )
1130
- test 'packed forward protocol' do |keys|
1131
- execute_test_with_source_hostname_key(*keys) { |events|
1132
- entries = ''
1133
- events.each { |tag, time, record|
1134
- Fluent::MessagePackFactory.msgpack_packer(entries).write([time, record]).flush
1135
- }
1136
- send_data Fluent::MessagePackFactory.msgpack_packer.write(["tag1", entries]).to_s
1137
- }
1138
- end
1139
- end
1140
-
1141
- def execute_test_with_source_hostname_key(*keys, &block)
1142
- conf = base_config.dup
1143
- if keys.include?(:hostname)
1144
- conf << <<EOL
1145
- source_hostname_key source_hostname
1146
- EOL
1147
- end
1148
- if keys.include?(:address)
1149
- conf << <<EOL
1150
- source_address_key source_address
1151
- EOL
1152
- end
1153
- @d = d = create_driver(conf)
1154
-
1155
- time = event_time("2011-01-02 13:14:15 UTC")
1156
- events = [
1157
- ["tag1", time, {"a"=>1}],
1158
- ["tag1", time, {"a"=>2}]
1159
- ]
1160
-
1161
- d.run(expect_records: events.size) do
1162
- block.call(events)
1163
- end
1164
-
1165
- d.events.each { |tag, _time, record|
1166
- if keys.include?(:hostname)
1167
- assert_true record.has_key?('source_hostname')
1168
- assert_equal DUMMY_SOCK.remote_host, record['source_hostname']
1169
- end
1170
- if keys.include?(:address)
1171
- assert_true record.has_key?('source_address')
1172
- assert_equal DUMMY_SOCK.remote_addr, record['source_address']
1173
- end
1174
- }
1175
- end
1176
-
1177
- # TODO heartbeat
1178
- end