fluentd 0.14.4 → 0.14.5

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (38) hide show
  1. checksums.yaml +4 -4
  2. data/ChangeLog +18 -0
  3. data/example/in_forward.conf +3 -0
  4. data/example/in_forward_client.conf +37 -0
  5. data/example/in_forward_shared_key.conf +15 -0
  6. data/example/in_forward_users.conf +24 -0
  7. data/example/out_forward.conf +13 -13
  8. data/example/out_forward_client.conf +109 -0
  9. data/example/out_forward_shared_key.conf +36 -0
  10. data/example/out_forward_users.conf +65 -0
  11. data/example/{out_buffered_null.conf → out_null.conf} +10 -6
  12. data/example/secondary_file.conf +41 -0
  13. data/lib/fluent/agent.rb +3 -1
  14. data/lib/fluent/plugin/buffer.rb +5 -1
  15. data/lib/fluent/plugin/in_forward.rb +300 -50
  16. data/lib/fluent/plugin/in_tail.rb +41 -85
  17. data/lib/fluent/plugin/multi_output.rb +4 -0
  18. data/lib/fluent/plugin/out_forward.rb +326 -209
  19. data/lib/fluent/plugin/out_null.rb +37 -0
  20. data/lib/fluent/plugin/out_secondary_file.rb +128 -0
  21. data/lib/fluent/plugin/out_stdout.rb +38 -2
  22. data/lib/fluent/plugin/output.rb +13 -5
  23. data/lib/fluent/root_agent.rb +1 -1
  24. data/lib/fluent/test/startup_shutdown.rb +33 -0
  25. data/lib/fluent/version.rb +1 -1
  26. data/test/plugin/test_in_forward.rb +906 -441
  27. data/test/plugin/test_in_monitor_agent.rb +4 -0
  28. data/test/plugin/test_in_tail.rb +681 -663
  29. data/test/plugin/test_out_forward.rb +150 -208
  30. data/test/plugin/test_out_null.rb +85 -9
  31. data/test/plugin/test_out_secondary_file.rb +432 -0
  32. data/test/plugin/test_out_stdout.rb +143 -45
  33. data/test/test_root_agent.rb +42 -0
  34. metadata +14 -9
  35. data/lib/fluent/plugin/out_buffered_null.rb +0 -59
  36. data/lib/fluent/plugin/out_buffered_stdout.rb +0 -70
  37. data/test/plugin/test_out_buffered_null.rb +0 -79
  38. data/test/plugin/test_out_buffered_stdout.rb +0 -122
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 529221a0c55683e4a49cc5916e9e81ee57d56a0f
4
- data.tar.gz: 1e3f9752361cbd8be54bb3477464e6b80fbe5d71
3
+ metadata.gz: 31d47738dce591e0ae2cad1a00f1d83809c0b986
4
+ data.tar.gz: 819627a4cb036aa554141de99baa6d52b32219a3
5
5
  SHA512:
6
- metadata.gz: c4522d4acb9917dfc69ee2d3ac0c71f19ab48fd918b837a284cc4ad9376d8661e7ed304ce3fa3fa70b29e42764daf659a1522509bfb6062c89e50b919e4d7354
7
- data.tar.gz: f8b8c42e975b412a0e42b65dbf3bf256756b7232407be7dc38345326f18154d0424c6e8e0e65ca4af8403f8bbc99f1ab0db349e34bc60c37f2975c5cb580d316
6
+ metadata.gz: 45a9621dcf0640691b33443e49cc6da6736205f94370b2787c6987d373c548bf3f396975b145858b7409fcd44b3368232d47b2c42b7d8ef52c34a235f7dcf080
7
+ data.tar.gz: 36f0133d0be03247818d4f7819870b208f466dfa56ccf0fa72135ead2a632cdea6d5c06acc9325ec1a3941fdfbc07f1d09e2a04b580807e7639d45d05a7cf14f
data/ChangeLog CHANGED
@@ -1,5 +1,23 @@
1
1
  # v0.14
2
2
 
3
+ ## Release v0.14.5 - 2016/09/06
4
+
5
+ ### New features / Enhancement
6
+
7
+ * Add authentication / authorization feature to forward protocol and in/out_forward plugins
8
+ https://github.com/fluent/fluentd/pull/1136
9
+ * Add a new plugin to dump buffers in retries as secondary plugin
10
+ https://github.com/fluent/fluentd/pull/1154
11
+ * Merge out_buffered_stdout and out_buffered_null into out_stdout and out_null
12
+ https://github.com/fluent/fluentd/pull/1200
13
+
14
+ ### Bug fixes
15
+
16
+ * Raise configuration errors to clearify what's wrong when "@type" is missing
17
+ https://github.com/fluent/fluentd/pull/1202
18
+ * Fix the bug not to launch Fluentd when v0.12 MultiOutput plugin is configured
19
+ https://github.com/fluent/fluentd/pull/1206
20
+
3
21
  ## Release v0.14.4 - 2016/08/31
4
22
 
5
23
  ### New features / Enhancement
@@ -8,4 +8,7 @@
8
8
 
9
9
  <match test>
10
10
  @type stdout
11
+ # <buffer>
12
+ # flush_interval 10s
13
+ # </buffer>
11
14
  </match>
@@ -0,0 +1,37 @@
1
+ <system>
2
+ rpc_endpoint 0.0.0.0:24444
3
+ </system>
4
+
5
+ <source>
6
+ @type forward
7
+ port 24224
8
+ bind 0.0.0.0
9
+ <security>
10
+ self_hostname input.testing.local
11
+ shared_key secure_communication_is_awesome
12
+ user_auth yes
13
+ allow_anonymous_source no
14
+ <user>
15
+ username user1
16
+ password yes_this_is_user1
17
+ </user>
18
+ <user>
19
+ username user2
20
+ password yes_this_is_really_user2
21
+ </user>
22
+ <user>
23
+ username user3
24
+ password noooooo_this_may_not_be_user3
25
+ </user>
26
+ <client>
27
+ # host 127.0.0.1
28
+ network 127.0.0.0/24
29
+ shared_key using_different_key_makes_us_secure
30
+ users user1,user2
31
+ </client>
32
+ </security>
33
+ </source>
34
+
35
+ <match {test,test2,test3,test4}>
36
+ @type stdout
37
+ </match>
@@ -0,0 +1,15 @@
1
+ <system>
2
+ rpc_endpoint 0.0.0.0:24444
3
+ </system>
4
+
5
+ <source>
6
+ @type forward
7
+ <security>
8
+ self_hostname input.testing.local
9
+ shared_key secure_communication_is_awesome
10
+ </security>
11
+ </source>
12
+
13
+ <match test>
14
+ @type stdout
15
+ </match>
@@ -0,0 +1,24 @@
1
+ <system>
2
+ rpc_endpoint 0.0.0.0:24444
3
+ </system>
4
+
5
+ <source>
6
+ @type forward
7
+ <security>
8
+ self_hostname input.testing.local
9
+ shared_key secure_communication_is_awesome
10
+ user_auth yes
11
+ <user>
12
+ username user1
13
+ password yes_this_is_user1
14
+ </user>
15
+ <user>
16
+ username user2
17
+ password yes_this_is_really_user2
18
+ </user>
19
+ </security>
20
+ </source>
21
+
22
+ <match {test,test2,test3}>
23
+ @type stdout
24
+ </match>
@@ -8,22 +8,22 @@
8
8
 
9
9
  <server>
10
10
  # first server
11
- host localhost
11
+ host 127.0.0.1
12
12
  port 24224
13
13
  </server>
14
- <server>
15
- # second server
16
- host localhost
17
- port 24225
18
- </server>
19
- <server>
20
- # second server
21
- host localhost
22
- port 24226
23
- standby
24
- </server>
14
+ # <server>
15
+ # # second server
16
+ # host localhost
17
+ # port 24225
18
+ # </server>
19
+ # <server>
20
+ # # second server
21
+ # host localhost
22
+ # port 24226
23
+ # standby
24
+ # </server>
25
25
 
26
- flush_interval 1
26
+ flush_interval 0
27
27
  send_timeout 60
28
28
  heartbeat_type udp
29
29
  heartbeat_interval 1
@@ -0,0 +1,109 @@
1
+ <source>
2
+ @type dummy
3
+ tag test
4
+ </source>
5
+ <source>
6
+ @type dummy
7
+ tag test2
8
+ </source>
9
+ <source>
10
+ @type dummy
11
+ tag test3
12
+ </source>
13
+ <source>
14
+ @type dummy
15
+ tag test4
16
+ </source>
17
+ <source>
18
+ @type dummy
19
+ tag test5
20
+ </source>
21
+
22
+ <match test>
23
+ @type forward
24
+ flush_interval 0
25
+ <security>
26
+ self_hostname output.testing.local
27
+ shared_key secure_communication_is_awesome
28
+ </security>
29
+ <server>
30
+ host 127.0.0.1
31
+ port 24224
32
+ username user1
33
+ password yes_this_is_user1
34
+ shared_key using_different_key_makes_us_secure
35
+ </server>
36
+ </match>
37
+
38
+ <match test2>
39
+ @type forward
40
+ flush_interval 0
41
+ <security>
42
+ self_hostname output-alt1.testing.local
43
+ shared_key using_different_key_makes_us_secure
44
+ </security>
45
+ <server>
46
+ host 127.0.0.1
47
+ port 24224
48
+ username user1
49
+ password yes_this_is_user1
50
+ </server>
51
+ <server>
52
+ host 127.0.0.1
53
+ port 24224
54
+ username user2
55
+ password yes_this_is_really_user2
56
+ </server>
57
+ </match>
58
+
59
+ <match test3>
60
+ @type forward
61
+ flush_interval 0
62
+ <security>
63
+ self_hostname output-fail1.testing.local
64
+ # default key: fail
65
+ shared_key secure_communication_is_awesome
66
+ </security>
67
+ <server>
68
+ host 127.0.0.1
69
+ port 24224
70
+ username user1
71
+ password yes_this_is_user1
72
+ # [warn]: Shared key mismatch address="127.0.0.1" hostname="output-fail1.testing.local"
73
+ </server>
74
+ </match>
75
+
76
+ <match test4>
77
+ @type forward
78
+ flush_interval 0
79
+ <security>
80
+ self_hostname output-fail2.testing.local
81
+ shared_key using_different_key_makes_us_secure
82
+ </security>
83
+ <server>
84
+ host 127.0.0.1
85
+ port 24224
86
+ username user3
87
+ # user3 (user denied): fail
88
+ password noooooo_this_may_not_be_user3
89
+ # [warn]: Authentication failed address="127.0.0.1" hostname="output-fail2.testing.local" username="user3"
90
+ </server>
91
+ </match>
92
+
93
+ <match test5>
94
+ @type forward
95
+ flush_interval 0
96
+ <security>
97
+ self_hostname output-fail3.testing.local
98
+ shared_key using_different_key_makes_us_secure
99
+ </security>
100
+ <server>
101
+ # another ip (host rejected): fail
102
+ # This pattern will work only with Ruby 2.3
103
+ host "#{Socket.getifaddrs.select{|i| i.addr.ipv4? }.reject{|i| i.addr.ip_address == '127.0.0.1' }.first.addr.ip_address}"
104
+ port 24224
105
+ username user1
106
+ password yes_this_is_user1
107
+ # [warn]: Anonymous client disallowed address="192.168.1.75" hostname="output-fail3.testing.local"
108
+ </server>
109
+ </match>
@@ -0,0 +1,36 @@
1
+ <source>
2
+ @type dummy
3
+ tag test
4
+ </source>
5
+ <source>
6
+ @type dummy
7
+ tag test2
8
+ </source>
9
+
10
+ <match test>
11
+ @type forward
12
+ flush_interval 0
13
+ <security>
14
+ self_hostname output.testing.local
15
+ shared_key secure_communication_is_awesome
16
+ </security>
17
+ <server>
18
+ host 127.0.0.1
19
+ port 24224
20
+ </server>
21
+ </match>
22
+
23
+ <match test2>
24
+ @type forward
25
+ flush_interval 0
26
+ <security>
27
+ self_hostname output-fail.testing.local
28
+ shared_key secure_communication_is_not_awesome
29
+ # input plugin shows warning for wrong shared_key
30
+ # 2016-08-08 16:27:00 +0900 [warn]: Shared key mismatch address="127.0.0.1" hostname="output-fail.testing.local"
31
+ </security>
32
+ <server>
33
+ host 127.0.0.1
34
+ port 24224
35
+ </server>
36
+ </match>
@@ -0,0 +1,65 @@
1
+ <source>
2
+ @type dummy
3
+ tag test
4
+ </source>
5
+ <source>
6
+ @type dummy
7
+ tag test2
8
+ </source>
9
+ <source>
10
+ @type dummy
11
+ tag test3
12
+ </source>
13
+
14
+ <match test>
15
+ @type forward
16
+ flush_interval 0
17
+ <security>
18
+ self_hostname output.testing.local
19
+ shared_key secure_communication_is_awesome
20
+ </security>
21
+ <server>
22
+ host 127.0.0.1
23
+ port 24224
24
+ username user1
25
+ password yes_this_is_user1
26
+ </server>
27
+ </match>
28
+
29
+ <match test2>
30
+ @type forward
31
+ flush_interval 0
32
+ <security>
33
+ self_hostname output-alt1.testing.local
34
+ shared_key secure_communication_is_awesome
35
+ </security>
36
+ <server>
37
+ host 127.0.0.1
38
+ port 24224
39
+ username user1
40
+ password yes_this_is_user1
41
+ </server>
42
+ <server>
43
+ host 127.0.0.1
44
+ port 24224
45
+ username user2
46
+ password yes_this_is_really_user2
47
+ </server>
48
+ </match>
49
+
50
+ <match test3>
51
+ @type forward
52
+ flush_interval 0
53
+ <security>
54
+ self_hostname output-fail.testing.local
55
+ shared_key secure_communication_is_awesome
56
+ </security>
57
+ <server>
58
+ host 127.0.0.1
59
+ port 24224
60
+ username user3
61
+ password no_there_are_not_such_user
62
+ # input plugin warns authentication erro:
63
+ # [warn]: Authentication failed address="127.0.0.1" hostname="output-fail.testing.local" username="user3"
64
+ </server>
65
+ </match>
@@ -13,16 +13,20 @@
13
13
  </source>
14
14
 
15
15
  <match dummy.**>
16
- @type buffered_null
17
- try_flush_interval 60
18
- flush_interval 60
19
- buffer_chunk_limit 1k
20
- buffer_queue_limit 2
16
+ @type null
17
+ <buffer>
18
+ flush_interval 60s
19
+ chunk_limit_size 1k
20
+ total_limit_size 4k
21
+ </buffer>
21
22
  </match>
22
23
 
23
24
  <label error_log>
24
25
  <match **>
25
- @type stdout # or buffered_stdout
26
+ @type stdout
27
+ # <buffer>
28
+ # flush_interval 1s
29
+ # </buffer>
26
30
  </match>
27
31
  </label>
28
32
 
@@ -0,0 +1,41 @@
1
+ <system>
2
+ rpc_endpoint 0.0.0.0:24444
3
+ </system>
4
+
5
+ <source>
6
+ @type dummy
7
+ tag test
8
+ </source>
9
+
10
+ <source>
11
+ @type forward
12
+ @label @raw
13
+ </source>
14
+
15
+ <label @raw>
16
+ <match>
17
+ @type stdout
18
+ </match>
19
+ </label>
20
+
21
+ <match test>
22
+ @type forward
23
+
24
+ <buffer time,tag,message>
25
+ type memory
26
+ timekey 2s
27
+ timekey_wait 1s
28
+ flush_interval 1s
29
+ </buffer>
30
+
31
+ <server>
32
+ host 0.0.0.0
33
+ port 24224
34
+ </server>
35
+
36
+ <secondary>
37
+ type secondary_file
38
+ directory log/secondary/
39
+ basename ${tag}_%Y%m%d%L_${message}
40
+ </secondary>
41
+ </match>
@@ -63,6 +63,7 @@ module Fluent
63
63
  conf.elements('filter', 'match').each { |e|
64
64
  pattern = e.arg.empty? ? '**' : e.arg
65
65
  type = e['@type']
66
+ raise ConfigError, "Missing '@type' parameter on <#{e.name}> directive" unless type
66
67
  if e.name == 'filter'
67
68
  add_filter(type, pattern, e)
68
69
  else
@@ -134,7 +135,8 @@ module Fluent
134
135
  output.router = @event_router if output.respond_to?(:router=)
135
136
  output.configure(conf)
136
137
  @outputs << output
137
- if output.respond_to?(:outputs) && (output.is_a?(Fluent::Plugin::MultiOutput) || output.is_a?(Fluent::MultiOutput))
138
+ if output.respond_to?(:outputs) && (output.respond_to?(:multi_output?) && output.multi_output? || output.is_a?(Fluent::MultiOutput))
139
+ # TODO: ruby 2.3 or later: replace `output.respond_to?(:multi_output?) && output.multi_output?` with output&.multi_output?
138
140
  @outputs.push(*output.outputs)
139
141
  end
140
142
  @event_router.add_rule(pattern, output)