fluentd 0.14.17-x86-mingw32 → 1.3.1-x86-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (159) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +16 -5
  3. data/ADOPTERS.md +5 -0
  4. data/{ChangeLog → CHANGELOG.md} +495 -6
  5. data/CONTRIBUTING.md +5 -2
  6. data/GOVERNANCE.md +55 -0
  7. data/LICENSE +202 -0
  8. data/MAINTAINERS.md +7 -5
  9. data/README.md +17 -10
  10. data/bin/fluent-ca-generate +6 -0
  11. data/example/counter.conf +18 -0
  12. data/example/secondary_file.conf +3 -2
  13. data/fluentd.gemspec +3 -3
  14. data/lib/fluent/agent.rb +1 -1
  15. data/lib/fluent/command/binlog_reader.rb +11 -2
  16. data/lib/fluent/command/ca_generate.rb +181 -0
  17. data/lib/fluent/command/cat.rb +28 -15
  18. data/lib/fluent/command/debug.rb +4 -4
  19. data/lib/fluent/command/fluentd.rb +2 -2
  20. data/lib/fluent/command/plugin_config_formatter.rb +24 -2
  21. data/lib/fluent/command/plugin_generator.rb +26 -8
  22. data/lib/fluent/config/configure_proxy.rb +7 -1
  23. data/lib/fluent/config/dsl.rb +8 -5
  24. data/lib/fluent/config/element.rb +5 -0
  25. data/lib/fluent/config/literal_parser.rb +7 -1
  26. data/lib/fluent/config/types.rb +28 -2
  27. data/lib/fluent/config/v1_parser.rb +1 -2
  28. data/lib/fluent/configurable.rb +1 -0
  29. data/lib/fluent/counter.rb +23 -0
  30. data/lib/fluent/counter/base_socket.rb +46 -0
  31. data/lib/fluent/counter/client.rb +297 -0
  32. data/lib/fluent/counter/error.rb +86 -0
  33. data/lib/fluent/counter/mutex_hash.rb +163 -0
  34. data/lib/fluent/counter/server.rb +273 -0
  35. data/lib/fluent/counter/store.rb +205 -0
  36. data/lib/fluent/counter/validator.rb +145 -0
  37. data/lib/fluent/env.rb +1 -0
  38. data/lib/fluent/event_router.rb +1 -1
  39. data/lib/fluent/log.rb +119 -29
  40. data/lib/fluent/plugin/base.rb +12 -0
  41. data/lib/fluent/plugin/buf_file.rb +20 -16
  42. data/lib/fluent/plugin/buffer.rb +130 -32
  43. data/lib/fluent/plugin/buffer/file_chunk.rb +23 -4
  44. data/lib/fluent/plugin/compressable.rb +1 -1
  45. data/lib/fluent/plugin/filter_grep.rb +135 -21
  46. data/lib/fluent/plugin/filter_parser.rb +13 -2
  47. data/lib/fluent/plugin/filter_record_transformer.rb +16 -14
  48. data/lib/fluent/plugin/formatter_stdout.rb +3 -2
  49. data/lib/fluent/plugin/formatter_tsv.rb +5 -1
  50. data/lib/fluent/plugin/in_debug_agent.rb +8 -1
  51. data/lib/fluent/plugin/in_forward.rb +1 -1
  52. data/lib/fluent/plugin/in_http.rb +84 -3
  53. data/lib/fluent/plugin/in_monitor_agent.rb +7 -1
  54. data/lib/fluent/plugin/in_syslog.rb +31 -10
  55. data/lib/fluent/plugin/in_tail.rb +142 -53
  56. data/lib/fluent/plugin/in_tcp.rb +5 -6
  57. data/lib/fluent/plugin/in_udp.rb +6 -2
  58. data/lib/fluent/plugin/in_unix.rb +1 -1
  59. data/lib/fluent/plugin/multi_output.rb +1 -0
  60. data/lib/fluent/plugin/out_copy.rb +25 -2
  61. data/lib/fluent/plugin/out_file.rb +26 -7
  62. data/lib/fluent/plugin/out_forward.rb +81 -42
  63. data/lib/fluent/plugin/out_secondary_file.rb +2 -2
  64. data/lib/fluent/plugin/out_stdout.rb +0 -1
  65. data/lib/fluent/plugin/out_stream.rb +1 -1
  66. data/lib/fluent/plugin/output.rb +221 -57
  67. data/lib/fluent/plugin/parser_apache.rb +1 -1
  68. data/lib/fluent/plugin/parser_apache2.rb +5 -1
  69. data/lib/fluent/plugin/parser_apache_error.rb +1 -1
  70. data/lib/fluent/plugin/parser_json.rb +10 -3
  71. data/lib/fluent/plugin/parser_ltsv.rb +7 -0
  72. data/lib/fluent/plugin/parser_multiline.rb +2 -1
  73. data/lib/fluent/plugin/parser_nginx.rb +1 -1
  74. data/lib/fluent/plugin/parser_none.rb +1 -0
  75. data/lib/fluent/plugin/parser_regexp.rb +15 -14
  76. data/lib/fluent/plugin/parser_syslog.rb +9 -5
  77. data/lib/fluent/plugin_helper.rb +2 -0
  78. data/lib/fluent/plugin_helper/cert_option.rb +28 -9
  79. data/lib/fluent/plugin_helper/compat_parameters.rb +3 -1
  80. data/lib/fluent/plugin_helper/counter.rb +51 -0
  81. data/lib/fluent/plugin_helper/event_loop.rb +9 -0
  82. data/lib/fluent/plugin_helper/record_accessor.rb +210 -0
  83. data/lib/fluent/plugin_helper/retry_state.rb +15 -7
  84. data/lib/fluent/plugin_helper/server.rb +87 -25
  85. data/lib/fluent/plugin_helper/socket_option.rb +5 -2
  86. data/lib/fluent/plugin_helper/timer.rb +8 -7
  87. data/lib/fluent/root_agent.rb +18 -9
  88. data/lib/fluent/supervisor.rb +63 -23
  89. data/lib/fluent/system_config.rb +30 -2
  90. data/lib/fluent/test/helpers.rb +1 -1
  91. data/lib/fluent/time.rb +15 -7
  92. data/lib/fluent/timezone.rb +26 -2
  93. data/lib/fluent/version.rb +1 -1
  94. data/templates/new_gem/README.md.erb +2 -2
  95. data/templates/new_gem/lib/fluent/plugin/filter.rb.erb +1 -1
  96. data/templates/new_gem/lib/fluent/plugin/input.rb.erb +1 -1
  97. data/templates/new_gem/lib/fluent/plugin/output.rb.erb +1 -1
  98. data/templates/new_gem/lib/fluent/plugin/parser.rb.erb +4 -4
  99. data/test/command/test_ca_generate.rb +70 -0
  100. data/test/command/test_fluentd.rb +2 -2
  101. data/test/command/test_plugin_config_formatter.rb +8 -7
  102. data/test/command/test_plugin_generator.rb +65 -39
  103. data/test/config/test_config_parser.rb +7 -2
  104. data/test/config/test_configurable.rb +7 -2
  105. data/test/config/test_configure_proxy.rb +41 -3
  106. data/test/config/test_dsl.rb +10 -10
  107. data/test/config/test_element.rb +10 -0
  108. data/test/config/test_literal_parser.rb +8 -0
  109. data/test/config/test_plugin_configuration.rb +56 -0
  110. data/test/config/test_system_config.rb +19 -1
  111. data/test/config/test_types.rb +37 -0
  112. data/test/counter/test_client.rb +559 -0
  113. data/test/counter/test_error.rb +44 -0
  114. data/test/counter/test_mutex_hash.rb +179 -0
  115. data/test/counter/test_server.rb +589 -0
  116. data/test/counter/test_store.rb +258 -0
  117. data/test/counter/test_validator.rb +137 -0
  118. data/test/plugin/test_buf_file.rb +124 -0
  119. data/test/plugin/test_buffer.rb +3 -2
  120. data/test/plugin/test_filter_grep.rb +580 -2
  121. data/test/plugin/test_filter_parser.rb +33 -2
  122. data/test/plugin/test_filter_record_transformer.rb +22 -1
  123. data/test/plugin/test_formatter_ltsv.rb +3 -0
  124. data/test/plugin/test_formatter_tsv.rb +68 -0
  125. data/test/plugin/test_in_debug_agent.rb +21 -0
  126. data/test/plugin/test_in_exec.rb +3 -5
  127. data/test/plugin/test_in_http.rb +178 -0
  128. data/test/plugin/test_in_monitor_agent.rb +1 -1
  129. data/test/plugin/test_in_syslog.rb +64 -0
  130. data/test/plugin/test_in_tail.rb +116 -6
  131. data/test/plugin/test_in_tcp.rb +21 -0
  132. data/test/plugin/test_in_udp.rb +78 -0
  133. data/test/plugin/test_metadata.rb +89 -0
  134. data/test/plugin/test_out_copy.rb +31 -0
  135. data/test/plugin/test_out_file.rb +108 -2
  136. data/test/plugin/test_out_forward.rb +195 -2
  137. data/test/plugin/test_out_secondary_file.rb +14 -0
  138. data/test/plugin/test_output.rb +159 -45
  139. data/test/plugin/test_output_as_buffered.rb +19 -0
  140. data/test/plugin/test_output_as_buffered_backup.rb +307 -0
  141. data/test/plugin/test_output_as_buffered_retries.rb +70 -0
  142. data/test/plugin/test_output_as_buffered_secondary.rb +1 -1
  143. data/test/plugin/test_parser_apache2.rb +1 -0
  144. data/test/plugin/test_parser_labeled_tsv.rb +17 -0
  145. data/test/plugin/test_parser_nginx.rb +40 -0
  146. data/test/plugin/test_parser_regexp.rb +6 -7
  147. data/test/plugin/test_parser_syslog.rb +155 -5
  148. data/test/plugin_helper/test_child_process.rb +4 -4
  149. data/test/plugin_helper/test_compat_parameters.rb +22 -0
  150. data/test/plugin_helper/test_record_accessor.rb +197 -0
  151. data/test/plugin_helper/test_retry_state.rb +20 -0
  152. data/test/plugin_helper/test_server.rb +30 -2
  153. data/test/test_config.rb +3 -3
  154. data/test/test_configdsl.rb +2 -2
  155. data/test/test_log.rb +51 -1
  156. data/test/test_root_agent.rb +33 -0
  157. data/test/test_supervisor.rb +105 -0
  158. metadata +68 -8
  159. data/COPYING +0 -14
@@ -21,7 +21,7 @@ module Fluent
21
21
  class ApacheParser < RegexpParser
22
22
  Plugin.register_parser("apache", self)
23
23
 
24
- config_set_default :expression, %q{/^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/}
24
+ config_set_default :expression, /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/
25
25
  config_set_default :time_format, "%d/%b/%Y:%H:%M:%S %z"
26
26
  end
27
27
  end
@@ -26,10 +26,14 @@ module Fluent
26
26
 
27
27
  def initialize
28
28
  super
29
- @time_parser = time_parser_create(format: TIME_FORMAT)
30
29
  @mutex = Mutex.new
31
30
  end
32
31
 
32
+ def configure(conf)
33
+ super
34
+ @time_parser = time_parser_create(format: TIME_FORMAT)
35
+ end
36
+
33
37
  def patterns
34
38
  {'format' => REGEXP, 'time_format' => TIME_FORMAT}
35
39
  end
@@ -20,7 +20,7 @@ module Fluent
20
20
  module Plugin
21
21
  class ApacheErrorParser < RegexpParser
22
22
  Plugin.register_parser("apache_error", self)
23
- config_set_default :expression, %q{/^\[[^ ]* (?<time>[^\]]*)\] \[(?<level>[^\]]*)\](?: \[pid (?<pid>[^\]]*)\])?( \[client (?<client>[^\]]*)\])? (?<message>.*)$/}
23
+ config_set_default :expression, /^\[[^ ]* (?<time>[^\]]*)\] \[(?<level>[^\]]*)\](?: \[pid (?<pid>[^\]]*)\])?( \[client (?<client>[^\]]*)\])? (?<message>.*)$/
24
24
  end
25
25
  end
26
26
  end
@@ -27,6 +27,7 @@ module Fluent
27
27
  Plugin.register_parser('json', self)
28
28
 
29
29
  config_set_default :time_key, 'time'
30
+ desc 'Set JSON parser'
30
31
  config_param :json_parser, :enum, list: [:oj, :yajl, :json], default: :oj
31
32
 
32
33
  config_set_default :time_type, :float
@@ -51,9 +52,15 @@ module Fluent
51
52
  else
52
53
  raise "BUG: unknown json parser specified: #{name}"
53
54
  end
54
- rescue LoadError
55
+ rescue LoadError => ex
55
56
  name = :yajl
56
- log.info "Oj is not installed, and failing back to Yajl for json parser" if log
57
+ if log
58
+ if /\boj\z/ =~ ex.message
59
+ log.info "Oj is not installed, and failing back to Yajl for json parser"
60
+ else
61
+ log.warn ex.message
62
+ end
63
+ end
57
64
  retry
58
65
  end
59
66
 
@@ -61,7 +68,7 @@ module Fluent
61
68
  r = @load_proc.call(text)
62
69
  time, record = convert_values(parse_time(r), r)
63
70
  yield time, record
64
- rescue @error_class
71
+ rescue @error_class, EncodingError # EncodingError is for oj 3.x or later
65
72
  yield nil, nil
66
73
  end
67
74
 
@@ -23,11 +23,18 @@ module Fluent
23
23
 
24
24
  desc 'The delimiter character (or string) of TSV values'
25
25
  config_param :delimiter, :string, default: "\t"
26
+ desc 'The delimiter pattern of TSV values'
27
+ config_param :delimiter_pattern, :regexp, default: nil
26
28
  desc 'The delimiter character between field name and value'
27
29
  config_param :label_delimiter, :string, default: ":"
28
30
 
29
31
  config_set_default :time_key, 'time'
30
32
 
33
+ def configure(conf)
34
+ super
35
+ @delimiter = @delimiter_pattern || @delimiter
36
+ end
37
+
31
38
  def parse(text)
32
39
  r = {}
33
40
  text.split(@delimiter).each do |pair|
@@ -22,6 +22,7 @@ module Fluent
22
22
  class MultilineParser < Parser
23
23
  Plugin.register_parser('multiline', self)
24
24
 
25
+ desc 'Specify regexp pattern for start line of multiple lines'
25
26
  config_param :format_firstline, :string, default: nil
26
27
 
27
28
  FORMAT_MAX_NUM = 20
@@ -35,7 +36,7 @@ module Fluent
35
36
  if regexp.named_captures.empty?
36
37
  raise "No named captures"
37
38
  end
38
- regexp_conf = Fluent::Config::Element.new("", "", { "expression" => "/#{formats}/", "multiline" => true }, [])
39
+ regexp_conf = Fluent::Config::Element.new("", "", { "expression" => "/#{formats}/m" }, [])
39
40
  @parser = Fluent::Plugin::RegexpParser.new
40
41
  @parser.configure(conf + regexp_conf)
41
42
  rescue => e
@@ -21,7 +21,7 @@ module Fluent
21
21
  class NginxParser < RegexpParser
22
22
  Plugin.register_parser("nginx", self)
23
23
 
24
- config_set_default :expression, %q{/^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/}
24
+ config_set_default :expression, /^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)"(?:\s+\"?(?<http_x_forwarded_for>[^\"]*)\"?)?)?$/
25
25
  config_set_default :time_format, "%d/%b/%Y:%H:%M:%S %z"
26
26
  end
27
27
  end
@@ -23,6 +23,7 @@ module Fluent
23
23
  class NoneParser < Parser
24
24
  Plugin.register_parser('none', self)
25
25
 
26
+ desc 'Field name to contain logs'
26
27
  config_param :message_key, :string, default: 'message'
27
28
 
28
29
  def parse(text)
@@ -21,28 +21,29 @@ module Fluent
21
21
  class RegexpParser < Parser
22
22
  Plugin.register_parser("regexp", self)
23
23
 
24
- config_param :expression, :string
25
- config_param :ignorecase, :bool, default: false
26
- config_param :multiline, :bool, default: false
24
+ desc 'Regular expression for matching logs'
25
+ config_param :expression, :regexp
26
+ desc 'Ignore case in matching'
27
+ config_param :ignorecase, :bool, default: false, deprecated: "Use /pattern/i instead, this option is no longer effective"
28
+ desc 'Build regular expression as a multline mode'
29
+ config_param :multiline, :bool, default: false, deprecated: "Use /pattern/m instead, this option is no longer effective"
27
30
 
28
31
  config_set_default :time_key, 'time'
29
32
 
30
33
  def configure(conf)
31
34
  super
32
-
33
- expr = if @expression[0] == "/" && @expression[-1] == "/"
34
- @expression[1..-2]
35
- else
36
- @expression
37
- end
38
- regexp_option = 0
39
- regexp_option |= Regexp::IGNORECASE if @ignorecase
40
- regexp_option |= Regexp::MULTILINE if @multiline
41
- @regexp = Regexp.new(expr, regexp_option)
35
+ # For compat layer
36
+ if @ignorecase || @multiline
37
+ options = 0
38
+ options |= Regexp::IGNORECASE if @ignorecase
39
+ options |= Regexp::MULTILINE if @multiline
40
+ @expression = Regexp.compile(@expression.source, options)
41
+ end
42
+ @regexp = @expression # For backward compatibility
42
43
  end
43
44
 
44
45
  def parse(text)
45
- m = @regexp.match(text)
46
+ m = @expression.match(text)
46
47
  unless m
47
48
  yield nil, nil
48
49
  return
@@ -24,15 +24,19 @@ module Fluent
24
24
  Plugin.register_parser('syslog', self)
25
25
 
26
26
  # From existence TextParser pattern
27
- REGEXP = /^(?<time>[^ ]*\s*[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
27
+ REGEXP = /^(?<time>[^ ]*\s*[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[^ :\[]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
28
28
  # From in_syslog default pattern
29
- REGEXP_WITH_PRI = /^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
30
- REGEXP_RFC5424 = /\A^\<(?<pri>[0-9]{1,3})\>[1-9]\d{0,2} (?<time>[^ ]+) (?<host>[^ ]+) (?<ident>[^ ]+) (?<pid>[-0-9]+) (?<msgid>[^ ]+) (?<extradata>(\[(.*)\]|[^ ])) (?<message>.+)$\z/
29
+ REGEXP_WITH_PRI = /^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[^ :\[]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
30
+ REGEXP_RFC5424 = /\A^(?<time>[^ ]+) (?<host>[!-~]{1,255}) (?<ident>[!-~]{1,48}) (?<pid>[!-~]{1,128}) (?<msgid>[!-~]{1,32}) (?<extradata>(?:\-|\[(.*)\]))(?: (?<message>.+))?$\z/
31
+ REGEXP_RFC5424_WITH_PRI = /\A^\<(?<pri>[0-9]{1,3})\>[1-9]\d{0,2} (?<time>[^ ]+) (?<host>[!-~]{1,255}) (?<ident>[!-~]{1,48}) (?<pid>[!-~]{1,128}) (?<msgid>[!-~]{1,32}) (?<extradata>(?:\-|\[(.*)\]))(?: (?<message>.+))?$\z/
31
32
  REGEXP_DETECT_RFC5424 = /^\<.*\>[1-9]\d{0,2}/
32
33
 
33
34
  config_set_default :time_format, "%b %d %H:%M:%S"
35
+ desc 'If the incoming logs have priority prefix, e.g. <9>, set true'
34
36
  config_param :with_priority, :bool, default: false
37
+ desc 'Specify protocol format'
35
38
  config_param :message_format, :enum, list: [:rfc3164, :rfc5424, :auto], default: :rfc3164
39
+ desc 'Specify time format for event time for rfc5424 protocol'
36
40
  config_param :rfc5424_time_format, :string, default: "%Y-%m-%dT%H:%M:%S.%L%z"
37
41
 
38
42
  def initialize
@@ -55,7 +59,7 @@ module Fluent
55
59
  alias_method :parse, :parse_plain
56
60
  end
57
61
  @time_format = @rfc5424_time_format unless conf.has_key?('time_format')
58
- REGEXP_RFC5424
62
+ @with_priority ? REGEXP_RFC5424_WITH_PRI : REGEXP_RFC5424
59
63
  when :auto
60
64
  class << self
61
65
  alias_method :parse, :parse_auto
@@ -77,7 +81,7 @@ module Fluent
77
81
 
78
82
  def parse_auto(text, &block)
79
83
  if REGEXP_DETECT_RFC5424.match(text)
80
- @regexp = REGEXP_RFC5424
84
+ @regexp = @with_priority ? REGEXP_RFC5424_WITH_PRI : REGEXP_RFC5424
81
85
  @time_parser = @time_parser_rfc5424
82
86
  else
83
87
  @regexp = @with_priority ? REGEXP_WITH_PRI : REGEXP
@@ -26,7 +26,9 @@ require 'fluent/plugin_helper/inject'
26
26
  require 'fluent/plugin_helper/extract'
27
27
  require 'fluent/plugin_helper/socket'
28
28
  require 'fluent/plugin_helper/server'
29
+ require 'fluent/plugin_helper/counter'
29
30
  require 'fluent/plugin_helper/retry_state'
31
+ require 'fluent/plugin_helper/record_accessor'
30
32
  require 'fluent/plugin_helper/compat_parameters'
31
33
 
32
34
  module Fluent
@@ -33,6 +33,11 @@ module Fluent
33
33
  ctx.ciphers = ciphers
34
34
  end
35
35
 
36
+ if conf.client_cert_auth
37
+ ctx.verify_mode = OpenSSL::SSL::VERIFY_PEER | OpenSSL::SSL::VERIFY_FAIL_IF_NO_PEER_CERT
38
+ end
39
+
40
+ ctx.ca_file = conf.ca_path
36
41
  ctx.cert = cert
37
42
  ctx.key = key
38
43
  if extra && !extra.empty?
@@ -104,18 +109,28 @@ module Fluent
104
109
  cert.not_before = Time.at(0)
105
110
  cert.not_after = Time.now + opts[:expiration]
106
111
  cert.public_key = key
107
- cert.serial = 1
112
+ cert.version = 2
113
+ cert.serial = rand(2**(8*10))
108
114
  cert.issuer = issuer
109
115
  cert.subject = subject
110
116
 
111
117
  return cert, key
112
118
  end
113
119
 
120
+ def cert_option_add_extensions(cert, extensions)
121
+ ef = OpenSSL::X509::ExtensionFactory.new
122
+ extensions.each do |ext|
123
+ oid, value = ext
124
+ cert.add_extension ef.create_extension(oid, value)
125
+ end
126
+ end
127
+
114
128
  def cert_option_generate_ca_pair_self_signed(generate_opts)
115
129
  cert, key = cert_option_generate_pair(generate_opts)
116
130
 
117
- # basicConstraints: this cert is for CA or not
118
- cert.add_extension OpenSSL::X509::Extension.new('basicConstraints', OpenSSL::ASN1.Sequence([OpenSSL::ASN1::Boolean(true)]))
131
+ cert_option_add_extensions(cert, [
132
+ ['basicConstraints', 'CA:TRUE']
133
+ ])
119
134
 
120
135
  cert.sign(key, generate_opts[:digest].to_s)
121
136
  return cert, key
@@ -127,9 +142,12 @@ module Fluent
127
142
  cert, key = cert_option_generate_pair(generate_opts, ca_cert.subject)
128
143
  raise "BUG: certificate digest algorithm not set" unless generate_opts[:digest]
129
144
 
130
- # basicConstraints: this cert is for CA or not
131
- cert.add_extension OpenSSL::X509::Extension.new('basicConstraints', OpenSSL::ASN1.Sequence([OpenSSL::ASN1::Boolean(false)]))
132
- cert.add_extension OpenSSL::X509::Extension.new('nsCertType', 'server')
145
+ cert_option_add_extensions(cert, [
146
+ ['basicConstraints', 'CA:FALSE'],
147
+ ['nsCertType', 'server'],
148
+ ['keyUsage', 'digitalSignature,keyEncipherment'],
149
+ ['extendedKeyUsage', 'serverAuth']
150
+ ])
133
151
 
134
152
  cert.sign(ca_key, generate_opts[:digest].to_s)
135
153
  return cert, key, nil
@@ -139,9 +157,10 @@ module Fluent
139
157
  cert, key = cert_option_generate_pair(generate_opts)
140
158
  raise "BUG: certificate digest algorithm not set" unless generate_opts[:digest]
141
159
 
142
- # basicConstraints: this cert is for CA or not
143
- cert.add_extension OpenSSL::X509::Extension.new('basicConstraints', OpenSSL::ASN1.Sequence([OpenSSL::ASN1::Boolean(false)]))
144
- cert.add_extension OpenSSL::X509::Extension.new('nsCertType', 'server')
160
+ cert_option_add_extensions(cert, [
161
+ ['basicConstraints', 'CA:FALSE'],
162
+ ['nsCertType', 'server']
163
+ ])
145
164
 
146
165
  cert.sign(key, generate_opts[:digest].to_s)
147
166
  return cert, key, nil
@@ -65,7 +65,9 @@ module Fluent
65
65
  "label_delimiter" => "label_delimiter", # LabeledTSVParser
66
66
  "format_firstline" => "format_firstline", # MultilineParser
67
67
  "message_key" => "message_key", # NoneParser
68
- "with_priority" => "with_priority", # SyslogParser
68
+ "with_priority" => "with_priority", # SyslogParser
69
+ "message_format" => "message_format", # SyslogParser
70
+ "rfc5424_time_format" => "rfc5424_time_format", # SyslogParser
69
71
  # There has been no parsers which can handle timezone in v0.12
70
72
  }
71
73
 
@@ -0,0 +1,51 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'fluent/plugin'
18
+ require 'fluent/counter/client'
19
+
20
+ module Fluent
21
+ module PluginHelper
22
+ module Counter
23
+ def counter_client_create(scope:, loop: Coolio::Loop.new)
24
+ client_conf = system_config.counter_client
25
+ raise Fluent::ConfigError, '<counter_client> is required in <system>' unless client_conf
26
+ counter_client = Fluent::Counter::Client.new(loop, port: client_conf.port, host: client_conf.host, log: log, timeout: client_conf.timeout)
27
+ counter_client.start
28
+ counter_client.establish(scope)
29
+ @_counter_client = counter_client
30
+ counter_client
31
+ end
32
+
33
+ attr_reader :_counter_client
34
+
35
+ def initialize
36
+ super
37
+ @_counter_client = nil
38
+ end
39
+
40
+ def stop
41
+ super
42
+ @_counter_client.stop
43
+ end
44
+
45
+ def terminate
46
+ @_counter_client = nil
47
+ super
48
+ end
49
+ end
50
+ end
51
+ end
@@ -43,6 +43,15 @@ module Fluent
43
43
  end
44
44
  end
45
45
 
46
+ def event_loop_detach(watcher)
47
+ if watcher.attached?
48
+ watcher.detach
49
+ end
50
+ @_event_loop_mutex.synchronize do
51
+ @_event_loop_attached_watchers.delete(watcher)
52
+ end
53
+ end
54
+
46
55
  def event_loop_wait_until_start
47
56
  sleep(0.1) until event_loop_running?
48
57
  end
@@ -0,0 +1,210 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'fluent/config/error'
18
+ unless {}.respond_to?(:dig)
19
+ begin
20
+ # backport_dig is faster than dig_rb so prefer backport_dig.
21
+ require 'backport_dig'
22
+ rescue LoadError
23
+ require 'dig_rb'
24
+ end
25
+ end
26
+
27
+ module Fluent
28
+ module PluginHelper
29
+ module RecordAccessor
30
+ def record_accessor_create(param)
31
+ Accessor.new(param)
32
+ end
33
+
34
+ class Accessor
35
+ attr_reader :keys
36
+
37
+ def initialize(param)
38
+ @keys = Accessor.parse_parameter(param)
39
+
40
+ if @keys.is_a?(Array)
41
+ @last_key = @keys.last
42
+ @dig_keys = @keys[0..-2]
43
+ if @dig_keys.empty?
44
+ @keys = @keys.first
45
+ mcall = method(:call_index)
46
+ mdelete = method(:delete_top)
47
+ else
48
+ mcall = method(:call_dig)
49
+ mdelete = method(:delete_nest)
50
+ end
51
+ else
52
+ # Call [] for single key to reduce dig overhead
53
+ mcall = method(:call_index)
54
+ mdelete = method(:delete_top)
55
+ end
56
+
57
+ singleton_class.module_eval do
58
+ define_method(:call, mcall)
59
+ define_method(:delete, mdelete)
60
+ end
61
+ end
62
+
63
+ def call(r)
64
+ end
65
+
66
+ # To optimize the performance, use class_eval with pre-expanding @keys
67
+ # See https://gist.github.com/repeatedly/ab553ed260cd080bd01ec71da9427312
68
+ def call_dig(r)
69
+ r.dig(*@keys)
70
+ end
71
+
72
+ def call_index(r)
73
+ r[@keys]
74
+ end
75
+
76
+ def delete(r)
77
+ end
78
+
79
+ def delete_nest(r)
80
+ if target = r.dig(*@dig_keys)
81
+ if target.is_a?(Array)
82
+ target.delete_at(@last_key)
83
+ else
84
+ target.delete(@last_key)
85
+ end
86
+ end
87
+ rescue
88
+ nil
89
+ end
90
+
91
+ def delete_top(r)
92
+ r.delete(@keys)
93
+ end
94
+
95
+ def self.parse_parameter(param)
96
+ if param.start_with?('$.')
97
+ parse_dot_notation(param)
98
+ elsif param.start_with?('$[')
99
+ parse_bracket_notation(param)
100
+ else
101
+ param
102
+ end
103
+ end
104
+
105
+ def self.parse_dot_notation(param)
106
+ result = []
107
+ keys = param[2..-1].split('.')
108
+ keys.each { |key|
109
+ if key.include?('[')
110
+ result.concat(parse_dot_array_op(key, param))
111
+ else
112
+ result << key
113
+ end
114
+ }
115
+
116
+ raise Fluent::ConfigError, "empty keys in dot notation" if result.empty?
117
+ validate_dot_keys(result)
118
+
119
+ result
120
+ end
121
+
122
+ def self.validate_dot_keys(keys)
123
+ keys.each { |key|
124
+ next unless key.is_a?(String)
125
+ if /\s+/.match(key)
126
+ raise Fluent::ConfigError, "whitespace character is not allowed in dot notation. Use bracket notation: #{key}"
127
+ end
128
+ }
129
+ end
130
+
131
+ def self.parse_dot_array_op(key, param)
132
+ start = key.index('[')
133
+ result = if start.zero?
134
+ []
135
+ else
136
+ [key[0..start - 1]]
137
+ end
138
+ key = key[start + 1..-1]
139
+ in_bracket = true
140
+
141
+ until key.empty?
142
+ if in_bracket
143
+ if i = key.index(']')
144
+ index_value = key[0..i - 1]
145
+ raise Fluent::ConfigError, "missing array index in '[]'. Invalid syntax: #{param}" if index_value == ']'
146
+ result << Integer(index_value)
147
+ key = key[i + 1..-1]
148
+ in_bracket = false
149
+ else
150
+ raise Fluent::ConfigError, "'[' found but ']' not found. Invalid syntax: #{param}"
151
+ end
152
+ else
153
+ if i = key.index('[')
154
+ key = key[i + 1..-1]
155
+ in_bracket = true
156
+ else
157
+ raise Fluent::ConfigError, "found more characters after ']'. Invalid syntax: #{param}"
158
+ end
159
+ end
160
+ end
161
+
162
+ result
163
+ end
164
+
165
+ def self.parse_bracket_notation(param)
166
+ orig_param = param
167
+ result = []
168
+ param = param[1..-1]
169
+ in_bracket = false
170
+
171
+ until param.empty?
172
+ if in_bracket
173
+ if param[0] == "'" || param[0] == '"'
174
+ if i = param.index("']") || param.index('"]')
175
+ raise Fluent::ConfigError, "Mismatched quotes. Invalid syntax: #{orig_param}" unless param[0] == param[i]
176
+ result << param[1..i - 1]
177
+ param = param[i + 2..-1]
178
+ in_bracket = false
179
+ else
180
+ raise Fluent::ConfigError, "Incomplete bracket. Invalid syntax: #{orig_param}"
181
+ end
182
+ else
183
+ if i = param.index(']')
184
+ index_value = param[0..i - 1]
185
+ raise Fluent::ConfigError, "missing array index in '[]'. Invalid syntax: #{param}" if index_value == ']'
186
+ result << Integer(index_value)
187
+ param = param[i + 1..-1]
188
+ in_bracket = false
189
+ else
190
+ raise Fluent::ConfigError, "'[' found but ']' not found. Invalid syntax: #{orig_param}"
191
+ end
192
+ end
193
+ else
194
+ if i = param.index('[')
195
+ param = param[i + 1..-1]
196
+ in_bracket = true
197
+ else
198
+ raise Fluent::ConfigError, "found more characters after ']'. Invalid syntax: #{orig_param}"
199
+ end
200
+ end
201
+ end
202
+
203
+ raise Fluent::ConfigError, "empty keys in bracket notation" if result.empty?
204
+
205
+ result
206
+ end
207
+ end
208
+ end
209
+ end
210
+ end