fluentd 0.14.7-x86-mingw32 → 0.14.10-x86-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (120) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +2 -0
  3. data/.travis.yml +2 -0
  4. data/CONTRIBUTING.md +6 -1
  5. data/ChangeLog +95 -0
  6. data/Rakefile +21 -0
  7. data/appveyor.yml +1 -0
  8. data/code-of-conduct.md +3 -0
  9. data/example/out_exec_filter.conf +42 -0
  10. data/fluentd.gemspec +1 -1
  11. data/lib/fluent/agent.rb +2 -2
  12. data/lib/fluent/command/binlog_reader.rb +1 -1
  13. data/lib/fluent/command/cat.rb +15 -4
  14. data/lib/fluent/compat/output.rb +14 -9
  15. data/lib/fluent/compat/parser.rb +141 -11
  16. data/lib/fluent/config/configure_proxy.rb +2 -11
  17. data/lib/fluent/config/section.rb +8 -1
  18. data/lib/fluent/configurable.rb +1 -3
  19. data/lib/fluent/env.rb +1 -1
  20. data/lib/fluent/log.rb +1 -1
  21. data/lib/fluent/plugin/base.rb +17 -0
  22. data/lib/fluent/plugin/filter_parser.rb +108 -0
  23. data/lib/fluent/plugin/filter_record_transformer.rb +14 -35
  24. data/lib/fluent/plugin/filter_stdout.rb +1 -1
  25. data/lib/fluent/plugin/formatter.rb +5 -0
  26. data/lib/fluent/plugin/formatter_msgpack.rb +4 -0
  27. data/lib/fluent/plugin/formatter_stdout.rb +3 -2
  28. data/lib/fluent/plugin/formatter_tsv.rb +34 -0
  29. data/lib/fluent/plugin/in_exec.rb +48 -93
  30. data/lib/fluent/plugin/in_forward.rb +66 -265
  31. data/lib/fluent/plugin/in_http.rb +68 -65
  32. data/lib/fluent/plugin/in_monitor_agent.rb +8 -4
  33. data/lib/fluent/plugin/in_syslog.rb +42 -58
  34. data/lib/fluent/plugin/in_tail.rb +29 -14
  35. data/lib/fluent/plugin/in_tcp.rb +54 -14
  36. data/lib/fluent/plugin/in_udp.rb +49 -13
  37. data/lib/fluent/plugin/multi_output.rb +1 -3
  38. data/lib/fluent/plugin/out_exec.rb +58 -71
  39. data/lib/fluent/plugin/out_exec_filter.rb +199 -279
  40. data/lib/fluent/plugin/out_file.rb +172 -81
  41. data/lib/fluent/plugin/out_forward.rb +229 -206
  42. data/lib/fluent/plugin/out_stdout.rb +6 -21
  43. data/lib/fluent/plugin/output.rb +90 -59
  44. data/lib/fluent/plugin/parser.rb +121 -61
  45. data/lib/fluent/plugin/parser_csv.rb +9 -3
  46. data/lib/fluent/plugin/parser_json.rb +37 -35
  47. data/lib/fluent/plugin/parser_ltsv.rb +11 -19
  48. data/lib/fluent/plugin/parser_msgpack.rb +50 -0
  49. data/lib/fluent/plugin/parser_regexp.rb +15 -42
  50. data/lib/fluent/plugin/parser_tsv.rb +8 -3
  51. data/lib/fluent/plugin_helper.rb +10 -1
  52. data/lib/fluent/plugin_helper/child_process.rb +139 -73
  53. data/lib/fluent/plugin_helper/compat_parameters.rb +93 -4
  54. data/lib/fluent/plugin_helper/event_emitter.rb +14 -1
  55. data/lib/fluent/plugin_helper/event_loop.rb +24 -6
  56. data/lib/fluent/plugin_helper/extract.rb +16 -4
  57. data/lib/fluent/plugin_helper/formatter.rb +9 -11
  58. data/lib/fluent/plugin_helper/inject.rb +16 -1
  59. data/lib/fluent/plugin_helper/parser.rb +3 -3
  60. data/lib/fluent/plugin_helper/server.rb +494 -0
  61. data/lib/fluent/plugin_helper/socket.rb +101 -0
  62. data/lib/fluent/plugin_helper/socket_option.rb +84 -0
  63. data/lib/fluent/plugin_helper/timer.rb +1 -0
  64. data/lib/fluent/root_agent.rb +1 -1
  65. data/lib/fluent/test/driver/base.rb +95 -49
  66. data/lib/fluent/test/driver/base_owner.rb +18 -8
  67. data/lib/fluent/test/driver/multi_output.rb +2 -1
  68. data/lib/fluent/test/driver/output.rb +29 -6
  69. data/lib/fluent/test/helpers.rb +3 -1
  70. data/lib/fluent/test/log.rb +4 -0
  71. data/lib/fluent/test/startup_shutdown.rb +13 -0
  72. data/lib/fluent/time.rb +14 -8
  73. data/lib/fluent/version.rb +1 -1
  74. data/lib/fluent/winsvc.rb +1 -1
  75. data/test/command/test_binlog_reader.rb +5 -1
  76. data/test/compat/test_parser.rb +10 -0
  77. data/test/config/test_configurable.rb +193 -0
  78. data/test/config/test_configure_proxy.rb +0 -43
  79. data/test/helper.rb +36 -1
  80. data/test/plugin/test_base.rb +16 -0
  81. data/test/plugin/test_filter_parser.rb +665 -0
  82. data/test/plugin/test_filter_record_transformer.rb +36 -100
  83. data/test/plugin/test_filter_stdout.rb +18 -27
  84. data/test/plugin/test_in_dummy.rb +1 -1
  85. data/test/plugin/test_in_exec.rb +206 -94
  86. data/test/plugin/test_in_forward.rb +268 -347
  87. data/test/plugin/test_in_http.rb +310 -186
  88. data/test/plugin/test_in_monitor_agent.rb +65 -35
  89. data/test/plugin/test_in_syslog.rb +39 -3
  90. data/test/plugin/test_in_tcp.rb +78 -62
  91. data/test/plugin/test_in_udp.rb +101 -80
  92. data/test/plugin/test_out_exec.rb +223 -68
  93. data/test/plugin/test_out_exec_filter.rb +520 -169
  94. data/test/plugin/test_out_file.rb +637 -177
  95. data/test/plugin/test_out_forward.rb +242 -234
  96. data/test/plugin/test_out_null.rb +1 -1
  97. data/test/plugin/test_out_secondary_file.rb +4 -2
  98. data/test/plugin/test_out_stdout.rb +14 -35
  99. data/test/plugin/test_output_as_buffered.rb +60 -2
  100. data/test/plugin/test_parser.rb +359 -0
  101. data/test/plugin/test_parser_csv.rb +1 -2
  102. data/test/plugin/test_parser_json.rb +3 -4
  103. data/test/plugin/test_parser_labeled_tsv.rb +1 -2
  104. data/test/plugin/test_parser_none.rb +1 -2
  105. data/test/plugin/test_parser_regexp.rb +8 -4
  106. data/test/plugin/test_parser_tsv.rb +4 -3
  107. data/test/plugin_helper/test_child_process.rb +184 -0
  108. data/test/plugin_helper/test_compat_parameters.rb +88 -1
  109. data/test/plugin_helper/test_extract.rb +0 -1
  110. data/test/plugin_helper/test_formatter.rb +5 -2
  111. data/test/plugin_helper/test_inject.rb +21 -0
  112. data/test/plugin_helper/test_parser.rb +6 -5
  113. data/test/plugin_helper/test_server.rb +905 -0
  114. data/test/test_event_time.rb +3 -1
  115. data/test/test_output.rb +53 -2
  116. data/test/test_plugin_classes.rb +20 -0
  117. data/test/test_root_agent.rb +139 -0
  118. data/test/test_test_drivers.rb +135 -0
  119. metadata +28 -8
  120. data/test/plugin/test_parser_base.rb +0 -32
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: beb3dc3b05f93e01bcfbd3bd5bba851613e7a56a
4
- data.tar.gz: 6e618a140882db9be00c10fb76b47e5eec9feced
3
+ metadata.gz: cc9fd1115013c31c251063f81dc0d91be3b5624c
4
+ data.tar.gz: 95d7e3af0d15244336c42b8d1bfe83f08afd8f36
5
5
  SHA512:
6
- metadata.gz: 91175a11ce3c0307b21ec2f12932038c5e626b10287de29a75a65c702c62c915f749132d87e959761bedb2b2dd1bb283c160c4e375a7091a61f690fd6ddb51d7
7
- data.tar.gz: 57d912a874804209343d0dbe3c0fbc856cf3cdd6f085d19d7cfdf3c0112a9e739001027ce2ee69f21cb511d085f2e8098709b912cc0eccc063430f48e47738e1
6
+ metadata.gz: 92f075a8834ebbcd7b5c1c43433afc8f9d62a53f68e26ef8e82519515dc7abf8b4a9ad03a43307c3a232a84f281a1f3968e2dda160659c54a01da164fe348224
7
+ data.tar.gz: 5d0f4bc6df7f9bfe63d7c5f9c4b8e8117111bcaa93f60a32a884e90e69523200a458732e452c964edf590af43da71e51789d01a2dff7a9e28c653413909f834a
data/.gitignore CHANGED
@@ -24,3 +24,5 @@ Gemfile.local
24
24
  *.swp
25
25
  coverage/*
26
26
  .vagrant/
27
+ cov-int/
28
+ cov-fluentd.tar.gz
@@ -1,6 +1,8 @@
1
1
  language: ruby
2
2
  cache: bundler
3
3
 
4
+ # script: bundle exec rake test TESTOPTS=-v
5
+
4
6
  # http://rubies.travis-ci.org/
5
7
  # See here for osx_image -> OSX versions: https://docs.travis-ci.com/user/languages/objective-c
6
8
  matrix:
@@ -7,8 +7,13 @@ We'd love your contribution. Here are the guidelines!
7
7
  RESOURCES of [Official site](http://www.fluentd.org/) and [Fluentd documentation](http://docs.fluentd.org/) may help you.
8
8
 
9
9
  If you have further questions about Fluentd and plugins, please direct these to [Mailing List](https://groups.google.com/forum/#!forum/fluentd).
10
+ Don't use Github issue for asking questions. Here are examples:
10
11
 
11
- Don't use Github issue for asking questions.
12
+ - I installed xxx plugin but it doesn't work. Why?
13
+ - Fluentd starts but logs are not sent to xxx. Am I wrong?
14
+ - I want to do xxx. How to realize it with plugins?
15
+
16
+ We may close such questions to keep clear repository for developers and users.
12
17
  Github issue is mainly for submitting a bug report or feature request. See below.
13
18
 
14
19
  ## Found a bug?
data/ChangeLog CHANGED
@@ -1,15 +1,108 @@
1
1
  # v0.14
2
2
 
3
+ ## Release v0.14.10 - 2016/12/14
4
+
5
+ ### New features / Enhancement
6
+
7
+ * Add socket/server plugin helper to write TCP/UDP clients/servers as Fluentd plugin
8
+ https://github.com/fluent/fluentd/pull/1312
9
+ https://github.com/fluent/fluentd/pull/1350
10
+ https://github.com/fluent/fluentd/pull/1356
11
+ https://github.com/fluent/fluentd/pull/1362
12
+ * Fix to raise errors when injected hostname is also specified as chunk key
13
+ https://github.com/fluent/fluentd/pull/1357
14
+ * in_tail: Optimize to read lines from file
15
+ https://github.com/fluent/fluentd/pull/1325
16
+ * in_monitor_agent: Add new parameter "include_config"(default: true)
17
+ https://github.com/fluent/fluentd/pull/1317
18
+ * in_syslog: Add "priority_key" and "facility_key" options
19
+ https://github.com/fluent/fluentd/pull/1351
20
+ * filter_record_transformer: Remove obsoleted syntax like "${message}" and not to dump records in logs
21
+ https://github.com/fluent/fluentd/pull/1328
22
+ * Add an option "--time-as-integer" to fluent-cat command to send events from v0.14 fluent-cat to v0.12 fluentd
23
+ https://github.com/fluent/fluentd/pull/1349
24
+
25
+ ### Bug fixes
26
+
27
+ * Specify correct Oj options for newer versions (Oj 2.18.0 or later)
28
+ https://github.com/fluent/fluentd/pull/1331
29
+ * TimeSlice output plugins (in v0.12 style) raise errors when "utc" parameter is specified
30
+ https://github.com/fluent/fluentd/pull/1319
31
+ * Parser plugins cannot use options for regular expressions
32
+ https://github.com/fluent/fluentd/pull/1326/files
33
+ * Fix bugs not to raise errors to use logger in v0.12 plugins
34
+ https://github.com/fluent/fluentd/pull/1344
35
+ https://github.com/fluent/fluentd/pull/1332
36
+ * Fix bug about shutting down Fluentd in Windows
37
+ https://github.com/fluent/fluentd/pull/1367
38
+ * in_tail: Close files explicitly in tests
39
+ https://github.com/fluent/fluentd/pull/1327
40
+ * in_forward: Fix bug not to convert buffer configurations into v0.14 parameters
41
+ https://github.com/fluent/fluentd/pull/1337
42
+ * out_forward: Fix bug to raise error when "expire_dns_cache" is specified
43
+ https://github.com/fluent/fluentd/pull/1346
44
+ * out_file: Fix bug to raise error about buffer chunking when it's configured as secondary
45
+ https://github.com/fluent/fluentd/pull/1338
46
+
47
+ ## Release v0.14.9 - 2016/11/15
48
+
49
+ ### New features / Enhancement
50
+
51
+ * filter_parser: Port fluent-plugin-parser into built-in plugin
52
+ https://github.com/fluent/fluentd/pull/1191
53
+ * parser/formatter plugin helpers with default @type in plugin side
54
+ https://github.com/fluent/fluentd/pull/1267
55
+ * parser: Reconstruct Parser related classes
56
+ https://github.com/fluent/fluentd/pull/1286
57
+ * filter_record_transformer: Remove old behaviours
58
+ https://github.com/fluent/fluentd/pull/1311
59
+ * Migrate some built-in plugins into v0.14 API
60
+ https://github.com/fluent/fluentd/pull/1257 (out_file)
61
+ https://github.com/fluent/fluentd/pull/1297 (out_exec, out_exec_filter)
62
+ https://github.com/fluent/fluentd/pull/1306 (in_forward, out_forward)
63
+ https://github.com/fluent/fluentd/pull/1308 (in_http)
64
+ * test: Improve test drivers
65
+ https://github.com/fluent/fluentd/pull/1302
66
+ https://github.com/fluent/fluentd/pull/1305
67
+
68
+ ### Bug fixes
69
+
70
+ * log: Avoid name conflict between Fluent::Logger
71
+ https://github.com/fluent/fluentd/pull/1274
72
+ * fluent-cat: Fix fluent-cat command to send sub-second precision time
73
+ https://github.com/fluent/fluentd/pull/1277
74
+ * config: Fix a bug not to overwrite default value with nil
75
+ https://github.com/fluent/fluentd/pull/1296
76
+ * output: Fix timezone for compat timesliced output plugins
77
+ https://github.com/fluent/fluentd/pull/1307
78
+ * out_forward: fix not to raise error when out_forward is initialized as secondary
79
+ https://github.com/fluent/fluentd/pull/1313
80
+ * output: Event router for secondary output
81
+ https://github.com/fluent/fluentd/pull/1283
82
+ * test: fix to return the block value as expected by many rubyists
83
+ https://github.com/fluent/fluentd/pull/1284
84
+
85
+ ## Release v0.14.8 - 2016/10/13
86
+
87
+ ### Bug fixes
88
+
89
+ * Add msgpack_each to buffer chunks in compat-layer output plugins
90
+ https://github.com/fluent/fluentd/pull/1273
91
+
3
92
  ## Release v0.14.7 - 2016/10/07
4
93
 
5
94
  ### New features / Enhancement
6
95
 
96
+ * Support data compression in buffer plugins
97
+ https://github.com/fluent/fluentd/pull/1172
7
98
  * in_forward: support to transfer compressed data
8
99
  https://github.com/fluent/fluentd/pull/1179
9
100
  * out_stdout: fix to show nanosecond resolution time
10
101
  https://github.com/fluent/fluentd/pull/1249
11
102
  * Add option to rotate Fluentd daemon's log
12
103
  https://github.com/fluent/fluentd/pull/1235
104
+ * Add extract plugin helper, with symmetric time parameter support in parser/formatter and inject/extract
105
+ https://github.com/fluent/fluentd/pull/1207
13
106
  * Add a feature to parse/format numeric time (unix time [+ subsecond value])
14
107
  https://github.com/fluent/fluentd/pull/1254
15
108
  * Raise configuration errors for inconsistent <label> configurations
@@ -22,6 +115,8 @@
22
115
  https://github.com/fluent/fluentd/pull/1227
23
116
  * Add some features to write plugins more easily
24
117
  https://github.com/fluent/fluentd/pull/1256
118
+ * Add a tool to load dumped events from file
119
+ https://github.com/fluent/fluentd/pull/1165
25
120
 
26
121
  ### Bug fixes
27
122
 
data/Rakefile CHANGED
@@ -24,6 +24,9 @@ end
24
24
 
25
25
  desc 'Run test_unit based test'
26
26
  Rake::TestTask.new(:base_test) do |t|
27
+ # To run test with dumping all test case names (to find never ending test case)
28
+ # $ bundle exec rake test TESTOPTS=-v
29
+ #
27
30
  # To run test for only one file (or file path pattern)
28
31
  # $ bundle exec rake base_test TEST=test/test_specified_path.rb
29
32
  # $ bundle exec rake base_test TEST=test/test_*.rb
@@ -50,4 +53,22 @@ task :coverage do |t|
50
53
  Rake::Task["test"].invoke
51
54
  end
52
55
 
56
+ desc 'Build Coverity tarball & upload it'
57
+ task :coverity do
58
+ # https://scan.coverity.com/projects/fluentd?tab=overview
59
+ # See "View Defects" after sign-in.
60
+ #
61
+ # Setup steps:
62
+ # 1. get coverity build tool and set PATH to bin/: https://scan.coverity.com/download
63
+ # 2. set environment variables:
64
+ # * $COVERITY_USER (your email address)
65
+ # * $COVERITY_TOKEN (token for Fluentd project: https://scan.coverity.com/projects/fluentd?tab=project_settings)
66
+ sh "cov-build --dir cov-int --no-command --fs-capture-search ./"
67
+ sh "tar czf cov-fluentd.tar.gz cov-int"
68
+ user = ENV['COVERITY_USER']
69
+ token = ENV['COVERITY_TOKEN']
70
+ sh "curl --form token=#{token} --form email=#{user} --form file=@cov-fluentd.tar.gz --form version=\"Master\" --form description=\"GIT Master\" https://scan.coverity.com/builds?project=Fluentd"
71
+ FileUtils.rm_rf(['./cov-int', 'cov-fluentd.tar.gz'])
72
+ end
73
+
53
74
  task default: [:test, :build]
@@ -12,6 +12,7 @@ install:
12
12
  build: off
13
13
  test_script:
14
14
  - bundle exec rake test
15
+ # - bundle exec rake test TESTOPTS=-v
15
16
 
16
17
  branches:
17
18
  only:
@@ -0,0 +1,3 @@
1
+ ## Fluentd Community Code of Conduct
2
+
3
+ Fluentd follows the [CNCF Code of Conduct](https://github.com/cncf/foundation/blob/master/code-of-conduct.md).
@@ -0,0 +1,42 @@
1
+ <source>
2
+ @type dummy
3
+ @label @exec
4
+ tag exec_input
5
+ rate 10
6
+ auto_increment_key num
7
+ dummy {"data":"mydata"}
8
+ </source>
9
+
10
+ <label @exec>
11
+ <match exec_input>
12
+ @type exec_filter
13
+ @label @stdout
14
+ tag result
15
+ command ruby -e 'STDOUT.sync = true; proc = ->(){line = STDIN.readline.chomp; puts line + "\t" + Process.pid.to_s}; 1000.times{ proc.call }'
16
+ num_children 3
17
+ child_respawn -1
18
+ <inject>
19
+ time_key time
20
+ time_type float
21
+ </inject>
22
+ <format>
23
+ @type tsv
24
+ keys data, num, time
25
+ </format>
26
+ <parse>
27
+ @type tsv
28
+ keys data, num, time, pid
29
+ </parse>
30
+ <extract>
31
+ time_key time
32
+ time_type float
33
+ </extract>
34
+ </match>
35
+ </label>
36
+
37
+ <label @stdout>
38
+ <match result>
39
+ @type stdout
40
+ </match>
41
+ </label>
42
+
@@ -22,7 +22,7 @@ Gem::Specification.new do |gem|
22
22
  gem.add_runtime_dependency("msgpack", [">= 0.7.0", "< 2.0.0"])
23
23
  gem.add_runtime_dependency("yajl-ruby", ["~> 1.0"])
24
24
  gem.add_runtime_dependency("cool.io", ["~> 1.4.5"])
25
- gem.add_runtime_dependency("serverengine", ["~> 2.0"])
25
+ gem.add_runtime_dependency("serverengine", [">= 2.0.4", "< 3.0.0"])
26
26
  gem.add_runtime_dependency("http_parser.rb", [">= 0.5.1", "< 0.7.0"])
27
27
  gem.add_runtime_dependency("sigdump", ["~> 0.2.2"])
28
28
  gem.add_runtime_dependency("tzinfo", ["~> 1.0"])
@@ -123,7 +123,7 @@ module Fluent
123
123
  log.info "adding match#{@context.nil? ? '' : " in #{@context}"}", pattern: pattern, type: type
124
124
 
125
125
  output = Plugin.new_output(type)
126
- output.router = @event_router if output.respond_to?(:router=)
126
+ output.context_router = @event_router
127
127
  output.configure(conf)
128
128
  @outputs << output
129
129
  if output.respond_to?(:outputs) && output.respond_to?(:multi_output?) && output.multi_output?
@@ -144,7 +144,7 @@ module Fluent
144
144
  log.info "adding filter#{@context.nil? ? '' : " in #{@context}"}", pattern: pattern, type: type
145
145
 
146
146
  filter = Plugin.new_filter(type)
147
- filter.router = @event_router
147
+ filter.context_router = @event_router
148
148
  filter.configure(conf)
149
149
  @filters << filter
150
150
  @event_router.add_rule(pattern, filter)
@@ -164,7 +164,7 @@ module BinlogReaderCommand
164
164
  def call
165
165
  @formatter = lookup_formatter(@options[:format], @options[:config_params])
166
166
 
167
- File.open(@path, 'r') do |io|
167
+ File.open(@path, 'rb') do |io|
168
168
  i = 1
169
169
  Fluent::MessagePackFactory.unpacker(io).each do |(time, record)|
170
170
  print @formatter.format(@path, time, record) # path is used for tag
@@ -16,6 +16,8 @@
16
16
 
17
17
  require 'optparse'
18
18
  require 'fluent/env'
19
+ require 'fluent/time'
20
+ require 'fluent/msgpack_factory'
19
21
 
20
22
  op = OptionParser.new
21
23
 
@@ -29,6 +31,7 @@ socket_path = Fluent::DEFAULT_SOCKET_PATH
29
31
  config_path = Fluent::DEFAULT_CONFIG_PATH
30
32
  format = 'json'
31
33
  message_key = 'message'
34
+ time_as_integer = false
32
35
 
33
36
  op.on('-p', '--port PORT', "fluent tcp port (default: #{port})", Integer) {|i|
34
37
  port = i
@@ -66,6 +69,10 @@ op.on('--message-key KEY', "key field for none format (default: #{message_key})"
66
69
  message_key = s
67
70
  }
68
71
 
72
+ op.on('--time-as-integer', "Send time as integer for v0.12 or earlier", TrueClass) { |b|
73
+ time_as_integer = true
74
+ }
75
+
69
76
  (class << self; self; end).module_eval do
70
77
  define_method(:usage) do |msg|
71
78
  puts op.to_s
@@ -121,7 +128,7 @@ class Writer
121
128
  end
122
129
  end
123
130
 
124
- def initialize(tag, connector)
131
+ def initialize(tag, connector, time_as_integer: false)
125
132
  @tag = tag
126
133
  @connector = connector
127
134
  @socket = false
@@ -134,6 +141,7 @@ class Writer
134
141
  @pending_limit = 1024 # TODO
135
142
  @retry_wait = 1
136
143
  @retry_limit = 5 # TODO
144
+ @time_as_integer = time_as_integer
137
145
 
138
146
  super()
139
147
  end
@@ -143,7 +151,9 @@ class Writer
143
151
  raise ArgumentError, "Input must be a map (got #{record.class})"
144
152
  end
145
153
 
146
- entry = [Time.now.to_i, record]
154
+ time = Fluent::EventTime.now
155
+ time = time.to_i if @time_as_integer
156
+ entry = [time, record]
147
157
  synchronize {
148
158
  unless write_impl([entry])
149
159
  # write failed
@@ -200,7 +210,8 @@ class Writer
200
210
  end
201
211
 
202
212
  begin
203
- socket.write [@tag, array].to_msgpack
213
+ packer = Fluent::MessagePackFactory.packer
214
+ socket.write packer.pack([@tag, array])
204
215
  socket.flush
205
216
  rescue
206
217
  $stderr.puts "write failed: #{$!}"
@@ -272,7 +283,7 @@ else
272
283
  }
273
284
  end
274
285
 
275
- w = Writer.new(tag, connector)
286
+ w = Writer.new(tag, connector, time_as_integer: time_as_integer)
276
287
  w.start
277
288
 
278
289
  case format
@@ -26,6 +26,7 @@ require 'fluent/compat/record_filter_mixin'
26
26
  require 'fluent/compat/output_chain'
27
27
  require 'fluent/timezone'
28
28
  require 'fluent/mixin'
29
+ require 'fluent/event'
29
30
  require 'fluent/process' # to load Fluent::DetachProcessMixin
30
31
 
31
32
  require 'fluent/plugin_helper/compat_parameters'
@@ -80,7 +81,7 @@ module Fluent
80
81
  end
81
82
 
82
83
  def assume_timekey!
83
- @_formatter = Fluent::Timezone.formatter(@_timezone, @_time_slice_format)
84
+ @_formatter = Fluent::TimeFormatter.new(@_time_slice_format, nil, @_timezone)
84
85
 
85
86
  return if self.metadata.timekey
86
87
  if self.respond_to?(:path) && self.path =~ /\.(\d+)\.(?:b|q)(?:[a-z0-9]+)/
@@ -123,6 +124,7 @@ module Fluent
123
124
  # prepend this module to BufferedOutput (including ObjectBufferedOutput) plugin singleton class
124
125
  def write(chunk)
125
126
  chunk.extend(ChunkSizeCompatMixin)
127
+ chunk.extend(ChunkMessagePackEventStreamer)
126
128
  chunk.extend(AddKeyToChunkMixin) if chunk.metadata.variables && chunk.metadata.variables.has_key?(:key)
127
129
  super
128
130
  end
@@ -132,6 +134,7 @@ module Fluent
132
134
  # prepend this module to TimeSlicedOutput plugin singleton class
133
135
  def write(chunk)
134
136
  chunk.extend(ChunkSizeCompatMixin)
137
+ chunk.extend(ChunkMessagePackEventStreamer)
135
138
  chunk.extend(AddTimeSliceKeyToChunkMixin)
136
139
  chunk.time_slice_format = @time_slice_format
137
140
  chunk.timekey = @_timekey
@@ -614,17 +617,19 @@ module Fluent
614
617
  end
615
618
 
616
619
  if conf['timezone']
617
- @timezone = conf['timezone']
618
- Fluent::Timezone.validate!(@timezone)
620
+ Fluent::Timezone.validate!(conf['timezone'])
619
621
  elsif conf['utc']
620
- @timezone = "+0000"
621
- @localtime = false
622
+ # v0.12 assumes UTC without any configuration
623
+ # 'localtime=false && no timezone key' means UTC
624
+ conf['localtime'] = "false"
625
+ conf.delete('utc')
622
626
  elsif conf['localtime']
623
- @timezone = Time.now.strftime('%z')
624
- @localtime = true
627
+ conf['timezone'] = Time.now.strftime('%z')
628
+ conf['localtime'] = "true"
625
629
  else
626
- @timezone = "+0000" # v0.12 assumes UTC without any configuration
627
- @localtime = false
630
+ # v0.12 assumes UTC without any configuration
631
+ # 'localtime=false && no timezone key' means UTC
632
+ conf['localtime'] = "false"
628
633
  end
629
634
 
630
635
  @_timekey = case conf['time_slice_format']
@@ -18,6 +18,9 @@ require 'fluent/plugin'
18
18
  require 'fluent/plugin/parser'
19
19
  require 'fluent/mixin'
20
20
 
21
+ require 'fluent/config'
22
+ require 'fluent/compat/type_converter'
23
+
21
24
  require 'fluent/plugin/parser_regexp'
22
25
  require 'fluent/plugin/parser_json'
23
26
  require 'fluent/plugin/parser_tsv'
@@ -56,13 +59,14 @@ module Fluent
56
59
  format = conf['format']
57
60
 
58
61
  @parser = TextParser.lookup(format)
59
- if ! @estimate_current_event.nil? && @parser.respond_to?(:'estimate_current_event=')
60
- @parser.estimate_current_event = @estimate_current_event
61
- end
62
62
 
63
63
  if @parser.respond_to?(:configure)
64
64
  @parser.configure(conf)
65
65
  end
66
+ if !@estimate_current_event.nil? && @parser.respond_to?(:'estimate_current_event=')
67
+ # external code sets parser.estimate_current_event = false
68
+ @parser.estimate_current_event = @estimate_current_event
69
+ end
66
70
 
67
71
  return true
68
72
  end
@@ -116,23 +120,59 @@ module Fluent
116
120
  end
117
121
  end
118
122
 
119
- class TimeParser < Fluent::Plugin::Parser::TimeParser
123
+ module TypeConverterCompatParameters
124
+ def convert_type_converter_parameters!(conf)
125
+ if conf["types"]
126
+ delimiter = conf["types_delimiter"] || ','
127
+ label_delimiter = conf["types_label_delimiter"] || ':'
128
+ types = {}
129
+ conf['types'].split(delimiter).each do |pair|
130
+ key, value = pair.split(label_delimiter, 2)
131
+ if value.start_with?("time#{label_delimiter}")
132
+ value = value.split(label_delimiter, 2).join(':')
133
+ elsif value.start_with?("array#{label_delimiter}")
134
+ value = value.split(label_delimiter, 2).join(':')
135
+ end
136
+ types[key] = value
137
+ end
138
+ conf["types"] = JSON.dump(types)
139
+ end
140
+ end
141
+ end
142
+
143
+ class TimeParser < Fluent::TimeParser
120
144
  # TODO: warn when deprecated
121
145
  end
122
146
 
123
147
  class RegexpParser < Fluent::Plugin::RegexpParser
148
+ include TypeConverterCompatParameters
149
+
124
150
  # TODO: warn when deprecated
125
151
  def initialize(regexp, conf = {})
126
152
  super()
127
153
 
154
+ @stored_regexp = regexp
155
+ @manually_configured = false
128
156
  unless conf.empty?
129
- unless conf.is_a?(Config::Element)
130
- conf = Config::Element.new('default_regexp_conf', '', conf, [])
131
- end
132
- configure(conf)
157
+ conf_init = if conf.is_a?(Fluent::Config::Element)
158
+ conf
159
+ else
160
+ Fluent::Config::Element.new('parse', '', conf, [])
161
+ end
162
+ self.configure(conf_init)
163
+ @manually_configured = true
133
164
  end
165
+ end
166
+
167
+ def configure(conf)
168
+ return if @manually_configured # not to run twice
169
+
170
+ conf['expression'] ||= @stored_regexp.source
171
+ conf['ignorecase'] ||= @stored_regexp.options & Regexp::IGNORECASE != 0
172
+ conf['multiline'] ||= @stored_regexp.options & Regexp::MULTILINE != 0
173
+ convert_type_converter_parameters!(conf)
134
174
 
135
- @regexp = regexp
175
+ super
136
176
  end
137
177
 
138
178
  def patterns
@@ -140,24 +180,114 @@ module Fluent
140
180
  end
141
181
  end
142
182
 
143
- class ValuesParser < Fluent::Plugin::ValuesParser
144
- # TODO: warn when deprecated
183
+ class ValuesParser < Parser
184
+ include Fluent::Compat::TypeConverter
185
+
186
+ config_param :keys, :array, default: []
187
+ config_param :time_key, :string, default: nil
188
+ config_param :null_value_pattern, :string, default: nil
189
+ config_param :null_empty_string, :bool, default: false
190
+
191
+ def configure(conf)
192
+ super
193
+
194
+ if @time_key && !@keys.include?(@time_key) && @estimate_current_event
195
+ raise Fluent::ConfigError, "time_key (#{@time_key.inspect}) is not included in keys (#{@keys.inspect})"
196
+ end
197
+
198
+ if @time_format && !@time_key
199
+ raise Fluent::ConfigError, "time_format parameter is ignored because time_key parameter is not set. at #{conf.inspect}"
200
+ end
201
+
202
+ @time_parser = time_parser_create
203
+
204
+ if @null_value_pattern
205
+ @null_value_pattern = Regexp.new(@null_value_pattern)
206
+ end
207
+
208
+ @mutex = Mutex.new
209
+ end
210
+
211
+ def values_map(values)
212
+ record = Hash[keys.zip(values.map { |value| convert_value_to_nil(value) })]
213
+
214
+ if @time_key
215
+ value = @keep_time_key ? record[@time_key] : record.delete(@time_key)
216
+ time = if value.nil?
217
+ if @estimate_current_event
218
+ Fluent::EventTime.now
219
+ else
220
+ nil
221
+ end
222
+ else
223
+ @mutex.synchronize { @time_parser.parse(value) }
224
+ end
225
+ elsif @estimate_current_event
226
+ time = Fluent::EventTime.now
227
+ else
228
+ time = nil
229
+ end
230
+
231
+ convert_field_type!(record) if @type_converters
232
+
233
+ return time, record
234
+ end
235
+
236
+ private
237
+
238
+ def convert_field_type!(record)
239
+ @type_converters.each_key { |key|
240
+ if value = record[key]
241
+ record[key] = convert_type(key, value)
242
+ end
243
+ }
244
+ end
245
+
246
+ def convert_value_to_nil(value)
247
+ if value and @null_empty_string
248
+ value = (value == '') ? nil : value
249
+ end
250
+ if value and @null_value_pattern
251
+ value = ::Fluent::StringUtil.match_regexp(@null_value_pattern, value) ? nil : value
252
+ end
253
+ value
254
+ end
145
255
  end
146
256
 
147
257
  class JSONParser < Fluent::Plugin::JSONParser
258
+ include TypeConverterCompatParameters
148
259
  # TODO: warn when deprecated
260
+ def configure(conf)
261
+ convert_type_converter_parameters!(conf)
262
+ super
263
+ end
149
264
  end
150
265
 
151
266
  class TSVParser < Fluent::Plugin::TSVParser
267
+ include TypeConverterCompatParameters
152
268
  # TODO: warn when deprecated
269
+ def configure(conf)
270
+ convert_type_converter_parameters!(conf)
271
+ super
272
+ end
153
273
  end
154
274
 
155
275
  class LabeledTSVParser < Fluent::Plugin::LabeledTSVParser
276
+ include TypeConverterCompatParameters
156
277
  # TODO: warn when deprecated
278
+ def configure(conf)
279
+ convert_type_converter_parameters!(conf)
280
+ super
281
+ end
157
282
  end
158
283
 
159
284
  class CSVParser < Fluent::Plugin::CSVParser
285
+ include TypeConverterCompatParameters
160
286
  # TODO: warn when deprecated
287
+ def configure(conf)
288
+ convert_type_converter_parameters!(conf)
289
+ super
290
+ end
161
291
  end
162
292
 
163
293
  class NoneParser < Fluent::Plugin::NoneParser