fluentd 0.14.7-x86-mingw32 → 0.14.10-x86-mingw32
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of fluentd might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/.gitignore +2 -0
- data/.travis.yml +2 -0
- data/CONTRIBUTING.md +6 -1
- data/ChangeLog +95 -0
- data/Rakefile +21 -0
- data/appveyor.yml +1 -0
- data/code-of-conduct.md +3 -0
- data/example/out_exec_filter.conf +42 -0
- data/fluentd.gemspec +1 -1
- data/lib/fluent/agent.rb +2 -2
- data/lib/fluent/command/binlog_reader.rb +1 -1
- data/lib/fluent/command/cat.rb +15 -4
- data/lib/fluent/compat/output.rb +14 -9
- data/lib/fluent/compat/parser.rb +141 -11
- data/lib/fluent/config/configure_proxy.rb +2 -11
- data/lib/fluent/config/section.rb +8 -1
- data/lib/fluent/configurable.rb +1 -3
- data/lib/fluent/env.rb +1 -1
- data/lib/fluent/log.rb +1 -1
- data/lib/fluent/plugin/base.rb +17 -0
- data/lib/fluent/plugin/filter_parser.rb +108 -0
- data/lib/fluent/plugin/filter_record_transformer.rb +14 -35
- data/lib/fluent/plugin/filter_stdout.rb +1 -1
- data/lib/fluent/plugin/formatter.rb +5 -0
- data/lib/fluent/plugin/formatter_msgpack.rb +4 -0
- data/lib/fluent/plugin/formatter_stdout.rb +3 -2
- data/lib/fluent/plugin/formatter_tsv.rb +34 -0
- data/lib/fluent/plugin/in_exec.rb +48 -93
- data/lib/fluent/plugin/in_forward.rb +66 -265
- data/lib/fluent/plugin/in_http.rb +68 -65
- data/lib/fluent/plugin/in_monitor_agent.rb +8 -4
- data/lib/fluent/plugin/in_syslog.rb +42 -58
- data/lib/fluent/plugin/in_tail.rb +29 -14
- data/lib/fluent/plugin/in_tcp.rb +54 -14
- data/lib/fluent/plugin/in_udp.rb +49 -13
- data/lib/fluent/plugin/multi_output.rb +1 -3
- data/lib/fluent/plugin/out_exec.rb +58 -71
- data/lib/fluent/plugin/out_exec_filter.rb +199 -279
- data/lib/fluent/plugin/out_file.rb +172 -81
- data/lib/fluent/plugin/out_forward.rb +229 -206
- data/lib/fluent/plugin/out_stdout.rb +6 -21
- data/lib/fluent/plugin/output.rb +90 -59
- data/lib/fluent/plugin/parser.rb +121 -61
- data/lib/fluent/plugin/parser_csv.rb +9 -3
- data/lib/fluent/plugin/parser_json.rb +37 -35
- data/lib/fluent/plugin/parser_ltsv.rb +11 -19
- data/lib/fluent/plugin/parser_msgpack.rb +50 -0
- data/lib/fluent/plugin/parser_regexp.rb +15 -42
- data/lib/fluent/plugin/parser_tsv.rb +8 -3
- data/lib/fluent/plugin_helper.rb +10 -1
- data/lib/fluent/plugin_helper/child_process.rb +139 -73
- data/lib/fluent/plugin_helper/compat_parameters.rb +93 -4
- data/lib/fluent/plugin_helper/event_emitter.rb +14 -1
- data/lib/fluent/plugin_helper/event_loop.rb +24 -6
- data/lib/fluent/plugin_helper/extract.rb +16 -4
- data/lib/fluent/plugin_helper/formatter.rb +9 -11
- data/lib/fluent/plugin_helper/inject.rb +16 -1
- data/lib/fluent/plugin_helper/parser.rb +3 -3
- data/lib/fluent/plugin_helper/server.rb +494 -0
- data/lib/fluent/plugin_helper/socket.rb +101 -0
- data/lib/fluent/plugin_helper/socket_option.rb +84 -0
- data/lib/fluent/plugin_helper/timer.rb +1 -0
- data/lib/fluent/root_agent.rb +1 -1
- data/lib/fluent/test/driver/base.rb +95 -49
- data/lib/fluent/test/driver/base_owner.rb +18 -8
- data/lib/fluent/test/driver/multi_output.rb +2 -1
- data/lib/fluent/test/driver/output.rb +29 -6
- data/lib/fluent/test/helpers.rb +3 -1
- data/lib/fluent/test/log.rb +4 -0
- data/lib/fluent/test/startup_shutdown.rb +13 -0
- data/lib/fluent/time.rb +14 -8
- data/lib/fluent/version.rb +1 -1
- data/lib/fluent/winsvc.rb +1 -1
- data/test/command/test_binlog_reader.rb +5 -1
- data/test/compat/test_parser.rb +10 -0
- data/test/config/test_configurable.rb +193 -0
- data/test/config/test_configure_proxy.rb +0 -43
- data/test/helper.rb +36 -1
- data/test/plugin/test_base.rb +16 -0
- data/test/plugin/test_filter_parser.rb +665 -0
- data/test/plugin/test_filter_record_transformer.rb +36 -100
- data/test/plugin/test_filter_stdout.rb +18 -27
- data/test/plugin/test_in_dummy.rb +1 -1
- data/test/plugin/test_in_exec.rb +206 -94
- data/test/plugin/test_in_forward.rb +268 -347
- data/test/plugin/test_in_http.rb +310 -186
- data/test/plugin/test_in_monitor_agent.rb +65 -35
- data/test/plugin/test_in_syslog.rb +39 -3
- data/test/plugin/test_in_tcp.rb +78 -62
- data/test/plugin/test_in_udp.rb +101 -80
- data/test/plugin/test_out_exec.rb +223 -68
- data/test/plugin/test_out_exec_filter.rb +520 -169
- data/test/plugin/test_out_file.rb +637 -177
- data/test/plugin/test_out_forward.rb +242 -234
- data/test/plugin/test_out_null.rb +1 -1
- data/test/plugin/test_out_secondary_file.rb +4 -2
- data/test/plugin/test_out_stdout.rb +14 -35
- data/test/plugin/test_output_as_buffered.rb +60 -2
- data/test/plugin/test_parser.rb +359 -0
- data/test/plugin/test_parser_csv.rb +1 -2
- data/test/plugin/test_parser_json.rb +3 -4
- data/test/plugin/test_parser_labeled_tsv.rb +1 -2
- data/test/plugin/test_parser_none.rb +1 -2
- data/test/plugin/test_parser_regexp.rb +8 -4
- data/test/plugin/test_parser_tsv.rb +4 -3
- data/test/plugin_helper/test_child_process.rb +184 -0
- data/test/plugin_helper/test_compat_parameters.rb +88 -1
- data/test/plugin_helper/test_extract.rb +0 -1
- data/test/plugin_helper/test_formatter.rb +5 -2
- data/test/plugin_helper/test_inject.rb +21 -0
- data/test/plugin_helper/test_parser.rb +6 -5
- data/test/plugin_helper/test_server.rb +905 -0
- data/test/test_event_time.rb +3 -1
- data/test/test_output.rb +53 -2
- data/test/test_plugin_classes.rb +20 -0
- data/test/test_root_agent.rb +139 -0
- data/test/test_test_drivers.rb +135 -0
- metadata +28 -8
- data/test/plugin/test_parser_base.rb +0 -32
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: cc9fd1115013c31c251063f81dc0d91be3b5624c
|
4
|
+
data.tar.gz: 95d7e3af0d15244336c42b8d1bfe83f08afd8f36
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 92f075a8834ebbcd7b5c1c43433afc8f9d62a53f68e26ef8e82519515dc7abf8b4a9ad03a43307c3a232a84f281a1f3968e2dda160659c54a01da164fe348224
|
7
|
+
data.tar.gz: 5d0f4bc6df7f9bfe63d7c5f9c4b8e8117111bcaa93f60a32a884e90e69523200a458732e452c964edf590af43da71e51789d01a2dff7a9e28c653413909f834a
|
data/.gitignore
CHANGED
data/.travis.yml
CHANGED
data/CONTRIBUTING.md
CHANGED
@@ -7,8 +7,13 @@ We'd love your contribution. Here are the guidelines!
|
|
7
7
|
RESOURCES of [Official site](http://www.fluentd.org/) and [Fluentd documentation](http://docs.fluentd.org/) may help you.
|
8
8
|
|
9
9
|
If you have further questions about Fluentd and plugins, please direct these to [Mailing List](https://groups.google.com/forum/#!forum/fluentd).
|
10
|
+
Don't use Github issue for asking questions. Here are examples:
|
10
11
|
|
11
|
-
|
12
|
+
- I installed xxx plugin but it doesn't work. Why?
|
13
|
+
- Fluentd starts but logs are not sent to xxx. Am I wrong?
|
14
|
+
- I want to do xxx. How to realize it with plugins?
|
15
|
+
|
16
|
+
We may close such questions to keep clear repository for developers and users.
|
12
17
|
Github issue is mainly for submitting a bug report or feature request. See below.
|
13
18
|
|
14
19
|
## Found a bug?
|
data/ChangeLog
CHANGED
@@ -1,15 +1,108 @@
|
|
1
1
|
# v0.14
|
2
2
|
|
3
|
+
## Release v0.14.10 - 2016/12/14
|
4
|
+
|
5
|
+
### New features / Enhancement
|
6
|
+
|
7
|
+
* Add socket/server plugin helper to write TCP/UDP clients/servers as Fluentd plugin
|
8
|
+
https://github.com/fluent/fluentd/pull/1312
|
9
|
+
https://github.com/fluent/fluentd/pull/1350
|
10
|
+
https://github.com/fluent/fluentd/pull/1356
|
11
|
+
https://github.com/fluent/fluentd/pull/1362
|
12
|
+
* Fix to raise errors when injected hostname is also specified as chunk key
|
13
|
+
https://github.com/fluent/fluentd/pull/1357
|
14
|
+
* in_tail: Optimize to read lines from file
|
15
|
+
https://github.com/fluent/fluentd/pull/1325
|
16
|
+
* in_monitor_agent: Add new parameter "include_config"(default: true)
|
17
|
+
https://github.com/fluent/fluentd/pull/1317
|
18
|
+
* in_syslog: Add "priority_key" and "facility_key" options
|
19
|
+
https://github.com/fluent/fluentd/pull/1351
|
20
|
+
* filter_record_transformer: Remove obsoleted syntax like "${message}" and not to dump records in logs
|
21
|
+
https://github.com/fluent/fluentd/pull/1328
|
22
|
+
* Add an option "--time-as-integer" to fluent-cat command to send events from v0.14 fluent-cat to v0.12 fluentd
|
23
|
+
https://github.com/fluent/fluentd/pull/1349
|
24
|
+
|
25
|
+
### Bug fixes
|
26
|
+
|
27
|
+
* Specify correct Oj options for newer versions (Oj 2.18.0 or later)
|
28
|
+
https://github.com/fluent/fluentd/pull/1331
|
29
|
+
* TimeSlice output plugins (in v0.12 style) raise errors when "utc" parameter is specified
|
30
|
+
https://github.com/fluent/fluentd/pull/1319
|
31
|
+
* Parser plugins cannot use options for regular expressions
|
32
|
+
https://github.com/fluent/fluentd/pull/1326/files
|
33
|
+
* Fix bugs not to raise errors to use logger in v0.12 plugins
|
34
|
+
https://github.com/fluent/fluentd/pull/1344
|
35
|
+
https://github.com/fluent/fluentd/pull/1332
|
36
|
+
* Fix bug about shutting down Fluentd in Windows
|
37
|
+
https://github.com/fluent/fluentd/pull/1367
|
38
|
+
* in_tail: Close files explicitly in tests
|
39
|
+
https://github.com/fluent/fluentd/pull/1327
|
40
|
+
* in_forward: Fix bug not to convert buffer configurations into v0.14 parameters
|
41
|
+
https://github.com/fluent/fluentd/pull/1337
|
42
|
+
* out_forward: Fix bug to raise error when "expire_dns_cache" is specified
|
43
|
+
https://github.com/fluent/fluentd/pull/1346
|
44
|
+
* out_file: Fix bug to raise error about buffer chunking when it's configured as secondary
|
45
|
+
https://github.com/fluent/fluentd/pull/1338
|
46
|
+
|
47
|
+
## Release v0.14.9 - 2016/11/15
|
48
|
+
|
49
|
+
### New features / Enhancement
|
50
|
+
|
51
|
+
* filter_parser: Port fluent-plugin-parser into built-in plugin
|
52
|
+
https://github.com/fluent/fluentd/pull/1191
|
53
|
+
* parser/formatter plugin helpers with default @type in plugin side
|
54
|
+
https://github.com/fluent/fluentd/pull/1267
|
55
|
+
* parser: Reconstruct Parser related classes
|
56
|
+
https://github.com/fluent/fluentd/pull/1286
|
57
|
+
* filter_record_transformer: Remove old behaviours
|
58
|
+
https://github.com/fluent/fluentd/pull/1311
|
59
|
+
* Migrate some built-in plugins into v0.14 API
|
60
|
+
https://github.com/fluent/fluentd/pull/1257 (out_file)
|
61
|
+
https://github.com/fluent/fluentd/pull/1297 (out_exec, out_exec_filter)
|
62
|
+
https://github.com/fluent/fluentd/pull/1306 (in_forward, out_forward)
|
63
|
+
https://github.com/fluent/fluentd/pull/1308 (in_http)
|
64
|
+
* test: Improve test drivers
|
65
|
+
https://github.com/fluent/fluentd/pull/1302
|
66
|
+
https://github.com/fluent/fluentd/pull/1305
|
67
|
+
|
68
|
+
### Bug fixes
|
69
|
+
|
70
|
+
* log: Avoid name conflict between Fluent::Logger
|
71
|
+
https://github.com/fluent/fluentd/pull/1274
|
72
|
+
* fluent-cat: Fix fluent-cat command to send sub-second precision time
|
73
|
+
https://github.com/fluent/fluentd/pull/1277
|
74
|
+
* config: Fix a bug not to overwrite default value with nil
|
75
|
+
https://github.com/fluent/fluentd/pull/1296
|
76
|
+
* output: Fix timezone for compat timesliced output plugins
|
77
|
+
https://github.com/fluent/fluentd/pull/1307
|
78
|
+
* out_forward: fix not to raise error when out_forward is initialized as secondary
|
79
|
+
https://github.com/fluent/fluentd/pull/1313
|
80
|
+
* output: Event router for secondary output
|
81
|
+
https://github.com/fluent/fluentd/pull/1283
|
82
|
+
* test: fix to return the block value as expected by many rubyists
|
83
|
+
https://github.com/fluent/fluentd/pull/1284
|
84
|
+
|
85
|
+
## Release v0.14.8 - 2016/10/13
|
86
|
+
|
87
|
+
### Bug fixes
|
88
|
+
|
89
|
+
* Add msgpack_each to buffer chunks in compat-layer output plugins
|
90
|
+
https://github.com/fluent/fluentd/pull/1273
|
91
|
+
|
3
92
|
## Release v0.14.7 - 2016/10/07
|
4
93
|
|
5
94
|
### New features / Enhancement
|
6
95
|
|
96
|
+
* Support data compression in buffer plugins
|
97
|
+
https://github.com/fluent/fluentd/pull/1172
|
7
98
|
* in_forward: support to transfer compressed data
|
8
99
|
https://github.com/fluent/fluentd/pull/1179
|
9
100
|
* out_stdout: fix to show nanosecond resolution time
|
10
101
|
https://github.com/fluent/fluentd/pull/1249
|
11
102
|
* Add option to rotate Fluentd daemon's log
|
12
103
|
https://github.com/fluent/fluentd/pull/1235
|
104
|
+
* Add extract plugin helper, with symmetric time parameter support in parser/formatter and inject/extract
|
105
|
+
https://github.com/fluent/fluentd/pull/1207
|
13
106
|
* Add a feature to parse/format numeric time (unix time [+ subsecond value])
|
14
107
|
https://github.com/fluent/fluentd/pull/1254
|
15
108
|
* Raise configuration errors for inconsistent <label> configurations
|
@@ -22,6 +115,8 @@
|
|
22
115
|
https://github.com/fluent/fluentd/pull/1227
|
23
116
|
* Add some features to write plugins more easily
|
24
117
|
https://github.com/fluent/fluentd/pull/1256
|
118
|
+
* Add a tool to load dumped events from file
|
119
|
+
https://github.com/fluent/fluentd/pull/1165
|
25
120
|
|
26
121
|
### Bug fixes
|
27
122
|
|
data/Rakefile
CHANGED
@@ -24,6 +24,9 @@ end
|
|
24
24
|
|
25
25
|
desc 'Run test_unit based test'
|
26
26
|
Rake::TestTask.new(:base_test) do |t|
|
27
|
+
# To run test with dumping all test case names (to find never ending test case)
|
28
|
+
# $ bundle exec rake test TESTOPTS=-v
|
29
|
+
#
|
27
30
|
# To run test for only one file (or file path pattern)
|
28
31
|
# $ bundle exec rake base_test TEST=test/test_specified_path.rb
|
29
32
|
# $ bundle exec rake base_test TEST=test/test_*.rb
|
@@ -50,4 +53,22 @@ task :coverage do |t|
|
|
50
53
|
Rake::Task["test"].invoke
|
51
54
|
end
|
52
55
|
|
56
|
+
desc 'Build Coverity tarball & upload it'
|
57
|
+
task :coverity do
|
58
|
+
# https://scan.coverity.com/projects/fluentd?tab=overview
|
59
|
+
# See "View Defects" after sign-in.
|
60
|
+
#
|
61
|
+
# Setup steps:
|
62
|
+
# 1. get coverity build tool and set PATH to bin/: https://scan.coverity.com/download
|
63
|
+
# 2. set environment variables:
|
64
|
+
# * $COVERITY_USER (your email address)
|
65
|
+
# * $COVERITY_TOKEN (token for Fluentd project: https://scan.coverity.com/projects/fluentd?tab=project_settings)
|
66
|
+
sh "cov-build --dir cov-int --no-command --fs-capture-search ./"
|
67
|
+
sh "tar czf cov-fluentd.tar.gz cov-int"
|
68
|
+
user = ENV['COVERITY_USER']
|
69
|
+
token = ENV['COVERITY_TOKEN']
|
70
|
+
sh "curl --form token=#{token} --form email=#{user} --form file=@cov-fluentd.tar.gz --form version=\"Master\" --form description=\"GIT Master\" https://scan.coverity.com/builds?project=Fluentd"
|
71
|
+
FileUtils.rm_rf(['./cov-int', 'cov-fluentd.tar.gz'])
|
72
|
+
end
|
73
|
+
|
53
74
|
task default: [:test, :build]
|
data/appveyor.yml
CHANGED
data/code-of-conduct.md
ADDED
@@ -0,0 +1,42 @@
|
|
1
|
+
<source>
|
2
|
+
@type dummy
|
3
|
+
@label @exec
|
4
|
+
tag exec_input
|
5
|
+
rate 10
|
6
|
+
auto_increment_key num
|
7
|
+
dummy {"data":"mydata"}
|
8
|
+
</source>
|
9
|
+
|
10
|
+
<label @exec>
|
11
|
+
<match exec_input>
|
12
|
+
@type exec_filter
|
13
|
+
@label @stdout
|
14
|
+
tag result
|
15
|
+
command ruby -e 'STDOUT.sync = true; proc = ->(){line = STDIN.readline.chomp; puts line + "\t" + Process.pid.to_s}; 1000.times{ proc.call }'
|
16
|
+
num_children 3
|
17
|
+
child_respawn -1
|
18
|
+
<inject>
|
19
|
+
time_key time
|
20
|
+
time_type float
|
21
|
+
</inject>
|
22
|
+
<format>
|
23
|
+
@type tsv
|
24
|
+
keys data, num, time
|
25
|
+
</format>
|
26
|
+
<parse>
|
27
|
+
@type tsv
|
28
|
+
keys data, num, time, pid
|
29
|
+
</parse>
|
30
|
+
<extract>
|
31
|
+
time_key time
|
32
|
+
time_type float
|
33
|
+
</extract>
|
34
|
+
</match>
|
35
|
+
</label>
|
36
|
+
|
37
|
+
<label @stdout>
|
38
|
+
<match result>
|
39
|
+
@type stdout
|
40
|
+
</match>
|
41
|
+
</label>
|
42
|
+
|
data/fluentd.gemspec
CHANGED
@@ -22,7 +22,7 @@ Gem::Specification.new do |gem|
|
|
22
22
|
gem.add_runtime_dependency("msgpack", [">= 0.7.0", "< 2.0.0"])
|
23
23
|
gem.add_runtime_dependency("yajl-ruby", ["~> 1.0"])
|
24
24
|
gem.add_runtime_dependency("cool.io", ["~> 1.4.5"])
|
25
|
-
gem.add_runtime_dependency("serverengine", ["
|
25
|
+
gem.add_runtime_dependency("serverengine", [">= 2.0.4", "< 3.0.0"])
|
26
26
|
gem.add_runtime_dependency("http_parser.rb", [">= 0.5.1", "< 0.7.0"])
|
27
27
|
gem.add_runtime_dependency("sigdump", ["~> 0.2.2"])
|
28
28
|
gem.add_runtime_dependency("tzinfo", ["~> 1.0"])
|
data/lib/fluent/agent.rb
CHANGED
@@ -123,7 +123,7 @@ module Fluent
|
|
123
123
|
log.info "adding match#{@context.nil? ? '' : " in #{@context}"}", pattern: pattern, type: type
|
124
124
|
|
125
125
|
output = Plugin.new_output(type)
|
126
|
-
output.
|
126
|
+
output.context_router = @event_router
|
127
127
|
output.configure(conf)
|
128
128
|
@outputs << output
|
129
129
|
if output.respond_to?(:outputs) && output.respond_to?(:multi_output?) && output.multi_output?
|
@@ -144,7 +144,7 @@ module Fluent
|
|
144
144
|
log.info "adding filter#{@context.nil? ? '' : " in #{@context}"}", pattern: pattern, type: type
|
145
145
|
|
146
146
|
filter = Plugin.new_filter(type)
|
147
|
-
filter.
|
147
|
+
filter.context_router = @event_router
|
148
148
|
filter.configure(conf)
|
149
149
|
@filters << filter
|
150
150
|
@event_router.add_rule(pattern, filter)
|
@@ -164,7 +164,7 @@ module BinlogReaderCommand
|
|
164
164
|
def call
|
165
165
|
@formatter = lookup_formatter(@options[:format], @options[:config_params])
|
166
166
|
|
167
|
-
File.open(@path, '
|
167
|
+
File.open(@path, 'rb') do |io|
|
168
168
|
i = 1
|
169
169
|
Fluent::MessagePackFactory.unpacker(io).each do |(time, record)|
|
170
170
|
print @formatter.format(@path, time, record) # path is used for tag
|
data/lib/fluent/command/cat.rb
CHANGED
@@ -16,6 +16,8 @@
|
|
16
16
|
|
17
17
|
require 'optparse'
|
18
18
|
require 'fluent/env'
|
19
|
+
require 'fluent/time'
|
20
|
+
require 'fluent/msgpack_factory'
|
19
21
|
|
20
22
|
op = OptionParser.new
|
21
23
|
|
@@ -29,6 +31,7 @@ socket_path = Fluent::DEFAULT_SOCKET_PATH
|
|
29
31
|
config_path = Fluent::DEFAULT_CONFIG_PATH
|
30
32
|
format = 'json'
|
31
33
|
message_key = 'message'
|
34
|
+
time_as_integer = false
|
32
35
|
|
33
36
|
op.on('-p', '--port PORT', "fluent tcp port (default: #{port})", Integer) {|i|
|
34
37
|
port = i
|
@@ -66,6 +69,10 @@ op.on('--message-key KEY', "key field for none format (default: #{message_key})"
|
|
66
69
|
message_key = s
|
67
70
|
}
|
68
71
|
|
72
|
+
op.on('--time-as-integer', "Send time as integer for v0.12 or earlier", TrueClass) { |b|
|
73
|
+
time_as_integer = true
|
74
|
+
}
|
75
|
+
|
69
76
|
(class << self; self; end).module_eval do
|
70
77
|
define_method(:usage) do |msg|
|
71
78
|
puts op.to_s
|
@@ -121,7 +128,7 @@ class Writer
|
|
121
128
|
end
|
122
129
|
end
|
123
130
|
|
124
|
-
def initialize(tag, connector)
|
131
|
+
def initialize(tag, connector, time_as_integer: false)
|
125
132
|
@tag = tag
|
126
133
|
@connector = connector
|
127
134
|
@socket = false
|
@@ -134,6 +141,7 @@ class Writer
|
|
134
141
|
@pending_limit = 1024 # TODO
|
135
142
|
@retry_wait = 1
|
136
143
|
@retry_limit = 5 # TODO
|
144
|
+
@time_as_integer = time_as_integer
|
137
145
|
|
138
146
|
super()
|
139
147
|
end
|
@@ -143,7 +151,9 @@ class Writer
|
|
143
151
|
raise ArgumentError, "Input must be a map (got #{record.class})"
|
144
152
|
end
|
145
153
|
|
146
|
-
|
154
|
+
time = Fluent::EventTime.now
|
155
|
+
time = time.to_i if @time_as_integer
|
156
|
+
entry = [time, record]
|
147
157
|
synchronize {
|
148
158
|
unless write_impl([entry])
|
149
159
|
# write failed
|
@@ -200,7 +210,8 @@ class Writer
|
|
200
210
|
end
|
201
211
|
|
202
212
|
begin
|
203
|
-
|
213
|
+
packer = Fluent::MessagePackFactory.packer
|
214
|
+
socket.write packer.pack([@tag, array])
|
204
215
|
socket.flush
|
205
216
|
rescue
|
206
217
|
$stderr.puts "write failed: #{$!}"
|
@@ -272,7 +283,7 @@ else
|
|
272
283
|
}
|
273
284
|
end
|
274
285
|
|
275
|
-
w = Writer.new(tag, connector)
|
286
|
+
w = Writer.new(tag, connector, time_as_integer: time_as_integer)
|
276
287
|
w.start
|
277
288
|
|
278
289
|
case format
|
data/lib/fluent/compat/output.rb
CHANGED
@@ -26,6 +26,7 @@ require 'fluent/compat/record_filter_mixin'
|
|
26
26
|
require 'fluent/compat/output_chain'
|
27
27
|
require 'fluent/timezone'
|
28
28
|
require 'fluent/mixin'
|
29
|
+
require 'fluent/event'
|
29
30
|
require 'fluent/process' # to load Fluent::DetachProcessMixin
|
30
31
|
|
31
32
|
require 'fluent/plugin_helper/compat_parameters'
|
@@ -80,7 +81,7 @@ module Fluent
|
|
80
81
|
end
|
81
82
|
|
82
83
|
def assume_timekey!
|
83
|
-
@_formatter = Fluent::
|
84
|
+
@_formatter = Fluent::TimeFormatter.new(@_time_slice_format, nil, @_timezone)
|
84
85
|
|
85
86
|
return if self.metadata.timekey
|
86
87
|
if self.respond_to?(:path) && self.path =~ /\.(\d+)\.(?:b|q)(?:[a-z0-9]+)/
|
@@ -123,6 +124,7 @@ module Fluent
|
|
123
124
|
# prepend this module to BufferedOutput (including ObjectBufferedOutput) plugin singleton class
|
124
125
|
def write(chunk)
|
125
126
|
chunk.extend(ChunkSizeCompatMixin)
|
127
|
+
chunk.extend(ChunkMessagePackEventStreamer)
|
126
128
|
chunk.extend(AddKeyToChunkMixin) if chunk.metadata.variables && chunk.metadata.variables.has_key?(:key)
|
127
129
|
super
|
128
130
|
end
|
@@ -132,6 +134,7 @@ module Fluent
|
|
132
134
|
# prepend this module to TimeSlicedOutput plugin singleton class
|
133
135
|
def write(chunk)
|
134
136
|
chunk.extend(ChunkSizeCompatMixin)
|
137
|
+
chunk.extend(ChunkMessagePackEventStreamer)
|
135
138
|
chunk.extend(AddTimeSliceKeyToChunkMixin)
|
136
139
|
chunk.time_slice_format = @time_slice_format
|
137
140
|
chunk.timekey = @_timekey
|
@@ -614,17 +617,19 @@ module Fluent
|
|
614
617
|
end
|
615
618
|
|
616
619
|
if conf['timezone']
|
617
|
-
|
618
|
-
Fluent::Timezone.validate!(@timezone)
|
620
|
+
Fluent::Timezone.validate!(conf['timezone'])
|
619
621
|
elsif conf['utc']
|
620
|
-
|
621
|
-
|
622
|
+
# v0.12 assumes UTC without any configuration
|
623
|
+
# 'localtime=false && no timezone key' means UTC
|
624
|
+
conf['localtime'] = "false"
|
625
|
+
conf.delete('utc')
|
622
626
|
elsif conf['localtime']
|
623
|
-
|
624
|
-
|
627
|
+
conf['timezone'] = Time.now.strftime('%z')
|
628
|
+
conf['localtime'] = "true"
|
625
629
|
else
|
626
|
-
|
627
|
-
|
630
|
+
# v0.12 assumes UTC without any configuration
|
631
|
+
# 'localtime=false && no timezone key' means UTC
|
632
|
+
conf['localtime'] = "false"
|
628
633
|
end
|
629
634
|
|
630
635
|
@_timekey = case conf['time_slice_format']
|
data/lib/fluent/compat/parser.rb
CHANGED
@@ -18,6 +18,9 @@ require 'fluent/plugin'
|
|
18
18
|
require 'fluent/plugin/parser'
|
19
19
|
require 'fluent/mixin'
|
20
20
|
|
21
|
+
require 'fluent/config'
|
22
|
+
require 'fluent/compat/type_converter'
|
23
|
+
|
21
24
|
require 'fluent/plugin/parser_regexp'
|
22
25
|
require 'fluent/plugin/parser_json'
|
23
26
|
require 'fluent/plugin/parser_tsv'
|
@@ -56,13 +59,14 @@ module Fluent
|
|
56
59
|
format = conf['format']
|
57
60
|
|
58
61
|
@parser = TextParser.lookup(format)
|
59
|
-
if ! @estimate_current_event.nil? && @parser.respond_to?(:'estimate_current_event=')
|
60
|
-
@parser.estimate_current_event = @estimate_current_event
|
61
|
-
end
|
62
62
|
|
63
63
|
if @parser.respond_to?(:configure)
|
64
64
|
@parser.configure(conf)
|
65
65
|
end
|
66
|
+
if !@estimate_current_event.nil? && @parser.respond_to?(:'estimate_current_event=')
|
67
|
+
# external code sets parser.estimate_current_event = false
|
68
|
+
@parser.estimate_current_event = @estimate_current_event
|
69
|
+
end
|
66
70
|
|
67
71
|
return true
|
68
72
|
end
|
@@ -116,23 +120,59 @@ module Fluent
|
|
116
120
|
end
|
117
121
|
end
|
118
122
|
|
119
|
-
|
123
|
+
module TypeConverterCompatParameters
|
124
|
+
def convert_type_converter_parameters!(conf)
|
125
|
+
if conf["types"]
|
126
|
+
delimiter = conf["types_delimiter"] || ','
|
127
|
+
label_delimiter = conf["types_label_delimiter"] || ':'
|
128
|
+
types = {}
|
129
|
+
conf['types'].split(delimiter).each do |pair|
|
130
|
+
key, value = pair.split(label_delimiter, 2)
|
131
|
+
if value.start_with?("time#{label_delimiter}")
|
132
|
+
value = value.split(label_delimiter, 2).join(':')
|
133
|
+
elsif value.start_with?("array#{label_delimiter}")
|
134
|
+
value = value.split(label_delimiter, 2).join(':')
|
135
|
+
end
|
136
|
+
types[key] = value
|
137
|
+
end
|
138
|
+
conf["types"] = JSON.dump(types)
|
139
|
+
end
|
140
|
+
end
|
141
|
+
end
|
142
|
+
|
143
|
+
class TimeParser < Fluent::TimeParser
|
120
144
|
# TODO: warn when deprecated
|
121
145
|
end
|
122
146
|
|
123
147
|
class RegexpParser < Fluent::Plugin::RegexpParser
|
148
|
+
include TypeConverterCompatParameters
|
149
|
+
|
124
150
|
# TODO: warn when deprecated
|
125
151
|
def initialize(regexp, conf = {})
|
126
152
|
super()
|
127
153
|
|
154
|
+
@stored_regexp = regexp
|
155
|
+
@manually_configured = false
|
128
156
|
unless conf.empty?
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
157
|
+
conf_init = if conf.is_a?(Fluent::Config::Element)
|
158
|
+
conf
|
159
|
+
else
|
160
|
+
Fluent::Config::Element.new('parse', '', conf, [])
|
161
|
+
end
|
162
|
+
self.configure(conf_init)
|
163
|
+
@manually_configured = true
|
133
164
|
end
|
165
|
+
end
|
166
|
+
|
167
|
+
def configure(conf)
|
168
|
+
return if @manually_configured # not to run twice
|
169
|
+
|
170
|
+
conf['expression'] ||= @stored_regexp.source
|
171
|
+
conf['ignorecase'] ||= @stored_regexp.options & Regexp::IGNORECASE != 0
|
172
|
+
conf['multiline'] ||= @stored_regexp.options & Regexp::MULTILINE != 0
|
173
|
+
convert_type_converter_parameters!(conf)
|
134
174
|
|
135
|
-
|
175
|
+
super
|
136
176
|
end
|
137
177
|
|
138
178
|
def patterns
|
@@ -140,24 +180,114 @@ module Fluent
|
|
140
180
|
end
|
141
181
|
end
|
142
182
|
|
143
|
-
class ValuesParser <
|
144
|
-
|
183
|
+
class ValuesParser < Parser
|
184
|
+
include Fluent::Compat::TypeConverter
|
185
|
+
|
186
|
+
config_param :keys, :array, default: []
|
187
|
+
config_param :time_key, :string, default: nil
|
188
|
+
config_param :null_value_pattern, :string, default: nil
|
189
|
+
config_param :null_empty_string, :bool, default: false
|
190
|
+
|
191
|
+
def configure(conf)
|
192
|
+
super
|
193
|
+
|
194
|
+
if @time_key && !@keys.include?(@time_key) && @estimate_current_event
|
195
|
+
raise Fluent::ConfigError, "time_key (#{@time_key.inspect}) is not included in keys (#{@keys.inspect})"
|
196
|
+
end
|
197
|
+
|
198
|
+
if @time_format && !@time_key
|
199
|
+
raise Fluent::ConfigError, "time_format parameter is ignored because time_key parameter is not set. at #{conf.inspect}"
|
200
|
+
end
|
201
|
+
|
202
|
+
@time_parser = time_parser_create
|
203
|
+
|
204
|
+
if @null_value_pattern
|
205
|
+
@null_value_pattern = Regexp.new(@null_value_pattern)
|
206
|
+
end
|
207
|
+
|
208
|
+
@mutex = Mutex.new
|
209
|
+
end
|
210
|
+
|
211
|
+
def values_map(values)
|
212
|
+
record = Hash[keys.zip(values.map { |value| convert_value_to_nil(value) })]
|
213
|
+
|
214
|
+
if @time_key
|
215
|
+
value = @keep_time_key ? record[@time_key] : record.delete(@time_key)
|
216
|
+
time = if value.nil?
|
217
|
+
if @estimate_current_event
|
218
|
+
Fluent::EventTime.now
|
219
|
+
else
|
220
|
+
nil
|
221
|
+
end
|
222
|
+
else
|
223
|
+
@mutex.synchronize { @time_parser.parse(value) }
|
224
|
+
end
|
225
|
+
elsif @estimate_current_event
|
226
|
+
time = Fluent::EventTime.now
|
227
|
+
else
|
228
|
+
time = nil
|
229
|
+
end
|
230
|
+
|
231
|
+
convert_field_type!(record) if @type_converters
|
232
|
+
|
233
|
+
return time, record
|
234
|
+
end
|
235
|
+
|
236
|
+
private
|
237
|
+
|
238
|
+
def convert_field_type!(record)
|
239
|
+
@type_converters.each_key { |key|
|
240
|
+
if value = record[key]
|
241
|
+
record[key] = convert_type(key, value)
|
242
|
+
end
|
243
|
+
}
|
244
|
+
end
|
245
|
+
|
246
|
+
def convert_value_to_nil(value)
|
247
|
+
if value and @null_empty_string
|
248
|
+
value = (value == '') ? nil : value
|
249
|
+
end
|
250
|
+
if value and @null_value_pattern
|
251
|
+
value = ::Fluent::StringUtil.match_regexp(@null_value_pattern, value) ? nil : value
|
252
|
+
end
|
253
|
+
value
|
254
|
+
end
|
145
255
|
end
|
146
256
|
|
147
257
|
class JSONParser < Fluent::Plugin::JSONParser
|
258
|
+
include TypeConverterCompatParameters
|
148
259
|
# TODO: warn when deprecated
|
260
|
+
def configure(conf)
|
261
|
+
convert_type_converter_parameters!(conf)
|
262
|
+
super
|
263
|
+
end
|
149
264
|
end
|
150
265
|
|
151
266
|
class TSVParser < Fluent::Plugin::TSVParser
|
267
|
+
include TypeConverterCompatParameters
|
152
268
|
# TODO: warn when deprecated
|
269
|
+
def configure(conf)
|
270
|
+
convert_type_converter_parameters!(conf)
|
271
|
+
super
|
272
|
+
end
|
153
273
|
end
|
154
274
|
|
155
275
|
class LabeledTSVParser < Fluent::Plugin::LabeledTSVParser
|
276
|
+
include TypeConverterCompatParameters
|
156
277
|
# TODO: warn when deprecated
|
278
|
+
def configure(conf)
|
279
|
+
convert_type_converter_parameters!(conf)
|
280
|
+
super
|
281
|
+
end
|
157
282
|
end
|
158
283
|
|
159
284
|
class CSVParser < Fluent::Plugin::CSVParser
|
285
|
+
include TypeConverterCompatParameters
|
160
286
|
# TODO: warn when deprecated
|
287
|
+
def configure(conf)
|
288
|
+
convert_type_converter_parameters!(conf)
|
289
|
+
super
|
290
|
+
end
|
161
291
|
end
|
162
292
|
|
163
293
|
class NoneParser < Fluent::Plugin::NoneParser
|