fluent-plugin-detect-exceptions-xiniaoyun 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/CONTRIBUTING +24 -0
- data/Gemfile +3 -0
- data/LICENSE +201 -0
- data/README.rdoc +120 -0
- data/Rakefile +48 -0
- data/fluent-plugin-detect-exceptions.gemspec +29 -0
- data/lib/fluent/plugin/exception_detector.rb +349 -0
- data/lib/fluent/plugin/out_detect_exceptions.rb +144 -0
- data/test/helper.rb +46 -0
- data/test/plugin/bench_exception_detector.rb +73 -0
- data/test/plugin/test_exception_detector.rb +768 -0
- data/test/plugin/test_out_detect_exceptions.rb +287 -0
- metadata +133 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 2d1058194be8d3fb52f3d199a9348f52810b836e2bae82842c17f59a7a922c44
|
4
|
+
data.tar.gz: c3b9dbef4a81ed1b601f3c3fc032cf4798c5170120cc1532280f0bbc000d765b
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 995bdd40207983c5576b97ab82ee94f3bba8b4900523d59ec4155c95261d24d65895a0d27af786d949f9a051b2c6798b6c0002f371fa6ee38759e54bd7f8e8b8
|
7
|
+
data.tar.gz: a3129ca850b705c26bd7879deb57fc8adac3367708aa48af2ec08037d8d857caeb2b0a59bbf3bcdc9153d2c02c5548f7445fcf6be72ff4b955aa3db2b9b5b103
|
data/CONTRIBUTING
ADDED
@@ -0,0 +1,24 @@
|
|
1
|
+
Want to contribute? Great! First, read this page (including the small print at the end).
|
2
|
+
|
3
|
+
### Before you contribute
|
4
|
+
Before we can use your code, you must sign the
|
5
|
+
[Google Individual Contributor License Agreement](https://developers.google.com/open-source/cla/individual?csw=1)
|
6
|
+
(CLA), which you can do online. The CLA is necessary mainly because you own the
|
7
|
+
copyright to your changes, even after your contribution becomes part of our
|
8
|
+
codebase, so we need your permission to use and distribute your code. We also
|
9
|
+
need to be sure of various other things—for instance that you'll tell us if you
|
10
|
+
know that your code infringes on other people's patents. You don't have to sign
|
11
|
+
the CLA until after you've submitted your code for review and a member has
|
12
|
+
approved it, but you must do it before we can put your code into our codebase.
|
13
|
+
Before you start working on a larger contribution, you should get in touch with
|
14
|
+
us first through the issue tracker with your idea so that we can help out and
|
15
|
+
possibly guide you. Coordinating up front makes it much easier to avoid
|
16
|
+
frustration later on.
|
17
|
+
|
18
|
+
### Code reviews
|
19
|
+
All submissions, including submissions by project members, require review. We
|
20
|
+
use Github pull requests for this purpose.
|
21
|
+
|
22
|
+
### The small print
|
23
|
+
Contributions made by corporations are covered by a different agreement than
|
24
|
+
the one above, the Software Grant and Corporate Contributor License Agreement.
|
data/Gemfile
ADDED
data/LICENSE
ADDED
@@ -0,0 +1,201 @@
|
|
1
|
+
Apache License
|
2
|
+
Version 2.0, January 2004
|
3
|
+
http://www.apache.org/licenses/
|
4
|
+
|
5
|
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
6
|
+
|
7
|
+
1. Definitions.
|
8
|
+
|
9
|
+
"License" shall mean the terms and conditions for use, reproduction,
|
10
|
+
and distribution as defined by Sections 1 through 9 of this document.
|
11
|
+
|
12
|
+
"Licensor" shall mean the copyright owner or entity authorized by
|
13
|
+
the copyright owner that is granting the License.
|
14
|
+
|
15
|
+
"Legal Entity" shall mean the union of the acting entity and all
|
16
|
+
other entities that control, are controlled by, or are under common
|
17
|
+
control with that entity. For the purposes of this definition,
|
18
|
+
"control" means (i) the power, direct or indirect, to cause the
|
19
|
+
direction or management of such entity, whether by contract or
|
20
|
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
21
|
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
22
|
+
|
23
|
+
"You" (or "Your") shall mean an individual or Legal Entity
|
24
|
+
exercising permissions granted by this License.
|
25
|
+
|
26
|
+
"Source" form shall mean the preferred form for making modifications,
|
27
|
+
including but not limited to software source code, documentation
|
28
|
+
source, and configuration files.
|
29
|
+
|
30
|
+
"Object" form shall mean any form resulting from mechanical
|
31
|
+
transformation or translation of a Source form, including but
|
32
|
+
not limited to compiled object code, generated documentation,
|
33
|
+
and conversions to other media types.
|
34
|
+
|
35
|
+
"Work" shall mean the work of authorship, whether in Source or
|
36
|
+
Object form, made available under the License, as indicated by a
|
37
|
+
copyright notice that is included in or attached to the work
|
38
|
+
(an example is provided in the Appendix below).
|
39
|
+
|
40
|
+
"Derivative Works" shall mean any work, whether in Source or Object
|
41
|
+
form, that is based on (or derived from) the Work and for which the
|
42
|
+
editorial revisions, annotations, elaborations, or other modifications
|
43
|
+
represent, as a whole, an original work of authorship. For the purposes
|
44
|
+
of this License, Derivative Works shall not include works that remain
|
45
|
+
separable from, or merely link (or bind by name) to the interfaces of,
|
46
|
+
the Work and Derivative Works thereof.
|
47
|
+
|
48
|
+
"Contribution" shall mean any work of authorship, including
|
49
|
+
the original version of the Work and any modifications or additions
|
50
|
+
to that Work or Derivative Works thereof, that is intentionally
|
51
|
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
52
|
+
or by an individual or Legal Entity authorized to submit on behalf of
|
53
|
+
the copyright owner. For the purposes of this definition, "submitted"
|
54
|
+
means any form of electronic, verbal, or written communication sent
|
55
|
+
to the Licensor or its representatives, including but not limited to
|
56
|
+
communication on electronic mailing lists, source code control systems,
|
57
|
+
and issue tracking systems that are managed by, or on behalf of, the
|
58
|
+
Licensor for the purpose of discussing and improving the Work, but
|
59
|
+
excluding communication that is conspicuously marked or otherwise
|
60
|
+
designated in writing by the copyright owner as "Not a Contribution."
|
61
|
+
|
62
|
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
63
|
+
on behalf of whom a Contribution has been received by Licensor and
|
64
|
+
subsequently incorporated within the Work.
|
65
|
+
|
66
|
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
67
|
+
this License, each Contributor hereby grants to You a perpetual,
|
68
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
69
|
+
copyright license to reproduce, prepare Derivative Works of,
|
70
|
+
publicly display, publicly perform, sublicense, and distribute the
|
71
|
+
Work and such Derivative Works in Source or Object form.
|
72
|
+
|
73
|
+
3. Grant of Patent License. Subject to the terms and conditions of
|
74
|
+
this License, each Contributor hereby grants to You a perpetual,
|
75
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
76
|
+
(except as stated in this section) patent license to make, have made,
|
77
|
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
78
|
+
where such license applies only to those patent claims licensable
|
79
|
+
by such Contributor that are necessarily infringed by their
|
80
|
+
Contribution(s) alone or by combination of their Contribution(s)
|
81
|
+
with the Work to which such Contribution(s) was submitted. If You
|
82
|
+
institute patent litigation against any entity (including a
|
83
|
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
84
|
+
or a Contribution incorporated within the Work constitutes direct
|
85
|
+
or contributory patent infringement, then any patent licenses
|
86
|
+
granted to You under this License for that Work shall terminate
|
87
|
+
as of the date such litigation is filed.
|
88
|
+
|
89
|
+
4. Redistribution. You may reproduce and distribute copies of the
|
90
|
+
Work or Derivative Works thereof in any medium, with or without
|
91
|
+
modifications, and in Source or Object form, provided that You
|
92
|
+
meet the following conditions:
|
93
|
+
|
94
|
+
(a) You must give any other recipients of the Work or
|
95
|
+
Derivative Works a copy of this License; and
|
96
|
+
|
97
|
+
(b) You must cause any modified files to carry prominent notices
|
98
|
+
stating that You changed the files; and
|
99
|
+
|
100
|
+
(c) You must retain, in the Source form of any Derivative Works
|
101
|
+
that You distribute, all copyright, patent, trademark, and
|
102
|
+
attribution notices from the Source form of the Work,
|
103
|
+
excluding those notices that do not pertain to any part of
|
104
|
+
the Derivative Works; and
|
105
|
+
|
106
|
+
(d) If the Work includes a "NOTICE" text file as part of its
|
107
|
+
distribution, then any Derivative Works that You distribute must
|
108
|
+
include a readable copy of the attribution notices contained
|
109
|
+
within such NOTICE file, excluding those notices that do not
|
110
|
+
pertain to any part of the Derivative Works, in at least one
|
111
|
+
of the following places: within a NOTICE text file distributed
|
112
|
+
as part of the Derivative Works; within the Source form or
|
113
|
+
documentation, if provided along with the Derivative Works; or,
|
114
|
+
within a display generated by the Derivative Works, if and
|
115
|
+
wherever such third-party notices normally appear. The contents
|
116
|
+
of the NOTICE file are for informational purposes only and
|
117
|
+
do not modify the License. You may add Your own attribution
|
118
|
+
notices within Derivative Works that You distribute, alongside
|
119
|
+
or as an addendum to the NOTICE text from the Work, provided
|
120
|
+
that such additional attribution notices cannot be construed
|
121
|
+
as modifying the License.
|
122
|
+
|
123
|
+
You may add Your own copyright statement to Your modifications and
|
124
|
+
may provide additional or different license terms and conditions
|
125
|
+
for use, reproduction, or distribution of Your modifications, or
|
126
|
+
for any such Derivative Works as a whole, provided Your use,
|
127
|
+
reproduction, and distribution of the Work otherwise complies with
|
128
|
+
the conditions stated in this License.
|
129
|
+
|
130
|
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
131
|
+
any Contribution intentionally submitted for inclusion in the Work
|
132
|
+
by You to the Licensor shall be under the terms and conditions of
|
133
|
+
this License, without any additional terms or conditions.
|
134
|
+
Notwithstanding the above, nothing herein shall supersede or modify
|
135
|
+
the terms of any separate license agreement you may have executed
|
136
|
+
with Licensor regarding such Contributions.
|
137
|
+
|
138
|
+
6. Trademarks. This License does not grant permission to use the trade
|
139
|
+
names, trademarks, service marks, or product names of the Licensor,
|
140
|
+
except as required for reasonable and customary use in describing the
|
141
|
+
origin of the Work and reproducing the content of the NOTICE file.
|
142
|
+
|
143
|
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
144
|
+
agreed to in writing, Licensor provides the Work (and each
|
145
|
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
146
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
147
|
+
implied, including, without limitation, any warranties or conditions
|
148
|
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
149
|
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
150
|
+
appropriateness of using or redistributing the Work and assume any
|
151
|
+
risks associated with Your exercise of permissions under this License.
|
152
|
+
|
153
|
+
8. Limitation of Liability. In no event and under no legal theory,
|
154
|
+
whether in tort (including negligence), contract, or otherwise,
|
155
|
+
unless required by applicable law (such as deliberate and grossly
|
156
|
+
negligent acts) or agreed to in writing, shall any Contributor be
|
157
|
+
liable to You for damages, including any direct, indirect, special,
|
158
|
+
incidental, or consequential damages of any character arising as a
|
159
|
+
result of this License or out of the use or inability to use the
|
160
|
+
Work (including but not limited to damages for loss of goodwill,
|
161
|
+
work stoppage, computer failure or malfunction, or any and all
|
162
|
+
other commercial damages or losses), even if such Contributor
|
163
|
+
has been advised of the possibility of such damages.
|
164
|
+
|
165
|
+
9. Accepting Warranty or Additional Liability. While redistributing
|
166
|
+
the Work or Derivative Works thereof, You may choose to offer,
|
167
|
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
168
|
+
or other liability obligations and/or rights consistent with this
|
169
|
+
License. However, in accepting such obligations, You may act only
|
170
|
+
on Your own behalf and on Your sole responsibility, not on behalf
|
171
|
+
of any other Contributor, and only if You agree to indemnify,
|
172
|
+
defend, and hold each Contributor harmless for any liability
|
173
|
+
incurred by, or claims asserted against, such Contributor by reason
|
174
|
+
of your accepting any such warranty or additional liability.
|
175
|
+
|
176
|
+
END OF TERMS AND CONDITIONS
|
177
|
+
|
178
|
+
APPENDIX: How to apply the Apache License to your work.
|
179
|
+
|
180
|
+
To apply the Apache License to your work, attach the following
|
181
|
+
boilerplate notice, with the fields enclosed by brackets "{}"
|
182
|
+
replaced with your own identifying information. (Don't include
|
183
|
+
the brackets!) The text should be enclosed in the appropriate
|
184
|
+
comment syntax for the file format. We also recommend that a
|
185
|
+
file or class name and description of purpose be included on the
|
186
|
+
same "printed page" as the copyright notice for easier
|
187
|
+
identification within third-party archives.
|
188
|
+
|
189
|
+
Copyright {yyyy} {name of copyright owner}
|
190
|
+
|
191
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
192
|
+
you may not use this file except in compliance with the License.
|
193
|
+
You may obtain a copy of the License at
|
194
|
+
|
195
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
196
|
+
|
197
|
+
Unless required by applicable law or agreed to in writing, software
|
198
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
199
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
200
|
+
See the License for the specific language governing permissions and
|
201
|
+
limitations under the License.
|
data/README.rdoc
ADDED
@@ -0,0 +1,120 @@
|
|
1
|
+
= Exception detector plugin for {fluentd}[http://github.com/fluent/fluentd]
|
2
|
+
|
3
|
+
fluent-plugin-detect-exceptions is an
|
4
|
+
{output plugin for fluentd}[http://docs.fluentd.org/articles/output-plugin-overview]
|
5
|
+
which scans a log stream text messages or JSON records for multi-line exception
|
6
|
+
stack traces: If a consecutive sequence of log messages forms an exception stack
|
7
|
+
trace, the log messages are forwarded as a single, combined log message.
|
8
|
+
Otherwise, the input log data is forwarded as is.
|
9
|
+
|
10
|
+
Text log messages are assumed to contain single lines and are combined by
|
11
|
+
concatenating them.
|
12
|
+
|
13
|
+
For JSON records, the log message is taken from a configurable message field
|
14
|
+
of the log record. The message field is assumed to contain a single line of
|
15
|
+
text. If the message fields of a consecutive sequence of logs records
|
16
|
+
form an exception stack, they are combined to a single log record by taking
|
17
|
+
the first log record of the sequence and replacing the content of the message
|
18
|
+
field with the concatenated content of all message fields in the sequence.
|
19
|
+
Note that the content of other fields in the records of the sequence are thus
|
20
|
+
not reflected in the combined output. This plugin is intended to be used in
|
21
|
+
cases where the content of log records that belong to a single exception stack
|
22
|
+
are so similar (e.g. because they contain the timestamp of the log entry) that
|
23
|
+
this loss of information is irrelevant.
|
24
|
+
|
25
|
+
This is NOT an official Google product.
|
26
|
+
|
27
|
+
{<img src="https://badge.fury.io/rb/fluent-plugin-detect-exceptions.svg" alt="Gem Version" />}[http://badge.fury.io/rb/fluent-plugin-detect-exceptions]
|
28
|
+
{<img src="https://secure.travis-ci.org/GoogleCloudPlatform/fluent-plugin-detect-exceptions.png" alt="Build Status" />}[https://travis-ci.org/GoogleCloudPlatform/fluent-plugin-detect-exceptions]
|
29
|
+
|
30
|
+
== Installation
|
31
|
+
|
32
|
+
This gem is hosted at
|
33
|
+
{RubyGems.org}[https://rubygems.org/gems/fluent-plugin-detect-exceptions]
|
34
|
+
and can be installed using:
|
35
|
+
|
36
|
+
$ gem install fluent-plugin-detect-exceptions
|
37
|
+
|
38
|
+
Installing {google-fluentd}[https://cloud.google.com/logging/docs/agent/]
|
39
|
+
will also install and configure the gem.
|
40
|
+
|
41
|
+
== Configuration
|
42
|
+
|
43
|
+
The plugin supports the following parameters:
|
44
|
+
|
45
|
+
[message] Name of the field in the JSON record that contains the
|
46
|
+
single-line log messages that shall be scanned for exceptions.
|
47
|
+
If this is set to '', the plugin will try 'message' and 'log',
|
48
|
+
in that order.
|
49
|
+
This parameter is only applicable to structured (JSON) log streams.
|
50
|
+
Default: ''.
|
51
|
+
|
52
|
+
[remove_tag_prefix] The prefix to remove from the input tag when outputting
|
53
|
+
a record. A prefix has to be a complete tag part.
|
54
|
+
Example: If remove_tag_prefix is set to 'foo', the input
|
55
|
+
tag foo.bar.baz is transformed to bar.baz and the input tag
|
56
|
+
'foofoo.bar' is not modified. Default: empty string.
|
57
|
+
|
58
|
+
[languages] A list of language for which exception stack traces shall be
|
59
|
+
detected. The values in the list can be separated by commas or
|
60
|
+
written as JSON list.
|
61
|
+
Supported values: java, js, csharp, python, go, ruby, php, all.
|
62
|
+
Default: all.
|
63
|
+
|
64
|
+
[multiline_flush_interval] Interval in seconds after which a (possibly not
|
65
|
+
yet complete) buffered exception stack shall be
|
66
|
+
forwarded. If not set, incomplete exceptions stacks
|
67
|
+
are not flushed.
|
68
|
+
|
69
|
+
[max_lines] Maximum number of lines in a detected exception stack trace.
|
70
|
+
If this maximum number is exceeded, the exception stack trace
|
71
|
+
that has been detected so far will be output as a single
|
72
|
+
log message and the detection will start from scratch.
|
73
|
+
This is intended as a safeguard against buffering very large
|
74
|
+
amounts of data before outputting them to the log stream.
|
75
|
+
Zero means no limit.
|
76
|
+
Default: 1000.
|
77
|
+
|
78
|
+
[max_bytes] Maximum number of bytes in a detected exception stack trace.
|
79
|
+
If this maximum number is exceeded, the exception stack trace
|
80
|
+
that has been detected so far will be output as a single
|
81
|
+
log message and the detection will start from scratch.
|
82
|
+
This is intended as a safeguard against buffering very large
|
83
|
+
amounts of data before outputting them to the log stream.
|
84
|
+
Zero means no limit.
|
85
|
+
Default: 0.
|
86
|
+
|
87
|
+
[stream] Name of the field in the JSON record that contains the name of a
|
88
|
+
logical log stream within the "real" log stream.
|
89
|
+
The exception detection is handled separately for each logical
|
90
|
+
log stream, i.e., exceptions will be detected even if the messages
|
91
|
+
for the logical log streams are interleaved in the "real" log stream.
|
92
|
+
Consequently, only records in the same logical stream will be
|
93
|
+
combined.
|
94
|
+
This parameter is ignored if set to ''.
|
95
|
+
This parameter is only applicable to structured (JSON) log streams.
|
96
|
+
Default: ''.
|
97
|
+
|
98
|
+
Example configuration:
|
99
|
+
|
100
|
+
<match **>
|
101
|
+
@type detect_exceptions
|
102
|
+
remove_tag_prefix foo
|
103
|
+
message log
|
104
|
+
languages java, python
|
105
|
+
multiline_flush_interval 0.1
|
106
|
+
</match>
|
107
|
+
|
108
|
+
== Extending language support
|
109
|
+
|
110
|
+
Supporting a new language requires new detection rules in this gem and
|
111
|
+
additional changes in the
|
112
|
+
{Stackdriver Error Reporting}[https://cloud.google.com/error-reporting/]
|
113
|
+
service. Please contact our product team by
|
114
|
+
{filing a support case}[https://cloud.google.com/support-hub/#google-cloud-platform]
|
115
|
+
if you'd like to see support for a new language.
|
116
|
+
|
117
|
+
== Copyright
|
118
|
+
|
119
|
+
Copyright:: Copyright 2016 Google Inc. All rights reserved.
|
120
|
+
License:: Apache License, Version 2.0
|
data/Rakefile
ADDED
@@ -0,0 +1,48 @@
|
|
1
|
+
#!/usr/bin/env rake
|
2
|
+
|
3
|
+
require 'bundler'
|
4
|
+
Bundler::GemHelper.install_tasks
|
5
|
+
|
6
|
+
require 'rake/testtask'
|
7
|
+
require 'rubocop/rake_task'
|
8
|
+
|
9
|
+
desc 'Run Rubocop to check for style violations'
|
10
|
+
RuboCop::RakeTask.new
|
11
|
+
|
12
|
+
desc 'Run benchmark tests'
|
13
|
+
Rake::TestTask.new(:bench) do |test|
|
14
|
+
test.libs << 'lib' << 'test'
|
15
|
+
test.test_files = FileList['test/plugin/bench*.rb']
|
16
|
+
test.verbose = true
|
17
|
+
end
|
18
|
+
|
19
|
+
desc 'Run unit tests'
|
20
|
+
Rake::TestTask.new(:test) do |test|
|
21
|
+
test.libs << 'lib' << 'test'
|
22
|
+
test.test_files = FileList['test/plugin/test*.rb']
|
23
|
+
test.verbose = true
|
24
|
+
end
|
25
|
+
|
26
|
+
# Building the gem will use the local file mode, so ensure it's world-readable.
|
27
|
+
# https://github.com/GoogleCloudPlatform/fluent-plugin-detect-exceptions/issues/32
|
28
|
+
desc 'Fix file permissions'
|
29
|
+
task :fix_perms do
|
30
|
+
files = [
|
31
|
+
'lib/fluent/plugin/*.rb'
|
32
|
+
].flat_map do |file|
|
33
|
+
file.include?('*') ? Dir.glob(file) : [file]
|
34
|
+
end
|
35
|
+
|
36
|
+
files.each do |file|
|
37
|
+
mode = File.stat(file).mode & 0o777
|
38
|
+
next unless mode & 0o444 != 0o444
|
39
|
+
puts "Changing mode of #{file} from #{mode.to_s(8)} to "\
|
40
|
+
"#{(mode | 0o444).to_s(8)}"
|
41
|
+
chmod mode | 0o444, file
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
desc 'Run unit tests and RuboCop to check for style violations'
|
46
|
+
task all: [:test, :rubocop, :fix_perms]
|
47
|
+
|
48
|
+
task default: :all
|
@@ -0,0 +1,29 @@
|
|
1
|
+
Gem::Specification.new do |gem|
|
2
|
+
gem.name = 'fluent-plugin-detect-exceptions-xiniaoyun'
|
3
|
+
gem.description = <<-eos
|
4
|
+
Fluentd output plugin which detects exception stack traces in a stream of
|
5
|
+
JSON log messages and combines all single-line messages that belong to the
|
6
|
+
same stack trace into one multi-line message.
|
7
|
+
This is an official Google Ruby gem.
|
8
|
+
eos
|
9
|
+
gem.summary = \
|
10
|
+
'fluentd output plugin for combining stack traces as multi-line JSON logs'
|
11
|
+
gem.homepage = \
|
12
|
+
'https://github.com/GoogleCloudPlatform/fluent-plugin-detect-exceptions'
|
13
|
+
gem.license = 'Apache-2.0'
|
14
|
+
gem.version = '0.0.1'
|
15
|
+
gem.authors = ['Stackdriver Agents']
|
16
|
+
gem.email = ['stackdriver-agents@google.com']
|
17
|
+
gem.required_ruby_version = Gem::Requirement.new('>= 2.0')
|
18
|
+
|
19
|
+
gem.files = Dir['**/*'].keep_if { |file| File.file?(file) }
|
20
|
+
gem.test_files = gem.files.grep(/^(test)/)
|
21
|
+
gem.require_paths = ['lib']
|
22
|
+
|
23
|
+
gem.add_runtime_dependency 'fluentd', '>= 0.10'
|
24
|
+
|
25
|
+
gem.add_development_dependency 'rake', '~> 10.3'
|
26
|
+
gem.add_development_dependency 'rubocop', '= 0.42.0'
|
27
|
+
gem.add_development_dependency 'test-unit', '~> 3.0'
|
28
|
+
gem.add_development_dependency 'flexmock', '~> 2.0'
|
29
|
+
end
|
@@ -0,0 +1,349 @@
|
|
1
|
+
# Copyright 2016 Google Inc. All rights reserved.
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
#
|
15
|
+
module Fluent
|
16
|
+
Struct.new('Rule', :from_state, :pattern, :to_state)
|
17
|
+
|
18
|
+
# Configuration of the state machine that detects exceptions.
|
19
|
+
module ExceptionDetectorConfig
|
20
|
+
# Rule for a state transition: if pattern matches go to the given state.
|
21
|
+
class RuleTarget
|
22
|
+
attr_accessor :pattern, :to_state
|
23
|
+
|
24
|
+
def initialize(p, s)
|
25
|
+
@pattern = p
|
26
|
+
@to_state = s
|
27
|
+
end
|
28
|
+
|
29
|
+
def ==(other)
|
30
|
+
other.class == self.class && other.state == state
|
31
|
+
end
|
32
|
+
|
33
|
+
alias eql? ==
|
34
|
+
|
35
|
+
def hash
|
36
|
+
state.hash
|
37
|
+
end
|
38
|
+
|
39
|
+
def state
|
40
|
+
[@pattern, @to_state]
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
def self.rule(from_state, pattern, to_state)
|
45
|
+
Struct::Rule.new(from_state, pattern, to_state)
|
46
|
+
end
|
47
|
+
|
48
|
+
def self.supported
|
49
|
+
RULES_BY_LANG.keys
|
50
|
+
end
|
51
|
+
|
52
|
+
JAVA_RULES = [
|
53
|
+
rule(:start_state,
|
54
|
+
/(?:Exception|Error|Throwable|V8 errors stack trace)[:\r\n]/,
|
55
|
+
:java),
|
56
|
+
rule(:java, /^[\t ]+(?:eval )?at /, :java),
|
57
|
+
rule(:java, /^[\t ]*(?:Caused by|Suppressed):/, :java),
|
58
|
+
rule(:java, /^[\t ]*... \d+\ more/, :java)
|
59
|
+
].freeze
|
60
|
+
|
61
|
+
PYTHON_RULES = [
|
62
|
+
rule(:start_state, /^Traceback \(most recent call last\):$/, :python),
|
63
|
+
rule(:python, /^[\t ]+File /, :python_code),
|
64
|
+
rule(:python_code, /[^\t ]/, :python),
|
65
|
+
rule(:python, /^(?:[^\s.():]+\.)*[^\s.():]+:/, :start_state)
|
66
|
+
].freeze
|
67
|
+
|
68
|
+
PHP_RULES = [
|
69
|
+
rule(:start_state, /
|
70
|
+
(?:PHP\ (?:Notice|Parse\ error|Fatal\ error|Warning):)|
|
71
|
+
(?:exception\ '[^']+'\ with\ message\ ')/x, :php_stack_begin),
|
72
|
+
rule(:php_stack_begin, /^Stack trace:/, :php_stack_frames),
|
73
|
+
rule(:php_stack_frames, /^#\d/, :php_stack_frames),
|
74
|
+
rule(:php_stack_frames, /^\s+thrown in /, :start_state)
|
75
|
+
].freeze
|
76
|
+
|
77
|
+
GO_RULES = [
|
78
|
+
rule(:start_state, /\bpanic: /, :go_after_panic),
|
79
|
+
rule(:go_after_panic, /^$/, :go_goroutine),
|
80
|
+
rule(:go_after_panic, /^\[signal /, :go_after_signal),
|
81
|
+
rule(:go_after_signal, /^$/, :go_goroutine),
|
82
|
+
rule(:go_goroutine, /^goroutine \d+ \[[^\]]+\]:$/, :go_frame_1),
|
83
|
+
rule(:go_frame_1, /^(?:[^\s.:]+\.)*[^\s.():]+\(|^created by /,
|
84
|
+
:go_frame_2),
|
85
|
+
rule(:go_frame_1, /^$/, :go_goroutine),
|
86
|
+
rule(:go_frame_2, /^\s/, :go_frame_1)
|
87
|
+
].freeze
|
88
|
+
|
89
|
+
RUBY_RULES = [
|
90
|
+
rule(:start_state, /Error \(.*\):$/, :ruby_before_rails_trace),
|
91
|
+
rule(:ruby_before_rails_trace, /^ $/, :ruby),
|
92
|
+
rule(:ruby_before_rails_trace, /^[\t ]+.*?\.rb:\d+:in `/, :ruby),
|
93
|
+
rule(:ruby, /^[\t ]+.*?\.rb:\d+:in `/, :ruby)
|
94
|
+
].freeze
|
95
|
+
|
96
|
+
DART_RULES = [
|
97
|
+
rule(:start_state, /^Unhandled exception:$/, :dart_exc),
|
98
|
+
rule(:dart_exc, /^Instance of/, :dart_stack),
|
99
|
+
rule(:dart_exc, /^Exception/, :dart_stack),
|
100
|
+
rule(:dart_exc, /^Bad state/, :dart_stack),
|
101
|
+
rule(:dart_exc, /^IntegerDivisionByZeroException/, :dart_stack),
|
102
|
+
rule(:dart_exc, /^Invalid argument/, :dart_stack),
|
103
|
+
rule(:dart_exc, /^RangeError/, :dart_stack),
|
104
|
+
rule(:dart_exc, /^Assertion failed/, :dart_stack),
|
105
|
+
rule(:dart_exc, /^Cannot instantiate/, :dart_stack),
|
106
|
+
rule(:dart_exc, /^Reading static variable/, :dart_stack),
|
107
|
+
rule(:dart_exc, /^UnimplementedError/, :dart_stack),
|
108
|
+
rule(:dart_exc, /^Unsupported operation/, :dart_stack),
|
109
|
+
rule(:dart_exc, /^Concurrent modification/, :dart_stack),
|
110
|
+
rule(:dart_exc, /^Out of Memory/, :dart_stack),
|
111
|
+
rule(:dart_exc, /^Stack Overflow/, :dart_stack),
|
112
|
+
rule(:dart_exc, /^'.+?':.+?$/, :dart_type_err_1),
|
113
|
+
rule(:dart_type_err_1, /^#\d+\s+.+?\(.+?\)$/, :dart_stack),
|
114
|
+
rule(:dart_type_err_1, /^.+?$/, :dart_type_err_2),
|
115
|
+
rule(:dart_type_err_2, /^.*?\^.*?$/, :dart_type_err_3),
|
116
|
+
rule(:dart_type_err_3, /^$/, :dart_type_err_4),
|
117
|
+
rule(:dart_type_err_4, /^$/, :dart_stack),
|
118
|
+
rule(:dart_exc, /^FormatException/, :dart_format_err_1),
|
119
|
+
rule(:dart_format_err_1, /^#\d+\s+.+?\(.+?\)$/, :dart_stack),
|
120
|
+
rule(:dart_format_err_1, /^./, :dart_format_err_2),
|
121
|
+
rule(:dart_format_err_2, /^.*?\^/, :dart_format_err_3),
|
122
|
+
rule(:dart_format_err_3, /^$/, :dart_stack),
|
123
|
+
rule(:dart_exc, /^NoSuchMethodError:/, :dart_method_err_1),
|
124
|
+
rule(:dart_method_err_1, /^Receiver:/, :dart_method_err_2),
|
125
|
+
rule(:dart_method_err_2, /^Tried calling:/, :dart_method_err_3),
|
126
|
+
rule(:dart_method_err_3, /^Found:/, :dart_stack),
|
127
|
+
rule(:dart_method_err_3, /^#\d+\s+.+?\(.+?\)$/, :dart_stack),
|
128
|
+
rule(:dart_stack, /^#\d+\s+.+?\(.+?\)$/, :dart_stack),
|
129
|
+
rule(:dart_stack, /^<asynchronous suspension>$/, :dart_stack)
|
130
|
+
].freeze
|
131
|
+
|
132
|
+
ALL_RULES = (
|
133
|
+
JAVA_RULES + PYTHON_RULES + PHP_RULES + GO_RULES + RUBY_RULES + DART_RULES
|
134
|
+
).freeze
|
135
|
+
|
136
|
+
RULES_BY_LANG = {
|
137
|
+
java: JAVA_RULES,
|
138
|
+
javascript: JAVA_RULES,
|
139
|
+
js: JAVA_RULES,
|
140
|
+
csharp: JAVA_RULES,
|
141
|
+
py: PYTHON_RULES,
|
142
|
+
python: PYTHON_RULES,
|
143
|
+
php: PHP_RULES,
|
144
|
+
go: GO_RULES,
|
145
|
+
rb: RUBY_RULES,
|
146
|
+
ruby: RUBY_RULES,
|
147
|
+
dart: DART_RULES,
|
148
|
+
all: ALL_RULES
|
149
|
+
}.freeze
|
150
|
+
|
151
|
+
DEFAULT_FIELDS = %w(message log).freeze
|
152
|
+
end
|
153
|
+
|
154
|
+
# State machine that consumes individual log lines and detects
|
155
|
+
# multi-line stack traces.
|
156
|
+
class ExceptionDetector
|
157
|
+
def initialize(*languages)
|
158
|
+
@state = :start_state
|
159
|
+
@rules = Hash.new { |h, k| h[k] = [] }
|
160
|
+
|
161
|
+
languages = [:all] if languages.empty?
|
162
|
+
|
163
|
+
languages.each do |lang|
|
164
|
+
rule_config =
|
165
|
+
ExceptionDetectorConfig::RULES_BY_LANG.fetch(lang.downcase) do |_k|
|
166
|
+
raise ArgumentError, "Unknown language: #{lang}"
|
167
|
+
end
|
168
|
+
|
169
|
+
rule_config.each do |r|
|
170
|
+
target = ExceptionDetectorConfig::RuleTarget.new(r[:pattern],
|
171
|
+
r[:to_state])
|
172
|
+
@rules[r[:from_state]] << target
|
173
|
+
end
|
174
|
+
end
|
175
|
+
|
176
|
+
@rules.each_value(&:uniq!)
|
177
|
+
end
|
178
|
+
|
179
|
+
# Updates the state machine and returns the trace detection status:
|
180
|
+
# - no_trace: 'line' does not belong to an exception trace,
|
181
|
+
# - start_trace: 'line' starts a detected exception trace,
|
182
|
+
# - inside: 'line' is part of a detected exception trace,
|
183
|
+
# - end: the detected exception trace ends after 'line'.
|
184
|
+
def update(line)
|
185
|
+
trace_seen_before = transition(line)
|
186
|
+
# If the state machine fell back to the start state because there is no
|
187
|
+
# defined transition for 'line', trigger another state transition because
|
188
|
+
# 'line' may contain the beginning of another exception.
|
189
|
+
transition(line) unless trace_seen_before
|
190
|
+
new_state = @state
|
191
|
+
trace_seen_after = new_state != :start_state
|
192
|
+
|
193
|
+
case [trace_seen_before, trace_seen_after]
|
194
|
+
when [true, true]
|
195
|
+
:inside_trace
|
196
|
+
when [true, false]
|
197
|
+
:end_trace
|
198
|
+
when [false, true]
|
199
|
+
:start_trace
|
200
|
+
else
|
201
|
+
:no_trace
|
202
|
+
end
|
203
|
+
end
|
204
|
+
|
205
|
+
def reset
|
206
|
+
@state = :start_state
|
207
|
+
end
|
208
|
+
|
209
|
+
private
|
210
|
+
|
211
|
+
# Executes a transition of the state machine for the given line.
|
212
|
+
# Returns false if the line does not match any transition rule and the
|
213
|
+
# state machine was reset to the initial state.
|
214
|
+
def transition(line)
|
215
|
+
@rules[@state].each do |r|
|
216
|
+
next unless line =~ r.pattern
|
217
|
+
@state = r.to_state
|
218
|
+
return true
|
219
|
+
end
|
220
|
+
@state = :start_state
|
221
|
+
false
|
222
|
+
end
|
223
|
+
end
|
224
|
+
|
225
|
+
# Buffers and groups log records if they contain exception stack traces.
|
226
|
+
class TraceAccumulator
|
227
|
+
attr_reader :buffer_start_time
|
228
|
+
|
229
|
+
# If message_field is nil, the instance is set up to accumulate
|
230
|
+
# records that are plain strings (i.e. the whole record is concatenated).
|
231
|
+
# Otherwise, the instance accepts records that are dictionaries (usually
|
232
|
+
# originating from structured JSON logs) and accumulates just the
|
233
|
+
# content of the given message field.
|
234
|
+
# message_field may contain the empty string. In this case, the
|
235
|
+
# TraceAccumulator 'learns' the field name from the first record by checking
|
236
|
+
# for some pre-defined common field names of text logs.
|
237
|
+
# The named parameters max_lines and max_bytes limit the maximum amount
|
238
|
+
# of data to be buffered. The default value 0 indicates 'no limit'.
|
239
|
+
def initialize(message_field, languages, max_lines: 0, max_bytes: 0,
|
240
|
+
&emit_callback)
|
241
|
+
@exception_detector = Fluent::ExceptionDetector.new(*languages)
|
242
|
+
@max_lines = max_lines
|
243
|
+
@max_bytes = max_bytes
|
244
|
+
@message_field = message_field
|
245
|
+
@messages = []
|
246
|
+
@buffer_start_time = Time.now
|
247
|
+
@buffer_size = 0
|
248
|
+
@first_record = nil
|
249
|
+
@first_timestamp = nil
|
250
|
+
@emit = emit_callback
|
251
|
+
end
|
252
|
+
|
253
|
+
def push(time_sec, record)
|
254
|
+
message = extract_message(record)
|
255
|
+
if message.nil?
|
256
|
+
@exception_detector.reset
|
257
|
+
detection_status = :no_trace
|
258
|
+
else
|
259
|
+
force_flush if @max_bytes > 0 &&
|
260
|
+
@buffer_size + message.length > @max_bytes
|
261
|
+
detection_status = @exception_detector.update(message)
|
262
|
+
end
|
263
|
+
|
264
|
+
update_buffer(detection_status, time_sec, record, message)
|
265
|
+
|
266
|
+
force_flush if @max_lines > 0 && @messages.length == @max_lines
|
267
|
+
end
|
268
|
+
|
269
|
+
def flush
|
270
|
+
case @messages.length
|
271
|
+
when 0
|
272
|
+
return
|
273
|
+
when 1
|
274
|
+
@emit.call(@first_timestamp, @first_record)
|
275
|
+
else
|
276
|
+
combined_message = @messages.each_with_object([]) do |line, memo|
|
277
|
+
memo << LINE_SEPARATOR unless memo.empty? || memo[-1].end_with?("\n")
|
278
|
+
memo << line
|
279
|
+
end.join
|
280
|
+
if @message_field.nil?
|
281
|
+
output_record = combined_message
|
282
|
+
else
|
283
|
+
output_record = @first_record
|
284
|
+
output_record[@message_field] = combined_message
|
285
|
+
end
|
286
|
+
@emit.call(@first_timestamp, output_record)
|
287
|
+
end
|
288
|
+
@messages = []
|
289
|
+
@first_record = nil
|
290
|
+
@first_timestamp = nil
|
291
|
+
@buffer_size = 0
|
292
|
+
end
|
293
|
+
|
294
|
+
def force_flush
|
295
|
+
flush
|
296
|
+
@exception_detector.reset
|
297
|
+
end
|
298
|
+
|
299
|
+
private
|
300
|
+
|
301
|
+
def extract_message(record)
|
302
|
+
if !@message_field.nil? && @message_field.empty?
|
303
|
+
ExceptionDetectorConfig::DEFAULT_FIELDS.each do |f|
|
304
|
+
if record.key?(f)
|
305
|
+
@message_field = f
|
306
|
+
break
|
307
|
+
end
|
308
|
+
end
|
309
|
+
end
|
310
|
+
@message_field.nil? ? record : record[@message_field]
|
311
|
+
end
|
312
|
+
|
313
|
+
def update_buffer(detection_status, time_sec, record, message)
|
314
|
+
trigger_emit = detection_status == :no_trace ||
|
315
|
+
detection_status == :end_trace
|
316
|
+
if @messages.empty? && trigger_emit
|
317
|
+
@emit.call(time_sec, record)
|
318
|
+
return
|
319
|
+
end
|
320
|
+
|
321
|
+
case detection_status
|
322
|
+
when :inside_trace
|
323
|
+
add(time_sec, record, message)
|
324
|
+
when :end_trace
|
325
|
+
add(time_sec, record, message)
|
326
|
+
flush
|
327
|
+
when :no_trace
|
328
|
+
flush
|
329
|
+
add(time_sec, record, message)
|
330
|
+
flush
|
331
|
+
when :start_trace
|
332
|
+
flush
|
333
|
+
add(time_sec, record, message)
|
334
|
+
end
|
335
|
+
end
|
336
|
+
|
337
|
+
def add(time_sec, record, message)
|
338
|
+
if @messages.empty?
|
339
|
+
@first_record = record
|
340
|
+
@first_timestamp = time_sec
|
341
|
+
@buffer_start_time = Time.now
|
342
|
+
end
|
343
|
+
unless message.nil?
|
344
|
+
@messages << message
|
345
|
+
@buffer_size += message.length
|
346
|
+
end
|
347
|
+
end
|
348
|
+
end
|
349
|
+
end
|