fluent-plugin-parser-cloudfoundry-syslog 0.1.1

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: 7ff07c83db3eb09a0d7558f467c23d61456138319964217939151c66356594b0
4
+ data.tar.gz: 94ee8161810875c519bc36dbbe67664cf56684e81dbf103e21a97e18f09a1739
5
+ SHA512:
6
+ metadata.gz: 5eeee36d0cf7bd24922d0633fee62d1aa185639031db03834dc73b33465a52c65ed4f306d013733fe05112572cbc7c112ea312c11da65e4accb148cd616720f5
7
+ data.tar.gz: 754d39b3d2e360877ac58aa713d7978868a80aa6832f2fa557fc29f9a297b5dea6bd501bb1954dea44887f932527f6f6c142281e2be0353d3f46da1383b1b207
@@ -0,0 +1,10 @@
1
+ FROM ruby:3.0.3-bullseye
2
+
3
+ RUN useradd -ms /bin/bash vscode \
4
+ && usermod -aG sudo vscode
5
+
6
+ RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
7
+
8
+ USER vscode
9
+ RUN gem install solargraph
10
+ RUN gem install rufo
@@ -0,0 +1,13 @@
1
+ {
2
+ "name": "Fluent Plugin Parser CloudFoundry",
3
+ "build": {
4
+ "dockerfile": "Dockerfile",
5
+ },
6
+ "extensions": [
7
+ "rebornix.Ruby",
8
+ "castwide.solargraph",
9
+ "jnbt.vscode-rufo"
10
+ ],
11
+ "remoteUser": "vscode",
12
+ "postCreateCommand": "bash -i .devcontainer/post-create.sh",
13
+ }
@@ -0,0 +1,6 @@
1
+ #!/bin/bash
2
+
3
+ set -e
4
+
5
+ gem install bundler
6
+ bundle install
@@ -0,0 +1,26 @@
1
+ name: Build and Test
2
+ on:
3
+ - push
4
+ - pull_request
5
+ jobs:
6
+ build:
7
+ runs-on: ${{ matrix.os }}
8
+ strategy:
9
+ fail-fast: false
10
+ matrix:
11
+ ruby: [ '2.6', '2.7', '3.0' ]
12
+ os:
13
+ - ubuntu-latest
14
+ name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
15
+ steps:
16
+ - uses: actions/checkout@v2
17
+ - uses: ruby/setup-ruby@v1
18
+ with:
19
+ ruby-version: ${{ matrix.ruby }}
20
+ - name: unit testing
21
+ env:
22
+ CI: true
23
+ run: |
24
+ gem install bundler rake
25
+ bundle install --jobs 4 --retry 3
26
+ bundle exec rake test
@@ -0,0 +1,23 @@
1
+ name: Publish Gem
2
+
3
+ on:
4
+ release:
5
+ types:
6
+ - created
7
+
8
+ jobs:
9
+ publish:
10
+ if: github.repository_owner == 'bitpatty'
11
+ runs-on: ubuntu-latest
12
+ environment: package-registries
13
+ permissions:
14
+ contents: read
15
+ packages: write
16
+ steps:
17
+ - name: Checkout
18
+ uses: actions/checkout@v2
19
+ - name: Publish gem
20
+ uses: dawidd6/action-publish-gem@v1
21
+ with:
22
+ api_key: ${{ secrets.RUBYGEMS_API_KEY }}
23
+ github_token: ${{ secrets.GITHUB_TOKEN }}
data/.gitignore ADDED
@@ -0,0 +1 @@
1
+ *.lock
@@ -0,0 +1,41 @@
1
+ {
2
+ "files.autoSave": "off",
3
+ "breadcrumbs.symbolSortOrder": "type",
4
+ "editor.codeLens": true,
5
+ "editor.detectIndentation": true,
6
+ "editor.formatOnSave": true,
7
+ "editor.minimap.maxColumn": 150,
8
+ "editor.tabSize": 2,
9
+ "explorer.confirmDragAndDrop": false,
10
+ "files.associations": {
11
+ "*.erb": "html",
12
+ "*.html.erb": "html"
13
+ },
14
+ "git.confirmSync": false,
15
+ "git.enableSmartCommit": true,
16
+ "html.format.wrapLineLength": 150,
17
+ "javascript.updateImportsOnFileMove.enabled": "always",
18
+ "search.exclude": {
19
+ "**/*.eot": true,
20
+ "**/*.png": true,
21
+ "**/*.svg": true,
22
+ "**/*.ttf": true,
23
+ "**/*.woff": true,
24
+ "**/*.woff2": true,
25
+ "**/.git": true,
26
+ "**/bower_components": true,
27
+ "**/dist/": true,
28
+ "**/node_modules": true,
29
+ "**/tmp": true
30
+ },
31
+ "todohighlight.keywords": [
32
+ "@TODO"
33
+ ],
34
+ "typescript.preferences.importModuleSpecifier": "relative",
35
+ "typescript.referencesCodeLens.enabled": true,
36
+ "typescript.referencesCodeLens.showOnAllFunctions": true,
37
+ "typescript.reportStyleChecksAsWarnings": true,
38
+ "typescript.updateImportsOnFileMove.enabled": "always",
39
+ "workbench.editor.enablePreview": false,
40
+ "workbench.editor.enablePreviewFromQuickOpen": false
41
+ }
data/Gemfile ADDED
@@ -0,0 +1,3 @@
1
+ source "https://rubygems.org"
2
+
3
+ gemspec
data/LICENSE ADDED
@@ -0,0 +1,202 @@
1
+
2
+ Apache License
3
+ Version 2.0, January 2004
4
+ http://www.apache.org/licenses/
5
+
6
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
7
+
8
+ 1. Definitions.
9
+
10
+ "License" shall mean the terms and conditions for use, reproduction,
11
+ and distribution as defined by Sections 1 through 9 of this document.
12
+
13
+ "Licensor" shall mean the copyright owner or entity authorized by
14
+ the copyright owner that is granting the License.
15
+
16
+ "Legal Entity" shall mean the union of the acting entity and all
17
+ other entities that control, are controlled by, or are under common
18
+ control with that entity. For the purposes of this definition,
19
+ "control" means (i) the power, direct or indirect, to cause the
20
+ direction or management of such entity, whether by contract or
21
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
22
+ outstanding shares, or (iii) beneficial ownership of such entity.
23
+
24
+ "You" (or "Your") shall mean an individual or Legal Entity
25
+ exercising permissions granted by this License.
26
+
27
+ "Source" form shall mean the preferred form for making modifications,
28
+ including but not limited to software source code, documentation
29
+ source, and configuration files.
30
+
31
+ "Object" form shall mean any form resulting from mechanical
32
+ transformation or translation of a Source form, including but
33
+ not limited to compiled object code, generated documentation,
34
+ and conversions to other media types.
35
+
36
+ "Work" shall mean the work of authorship, whether in Source or
37
+ Object form, made available under the License, as indicated by a
38
+ copyright notice that is included in or attached to the work
39
+ (an example is provided in the Appendix below).
40
+
41
+ "Derivative Works" shall mean any work, whether in Source or Object
42
+ form, that is based on (or derived from) the Work and for which the
43
+ editorial revisions, annotations, elaborations, or other modifications
44
+ represent, as a whole, an original work of authorship. For the purposes
45
+ of this License, Derivative Works shall not include works that remain
46
+ separable from, or merely link (or bind by name) to the interfaces of,
47
+ the Work and Derivative Works thereof.
48
+
49
+ "Contribution" shall mean any work of authorship, including
50
+ the original version of the Work and any modifications or additions
51
+ to that Work or Derivative Works thereof, that is intentionally
52
+ submitted to Licensor for inclusion in the Work by the copyright owner
53
+ or by an individual or Legal Entity authorized to submit on behalf of
54
+ the copyright owner. For the purposes of this definition, "submitted"
55
+ means any form of electronic, verbal, or written communication sent
56
+ to the Licensor or its representatives, including but not limited to
57
+ communication on electronic mailing lists, source code control systems,
58
+ and issue tracking systems that are managed by, or on behalf of, the
59
+ Licensor for the purpose of discussing and improving the Work, but
60
+ excluding communication that is conspicuously marked or otherwise
61
+ designated in writing by the copyright owner as "Not a Contribution."
62
+
63
+ "Contributor" shall mean Licensor and any individual or Legal Entity
64
+ on behalf of whom a Contribution has been received by Licensor and
65
+ subsequently incorporated within the Work.
66
+
67
+ 2. Grant of Copyright License. Subject to the terms and conditions of
68
+ this License, each Contributor hereby grants to You a perpetual,
69
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
70
+ copyright license to reproduce, prepare Derivative Works of,
71
+ publicly display, publicly perform, sublicense, and distribute the
72
+ Work and such Derivative Works in Source or Object form.
73
+
74
+ 3. Grant of Patent License. Subject to the terms and conditions of
75
+ this License, each Contributor hereby grants to You a perpetual,
76
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
77
+ (except as stated in this section) patent license to make, have made,
78
+ use, offer to sell, sell, import, and otherwise transfer the Work,
79
+ where such license applies only to those patent claims licensable
80
+ by such Contributor that are necessarily infringed by their
81
+ Contribution(s) alone or by combination of their Contribution(s)
82
+ with the Work to which such Contribution(s) was submitted. If You
83
+ institute patent litigation against any entity (including a
84
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
85
+ or a Contribution incorporated within the Work constitutes direct
86
+ or contributory patent infringement, then any patent licenses
87
+ granted to You under this License for that Work shall terminate
88
+ as of the date such litigation is filed.
89
+
90
+ 4. Redistribution. You may reproduce and distribute copies of the
91
+ Work or Derivative Works thereof in any medium, with or without
92
+ modifications, and in Source or Object form, provided that You
93
+ meet the following conditions:
94
+
95
+ (a) You must give any other recipients of the Work or
96
+ Derivative Works a copy of this License; and
97
+
98
+ (b) You must cause any modified files to carry prominent notices
99
+ stating that You changed the files; and
100
+
101
+ (c) You must retain, in the Source form of any Derivative Works
102
+ that You distribute, all copyright, patent, trademark, and
103
+ attribution notices from the Source form of the Work,
104
+ excluding those notices that do not pertain to any part of
105
+ the Derivative Works; and
106
+
107
+ (d) If the Work includes a "NOTICE" text file as part of its
108
+ distribution, then any Derivative Works that You distribute must
109
+ include a readable copy of the attribution notices contained
110
+ within such NOTICE file, excluding those notices that do not
111
+ pertain to any part of the Derivative Works, in at least one
112
+ of the following places: within a NOTICE text file distributed
113
+ as part of the Derivative Works; within the Source form or
114
+ documentation, if provided along with the Derivative Works; or,
115
+ within a display generated by the Derivative Works, if and
116
+ wherever such third-party notices normally appear. The contents
117
+ of the NOTICE file are for informational purposes only and
118
+ do not modify the License. You may add Your own attribution
119
+ notices within Derivative Works that You distribute, alongside
120
+ or as an addendum to the NOTICE text from the Work, provided
121
+ that such additional attribution notices cannot be construed
122
+ as modifying the License.
123
+
124
+ You may add Your own copyright statement to Your modifications and
125
+ may provide additional or different license terms and conditions
126
+ for use, reproduction, or distribution of Your modifications, or
127
+ for any such Derivative Works as a whole, provided Your use,
128
+ reproduction, and distribution of the Work otherwise complies with
129
+ the conditions stated in this License.
130
+
131
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
132
+ any Contribution intentionally submitted for inclusion in the Work
133
+ by You to the Licensor shall be under the terms and conditions of
134
+ this License, without any additional terms or conditions.
135
+ Notwithstanding the above, nothing herein shall supersede or modify
136
+ the terms of any separate license agreement you may have executed
137
+ with Licensor regarding such Contributions.
138
+
139
+ 6. Trademarks. This License does not grant permission to use the trade
140
+ names, trademarks, service marks, or product names of the Licensor,
141
+ except as required for reasonable and customary use in describing the
142
+ origin of the Work and reproducing the content of the NOTICE file.
143
+
144
+ 7. Disclaimer of Warranty. Unless required by applicable law or
145
+ agreed to in writing, Licensor provides the Work (and each
146
+ Contributor provides its Contributions) on an "AS IS" BASIS,
147
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
148
+ implied, including, without limitation, any warranties or conditions
149
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
150
+ PARTICULAR PURPOSE. You are solely responsible for determining the
151
+ appropriateness of using or redistributing the Work and assume any
152
+ risks associated with Your exercise of permissions under this License.
153
+
154
+ 8. Limitation of Liability. In no event and under no legal theory,
155
+ whether in tort (including negligence), contract, or otherwise,
156
+ unless required by applicable law (such as deliberate and grossly
157
+ negligent acts) or agreed to in writing, shall any Contributor be
158
+ liable to You for damages, including any direct, indirect, special,
159
+ incidental, or consequential damages of any character arising as a
160
+ result of this License or out of the use or inability to use the
161
+ Work (including but not limited to damages for loss of goodwill,
162
+ work stoppage, computer failure or malfunction, or any and all
163
+ other commercial damages or losses), even if such Contributor
164
+ has been advised of the possibility of such damages.
165
+
166
+ 9. Accepting Warranty or Additional Liability. While redistributing
167
+ the Work or Derivative Works thereof, You may choose to offer,
168
+ and charge a fee for, acceptance of support, warranty, indemnity,
169
+ or other liability obligations and/or rights consistent with this
170
+ License. However, in accepting such obligations, You may act only
171
+ on Your own behalf and on Your sole responsibility, not on behalf
172
+ of any other Contributor, and only if You agree to indemnify,
173
+ defend, and hold each Contributor harmless for any liability
174
+ incurred by, or claims asserted against, such Contributor by reason
175
+ of your accepting any such warranty or additional liability.
176
+
177
+ END OF TERMS AND CONDITIONS
178
+
179
+ APPENDIX: How to apply the Apache License to your work.
180
+
181
+ To apply the Apache License to your work, attach the following
182
+ boilerplate notice, with the fields enclosed by brackets "[]"
183
+ replaced with your own identifying information. (Don't include
184
+ the brackets!) The text should be enclosed in the appropriate
185
+ comment syntax for the file format. We also recommend that a
186
+ file or class name and description of purpose be included on the
187
+ same "printed page" as the copyright notice for easier
188
+ identification within third-party archives.
189
+
190
+ Copyright 2011-2018 Fluentd Authors
191
+
192
+ Licensed under the Apache License, Version 2.0 (the "License");
193
+ you may not use this file except in compliance with the License.
194
+ You may obtain a copy of the License at
195
+
196
+ http://www.apache.org/licenses/LICENSE-2.0
197
+
198
+ Unless required by applicable law or agreed to in writing, software
199
+ distributed under the License is distributed on an "AS IS" BASIS,
200
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
201
+ See the License for the specific language governing permissions and
202
+ limitations under the License.
data/README.md ADDED
@@ -0,0 +1,123 @@
1
+ # fluent-plugin-parser-cloudfoundry-syslog
2
+
3
+ A experimental, work-in-progress [fluentd](https://www.fluentd.org/) parser for CloudFoundry specific syslog drains - basically the opposite end of their [output formatter](https://github.com/cloudfoundry/fluent-plugin-syslog_rfc5424).
4
+
5
+ This plugin should make CF metrics more accessible than the current [syslog parser for fluent](https://docs.fluentd.org/parser/syslog) allows for.
6
+
7
+
8
+ ## Sample
9
+
10
+ An access log in the format
11
+
12
+ ```
13
+ <14>1 2021-12-24T22:20:01.438069+00:00 some-hostname some-appname [RTR/0] - [tags@47450 __v1_type="LogMessage" app_id="some-app-id" app_name="some-appname" component="route-emitter" deployment="eu-gb-prod" index="some-index" instance_id="0" ip="some-ip" job="router" organization_id="some-org-id" organization_name="some-org-name" origin="gorouter" process_id="some-process-id" process_instance_id="some-process-instance-id" process_type="web" source_type="RTR" space_id="some-space-id" space_name="dev"] example.com - [2021-12-24T22:20:01.429164095Z] "GET /styles.css HTTP/1.1" 304 0 0 "https://example.com/" "Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:95.0) Gecko/20100101 Firefox/95.0" "some-remote-host" "some-backend-host" x_forwarded_for:"a, b" x_forwarded_proto:"https" vcap_request_id:"some-request-id" response_time:0.008452 gorouter_time:0.000625 app_id:"some-app-id" app_index:"0" instance_id:"some-instance-id" x_cf_routererror:"-" x_global_transaction_id:"some-global-transaction-id" true_client_ip:"-" x_b3_traceid:"some-trace-id" x_b3_spanid:"some-span-id" x_b3_parentspanid:"-" b3:"some-b3"
14
+ ```
15
+
16
+ .. is turned into ..
17
+
18
+ ```ruby
19
+ {
20
+ "header" => {
21
+ "pri" => {
22
+ "facility" => 1,
23
+ "severity" => "info",
24
+ },
25
+ "version" => 1,
26
+ "timestamp" => "2021-12-24T22:20:01.438069+00:00",
27
+ "hostname" => "some-hostname",
28
+ "app_name" => "some-appname",
29
+ "proc_id" => "[RTR/0]",
30
+ "msg_id" => "-",
31
+ },
32
+ "sd" => {
33
+ "tags@47450" => {
34
+ "__v1_type" => "LogMessage",
35
+ "app_id" => "some-app-id",
36
+ "app_name" => "some-appname",
37
+ "component" => "route-emitter",
38
+ "deployment" => "eu-gb-prod",
39
+ "index" => "some-index",
40
+ "instance_id" => "0",
41
+ "ip" => "some-ip",
42
+ "job" => "router",
43
+ "organization_id" => "some-org-id",
44
+ "organization_name" => "some-org-name",
45
+ "origin" => "gorouter",
46
+ "process_id" => "some-process-id",
47
+ "process_instance_id" => "some-process-instance-id",
48
+ "process_type" => "web",
49
+ "source_type" => "RTR",
50
+ "space_id" => "some-space-id",
51
+ "space_name" => "dev",
52
+ },
53
+ },
54
+ "gorouter" => {
55
+ "host" => "example.com",
56
+ "timestamp" => "2021-12-24T22:20:01.429164095Z",
57
+ "method" => "GET",
58
+ "pathname" => "/styles.css",
59
+ "protocol" => "HTTP/1.1",
60
+ "status" => "304",
61
+ "bytes_received" => "0",
62
+ "bytes_sent" => "0",
63
+ "referer" => "https://example.com/",
64
+ "user_agent" => "Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:95.0) Gecko/20100101 Firefox/95.0",
65
+ "remote_address" => "some-remote-host",
66
+ "backend_address" => "some-backend-host",
67
+ "x_forwarded_for" => "a, b",
68
+ "x_forwarded_proto" => "https",
69
+ "vcap_request_id" => "some-request-id",
70
+ "response_time" => 0.008452,
71
+ "gorouter_time" => 0.000625,
72
+ "app_id" => "some-app-id",
73
+ "app_index" => "0",
74
+ "instance_id" => "some-instance-id",
75
+ "x_cf_routererror" => "-",
76
+ "x_global_transaction_id" => "some-global-transaction-id",
77
+ "true_client_ip" => "-",
78
+ "x_b3_traceid" => "some-trace-id",
79
+ "x_b3_spanid" => "some-span-id",
80
+ "x_b3_parentspanid" => "-",
81
+ "b3" => "some-b3",
82
+ },
83
+ "message" => 'example.com - [2021-12-24T22:20:01.429164095Z] "GET /styles.css HTTP/1.1" 304 0 0 "https://example.com/" "Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:95.0) Gecko/20100101 Firefox/95.0" "some-remote-host" "some-backend-host" x_forwarded_for:"a, b" x_forwarded_proto:"https" vcap_request_id:"some-request-id" response_time:0.008452 gorouter_time:0.000625 app_id:"some-app-id" app_index:"0" instance_id:"some-instance-id" x_cf_routererror:"-" x_global_transaction_id:"some-global-transaction-id" true_client_ip:"-" x_b3_traceid:"some-trace-id" x_b3_spanid:"some-span-id" x_b3_parentspanid:"-" b3:"some-b3"',
84
+ }
85
+ ```
86
+
87
+ ## Usage
88
+
89
+ Create a logdrain and update your fluent configuration:
90
+
91
+ ```conf
92
+ <source>
93
+ @type http
94
+
95
+ # HTTP Ports are provided by the CF environment
96
+ port "#{ENV['PORT']}"
97
+ bind 0.0.0.0
98
+
99
+ body_size_limit 32m
100
+ keepalive_timeout 10s
101
+
102
+ <parse>
103
+ @type cloudfoundry_syslog
104
+
105
+ # Set this to true if access log messages should be parsed.
106
+ # Defaults to false
107
+ parse_gorouter_access_log true
108
+
109
+ # Set this to true if you want the raw message to be available
110
+ # under the key `raw`. Defaults to false
111
+ include_raw_message true
112
+ </parse>
113
+ </source>
114
+ ```
115
+
116
+ ## Limitations
117
+
118
+ - Values in `STRUCTURED-DATA`, such as app names, may not contain quotes since they're not being escaped on CloudFoundry's side. See https://github.com/cloudfoundry/loggregator-agent-release/issues/69
119
+
120
+ ## Credit
121
+
122
+ - [fluent-plugin-elasticsearch](https://github.com/uken/fluent-plugin-elasticsearch) used as reference for boilerplating the codebase and GH workflows
123
+ - [fluentd/parser_syslog](https://github.com/fluent/fluentd/blob/master/lib/fluent/plugin/parser_syslog.rb) used as reference on the current builtin syslog parser
data/Rakefile ADDED
@@ -0,0 +1,13 @@
1
+ require "bundler"
2
+ Bundler::GemHelper.install_tasks
3
+
4
+ require "rake/testtask"
5
+
6
+ Rake::TestTask.new(:test) do |t|
7
+ t.libs.push("lib", "test")
8
+ t.test_files = FileList["test/**/test_*.rb"]
9
+ t.verbose = true
10
+ t.warning = true
11
+ end
12
+
13
+ task default: [:test]
@@ -0,0 +1,28 @@
1
+ lib = File.expand_path("../lib", __FILE__)
2
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
3
+
4
+ Gem::Specification.new do |spec|
5
+ spec.name = "fluent-plugin-parser-cloudfoundry-syslog"
6
+ spec.version = "0.1.1"
7
+ spec.authors = ["Matteias Collet"]
8
+ spec.email = ["matteias.collet@bluewin.ch"]
9
+
10
+ spec.summary = %q{CloudFoundry log parser for Fluentd}
11
+ spec.description = spec.summary
12
+ spec.homepage = "https://github.com/bitpatty/fluent-plugin-parser-cloudfoundry-syslog"
13
+ spec.license = "Apache-2.0"
14
+
15
+ test_files, files = `git ls-files -z`.split("\x0").partition do |f|
16
+ f.match(%r{^(test|spec|features)/})
17
+ end
18
+
19
+ spec.files = files
20
+ spec.executables = files.grep(%r{^bin/}) { |f| File.basename(f) }
21
+ spec.test_files = test_files
22
+ spec.require_paths = ["lib"]
23
+
24
+ spec.add_development_dependency "bundler", "~> 2.1"
25
+ spec.add_development_dependency "rake", "~> 13.0"
26
+ spec.add_development_dependency "test-unit", "~> 3.3"
27
+ spec.add_runtime_dependency "fluentd", ">= 1"
28
+ end
@@ -0,0 +1,236 @@
1
+ require "fluent/plugin/parser"
2
+ require "fluent/time"
3
+
4
+ module Fluent
5
+ module Plugin
6
+ class CloudFoundrySyslogParser < Parser
7
+ Plugin.register_parser("cloudfoundry_syslog", self)
8
+
9
+ # https://www.iana.org/assignments/enterprise-numbers/enterprise-numbers
10
+ CF_IANA_ENTERPRISE_ID = 47450.freeze
11
+
12
+ # Syslog Constants https://datatracker.ietf.org/doc/html/rfc5424#section-6
13
+ SYSLOG_HEADER_SPLIT_CHAR = " ".freeze # See SP
14
+ SYSLOG_NILVALUE = "-".freeze # See NILVALUE
15
+ SYSLOG_PRI_DELIMITER_START = "<".freeze # See PRI
16
+ SYSLOG_PRI_DELIMITER_END = ">".freeze # See PRI
17
+ SYSLOG_SD_DELIMITER_START = "[".freeze # See STRUCTURED-DATA
18
+ SYSLOG_SD_DELIMITER_END = "]".freeze # See STRUCTURED-DATA
19
+
20
+ # https://datatracker.ietf.org/doc/html/rfc5424#section-6.2
21
+ # PRI is parsed separately (https://datatracker.ietf.org/doc/html/rfc5424#section-6.2.1)
22
+ SYSLOG_HEADER_FIELDS = [
23
+ "version", # https://datatracker.ietf.org/doc/html/rfc5424#section-6.2.2
24
+ "timestamp", # https://datatracker.ietf.org/doc/html/rfc5424#section-6.2.3
25
+ "hostname", # https://datatracker.ietf.org/doc/html/rfc5424#section-6.2.4
26
+ "app_name", # https://datatracker.ietf.org/doc/html/rfc5424#section-6.2.5
27
+ "proc_id", # https://datatracker.ietf.org/doc/html/rfc5424#section-6.2.6
28
+ "msg_id", # https://datatracker.ietf.org/doc/html/rfc5424#section-6.2.7
29
+ ]
30
+
31
+ # https://datatracker.ietf.org/doc/html/rfc5424#appendix-A.3
32
+ # https://resources.docs.pivotal.io/pdfs/tiledev-guide-2.1.pdf
33
+ SYSLOG_SEVERITY_CODES = [
34
+ "emergency",
35
+ "alert",
36
+ "critical",
37
+ "error",
38
+ "warning",
39
+ "notice",
40
+ "info",
41
+ "debug",
42
+ ]
43
+
44
+ # Regexes to extract information from STRUCTURED-DATA
45
+ # Matches a whole STRUCTURED-DATA block including the delimiters '[', ']'
46
+ SYSLOG_STRUCTURED_DATA_MATCH_REGEX = Regexp.new(/^(\[(?:[a-zA-Z_-]+(?:\@[0-9]+)?)*(?:[a-zA-Z0-9_-]+="(?:[^\\\]\"]|\\"|\\\]|\\\\|\\[^"\]\\])*"| )*\])/)
47
+ # Matches the SD-ID if applied to the SYSLOG_STRUCTURED_DATA_MATCH_REGEX match
48
+ SYSLOG_SD_ID_MATCH_REGEX = Regexp.new(/^\[([a-zA-Z0-9_-]+(?:\@[0-9]+)?)/)
49
+ # Matches an SD-PARAM (SD-NAME=SD-VALUE)
50
+ SYSLOG_SD_PARAM_MATCH_REGEX = Regexp.new(/([a-zA-Z0-9_-]+="(?:[^\\\]\"]|\\"|\\\]|\\\\|\\[^"\]\\])*")/)
51
+
52
+ # Regex to extract information from Gorouter access logs
53
+ # https://github.com/cloudfoundry/gorouter#access-logs
54
+ # <Request Host> - [<Start Date>] "<Request Method> <Request URL> <Request Protocol>" <Status Code> <Bytes Received> <Bytes Sent> "<Referer>" "<User-Agent>" <Remote Address> <Backend Address> x_forwarded_for:"<X-Forwarded-For>" x_forwarded_proto:"<X-Forwarded-Proto>" vcap_request_id:<X-Vcap-Request-ID> response_time:<Response Time> gorouter_time:<Gorouter Time> app_id:<Application ID> app_index:<Application Index> x_cf_routererror:<X-Cf-RouterError> <Extra Headers>
55
+ GOROUTER_MESSAGE_STATIC_REGEX = Regexp.new(/^(?<host>[^ ]+) - \[(?<timestamp>[^\]]+)\] "(?<method>[^ ]+) (?<pathname>[^ ]+) (?<protocol>[^"]+)" (?<status>[^ ]+) (?<bytes_received>[^ ]+) (?<bytes_sent>[^ ]+) "(?<referer>[^"]+)" "(?<user_agent>[^"]+)" "(?<remote_address>[^"]+)" "(?<backend_address>[^"]+)"/)
56
+ GOROUTER_MESSAGE_EXTRADATA_REGEX = Regexp.new(/(?<param>[a-zA-Z0-9_-]+):(?<value>(?:[0-9\.]+|"[^"]+"|-))(?: |\n|\\|$)/)
57
+
58
+ config_param :parse_gorouter_access_log, :bool, default: false
59
+ config_param :include_raw_message, :bool, default: false
60
+
61
+ @time_parser
62
+
63
+ def initialize
64
+ super
65
+ end
66
+
67
+ def configure(conf)
68
+ super
69
+ @time_parser = time_parser_create(format: "%Y-%m-%dT%H:%M:%S.%L%z")
70
+ end
71
+
72
+ def parse(text)
73
+ if text.nil? or not text.start_with?(SYSLOG_PRI_DELIMITER_START)
74
+ yield nil
75
+ return
76
+ end
77
+
78
+ cursor = 0
79
+ record = {}
80
+
81
+ if @include_raw_message
82
+ record["raw"] = text
83
+ end
84
+
85
+ # RFC 5424 currently only defines version 1
86
+ # https://datatracker.ietf.org/doc/html/rfc5424#section-6.2.2
87
+ record["header"], cursor = parse_header(text)
88
+ if cursor.nil? or record.dig("header", "version") != "1"
89
+ yield nil
90
+ return
91
+ end
92
+
93
+ # Convert to integer for convenience
94
+ record["header"]["version"] = 1
95
+
96
+ # https://datatracker.ietf.org/doc/html/rfc5424#section-6.2.3
97
+ time = @time_parser.parse(record["header"]["timestamp"]) rescue nil
98
+
99
+ if time.nil?
100
+ yield nil
101
+ return
102
+ end
103
+
104
+ # Parse STRUCTURED_DATA
105
+ record["sd"], cursor = parse_structured_data(text, cursor)
106
+ if (cursor.nil?)
107
+ yield nil
108
+ return
109
+ end
110
+
111
+ # Parse MESSAGE
112
+ msg = text.slice(cursor, text.length - cursor)
113
+
114
+ if msg.nil?
115
+ record["message"] = nil
116
+ else
117
+ record["message"] = msg.strip
118
+
119
+ if @parse_gorouter_access_log and
120
+ record.dig("sd", "tags@#{CF_IANA_ENTERPRISE_ID}", "origin") == "gorouter"
121
+ record["gorouter"] = parse_gorouter_access_logs(record["message"])
122
+ end
123
+ end
124
+
125
+ yield time, record
126
+ end
127
+
128
+ def parse_integer(str)
129
+ return Integer(str || "")
130
+ rescue ArgumentError
131
+ return
132
+ end
133
+
134
+ def parse_header_block(text, startIdx)
135
+ i = text.index(SYSLOG_HEADER_SPLIT_CHAR, startIdx)
136
+ if i.nil? or i - startIdx < 1 then return end
137
+ return text.slice(startIdx, i - startIdx), i + 1
138
+ end
139
+
140
+ # https://datatracker.ietf.org/doc/html/rfc5424#section-6.2.1
141
+ def parse_pri(text)
142
+ unless text.start_with?(SYSLOG_PRI_DELIMITER_START) then return end
143
+ endIdx = text.index(SYSLOG_PRI_DELIMITER_END, 1)
144
+ if endIdx.nil? or endIdx < 2 then return end
145
+ v_pri = parse_integer(text.slice(1, endIdx - 1))
146
+ if v_pri.nil? or v_pri < 0 then return end
147
+ return v_pri >> 3, SYSLOG_SEVERITY_CODES[v_pri & 0b111], endIdx
148
+ end
149
+
150
+ # https://datatracker.ietf.org/doc/html/rfc5424#section-6.2
151
+ def parse_header(text)
152
+ facility, severity, c = parse_pri(text)
153
+ if (c.nil?) then return end
154
+ c = c + 1
155
+
156
+ r = {
157
+ "pri" => {
158
+ "facility" => facility,
159
+ "severity" => severity,
160
+ },
161
+ }
162
+
163
+ SYSLOG_HEADER_FIELDS.each { |field|
164
+ block, endIdx = parse_header_block(text, c)
165
+ if block.nil? then return end
166
+ r[field] = block
167
+ c = endIdx
168
+ }
169
+
170
+ return r, c
171
+ end
172
+
173
+ # https://datatracker.ietf.org/doc/html/rfc5424#section-6.3.1
174
+ def parse_sd_element(sd_element)
175
+ sd_params = {}
176
+
177
+ # https://datatracker.ietf.org/doc/html/rfc5424#section-6.3.2
178
+ sd_id = sd_element[SYSLOG_SD_ID_MATCH_REGEX]
179
+ if sd_id.nil? then return end
180
+ sd_id = sd_id[1..-1]
181
+
182
+ # https://datatracker.ietf.org/doc/html/rfc5424#section-6.3.3
183
+ sd_element.scan(SYSLOG_SD_PARAM_MATCH_REGEX).each { |match|
184
+ arr = match[0].strip.split("=", 2)
185
+ sd_params[arr[0]] = arr[1][1..-2]
186
+ }
187
+
188
+ return sd_id, sd_params
189
+ end
190
+
191
+ # https://datatracker.ietf.org/doc/html/rfc5424#section-6.3
192
+ def parse_structured_data(text, startIdx)
193
+ if text[startIdx] == SYSLOG_NILVALUE then return {}, startIdx + 1 end
194
+ unless text[startIdx] == SYSLOG_SD_DELIMITER_START then return end
195
+
196
+ sd = text[startIdx..-1][SYSLOG_STRUCTURED_DATA_MATCH_REGEX]
197
+ if sd.nil? then return end
198
+
199
+ r = {}
200
+ len = 0
201
+ loop do
202
+ len += sd.length
203
+ sd_id, sd_params = parse_sd_element(sd)
204
+ if sd_id.nil? then return end
205
+ r[sd_id] = sd_params
206
+ sd = text[startIdx + len..-1][SYSLOG_STRUCTURED_DATA_MATCH_REGEX]
207
+ break if sd.nil?
208
+ end
209
+
210
+ return r, startIdx + len
211
+ end
212
+
213
+ # https://github.com/cloudfoundry/gorouter#access-logs
214
+ def parse_gorouter_access_logs(msg)
215
+ r = msg.match(GOROUTER_MESSAGE_STATIC_REGEX)
216
+ if r.nil? then return end
217
+ extra_headers = msg[r.to_s.length..-1]
218
+ r = r.named_captures
219
+ if extra_headers.nil? then return r end
220
+ extra_headers.strip.scan(GOROUTER_MESSAGE_EXTRADATA_REGEX) { |match|
221
+ unless match.length == 2 then next end
222
+ if match[1].start_with?('"') and match[1].end_with?('"')
223
+ # Strings
224
+ r[match[0]] = match[1][1..-2]
225
+ elsif match[1].match(/^[0-9]+(?:\.[0-9+]+)?$/)
226
+ # Numbers
227
+ r[match[0]] = match[1].to_f
228
+ else
229
+ r[match[0]] = match[1]
230
+ end
231
+ }
232
+ return r
233
+ end
234
+ end
235
+ end
236
+ end
@@ -0,0 +1,88 @@
1
+ require "test-unit"
2
+ require "fluent/test"
3
+ require "fluent/test/helpers"
4
+ require "fluent/test/driver/parser"
5
+ require "fluent/plugin/parser_cloudfoundry_syslog.rb"
6
+
7
+ class GorouterParserLog < Test::Unit::TestCase
8
+ include Fluent::Test::Helpers
9
+
10
+ def setup
11
+ Fluent::Test.setup
12
+ @parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::CloudFoundrySyslogParser)
13
+
14
+ @parser.configure({
15
+ "parse_gorouter_access_log" => true,
16
+ })
17
+ end
18
+
19
+ def test_a
20
+ log = %{
21
+ <14>1 2021-12-24T22:20:01.438069+00:00 some-hostname some-appname [RTR/0] -
22
+ [tags@47450
23
+ __v1_type="LogMessage"
24
+ app_id="some-app-id"
25
+ app_name="some-appname"
26
+ component="route-emitter"
27
+ deployment="eu-gb-prod"
28
+ index="some-index"
29
+ instance_id="0"
30
+ ip="some-ip"
31
+ job="router"
32
+ organization_id="some-org-id"
33
+ organization_name="some-org-name"
34
+ origin="gorouter"
35
+ process_id="some-process-id"
36
+ process_instance_id="some-process-instance-id"
37
+ process_type="web"
38
+ source_type="RTR"
39
+ space_id="some-space-id"
40
+ space_name="dev"]
41
+ example.com
42
+ -
43
+ [2021-12-24T22:20:01.429164095Z]
44
+ "GET /styles.css HTTP/1.1"
45
+ 304
46
+ 0
47
+ 0
48
+ "https://example.com/"
49
+ "Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:95.0) Gecko/20100101 Firefox/95.0"
50
+ "some-remote-host"
51
+ "some-backend-host"
52
+ x_forwarded_for:"a, b"
53
+ x_forwarded_proto:"https"
54
+ vcap_request_id:"some-request-id"
55
+ response_time:0.008452
56
+ gorouter_time:0.000625
57
+ app_id:"some-app-id"
58
+ app_index:"0"
59
+ instance_id:"some-instance-id"
60
+ x_cf_routererror:"-"
61
+ x_global_transaction_id:"some-global-transaction-id"
62
+ true_client_ip:"-"
63
+ x_b3_traceid:"some-trace-id"
64
+ x_b3_spanid:"some-span-id"
65
+ x_b3_parentspanid:"-"
66
+ b3:"some-b3"
67
+ }.gsub(/\s+/, " ").strip
68
+ @parser.instance.parse(log) { |time, record|
69
+ assert_not_nil(time)
70
+ assert_not_nil(record)
71
+ assert_not_nil(record["gorouter"])
72
+ assert_equal("LogMessage", record.dig("sd", "tags@47450", "__v1_type"))
73
+ assert_equal("2021-12-24T22:20:01.429164095Z", record.dig("gorouter", "timestamp"))
74
+ assert_equal("GET", record.dig("gorouter", "method"))
75
+ assert_equal("/styles.css", record.dig("gorouter", "pathname"))
76
+ assert_equal("HTTP/1.1", record.dig("gorouter", "protocol"))
77
+ assert_equal("304", record.dig("gorouter", "status"))
78
+ assert_equal("0", record.dig("gorouter", "bytes_received"))
79
+ assert_equal("0", record.dig("gorouter", "bytes_sent"))
80
+ assert_equal("https://example.com/", record.dig("gorouter", "referer"))
81
+ assert_equal("Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:95.0) Gecko/20100101 Firefox/95.0", record.dig("gorouter", "user_agent"))
82
+ assert_equal("some-remote-host", record.dig("gorouter", "remote_address"))
83
+ assert_equal("some-backend-host", record.dig("gorouter", "backend_address"))
84
+ assert_equal("a, b", record.dig("gorouter", "x_forwarded_for"))
85
+ assert_equal(0.000625, record.dig("gorouter", "gorouter_time"))
86
+ }
87
+ end
88
+ end
@@ -0,0 +1,216 @@
1
+ require "test-unit"
2
+ require "fluent/test"
3
+ require "fluent/test/helpers"
4
+ require "fluent/test/driver/parser"
5
+ require "fluent/plugin/parser_cloudfoundry_syslog.rb"
6
+
7
+ class SyslogRFCTest < Test::Unit::TestCase
8
+ include Fluent::Test::Helpers
9
+
10
+ def setup
11
+ Fluent::Test.setup
12
+ @parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::CloudFoundrySyslogParser)
13
+ @parser.configure({})
14
+ end
15
+
16
+ def test_nil
17
+ @parser.instance.parse(nil) { |time, record|
18
+ assert_nil(time)
19
+ assert_nil(record)
20
+ }
21
+ end
22
+
23
+ def test_empty_string
24
+ @parser.instance.parse("") { |time, record|
25
+ assert_nil(time)
26
+ assert_nil(record)
27
+ }
28
+ end
29
+
30
+ def test_words
31
+ @parser.instance.parse("foo bar") { |time, record|
32
+ assert_nil(time)
33
+ assert_nil(record)
34
+ }
35
+ end
36
+
37
+ def test_parse_syslog
38
+ log = '<13>1 1985-04-12T23:20:50.52Z some-hostname some-appname some-procid some-msgid [instance@47450 paramA="def123" paramB="j k l" paramC=""] some foo bar'
39
+ @parser.instance.parse(log) { |time, record|
40
+ assert_not_nil(time)
41
+ assert_not_nil(record)
42
+ assert_equal(1, record.dig("header", "pri", "facility"))
43
+ assert_equal(Fluent::Plugin::CloudFoundrySyslogParser::SYSLOG_SEVERITY_CODES[5], record.dig("header", "pri", "severity"))
44
+ assert_equal("some-hostname", record.dig("header", "hostname"))
45
+ assert_equal("some-appname", record.dig("header", "app_name"))
46
+ assert_equal("some-procid", record.dig("header", "proc_id"))
47
+ assert_equal("some-msgid", record.dig("header", "msg_id"))
48
+ assert_equal({
49
+ "paramA" => "def123",
50
+ "paramB" => "j k l",
51
+ "paramC" => "",
52
+ }, record.dig("sd", "instance@47450"))
53
+ assert_equal("some foo bar", record.dig("message"))
54
+ }
55
+ end
56
+
57
+ def test_parse_syslog_with_nilvalue_structured_data
58
+ log = "<13>1 1985-04-12T23:20:50.52Z some-hostname some-appname some-procid some-msgid - some foo bar"
59
+ @parser.instance.parse(log) { |time, record|
60
+ assert_not_nil(time)
61
+ assert_not_nil(record)
62
+ assert_equal(1, record.dig("header", "pri", "facility"))
63
+ assert_equal(Fluent::Plugin::CloudFoundrySyslogParser::SYSLOG_SEVERITY_CODES[5], record.dig("header", "pri", "severity"))
64
+ assert_equal("some-hostname", record.dig("header", "hostname"))
65
+ assert_equal("some-appname", record.dig("header", "app_name"))
66
+ assert_equal("some-procid", record.dig("header", "proc_id"))
67
+ assert_equal("some-msgid", record.dig("header", "msg_id"))
68
+ assert_equal({}, record.dig("sd"))
69
+ }
70
+ end
71
+
72
+ def test_parse_syslog_with_sd_element_without_sd_params
73
+ log = "<13>1 1985-04-12T23:20:50.52Z some-hostname some-appname some-procid some-msgid [instance] some foo bar"
74
+ @parser.instance.parse(log) { |time, record|
75
+ assert_not_nil(time)
76
+ assert_not_nil(record)
77
+ assert_equal(1, record.dig("header", "pri", "facility"))
78
+ assert_equal(Fluent::Plugin::CloudFoundrySyslogParser::SYSLOG_SEVERITY_CODES[5], record.dig("header", "pri", "severity"))
79
+ assert_equal("some-hostname", record.dig("header", "hostname"))
80
+ assert_equal("some-appname", record.dig("header", "app_name"))
81
+ assert_equal("some-procid", record.dig("header", "proc_id"))
82
+ assert_equal("some-msgid", record.dig("header", "msg_id"))
83
+ assert_equal({ "instance" => {} }, record.dig("sd"))
84
+ }
85
+ end
86
+
87
+ def test_parse_syslog_with_sd_element_with_registered_id
88
+ log = "<13>1 1985-04-12T23:20:50.52Z some-hostname some-appname some-procid some-msgid [instance@1234] some foo bar"
89
+ @parser.instance.parse(log) { |time, record|
90
+ assert_not_nil(time)
91
+ assert_not_nil(record)
92
+ assert_equal(1, record.dig("header", "pri", "facility"))
93
+ assert_equal(Fluent::Plugin::CloudFoundrySyslogParser::SYSLOG_SEVERITY_CODES[5], record.dig("header", "pri", "severity"))
94
+ assert_equal("some-hostname", record.dig("header", "hostname"))
95
+ assert_equal("some-appname", record.dig("header", "app_name"))
96
+ assert_equal("some-procid", record.dig("header", "proc_id"))
97
+ assert_equal("some-msgid", record.dig("header", "msg_id"))
98
+ assert_equal({ "instance@1234" => {} }, record.dig("sd"))
99
+ }
100
+ end
101
+
102
+ def test_parse_syslog_with_escapes_in_sd_value
103
+ log = '<13>1 1985-04-12T23:20:50.52Z some-hostname some-appname some-procid some-msgid [instance@47450 paramA="def\\12\\3" paramB="j k l\\"" paramC="\\"\\]"] some foo bar'
104
+ @parser.instance.parse(log) { |time, record|
105
+ assert_not_nil(time)
106
+ assert_not_nil(record)
107
+ assert_equal(1, record.dig("header", "pri", "facility"))
108
+ assert_equal(Fluent::Plugin::CloudFoundrySyslogParser::SYSLOG_SEVERITY_CODES[5], record.dig("header", "pri", "severity"))
109
+ assert_equal("some-hostname", record.dig("header", "hostname"))
110
+ assert_equal("some-appname", record.dig("header", "app_name"))
111
+ assert_equal("some-procid", record.dig("header", "proc_id"))
112
+ assert_equal("some-msgid", record.dig("header", "msg_id"))
113
+ assert_equal({
114
+ "paramA" => "def\\12\\3",
115
+ "paramB" => "j k l\\\"",
116
+ "paramC" => "\\\"\\]",
117
+ }, record.dig("sd", "instance@47450"))
118
+ assert_equal("some foo bar", record.dig("message"))
119
+ }
120
+ end
121
+
122
+ def test_parse_syslog_with_invalid_escapes_in_sd_value
123
+ log = '<13>1 1985-04-12T23:20:50.52Z some-hostname some-appname some-procid some-msgid [instance@47450 paramB="j k l\\\\""] some foo bar'
124
+ @parser.instance.parse(log) { |time, record|
125
+ assert_nil(time)
126
+ assert_nil(record)
127
+ }
128
+ end
129
+
130
+ def test_parse_syslog_with_invalid_escaped_sd_value_delimiter
131
+ log = '<13>1 1985-04-12T23:20:50.52Z some-hostname some-appname some-procid some-msgid [instance@47450 paramC="\\"] some foo bar'
132
+ @parser.instance.parse(log) { |time, record|
133
+ assert_nil(time)
134
+ assert_nil(record)
135
+ }
136
+ end
137
+
138
+ def test_parse_syslog_with_multiple_sd_elements
139
+ log = '<13>1 1985-04-12T23:20:50.52Z some-hostname some-appname some-procid some-msgid [elementA fooA="barA"][elementB fooB="barB"] some foo bar'
140
+ @parser.instance.parse(log) { |time, record|
141
+ assert_not_nil(time)
142
+ assert_not_nil(record)
143
+ assert_equal(1, record.dig("header", "pri", "facility"))
144
+ assert_equal(Fluent::Plugin::CloudFoundrySyslogParser::SYSLOG_SEVERITY_CODES[5], record.dig("header", "pri", "severity"))
145
+ assert_equal("some-hostname", record.dig("header", "hostname"))
146
+ assert_equal("some-appname", record.dig("header", "app_name"))
147
+ assert_equal("some-procid", record.dig("header", "proc_id"))
148
+ assert_equal("some-msgid", record.dig("header", "msg_id"))
149
+ assert_equal({
150
+ "elementA" => {
151
+ "fooA" => "barA",
152
+ },
153
+ "elementB" => {
154
+ "fooB" => "barB",
155
+ },
156
+ }, record.dig("sd"))
157
+ assert_equal("some foo bar", record.dig("message"))
158
+ }
159
+ end
160
+
161
+ def test_parse_syslog_with_seperated_sd_elements
162
+ log = '<13>1 1985-04-12T23:20:50.52Z some-hostname some-appname some-procid some-msgid [elementA fooA="barA"] [elementB fooB="barB"] some foo bar'
163
+ @parser.instance.parse(log) { |time, record|
164
+ assert_not_nil(time)
165
+ assert_not_nil(record)
166
+ assert_equal(1, record.dig("header", "pri", "facility"))
167
+ assert_equal(Fluent::Plugin::CloudFoundrySyslogParser::SYSLOG_SEVERITY_CODES[5], record.dig("header", "pri", "severity"))
168
+ assert_equal("some-hostname", record.dig("header", "hostname"))
169
+ assert_equal("some-appname", record.dig("header", "app_name"))
170
+ assert_equal("some-procid", record.dig("header", "proc_id"))
171
+ assert_equal("some-msgid", record.dig("header", "msg_id"))
172
+ assert_equal({
173
+ "elementA" => {
174
+ "fooA" => "barA",
175
+ },
176
+ }, record.dig("sd"))
177
+ assert_equal('[elementB fooB="barB"] some foo bar', record.dig("message"))
178
+ }
179
+ end
180
+
181
+ def test_parse_syslog_without_message
182
+ log = '<13>1 1985-04-12T23:20:50.52Z some-hostname some-appname some-procid some-msgid [instance@47450 paramA="def123" paramB="j k l" paramC=""]'
183
+ @parser.instance.parse(log) { |time, record|
184
+ assert_not_nil(time)
185
+ assert_not_nil(record)
186
+ assert_equal(1, record.dig("header", "pri", "facility"))
187
+ assert_equal(Fluent::Plugin::CloudFoundrySyslogParser::SYSLOG_SEVERITY_CODES[5], record.dig("header", "pri", "severity"))
188
+ assert_equal("some-hostname", record.dig("header", "hostname"))
189
+ assert_equal("some-appname", record.dig("header", "app_name"))
190
+ assert_equal("some-procid", record.dig("header", "proc_id"))
191
+ assert_equal("some-msgid", record.dig("header", "msg_id"))
192
+ assert_equal({
193
+ "paramA" => "def123",
194
+ "paramB" => "j k l",
195
+ "paramC" => "",
196
+ }, record.dig("sd", "instance@47450"))
197
+ assert_equal("", record.dig("message"))
198
+ }
199
+ end
200
+
201
+ def test_parse_syslog_with_nilvalue_sd_without_message
202
+ log = "<13>1 1985-04-12T23:20:50.52Z some-hostname some-appname some-procid some-msgid -"
203
+ @parser.instance.parse(log) { |time, record|
204
+ assert_not_nil(time)
205
+ assert_not_nil(record)
206
+ assert_equal(1, record.dig("header", "pri", "facility"))
207
+ assert_equal(Fluent::Plugin::CloudFoundrySyslogParser::SYSLOG_SEVERITY_CODES[5], record.dig("header", "pri", "severity"))
208
+ assert_equal("some-hostname", record.dig("header", "hostname"))
209
+ assert_equal("some-appname", record.dig("header", "app_name"))
210
+ assert_equal("some-procid", record.dig("header", "proc_id"))
211
+ assert_equal("some-msgid", record.dig("header", "msg_id"))
212
+ assert_equal({}, record.dig("sd"))
213
+ assert_equal("", record.dig("message"))
214
+ }
215
+ end
216
+ end
metadata ADDED
@@ -0,0 +1,116 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: fluent-plugin-parser-cloudfoundry-syslog
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.1
5
+ platform: ruby
6
+ authors:
7
+ - Matteias Collet
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2021-12-29 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: bundler
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: '2.1'
20
+ type: :development
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - "~>"
25
+ - !ruby/object:Gem::Version
26
+ version: '2.1'
27
+ - !ruby/object:Gem::Dependency
28
+ name: rake
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - "~>"
32
+ - !ruby/object:Gem::Version
33
+ version: '13.0'
34
+ type: :development
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - "~>"
39
+ - !ruby/object:Gem::Version
40
+ version: '13.0'
41
+ - !ruby/object:Gem::Dependency
42
+ name: test-unit
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - "~>"
46
+ - !ruby/object:Gem::Version
47
+ version: '3.3'
48
+ type: :development
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - "~>"
53
+ - !ruby/object:Gem::Version
54
+ version: '3.3'
55
+ - !ruby/object:Gem::Dependency
56
+ name: fluentd
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - ">="
60
+ - !ruby/object:Gem::Version
61
+ version: '1'
62
+ type: :runtime
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - ">="
67
+ - !ruby/object:Gem::Version
68
+ version: '1'
69
+ description: CloudFoundry log parser for Fluentd
70
+ email:
71
+ - matteias.collet@bluewin.ch
72
+ executables: []
73
+ extensions: []
74
+ extra_rdoc_files: []
75
+ files:
76
+ - ".devcontainer/Dockerfile"
77
+ - ".devcontainer/devcontainer.json"
78
+ - ".devcontainer/post-create.sh"
79
+ - ".github/workflows/build-and-test.yml"
80
+ - ".github/workflows/publish.yml"
81
+ - ".gitignore"
82
+ - ".vscode/settings.json"
83
+ - Gemfile
84
+ - LICENSE
85
+ - README.md
86
+ - Rakefile
87
+ - fluent-plugin-parser-cloudfoundry-syslog.gemspec
88
+ - lib/fluent/plugin/parser_cloudfoundry_syslog.rb
89
+ - test/test_parser_router_log.rb
90
+ - test/test_syslog_rfc.rb
91
+ homepage: https://github.com/bitpatty/fluent-plugin-parser-cloudfoundry-syslog
92
+ licenses:
93
+ - Apache-2.0
94
+ metadata: {}
95
+ post_install_message:
96
+ rdoc_options: []
97
+ require_paths:
98
+ - lib
99
+ required_ruby_version: !ruby/object:Gem::Requirement
100
+ requirements:
101
+ - - ">="
102
+ - !ruby/object:Gem::Version
103
+ version: '0'
104
+ required_rubygems_version: !ruby/object:Gem::Requirement
105
+ requirements:
106
+ - - ">="
107
+ - !ruby/object:Gem::Version
108
+ version: '0'
109
+ requirements: []
110
+ rubygems_version: 3.1.2
111
+ signing_key:
112
+ specification_version: 4
113
+ summary: CloudFoundry log parser for Fluentd
114
+ test_files:
115
+ - test/test_parser_router_log.rb
116
+ - test/test_syslog_rfc.rb