logstash-output-newrelic 0.9.1 → 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/CHANGELOG.md +2 -5
- data/CONTRIBUTORS +10 -0
- data/DEVELOPER.md +37 -0
- data/Gemfile +1 -0
- data/LICENSE +196 -17
- data/README.md +60 -74
- data/lib/logstash/outputs/newrelic_internal.rb +126 -0
- data/lib/logstash/outputs/newrelic_internal_version/version.rb +7 -0
- data/logstash-output-newrelic.gemspec +22 -17
- data/spec/outputs/newrelic_internal_spec.rb +284 -0
- metadata +54 -38
- data/.github/CONTRIBUTING.md +0 -65
- data/.github/ISSUE_TEMPLATE.md +0 -9
- data/.github/PULL_REQUEST_TEMPLATE.md +0 -1
- data/.gitignore +0 -2
- data/.travis.yml +0 -11
- data/NOTICE.TXT +0 -5
- data/Rakefile +0 -7
- data/bin/logstash-newrelic +0 -2
- data/bin/logstash-newrelic-debug +0 -2
- data/bin/logstash-newrelic.conf.template +0 -10
- data/lib/logstash/outputs/newrelic.rb +0 -220
- data/spec/outputs/newrelic_spec.rb +0 -66
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 9ee56614832b2c9301c8c3b63748c276ee40f5a6a5eb8fd641ed492f1e37c236
|
4
|
+
data.tar.gz: 05d3654fe08c97178690ee79f779249f9b141aa3201b6c8862364ff30f07ae2a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 74e759723023a155f8b278b19c7e1cf28018854dea57bbd243a6b67758f180011dd7878ae64aff8b68381dd225f352f2e03ca045a14eca724774476c1c90d706
|
7
|
+
data.tar.gz: 5e3c021c98de4960ee1c3782d82a27beaa9d5951e1dfb39039de93d7dd5bfa31c41df8a758152f52a2caa4e3db29212cfa54a9035f6612696fefeadc56ff7912
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,2 @@
|
|
1
|
-
##
|
2
|
-
|
3
|
-
instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895
|
4
|
-
- Dependency on logstash-core update to 2.0
|
5
|
-
|
1
|
+
## 0.1.0
|
2
|
+
- Plugin created with the logstash plugin generator
|
data/CONTRIBUTORS
ADDED
@@ -0,0 +1,10 @@
|
|
1
|
+
The following is a list of people who have contributed ideas, code, bug
|
2
|
+
reports, or in general have helped logstash along its way.
|
3
|
+
|
4
|
+
Contributors:
|
5
|
+
* -
|
6
|
+
|
7
|
+
Note: If you've sent us patches, bug reports, or otherwise contributed to
|
8
|
+
Logstash, and you aren't on the list above and want to be, please let us know
|
9
|
+
and we'll make sure you're here. Contributions from folks like you are what make
|
10
|
+
open source awesome.
|
data/DEVELOPER.md
ADDED
@@ -0,0 +1,37 @@
|
|
1
|
+
# Developing the plugin
|
2
|
+
|
3
|
+
# Getting started
|
4
|
+
|
5
|
+
* Install JRuby: `rbenv install jruby-9.2.5.0`
|
6
|
+
* Use that JRuby: `rbenv local jruby-9.2.5.0`
|
7
|
+
* Install Bundler gem: `jruby -S gem install bundler`
|
8
|
+
|
9
|
+
# Developing
|
10
|
+
|
11
|
+
* Install dependencies: `jruby -S bundle install`
|
12
|
+
* Write tests and production code!
|
13
|
+
* Bump version: edit version file `version.rb`
|
14
|
+
* Run tests: `jruby -S bundle exec rspec`
|
15
|
+
* Build the gem: `jruby -S gem build logstash-output-newrelic-internal.gemspec`
|
16
|
+
|
17
|
+
## Pushing changes to the public repo
|
18
|
+
After updating the New Relic repo with changes, changes will need to be pushed to the public GitHub repo at: https://github.com/newrelic/newrelic-fluent-bit-output
|
19
|
+
|
20
|
+
* `git remote add public git@github.com:newrelic/newrelic-fluent-bit-output.git`
|
21
|
+
* `git push public master:name-of-branch-to-create`
|
22
|
+
* Create a PR from that branch in https://github.com/newrelic/newrelic-fluent-bit-output
|
23
|
+
* Get the PR reviewed, merged, and delete the branch!
|
24
|
+
|
25
|
+
# Testing it with a local Logstash install
|
26
|
+
|
27
|
+
* Remove previous version: `logstash-plugin remove logstash-output-newrelic-internal`
|
28
|
+
* Add new version: `logstash-plugin install logstash-output-newrelic-internal-<version>.gem `
|
29
|
+
* Restart logstash: For Homebrew: `brew services restart logstash`
|
30
|
+
* Cause a change that you've configured Logstash to pick up (for instance, append to a file you're having it monitor)
|
31
|
+
* Look in `https://staging-one.newrelic.com/launcher/logger.log-launcher` for your log message
|
32
|
+
|
33
|
+
# Push changes to RubyGems
|
34
|
+
After updating the source code and gem version in `version.rb`, push the changes to RubyGems. Note, you must be a gem owner to publish changes on [RubyGems.org](https://rubygems.org/profiles/NR-LOGGING)
|
35
|
+
|
36
|
+
* Build the gem: `gem build logstash-output-newrelic.gemspec`
|
37
|
+
* Publish the gem: `gem push logstash-output-newrelic-logging-<VERSION>.gem` with the updated version (ex: `gem push logstash-output-newrelic-logging-0.2.16.gem`)
|
data/Gemfile
CHANGED
data/LICENSE
CHANGED
@@ -1,22 +1,201 @@
|
|
1
|
-
|
1
|
+
Apache License
|
2
|
+
Version 2.0, January 2004
|
3
|
+
http://www.apache.org/licenses/
|
2
4
|
|
3
|
-
|
5
|
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
4
6
|
|
5
|
-
|
6
|
-
of this software and associated documentation files (the "Software"), to deal
|
7
|
-
in the Software without restriction, including without limitation the rights
|
8
|
-
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
-
copies of the Software, and to permit persons to whom the Software is
|
10
|
-
furnished to do so, subject to the following conditions:
|
7
|
+
1. Definitions.
|
11
8
|
|
12
|
-
|
13
|
-
|
9
|
+
"License" shall mean the terms and conditions for use, reproduction,
|
10
|
+
and distribution as defined by Sections 1 through 9 of this document.
|
14
11
|
|
15
|
-
|
16
|
-
|
17
|
-
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
-
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
-
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
-
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
21
|
-
SOFTWARE.
|
12
|
+
"Licensor" shall mean the copyright owner or entity authorized by
|
13
|
+
the copyright owner that is granting the License.
|
22
14
|
|
15
|
+
"Legal Entity" shall mean the union of the acting entity and all
|
16
|
+
other entities that control, are controlled by, or are under common
|
17
|
+
control with that entity. For the purposes of this definition,
|
18
|
+
"control" means (i) the power, direct or indirect, to cause the
|
19
|
+
direction or management of such entity, whether by contract or
|
20
|
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
21
|
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
22
|
+
|
23
|
+
"You" (or "Your") shall mean an individual or Legal Entity
|
24
|
+
exercising permissions granted by this License.
|
25
|
+
|
26
|
+
"Source" form shall mean the preferred form for making modifications,
|
27
|
+
including but not limited to software source code, documentation
|
28
|
+
source, and configuration files.
|
29
|
+
|
30
|
+
"Object" form shall mean any form resulting from mechanical
|
31
|
+
transformation or translation of a Source form, including but
|
32
|
+
not limited to compiled object code, generated documentation,
|
33
|
+
and conversions to other media types.
|
34
|
+
|
35
|
+
"Work" shall mean the work of authorship, whether in Source or
|
36
|
+
Object form, made available under the License, as indicated by a
|
37
|
+
copyright notice that is included in or attached to the work
|
38
|
+
(an example is provided in the Appendix below).
|
39
|
+
|
40
|
+
"Derivative Works" shall mean any work, whether in Source or Object
|
41
|
+
form, that is based on (or derived from) the Work and for which the
|
42
|
+
editorial revisions, annotations, elaborations, or other modifications
|
43
|
+
represent, as a whole, an original work of authorship. For the purposes
|
44
|
+
of this License, Derivative Works shall not include works that remain
|
45
|
+
separable from, or merely link (or bind by name) to the interfaces of,
|
46
|
+
the Work and Derivative Works thereof.
|
47
|
+
|
48
|
+
"Contribution" shall mean any work of authorship, including
|
49
|
+
the original version of the Work and any modifications or additions
|
50
|
+
to that Work or Derivative Works thereof, that is intentionally
|
51
|
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
52
|
+
or by an individual or Legal Entity authorized to submit on behalf of
|
53
|
+
the copyright owner. For the purposes of this definition, "submitted"
|
54
|
+
means any form of electronic, verbal, or written communication sent
|
55
|
+
to the Licensor or its representatives, including but not limited to
|
56
|
+
communication on electronic mailing lists, source code control systems,
|
57
|
+
and issue tracking systems that are managed by, or on behalf of, the
|
58
|
+
Licensor for the purpose of discussing and improving the Work, but
|
59
|
+
excluding communication that is conspicuously marked or otherwise
|
60
|
+
designated in writing by the copyright owner as "Not a Contribution."
|
61
|
+
|
62
|
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
63
|
+
on behalf of whom a Contribution has been received by Licensor and
|
64
|
+
subsequently incorporated within the Work.
|
65
|
+
|
66
|
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
67
|
+
this License, each Contributor hereby grants to You a perpetual,
|
68
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
69
|
+
copyright license to reproduce, prepare Derivative Works of,
|
70
|
+
publicly display, publicly perform, sublicense, and distribute the
|
71
|
+
Work and such Derivative Works in Source or Object form.
|
72
|
+
|
73
|
+
3. Grant of Patent License. Subject to the terms and conditions of
|
74
|
+
this License, each Contributor hereby grants to You a perpetual,
|
75
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
76
|
+
(except as stated in this section) patent license to make, have made,
|
77
|
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
78
|
+
where such license applies only to those patent claims licensable
|
79
|
+
by such Contributor that are necessarily infringed by their
|
80
|
+
Contribution(s) alone or by combination of their Contribution(s)
|
81
|
+
with the Work to which such Contribution(s) was submitted. If You
|
82
|
+
institute patent litigation against any entity (including a
|
83
|
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
84
|
+
or a Contribution incorporated within the Work constitutes direct
|
85
|
+
or contributory patent infringement, then any patent licenses
|
86
|
+
granted to You under this License for that Work shall terminate
|
87
|
+
as of the date such litigation is filed.
|
88
|
+
|
89
|
+
4. Redistribution. You may reproduce and distribute copies of the
|
90
|
+
Work or Derivative Works thereof in any medium, with or without
|
91
|
+
modifications, and in Source or Object form, provided that You
|
92
|
+
meet the following conditions:
|
93
|
+
|
94
|
+
(a) You must give any other recipients of the Work or
|
95
|
+
Derivative Works a copy of this License; and
|
96
|
+
|
97
|
+
(b) You must cause any modified files to carry prominent notices
|
98
|
+
stating that You changed the files; and
|
99
|
+
|
100
|
+
(c) You must retain, in the Source form of any Derivative Works
|
101
|
+
that You distribute, all copyright, patent, trademark, and
|
102
|
+
attribution notices from the Source form of the Work,
|
103
|
+
excluding those notices that do not pertain to any part of
|
104
|
+
the Derivative Works; and
|
105
|
+
|
106
|
+
(d) If the Work includes a "NOTICE" text file as part of its
|
107
|
+
distribution, then any Derivative Works that You distribute must
|
108
|
+
include a readable copy of the attribution notices contained
|
109
|
+
within such NOTICE file, excluding those notices that do not
|
110
|
+
pertain to any part of the Derivative Works, in at least one
|
111
|
+
of the following places: within a NOTICE text file distributed
|
112
|
+
as part of the Derivative Works; within the Source form or
|
113
|
+
documentation, if provided along with the Derivative Works; or,
|
114
|
+
within a display generated by the Derivative Works, if and
|
115
|
+
wherever such third-party notices normally appear. The contents
|
116
|
+
of the NOTICE file are for informational purposes only and
|
117
|
+
do not modify the License. You may add Your own attribution
|
118
|
+
notices within Derivative Works that You distribute, alongside
|
119
|
+
or as an addendum to the NOTICE text from the Work, provided
|
120
|
+
that such additional attribution notices cannot be construed
|
121
|
+
as modifying the License.
|
122
|
+
|
123
|
+
You may add Your own copyright statement to Your modifications and
|
124
|
+
may provide additional or different license terms and conditions
|
125
|
+
for use, reproduction, or distribution of Your modifications, or
|
126
|
+
for any such Derivative Works as a whole, provided Your use,
|
127
|
+
reproduction, and distribution of the Work otherwise complies with
|
128
|
+
the conditions stated in this License.
|
129
|
+
|
130
|
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
131
|
+
any Contribution intentionally submitted for inclusion in the Work
|
132
|
+
by You to the Licensor shall be under the terms and conditions of
|
133
|
+
this License, without any additional terms or conditions.
|
134
|
+
Notwithstanding the above, nothing herein shall supersede or modify
|
135
|
+
the terms of any separate license agreement you may have executed
|
136
|
+
with Licensor regarding such Contributions.
|
137
|
+
|
138
|
+
6. Trademarks. This License does not grant permission to use the trade
|
139
|
+
names, trademarks, service marks, or product names of the Licensor,
|
140
|
+
except as required for reasonable and customary use in describing the
|
141
|
+
origin of the Work and reproducing the content of the NOTICE file.
|
142
|
+
|
143
|
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
144
|
+
agreed to in writing, Licensor provides the Work (and each
|
145
|
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
146
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
147
|
+
implied, including, without limitation, any warranties or conditions
|
148
|
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
149
|
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
150
|
+
appropriateness of using or redistributing the Work and assume any
|
151
|
+
risks associated with Your exercise of permissions under this License.
|
152
|
+
|
153
|
+
8. Limitation of Liability. In no event and under no legal theory,
|
154
|
+
whether in tort (including negligence), contract, or otherwise,
|
155
|
+
unless required by applicable law (such as deliberate and grossly
|
156
|
+
negligent acts) or agreed to in writing, shall any Contributor be
|
157
|
+
liable to You for damages, including any direct, indirect, special,
|
158
|
+
incidental, or consequential damages of any character arising as a
|
159
|
+
result of this License or out of the use or inability to use the
|
160
|
+
Work (including but not limited to damages for loss of goodwill,
|
161
|
+
work stoppage, computer failure or malfunction, or any and all
|
162
|
+
other commercial damages or losses), even if such Contributor
|
163
|
+
has been advised of the possibility of such damages.
|
164
|
+
|
165
|
+
9. Accepting Warranty or Additional Liability. While redistributing
|
166
|
+
the Work or Derivative Works thereof, You may choose to offer,
|
167
|
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
168
|
+
or other liability obligations and/or rights consistent with this
|
169
|
+
License. However, in accepting such obligations, You may act only
|
170
|
+
on Your own behalf and on Your sole responsibility, not on behalf
|
171
|
+
of any other Contributor, and only if You agree to indemnify,
|
172
|
+
defend, and hold each Contributor harmless for any liability
|
173
|
+
incurred by, or claims asserted against, such Contributor by reason
|
174
|
+
of your accepting any such warranty or additional liability.
|
175
|
+
|
176
|
+
END OF TERMS AND CONDITIONS
|
177
|
+
|
178
|
+
APPENDIX: How to apply the Apache License to your work.
|
179
|
+
|
180
|
+
To apply the Apache License to your work, attach the following
|
181
|
+
boilerplate notice, with the fields enclosed by brackets "[]"
|
182
|
+
replaced with your own identifying information. (Don't include
|
183
|
+
the brackets!) The text should be enclosed in the appropriate
|
184
|
+
comment syntax for the file format. We also recommend that a
|
185
|
+
file or class name and description of purpose be included on the
|
186
|
+
same "printed page" as the copyright notice for easier
|
187
|
+
identification within third-party archives.
|
188
|
+
|
189
|
+
Copyright 2019 New Relic, Inc.
|
190
|
+
|
191
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
192
|
+
you may not use this file except in compliance with the License.
|
193
|
+
You may obtain a copy of the License at
|
194
|
+
|
195
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
196
|
+
|
197
|
+
Unless required by applicable law or agreed to in writing, software
|
198
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
199
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
200
|
+
See the License for the specific language governing permissions and
|
201
|
+
limitations under the License.
|
data/README.md
CHANGED
@@ -1,99 +1,85 @@
|
|
1
|
-
# Logstash Plugin
|
1
|
+
# New Relic Logstash Output Plugin
|
2
2
|
|
3
|
-
[
|
4
|
-
Status](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Outputs/job/logstash-plugin-output-newrelic-unit/badge/icon)](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Outputs/job/logstash-plugin-output-newrelic-unit/)
|
3
|
+
This is a plugin for [Logstash](https://github.com/elastic/logstash) that outputs logs to New Relic.
|
5
4
|
|
6
|
-
|
5
|
+
## Installation
|
6
|
+
Install the New Relic Logstash plugin using the following command:</br>
|
7
|
+
`logstash-plugin install logstash-output-newrelic-logging`
|
7
8
|
|
8
|
-
|
9
|
+
## Configuration
|
9
10
|
|
10
|
-
|
11
|
+
Add the following block to your logstash.conf (with your specific API Insert key), then restart Logstash.
|
12
|
+
There are other optional configuration properties, see below.
|
11
13
|
|
12
|
-
|
14
|
+
Get your API Insert Key:
|
15
|
+
`https://insights.newrelic.com/accounts/<ACCOUNT_ID>/manage/api_keys`
|
13
16
|
|
14
|
-
|
15
|
-
|
17
|
+
Example:
|
18
|
+
```rb
|
19
|
+
output {
|
20
|
+
newrelic_internal {
|
21
|
+
api_key => "<API_INSERT_KEY>"
|
22
|
+
}
|
23
|
+
}
|
24
|
+
```
|
16
25
|
|
17
|
-
## Need Help?
|
18
26
|
|
19
|
-
|
27
|
+
### Required plugin configuration
|
20
28
|
|
21
|
-
|
29
|
+
| Property | Description |
|
30
|
+
|---|---|
|
31
|
+
| api_key | your New Relic API Insert key |
|
22
32
|
|
23
|
-
###
|
33
|
+
### Optional plugin configuration
|
24
34
|
|
25
|
-
|
26
|
-
|
35
|
+
| Property | Description | Default value |
|
36
|
+
|---|---|---|
|
37
|
+
| concurrent_requests | The number of threads to make requests from | 1 |
|
38
|
+
| retries | The maximum number of times to retry a failed request, exponentially increasing delay between each retry | 5 |
|
39
|
+
| retry_seconds | The inital delay between retries, in seconds | 5 |
|
40
|
+
| max_delay | The maximum delay between retries, in seconds | 30 |
|
41
|
+
| base_uri | New Relic ingestion endpoint | https://log-api.newrelic.com/log/v1 |
|
27
42
|
|
28
|
-
|
43
|
+
## Testing
|
29
44
|
|
30
|
-
|
31
|
-
```sh
|
32
|
-
bundle install
|
45
|
+
An easy way to test the plugin is to make sure Logstash is getting input from a log file you can write to. Something like this in your logstash.conf:
|
33
46
|
```
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
```sh
|
40
|
-
bundle install
|
47
|
+
input {
|
48
|
+
file {
|
49
|
+
path => "/path/to/your/log/file"
|
50
|
+
}
|
51
|
+
}
|
41
52
|
```
|
53
|
+
* Restart Logstash
|
54
|
+
* Append a test log message to your log file: `echo "test message" >> /path/to/your/log/file`
|
55
|
+
* Search New Relic Logs for `"test message"`
|
42
56
|
|
43
|
-
|
57
|
+
## Notes
|
44
58
|
|
45
|
-
|
46
|
-
bundle exec rspec
|
47
|
-
```
|
59
|
+
This plugin will attempt to parse any 'message' attribute as JSON -- if it is JSON, its JSON attributes will be added to the event.
|
48
60
|
|
49
|
-
|
50
|
-
|
51
|
-
#### 2.1 Run in a local Logstash clone
|
52
|
-
|
53
|
-
- Edit Logstash `Gemfile` and add the local plugin path, for example:
|
54
|
-
```ruby
|
55
|
-
gem "logstash-filter-awesome", :path => "/your/local/logstash-filter-awesome"
|
61
|
+
For example, the events:
|
56
62
|
```
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
63
|
+
[{
|
64
|
+
"message": "some message",
|
65
|
+
"timestamp": 1531414060739
|
66
|
+
},
|
67
|
+
{
|
68
|
+
{"message":"some_message","timestamp":"12897439", "compound" :"{\"a\":111, \"b\":222}"},
|
69
|
+
}]
|
65
70
|
```
|
66
|
-
- Run Logstash with your plugin
|
67
|
-
```sh
|
68
|
-
bin/logstash -e 'filter {awesome {}}'
|
69
|
-
```
|
70
|
-
At this point any modifications to the plugin code will be applied to this local Logstash setup. After modifying the plugin, simply rerun Logstash.
|
71
|
-
|
72
|
-
#### 2.2 Run in an installed Logstash
|
73
71
|
|
74
|
-
|
75
|
-
|
76
|
-
- Build your plugin gem
|
77
|
-
```sh
|
78
|
-
gem build logstash-filter-awesome.gemspec
|
72
|
+
Will be output as:
|
79
73
|
```
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
74
|
+
[{ "message": "{\"key\": \"value1\", \"compound\": {\"sub_key\": \"value2\"}}",
|
75
|
+
"key": "value1",
|
76
|
+
"compound": {
|
77
|
+
"sub_key": "value2"
|
78
|
+
},
|
79
|
+
"other": "other value"
|
80
|
+
}]
|
88
81
|
```
|
89
|
-
- Start Logstash and proceed to test the plugin
|
90
|
-
|
91
|
-
## Contributing
|
92
|
-
|
93
|
-
All contributions are welcome: ideas, patches, documentation, bug reports, complaints, and even something you drew up on a napkin.
|
94
|
-
|
95
|
-
Programming is not a required skill. Whatever you've seen about open source and maintainers or community members saying "send patches or die" - you will not see that here.
|
96
82
|
|
97
|
-
|
83
|
+
## Development
|
98
84
|
|
99
|
-
|
85
|
+
See [DEVELOPER.md](DEVELOPER.md)
|
@@ -0,0 +1,126 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require "logstash/outputs/base"
|
3
|
+
require "logstash/outputs/newrelic_internal_version/version"
|
4
|
+
require 'net/http'
|
5
|
+
require 'uri'
|
6
|
+
require 'zlib'
|
7
|
+
require 'json'
|
8
|
+
require 'java'
|
9
|
+
|
10
|
+
class LogStash::Outputs::NewRelicInternal < LogStash::Outputs::Base
|
11
|
+
java_import java.util.concurrent.Executors;
|
12
|
+
java_import java.util.concurrent.Semaphore;
|
13
|
+
|
14
|
+
config_name "newrelic_internal"
|
15
|
+
|
16
|
+
config :api_key, :validate => :password, :required => true
|
17
|
+
config :retry_seconds, :validate => :number, :default => 5
|
18
|
+
config :max_delay, :validate => :number, :default => 30
|
19
|
+
config :retries, :validate => :number, :default => 5
|
20
|
+
config :concurrent_requests, :validate => :number, :default => 1
|
21
|
+
config :base_uri, :validate => :string, :default => "https://insights-collector.newrelic.com/logs/v1"
|
22
|
+
|
23
|
+
# TODO: do we need to define "concurrency"? https://www.elastic.co/guide/en/logstash/current/_how_to_write_a_logstash_output_plugin.html
|
24
|
+
|
25
|
+
public
|
26
|
+
|
27
|
+
def register
|
28
|
+
@end_point = URI.parse(@base_uri)
|
29
|
+
@header = {
|
30
|
+
'X-Insert-Key' => @api_key.value,
|
31
|
+
'X-Event-Source' => 'logs',
|
32
|
+
'Content-Encoding' => 'gzip'
|
33
|
+
}.freeze
|
34
|
+
@executor = java.util.concurrent.Executors.newFixedThreadPool(@concurrent_requests)
|
35
|
+
@semaphor = java.util.concurrent.Semaphore.new(@concurrent_requests)
|
36
|
+
end
|
37
|
+
|
38
|
+
# Used by tests so that the test run can complete (background threads prevent JVM exit)
|
39
|
+
def shutdown
|
40
|
+
@executor&.shutdown
|
41
|
+
end
|
42
|
+
|
43
|
+
def encode(event_hash)
|
44
|
+
|
45
|
+
event_hash.delete('@timestamp')
|
46
|
+
event_hash = maybe_parse_message_json(event_hash)
|
47
|
+
event_hash
|
48
|
+
end
|
49
|
+
|
50
|
+
def maybe_parse_message_json(event_hash)
|
51
|
+
if event_hash.has_key?('message')
|
52
|
+
message = event_hash['message']
|
53
|
+
event_hash = event_hash.merge(maybe_parse_json(message))
|
54
|
+
end
|
55
|
+
event_hash
|
56
|
+
end
|
57
|
+
|
58
|
+
def maybe_parse_json(message)
|
59
|
+
begin
|
60
|
+
parsed = JSON.parse(message)
|
61
|
+
if Hash === parsed
|
62
|
+
return parsed
|
63
|
+
end
|
64
|
+
rescue JSON::ParserError
|
65
|
+
end
|
66
|
+
return {}
|
67
|
+
end
|
68
|
+
|
69
|
+
def multi_receive(events)
|
70
|
+
payload = []
|
71
|
+
events.each do |event|
|
72
|
+
payload.push(encode(event.to_hash))
|
73
|
+
end
|
74
|
+
payload = {
|
75
|
+
'logs' => payload,
|
76
|
+
'common' => {
|
77
|
+
'attributes' => {
|
78
|
+
'plugin' => {
|
79
|
+
'type' => 'logstash',
|
80
|
+
'version' => LogStash::Outputs::NewRelicInternalVersion::VERSION,
|
81
|
+
}
|
82
|
+
}
|
83
|
+
}
|
84
|
+
}
|
85
|
+
@semaphor.acquire()
|
86
|
+
execute = @executor.java_method :submit, [java.lang.Runnable]
|
87
|
+
execute.call do
|
88
|
+
begin
|
89
|
+
io = StringIO.new
|
90
|
+
gzip = Zlib::GzipWriter.new(io)
|
91
|
+
gzip << [payload].to_json
|
92
|
+
gzip.close
|
93
|
+
attempt_send(io.string, 0)
|
94
|
+
ensure
|
95
|
+
@semaphor.release()
|
96
|
+
end
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
def should_retry?(attempt)
|
101
|
+
attempt < retries
|
102
|
+
end
|
103
|
+
|
104
|
+
def sleep_duration(attempt)
|
105
|
+
[max_delay, (2 ** attempt) * retry_seconds].min
|
106
|
+
end
|
107
|
+
|
108
|
+
def attempt_send(payload, attempt)
|
109
|
+
sleep sleep_duration(attempt)
|
110
|
+
attempt_send(payload, attempt + 1) unless was_successful?(nr_send(payload)) if should_retry?(attempt)
|
111
|
+
end
|
112
|
+
|
113
|
+
def was_successful?(response)
|
114
|
+
200 <= response.code.to_i && response.code.to_i < 300
|
115
|
+
end
|
116
|
+
|
117
|
+
def nr_send(payload)
|
118
|
+
http = Net::HTTP.new(@end_point.host, 443)
|
119
|
+
request = Net::HTTP::Post.new(@end_point.request_uri)
|
120
|
+
http.use_ssl = true
|
121
|
+
http.verify_mode = OpenSSL::SSL::VERIFY_PEER
|
122
|
+
@header.each {|k, v| request[k] = v}
|
123
|
+
request.body = payload
|
124
|
+
http.request(request)
|
125
|
+
end
|
126
|
+
end # class LogStash::Outputs::NewRelicInternal
|
@@ -1,27 +1,32 @@
|
|
1
|
+
lib = File.expand_path('../lib/', __FILE__)
|
2
|
+
$:.unshift lib unless $:.include?(lib)
|
3
|
+
|
4
|
+
require 'logstash/outputs/newrelic_internal_version/version'
|
5
|
+
|
1
6
|
Gem::Specification.new do |s|
|
7
|
+
s.name = 'logstash-output-newrelic'
|
8
|
+
s.version = LogStash::Outputs::NewRelicInternalVersion::VERSION
|
9
|
+
s.licenses = ['Apache-2.0']
|
10
|
+
s.summary = 'Forwards logs as custom events to insights'
|
11
|
+
s.description = 'Gzips and decorates logstash events to be properly formatted as custom events'
|
12
|
+
s.homepage = 'https://source.datanerd.us/logging/logstash-output-newrelic'
|
13
|
+
s.authors = ['New Relic Logging Team']
|
14
|
+
s.email = 'logging-team@newrelic.com'
|
15
|
+
s.require_paths = ['lib']
|
2
16
|
|
3
|
-
s.name = "logstash-output-newrelic"
|
4
|
-
s.version = "0.9.1"
|
5
|
-
s.summary = "New Relic Insights output plugin for Logstash"
|
6
|
-
s.description = "Use logstash to ship log events to New Relic Insights. This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program."
|
7
|
-
s.authors = ["The Chocolate Factory"]
|
8
|
-
s.email = "info@chocolatefirm.com"
|
9
|
-
s.homepage = "http://www.chocolatefirm.com"
|
10
|
-
s.licenses = ["MIT"]
|
11
|
-
s.require_paths = ["lib"]
|
12
|
-
|
13
17
|
# Files
|
14
|
-
s.files =
|
15
|
-
|
16
|
-
# Tests
|
18
|
+
s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
|
19
|
+
# Tests
|
17
20
|
s.test_files = s.files.grep(%r{^(test|spec|features)/})
|
18
21
|
|
19
22
|
# Special flag to let us know this is actually a logstash plugin
|
20
23
|
s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
|
21
|
-
|
24
|
+
|
22
25
|
# Gem dependencies
|
23
|
-
s.add_runtime_dependency "logstash-core-plugin-api", "
|
24
|
-
s.add_runtime_dependency
|
25
|
-
s.add_development_dependency
|
26
|
+
s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0"
|
27
|
+
s.add_runtime_dependency "logstash-codec-plain"
|
28
|
+
s.add_development_dependency "logstash-devutils"
|
29
|
+
s.add_development_dependency "webmock"
|
30
|
+
s.add_development_dependency "rspec-wait"
|
26
31
|
|
27
32
|
end
|