logstash-output-lmlogs 1.3.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +6 -2
- data/README.md +11 -9
- data/lib/logstash/outputs/lmlogs.rb +52 -26
- data/lib/logstash/outputs/version.rb +5 -0
- data/logstash-output-lmlogs.gemspec +10 -1
- data/spec/outputs/metadata_spec.rb +86 -0
- metadata +19 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 46b4784b47b720c638e2f6cb53ace887c3ff3bd821cc27ca675fc3bf6b5b3325
|
4
|
+
data.tar.gz: 2dd4c277e4b769d4580ea1474d00f058d2f08f1d718da6e715c51ab16f722679
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 118161034fbd6b10b946449a75ccabc6fcce733006824711bdbcde2fff864f65ac249bd1de2f78600e975e71f47740881e946c4f2335a82956e6a8b130f44af5
|
7
|
+
data.tar.gz: f28a74d295e3f14bca4bfed4389c5dd32cfaaecb8f5b550168f2542494b0876c50b95d42661d101879284f9a7f532a74551b7fa1989abf83289b046f210e33de
|
data/CHANGELOG.md
CHANGED
@@ -7,8 +7,12 @@
|
|
7
7
|
## 1.1.0
|
8
8
|
- Disable mfa for gem push
|
9
9
|
## 1.2.0
|
10
|
-
- Add metadata by default to every log
|
10
|
+
- Add metadata by default to every log
|
11
11
|
## 1.2.1
|
12
12
|
- Fix ensuring metadata exists before deleting the original while forwarding to lm-logs
|
13
13
|
## 1.3.0
|
14
|
-
- Add bearer token support for authentication with Logicmonitor
|
14
|
+
- Add bearer token support for authentication with Logicmonitor
|
15
|
+
## 1.3.1
|
16
|
+
- Fix user agent populated in headers
|
17
|
+
## 2.0.0
|
18
|
+
- Dont send event metadata by default. Add config include_metadata_keys for including custom metadata
|
data/README.md
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
[](https://badge.fury.io/rb/logstash-output-lmlogs)
|
2
2
|
|
3
|
-
This plugin sends Logstash events to the [Logicmonitor Logs](https://www.logicmonitor.com)
|
3
|
+
This plugin sends Logstash events to the [Logicmonitor Logs](https://www.logicmonitor.com)
|
4
4
|
|
5
5
|
# Getting started
|
6
6
|
|
@@ -20,6 +20,7 @@ output {
|
|
20
20
|
}
|
21
21
|
}
|
22
22
|
```
|
23
|
+
You would need either `access_id` and `access_id` both or `bearer_token` for authentication with Logicmonitor.
|
23
24
|
|
24
25
|
|
25
26
|
|
@@ -27,27 +28,28 @@ output {
|
|
27
28
|
|
28
29
|
| Option | Description| Default |
|
29
30
|
| --- | --- | --- |
|
30
|
-
| batch_size | Event batch size to send to LM Logs.| 100 |
|
31
|
+
| batch_size | Event batch size to send to LM Logs.| 100 |
|
31
32
|
| message_key | Key that will be used by the plugin as the system key | "message" |
|
32
33
|
| lm_property | Key that will be used by LM to match resource based on property | "system.hostname" |
|
33
34
|
| keep_timestamp | If false, LM Logs will use the ingestion timestamp as the event timestamp | true |
|
34
35
|
| timestamp_is_key | If true, LM Logs will use a specified key as the event timestamp | false |
|
35
36
|
| timestamp_key | If timestamp_is_key is set, LM Logs will use this key in the event as the timestamp | "logtimestamp" |
|
36
|
-
| include_metadata | If
|
37
|
+
| include_metadata | If true, all metadata fields will be sent to LM Logs | false |
|
38
|
+
| include_metadata_keys | Array of json keys for which plugin looks for these keys and adds as event meatadata. A dot "." can be used to add nested subjson. If config `include_metadata` is set to true, all metadata will be sent regardless of this config. | [] |
|
37
39
|
|
38
40
|
See the [source code](lib/logstash/outputs/lmlogs.rb) for the full list of options
|
39
41
|
|
40
42
|
The syntax for `message_key` and `source_key` values are available in the [Logstash Event API Documentation](https://www.elastic.co/guide/en/logstash/current/event-api.html)
|
41
43
|
|
42
|
-
## Known issues
|
44
|
+
## Known issues
|
43
45
|
- Installation of the plugin fails on Logstash 6.2.1.
|
44
|
-
|
45
|
-
|
46
|
+
|
47
|
+
|
46
48
|
## Contributing
|
47
|
-
|
49
|
+
|
48
50
|
Bug reports and pull requests are welcome. This project is intended to
|
49
51
|
be a safe, welcoming space for collaboration.
|
50
|
-
|
52
|
+
|
51
53
|
## Development
|
52
|
-
|
54
|
+
|
53
55
|
We use docker to build the plugin. You can build it by running `docker-compose run jruby gem build logstash-output-lmlogs.gemspec `
|
@@ -8,6 +8,7 @@ require 'date'
|
|
8
8
|
require 'base64'
|
9
9
|
require 'openssl'
|
10
10
|
require 'manticore'
|
11
|
+
require_relative "version"
|
11
12
|
|
12
13
|
# An example output that does nothing.
|
13
14
|
class LogStash::Outputs::LMLogs < LogStash::Outputs::Base
|
@@ -94,7 +95,7 @@ class LogStash::Outputs::LMLogs < LogStash::Outputs::Base
|
|
94
95
|
config :access_id, :validate => :string, :required => false, :default => nil
|
95
96
|
|
96
97
|
# Include/Exclude metadata from sending to LM Logs
|
97
|
-
config :include_metadata, :validate => :boolean, :default =>
|
98
|
+
config :include_metadata, :validate => :boolean, :default => false
|
98
99
|
|
99
100
|
# Password to use for HTTP auth
|
100
101
|
config :access_key, :validate => :password, :required => false, :default => nil
|
@@ -102,6 +103,9 @@ class LogStash::Outputs::LMLogs < LogStash::Outputs::Base
|
|
102
103
|
# Use bearer token instead of access key/id for authentication.
|
103
104
|
config :bearer_token, :validate => :password, :required => false, :default => nil
|
104
105
|
|
106
|
+
# json keys for which plugin looks for these keys and adds as event meatadata. A dot "." can be used to add nested subjson.
|
107
|
+
config :include_metadata_keys, :validate => :array, :required => false, :default => []
|
108
|
+
|
105
109
|
@@MAX_PAYLOAD_SIZE = 8*1024*1024
|
106
110
|
|
107
111
|
# For developer debugging.
|
@@ -116,6 +120,14 @@ class LogStash::Outputs::LMLogs < LogStash::Outputs::Base
|
|
116
120
|
logger.info("Max Payload Size: ",
|
117
121
|
:size => @@MAX_PAYLOAD_SIZE)
|
118
122
|
configure_auth
|
123
|
+
|
124
|
+
@final_metadata_keys = Hash.new
|
125
|
+
if @include_metadata_keys.any?
|
126
|
+
include_metadata_keys.each do | nested_key |
|
127
|
+
@final_metadata_keys[nested_key] = nested_key.to_s.split('.')
|
128
|
+
end
|
129
|
+
end
|
130
|
+
|
119
131
|
end # def register
|
120
132
|
|
121
133
|
def client_config
|
@@ -164,12 +176,12 @@ class LogStash::Outputs::LMLogs < LogStash::Outputs::Base
|
|
164
176
|
if @access_id == nil || @access_key.value == nil
|
165
177
|
@logger.info "Access Id or access key null. Using bearer token for authentication."
|
166
178
|
@use_bearer_instead_of_lmv1 = true
|
167
|
-
end
|
168
|
-
if @use_bearer_instead_of_lmv1 && @bearer_token.value == nil
|
179
|
+
end
|
180
|
+
if @use_bearer_instead_of_lmv1 && @bearer_token.value == nil
|
169
181
|
@logger.error "Bearer token not specified. Either access_id and access_key both or bearer_token must be specified for authentication with Logicmonitor."
|
170
182
|
raise LogStash::ConfigurationError, 'No valid authentication specified. Either access_id and access_key both or bearer_token must be specified for authentication with Logicmonitor.'
|
171
183
|
end
|
172
|
-
end
|
184
|
+
end
|
173
185
|
def generate_auth_string(body)
|
174
186
|
if @use_bearer_instead_of_lmv1
|
175
187
|
return "Bearer #{@bearer_token.value}"
|
@@ -196,7 +208,7 @@ class LogStash::Outputs::LMLogs < LogStash::Outputs::Base
|
|
196
208
|
:body => body,
|
197
209
|
:headers => {
|
198
210
|
"Content-Type" => "application/json",
|
199
|
-
"User-Agent" => "
|
211
|
+
"User-Agent" => "lm-logs-logstash/" + LmLogsLogstashPlugin::VERSION,
|
200
212
|
"Authorization" => "#{auth_string}"
|
201
213
|
}
|
202
214
|
})
|
@@ -265,34 +277,48 @@ class LogStash::Outputs::LMLogs < LogStash::Outputs::Base
|
|
265
277
|
events.each_slice(@batch_size) do |chunk|
|
266
278
|
documents = []
|
267
279
|
chunk.each do |event|
|
268
|
-
event_json = JSON.parse(event.to_json)
|
269
|
-
lmlogs_event = {}
|
270
|
-
|
271
|
-
if @include_metadata
|
272
|
-
lmlogs_event = event_json
|
273
|
-
lmlogs_event.delete("@timestamp") # remove redundant timestamp field
|
274
|
-
if lmlogs_event.dig("event", "original") != nil
|
275
|
-
lmlogs_event["event"].delete("original") # remove redundant log field
|
276
|
-
end
|
277
|
-
end
|
278
280
|
|
279
|
-
|
280
|
-
|
281
|
-
|
281
|
+
documents = isValidPayloadSize(documents, processEvent(event), @@MAX_PAYLOAD_SIZE)
|
282
|
+
end
|
283
|
+
send_batch(documents)
|
284
|
+
end
|
285
|
+
end
|
282
286
|
|
283
|
-
if @keep_timestamp
|
284
|
-
lmlogs_event["timestamp"] = event.get("@timestamp")
|
285
|
-
end
|
286
287
|
|
287
|
-
|
288
|
-
|
288
|
+
def processEvent(event)
|
289
|
+
event_json = JSON.parse(event.to_json)
|
290
|
+
lmlogs_event = {}
|
291
|
+
|
292
|
+
if @include_metadata
|
293
|
+
lmlogs_event = event_json
|
294
|
+
lmlogs_event.delete("@timestamp") # remove redundant timestamp field
|
295
|
+
if lmlogs_event.dig("event", "original") != nil
|
296
|
+
lmlogs_event["event"].delete("original") # remove redundant log field
|
297
|
+
end
|
298
|
+
elsif @final_metadata_keys
|
299
|
+
@final_metadata_keys.each do | key, value |
|
300
|
+
nestedVal = event_json
|
301
|
+
value.each { |x| nestedVal = nestedVal[x] }
|
302
|
+
if nestedVal != nil
|
303
|
+
lmlogs_event[key] = nestedVal
|
289
304
|
end
|
305
|
+
end
|
306
|
+
end
|
290
307
|
|
291
|
-
|
308
|
+
lmlogs_event["message"] = event.get(@message_key).to_s
|
309
|
+
lmlogs_event["_lm.resourceId"] = {}
|
310
|
+
lmlogs_event["_lm.resourceId"]["#{@lm_property}"] = event.get(@property_key.to_s)
|
292
311
|
|
293
|
-
|
294
|
-
|
312
|
+
if @keep_timestamp
|
313
|
+
lmlogs_event["timestamp"] = event.get("@timestamp")
|
314
|
+
end
|
315
|
+
|
316
|
+
if @timestamp_is_key
|
317
|
+
lmlogs_event["timestamp"] = event.get(@timestamp_key.to_s)
|
295
318
|
end
|
319
|
+
|
320
|
+
return lmlogs_event
|
321
|
+
|
296
322
|
end
|
297
323
|
|
298
324
|
def log_failure(message, opts)
|
@@ -1,6 +1,14 @@
|
|
1
|
+
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
|
5
|
+
require "logstash/outputs/version.rb"
|
6
|
+
|
7
|
+
|
8
|
+
|
1
9
|
Gem::Specification.new do |s|
|
2
10
|
s.name = 'logstash-output-lmlogs'
|
3
|
-
s.version =
|
11
|
+
s.version = LmLogsLogstashPlugin::VERSION
|
4
12
|
s.licenses = ['Apache-2.0']
|
5
13
|
s.summary = "Logstash output plugin for LM Logs"
|
6
14
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
@@ -25,4 +33,5 @@ Gem::Specification.new do |s|
|
|
25
33
|
s.add_runtime_dependency 'manticore', '>= 0.5.2', '< 1.0.0'
|
26
34
|
|
27
35
|
s.add_development_dependency 'logstash-devutils'
|
36
|
+
s.add_development_dependency 'hashdiff', '>= 1.0.0'
|
28
37
|
end
|
@@ -0,0 +1,86 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require "logstash/devutils/rspec/spec_helper"
|
3
|
+
require "logstash/outputs/lmlogs"
|
4
|
+
require "logstash/event"
|
5
|
+
require "hashdiff"
|
6
|
+
|
7
|
+
|
8
|
+
describe LogStash::Outputs::LMLogs do
|
9
|
+
|
10
|
+
|
11
|
+
let(:logstash_event) {LogStash::Event.new("message" => "hello this is log 1",
|
12
|
+
"host" => "host1",
|
13
|
+
"nested1" => {"nested2" => {"nested3" => "value"},
|
14
|
+
"nested2a" => {"nested3a" => {"nested4" => "valueA"}},
|
15
|
+
"nested2b" => {"nested3b" => "value"}
|
16
|
+
},
|
17
|
+
"nested1_" => "value",
|
18
|
+
"nested" => {"nested2" => {"nested3" => "value",
|
19
|
+
"nested3b" => "value"},
|
20
|
+
"nested_ignored" => "somevalue"
|
21
|
+
}
|
22
|
+
)}
|
23
|
+
let(:sample_lm_logs_event){{"message" => "hello this is log 1", "_lm.resourceId" => {"test.property" => "host1"}, "timestamp" => "2021-03-22T04:28:55.907121106Z"}}
|
24
|
+
let(:include_metadata_keys) {["host", "nested1.nested2.nested3", "nested1.nested2a", "nested.nested2" ]}
|
25
|
+
|
26
|
+
def create_output_plugin_with_conf(conf)
|
27
|
+
return LogStash::Outputs::LMLogs.new(conf)
|
28
|
+
end
|
29
|
+
|
30
|
+
def check_same_hash(h1,h2)
|
31
|
+
|
32
|
+
end
|
33
|
+
|
34
|
+
it "default behaviour" do
|
35
|
+
puts "default behaviour"
|
36
|
+
plugin = create_output_plugin_with_conf({
|
37
|
+
"portal_name" => "localhost",
|
38
|
+
"access_id" => "abcd",
|
39
|
+
"access_key" => "abcd",
|
40
|
+
"lm_property" => "system.hostname",
|
41
|
+
"property_key" => "host"
|
42
|
+
})
|
43
|
+
constructed_event = plugin.processEvent(logstash_event)
|
44
|
+
expected_event = {
|
45
|
+
"message" => "hello this is log 1",
|
46
|
+
"timestamp" => logstash_event.timestamp,
|
47
|
+
"_lm.resourceId" => {"system.hostname" => "host1"},
|
48
|
+
|
49
|
+
}
|
50
|
+
puts " actual : #{constructed_event} \n expected : #{expected_event}"
|
51
|
+
|
52
|
+
expect(Hashdiff.diff(constructed_event,expected_event)).to eq([])
|
53
|
+
end
|
54
|
+
|
55
|
+
it "with include_metadata set to true" do
|
56
|
+
puts "with include_metadata set to true"
|
57
|
+
plugin = create_output_plugin_with_conf({
|
58
|
+
"portal_name" => "localhost",
|
59
|
+
"access_id" => "abcd",
|
60
|
+
"access_key" => "abcd",
|
61
|
+
"lm_property" => "system.hostname",
|
62
|
+
"property_key" => "host",
|
63
|
+
"include_metadata" => true
|
64
|
+
})
|
65
|
+
constructed_event = plugin.processEvent(logstash_event)
|
66
|
+
expected_event = {
|
67
|
+
"message" => "hello this is log 1",
|
68
|
+
"timestamp" => logstash_event.timestamp,
|
69
|
+
"@version" => "1",
|
70
|
+
"_lm.resourceId" => {"system.hostname" => "host1"},
|
71
|
+
"host" => "host1",
|
72
|
+
"nested1" => {"nested2" => {"nested3" => "value"},
|
73
|
+
"nested2a" => {"nested3a" => {"nested4" => "valueA"}},
|
74
|
+
"nested2b" => {"nested3b" => "value"}
|
75
|
+
},
|
76
|
+
"nested1_" => "value",
|
77
|
+
"nested" => {"nested2" => {"nested3" => "value",
|
78
|
+
"nested3b" => "value"},
|
79
|
+
"nested_ignored" => "somevalue"
|
80
|
+
}
|
81
|
+
}
|
82
|
+
puts " actual : #{constructed_event} \n expected : #{expected_event}"
|
83
|
+
puts " hash diff : #{Hashdiff.diff(constructed_event,expected_event)}"
|
84
|
+
expect(Hashdiff.diff(constructed_event,expected_event)).to eq([])
|
85
|
+
end
|
86
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-output-lmlogs
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 2.0.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- LogicMonitor
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2023-
|
11
|
+
date: 2023-11-24 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -78,6 +78,20 @@ dependencies:
|
|
78
78
|
- - ">="
|
79
79
|
- !ruby/object:Gem::Version
|
80
80
|
version: '0'
|
81
|
+
- !ruby/object:Gem::Dependency
|
82
|
+
requirement: !ruby/object:Gem::Requirement
|
83
|
+
requirements:
|
84
|
+
- - ">="
|
85
|
+
- !ruby/object:Gem::Version
|
86
|
+
version: 1.0.0
|
87
|
+
name: hashdiff
|
88
|
+
prerelease: false
|
89
|
+
type: :development
|
90
|
+
version_requirements: !ruby/object:Gem::Requirement
|
91
|
+
requirements:
|
92
|
+
- - ">="
|
93
|
+
- !ruby/object:Gem::Version
|
94
|
+
version: 1.0.0
|
81
95
|
description: This gem is a Logstash plugin required to be installed on top of the
|
82
96
|
Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
|
83
97
|
gem is not a stand-alone program
|
@@ -90,9 +104,11 @@ files:
|
|
90
104
|
- Gemfile
|
91
105
|
- README.md
|
92
106
|
- lib/logstash/outputs/lmlogs.rb
|
107
|
+
- lib/logstash/outputs/version.rb
|
93
108
|
- logstash-output-lmlogs.gemspec
|
94
109
|
- spec/outputs/auth_spec.rb
|
95
110
|
- spec/outputs/lmlogs_spec.rb
|
111
|
+
- spec/outputs/metadata_spec.rb
|
96
112
|
homepage: https://www.logicmonitor.com
|
97
113
|
licenses:
|
98
114
|
- Apache-2.0
|
@@ -122,3 +138,4 @@ summary: Logstash output plugin for LM Logs
|
|
122
138
|
test_files:
|
123
139
|
- spec/outputs/auth_spec.rb
|
124
140
|
- spec/outputs/lmlogs_spec.rb
|
141
|
+
- spec/outputs/metadata_spec.rb
|