logstash-output-lmlogs 1.0.0 → 1.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +1 -1
- data/README.md +10 -11
- data/lib/logstash/outputs/lmlogs.rb +33 -17
- data/logstash-output-lmlogs.gemspec +3 -3
- data/spec/outputs/lmlogs_spec.rb +9 -20
- metadata +5 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: d0215e007f329ca2c6538f6ec2d65932fe43d87c1396772e03c98ca132bc932e
|
4
|
+
data.tar.gz: 288bf0b9b00ab06c19a02d4f540daed77d50a797fcaaa4c96912b2ca1cc59140
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 7703c1da2e96bb1fbf1f55e72c04f8d3c22c0fe164f07496d70535b7eb358f67064b0021c3857b551e46419a6a1a62f5a26a88efed83c4b3135be1a48da943be
|
7
|
+
data.tar.gz: a49bbfa5e88c5dc18d72d7d37f5141da160ee020802a2812680b5433c343cab5a70d11a5af94f2ecde638455efc836d8b47aca798411dba648b369322b8cb25d
|
data/CHANGELOG.md
CHANGED
@@ -1,2 +1,2 @@
|
|
1
|
-
##
|
1
|
+
## 1.0.0
|
2
2
|
- First version of the plugin
|
data/README.md
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
[![
|
1
|
+
[![Gem Version](https://badge.fury.io/rb/logstash-output-lmlogs.svg)](https://badge.fury.io/rb/logstash-output-lmlogs)
|
2
2
|
|
3
3
|
This plugin sends Logstash events to the [Logicmonitor Logs](https://www.logicmonitor.com)
|
4
4
|
|
@@ -21,19 +21,18 @@ output {
|
|
21
21
|
}
|
22
22
|
```
|
23
23
|
|
24
|
-
```
|
25
24
|
|
26
25
|
|
27
|
-
|
26
|
+
## Important options
|
28
27
|
|
29
|
-
| Option
|
30
|
-
|
31
|
-
| batch_size
|
32
|
-
| message_key
|
33
|
-
| lm_property
|
34
|
-
| keep_timestamp
|
35
|
-
| timestamp_is_key
|
36
|
-
| timestamp_key
|
28
|
+
| Option | Description| Default |
|
29
|
+
| --- | --- | --- |
|
30
|
+
| batch_size | Event batch size to send to LM Logs.| 100 |
|
31
|
+
| message_key | Key that will be used by the plugin as the system key | "message" |
|
32
|
+
| lm_property | Key that will be used by LM to match resource based on property | "system.hostname" |
|
33
|
+
| keep_timestamp | If false, LM Logs will use the ingestion timestamp as the event timestamp | true |
|
34
|
+
| timestamp_is_key | If true, LM Logs will use a specified key as the event timestamp | false |
|
35
|
+
| timestamp_key | If timestamp_is_key is set, LM Logs will use this key in the event as the timestamp | "logtimestamp" |
|
37
36
|
|
38
37
|
See the [source code](lib/logstash/outputs/lmlogs.rb) for the full list of options
|
39
38
|
|
@@ -98,12 +98,17 @@ class LogStash::Outputs::LMLogs < LogStash::Outputs::Base
|
|
98
98
|
|
99
99
|
@@MAX_PAYLOAD_SIZE = 8*1024*1024
|
100
100
|
|
101
|
+
# For developer debugging.
|
102
|
+
@@CONSOLE_LOGS = false
|
103
|
+
|
101
104
|
public
|
102
105
|
def register
|
103
106
|
@total = 0
|
104
107
|
@total_failed = 0
|
105
108
|
logger.info("Initialized LogicMonitor output plugin with configuration",
|
106
109
|
:host => @host)
|
110
|
+
logger.info("Max Payload Size: ",
|
111
|
+
:size => @@MAX_PAYLOAD_SIZE)
|
107
112
|
|
108
113
|
end # def register
|
109
114
|
|
@@ -135,17 +140,18 @@ class LogStash::Outputs::LMLogs < LogStash::Outputs::Base
|
|
135
140
|
end
|
136
141
|
|
137
142
|
# Symbolize keys if necessary
|
138
|
-
c[:auth] = {
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
}
|
143
|
+
# c[:auth] = {
|
144
|
+
# :user => @access_id,
|
145
|
+
# :password => @access_key.value,
|
146
|
+
# :eager => true
|
147
|
+
# }
|
143
148
|
end
|
149
|
+
log_debug("manticore client config: ", :client => c)
|
150
|
+
return c
|
144
151
|
end
|
145
152
|
|
146
153
|
private
|
147
154
|
def make_client
|
148
|
-
puts client_config
|
149
155
|
Manticore::Client.new(client_config)
|
150
156
|
end
|
151
157
|
|
@@ -173,6 +179,8 @@ class LogStash::Outputs::LMLogs < LogStash::Outputs::Base
|
|
173
179
|
end
|
174
180
|
|
175
181
|
def send_batch(events)
|
182
|
+
log_debug("Started sending logs to LM: ",
|
183
|
+
:time => Time::now.utc)
|
176
184
|
url = "https://" + @portal_name + ".logicmonitor.com/rest/log/ingest"
|
177
185
|
body = events.to_json
|
178
186
|
auth_string = generate_auth_string(body)
|
@@ -188,7 +196,7 @@ class LogStash::Outputs::LMLogs < LogStash::Outputs::Base
|
|
188
196
|
request.on_success do |response|
|
189
197
|
if response.code == 202
|
190
198
|
@total += events.length
|
191
|
-
|
199
|
+
log_debug("Successfully sent ",
|
192
200
|
:response_code => response.code,
|
193
201
|
:batch_size => events.length,
|
194
202
|
:total_sent => @total,
|
@@ -213,10 +221,9 @@ class LogStash::Outputs::LMLogs < LogStash::Outputs::Base
|
|
213
221
|
|
214
222
|
request.on_failure do |exception|
|
215
223
|
@total_failed += 1
|
216
|
-
log_failure("
|
224
|
+
log_failure("The request failed. ",
|
217
225
|
:url => url,
|
218
226
|
:method => @http_method,
|
219
|
-
:body => body,
|
220
227
|
:message => exception.message,
|
221
228
|
:class => exception.class.name,
|
222
229
|
:backtrace => exception.backtrace,
|
@@ -224,7 +231,7 @@ class LogStash::Outputs::LMLogs < LogStash::Outputs::Base
|
|
224
231
|
)
|
225
232
|
end
|
226
233
|
|
227
|
-
|
234
|
+
log_debug("Completed sending logs to LM",
|
228
235
|
:total => @total,
|
229
236
|
:time => Time::now.utc)
|
230
237
|
request.call
|
@@ -233,20 +240,29 @@ class LogStash::Outputs::LMLogs < LogStash::Outputs::Base
|
|
233
240
|
@logger.error("[Exception=] #{e.message} #{e.backtrace}")
|
234
241
|
end
|
235
242
|
|
243
|
+
def log_debug(message, *opts)
|
244
|
+
if @@CONSOLE_LOGS
|
245
|
+
puts "[#{DateTime::now}] [logstash.outputs.lmlogs] [DEBUG] #{message} #{opts.to_s}"
|
246
|
+
elsif debug
|
247
|
+
@logger.debug(message, *opts)
|
248
|
+
end
|
249
|
+
end
|
236
250
|
|
237
251
|
public
|
238
252
|
def multi_receive(events)
|
239
|
-
|
240
|
-
|
241
|
-
|
242
|
-
end
|
253
|
+
if events.length() > 0
|
254
|
+
log_debug(events.to_json)
|
255
|
+
end
|
243
256
|
|
244
257
|
events.each_slice(@batch_size) do |chunk|
|
245
258
|
documents = []
|
246
259
|
chunk.each do |event|
|
247
|
-
|
248
|
-
|
249
|
-
|
260
|
+
event_json = JSON.parse(event.to_json)
|
261
|
+
lmlogs_event = event_json
|
262
|
+
lmlogs_event.delete("@timestamp") # remove redundant timestamp field
|
263
|
+
lmlogs_event["event"].delete("original") # remove redundant log field
|
264
|
+
|
265
|
+
lmlogs_event["message"] = event.get(@message_key).to_s
|
250
266
|
|
251
267
|
lmlogs_event["_lm.resourceId"] = {}
|
252
268
|
lmlogs_event["_lm.resourceId"]["#{@lm_property}"] = event.get(@property_key.to_s)
|
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'logstash-output-lmlogs'
|
3
|
-
s.version = '1.
|
4
|
-
s.licenses = ['Apache
|
3
|
+
s.version = '1.1.0'
|
4
|
+
s.licenses = ['Apache-2.0']
|
5
5
|
s.summary = "Logstash output plugin for LM Logs"
|
6
6
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
7
7
|
s.authors = ["LogicMonitor"]
|
@@ -15,7 +15,7 @@ Gem::Specification.new do |s|
|
|
15
15
|
s.test_files = s.files.grep(%r{^(test|spec|features)/})
|
16
16
|
|
17
17
|
# Special flag to let us know this is actually a logstash plugin
|
18
|
-
s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
|
18
|
+
s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" , "rubygems_mfa_required" => "false"}
|
19
19
|
|
20
20
|
# Gem dependencies
|
21
21
|
#
|
data/spec/outputs/lmlogs_spec.rb
CHANGED
@@ -6,6 +6,7 @@ require "logstash/event"
|
|
6
6
|
describe LogStash::Outputs::LMLogs do
|
7
7
|
let(:sample_event) { LogStash::Event.new("message" => "hello this is log") }
|
8
8
|
let(:client) { @lmlogs.client }
|
9
|
+
let(:sample_lm_logs_event){{"message" => "hello this is log 1", "_lm.resourceId" => {"test.property" => "host1"}, "timestamp" => "2021-03-22T04:28:55.907121106Z"}}
|
9
10
|
|
10
11
|
before do
|
11
12
|
@lmlogs = LogStash::Outputs::LMLogs.new(
|
@@ -47,20 +48,14 @@ describe LogStash::Outputs::LMLogs do
|
|
47
48
|
@lmlogs.multi_receive([sample_event, sample_event, sample_event, sample_event, sample_event])
|
48
49
|
end
|
49
50
|
|
50
|
-
it "max payload exceeded
|
51
|
+
it "max payload exceeded" do
|
51
52
|
|
52
|
-
document = [
|
53
|
-
{"message" => "hello this is log 2", "_lm.resourceId" => {"test.property" => "host2"}, "timestamp" => "2021-03-22T04:28:55.907321106Z"},
|
54
|
-
{"message" => "hello this is log 3", "_lm.resourceId" => {"test.property" => "host3"}, "timestamp" => "2021-03-22T04:28:55.907421106Z"}
|
55
|
-
]
|
53
|
+
document = [sample_lm_logs_event,sample_lm_logs_event]
|
56
54
|
|
57
|
-
lm_logs_event = {"message" => "hello this is log
|
58
|
-
document_expected = [
|
55
|
+
lm_logs_event = {"message" => "hello this is log 3", "_lm.resourceId" => {"test.property" => "host3"}, "timestamp" => "2021-03-22T04:28:55.909421106Z"}
|
56
|
+
document_expected = [lm_logs_event]
|
59
57
|
expect(client).to receive(:post).once.with("https://localhost.logicmonitor.com/rest/log/ingest",hash_including(:body => LogStash::Json.dump(
|
60
|
-
[
|
61
|
-
{"message" => "hello this is log 2", "_lm.resourceId" => {"test.property" => "host2"}, "timestamp" => "2021-03-22T04:28:55.907321106Z"},
|
62
|
-
{"message" => "hello this is log 3", "_lm.resourceId" => {"test.property" => "host3"}, "timestamp" => "2021-03-22T04:28:55.907421106Z"}
|
63
|
-
]
|
58
|
+
[sample_lm_logs_event,sample_lm_logs_event]
|
64
59
|
))).and_call_original
|
65
60
|
|
66
61
|
document_result = @lmlogs.isValidPayloadSize(document,lm_logs_event,document.to_json.bytesize)
|
@@ -68,17 +63,11 @@ describe LogStash::Outputs::LMLogs do
|
|
68
63
|
end
|
69
64
|
it "max payload in limit" do
|
70
65
|
|
71
|
-
document = [
|
72
|
-
{"message" => "hello this is log 2", "_lm.resourceId" => {"test.property" => "host2"}, "timestamp" => "2021-03-22T04:28:55.907321106Z"},
|
73
|
-
{"message" => "hello this is log 3", "_lm.resourceId" => {"test.property" => "host3"}, "timestamp" => "2021-03-22T04:28:55.907421106Z"}
|
74
|
-
]
|
66
|
+
document = [sample_lm_logs_event]
|
75
67
|
|
76
|
-
lm_logs_event = {"message" => "hello this is log
|
68
|
+
lm_logs_event = {"message" => "hello this is log 2", "_lm.resourceId" => {"test.property" => "host3"}, "timestamp" => "2021-03-22T04:28:55.909421106Z"}
|
77
69
|
|
78
|
-
document_expected = [
|
79
|
-
{"message" => "hello this is log 2", "_lm.resourceId" => {"test.property" => "host2"}, "timestamp" => "2021-03-22T04:28:55.907321106Z"},
|
80
|
-
{"message" => "hello this is log 3", "_lm.resourceId" => {"test.property" => "host3"}, "timestamp" => "2021-03-22T04:28:55.907421106Z"},
|
81
|
-
{"message" => "hello this is log 4", "_lm.resourceId" => {"test.property" => "host3"}, "timestamp" => "2021-03-22T04:28:55.909421106Z"}]
|
70
|
+
document_expected = [sample_lm_logs_event,lm_logs_event]
|
82
71
|
expect(client).to receive(:post).exactly(0).times.and_call_original
|
83
72
|
|
84
73
|
document_result = @lmlogs.isValidPayloadSize(document,lm_logs_event,document.to_json.bytesize + lm_logs_event.to_json.bytesize)
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-output-lmlogs
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- LogicMonitor
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2022-11-29 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -94,10 +94,11 @@ files:
|
|
94
94
|
- spec/outputs/lmlogs_spec.rb
|
95
95
|
homepage: https://www.logicmonitor.com
|
96
96
|
licenses:
|
97
|
-
- Apache
|
97
|
+
- Apache-2.0
|
98
98
|
metadata:
|
99
99
|
logstash_plugin: 'true'
|
100
100
|
logstash_group: output
|
101
|
+
rubygems_mfa_required: 'false'
|
101
102
|
post_install_message:
|
102
103
|
rdoc_options: []
|
103
104
|
require_paths:
|
@@ -113,7 +114,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
113
114
|
- !ruby/object:Gem::Version
|
114
115
|
version: '0'
|
115
116
|
requirements: []
|
116
|
-
rubygems_version: 3.
|
117
|
+
rubygems_version: 3.3.25
|
117
118
|
signing_key:
|
118
119
|
specification_version: 4
|
119
120
|
summary: Logstash output plugin for LM Logs
|