logstash-output-lmlogs 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/CHANGELOG.md +2 -0
- data/Gemfile +11 -0
- data/README.md +53 -0
- data/lib/logstash/outputs/lmlogs.rb +282 -0
- data/logstash-output-lmlogs.gemspec +28 -0
- data/spec/outputs/lmlogs_spec.rb +88 -0
- metadata +121 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: a4781f24993d21cefddbbdfdca25641f5382a4db7243a2ed0f06a15e58fdb870
|
4
|
+
data.tar.gz: cfaca0ff01103adbb6df7c6cf693c6db7eb06f435adfb0dd62ac7baf57de4d74
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: b79cb9b9306d41b5d1faf705208a1e3cacfc227bfb1ba1acf7b3580ed1f517e89ff148e05ea6cf892651d8997279a1ffad3c96bac4459a4bdacbcfc3fa1d3476
|
7
|
+
data.tar.gz: fff0d6826fa34cf0b6867c6406f5ca99d5c6e1244adfa1293a8d09e2d2393338c4c70d411b0f929365d4eb57eac5113f9537b9bfee815bf34bc68ec216527f20
|
data/CHANGELOG.md
ADDED
data/Gemfile
ADDED
@@ -0,0 +1,11 @@
|
|
1
|
+
source 'https://rubygems.org'
|
2
|
+
|
3
|
+
gemspec
|
4
|
+
|
5
|
+
logstash_path = ENV["LOGSTASH_PATH"] || "../../logstash"
|
6
|
+
use_logstash_source = ENV["LOGSTASH_SOURCE"] && ENV["LOGSTASH_SOURCE"].to_s == "1"
|
7
|
+
|
8
|
+
if Dir.exist?(logstash_path) && use_logstash_source
|
9
|
+
gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
|
10
|
+
gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
|
11
|
+
end
|
data/README.md
ADDED
@@ -0,0 +1,53 @@
|
|
1
|
+
[![Build Status](https://travis-ci.org/unomaly/logstash-output-unomaly.svg?branch=master)](https://travis-ci.org/unomaly/logstash-output-unomaly)
|
2
|
+
|
3
|
+
This plugin sends Logstash events to the [Logicmonitor Logs](https://www.logicmonitor.com)
|
4
|
+
|
5
|
+
# Getting started
|
6
|
+
|
7
|
+
## Installing through rubygems
|
8
|
+
|
9
|
+
Run the following on your Logstash instance
|
10
|
+
|
11
|
+
`logstash-plugin install logstash-output-lmlogs`
|
12
|
+
|
13
|
+
## Minimal configuration
|
14
|
+
```
|
15
|
+
output {
|
16
|
+
lmlogs {
|
17
|
+
portal_name => "your company name"
|
18
|
+
access_id => "your lm access id"
|
19
|
+
access_key => "your access key"
|
20
|
+
}
|
21
|
+
}
|
22
|
+
```
|
23
|
+
|
24
|
+
```
|
25
|
+
|
26
|
+
|
27
|
+
# Important options
|
28
|
+
|
29
|
+
| Option | Description | Default |
|
30
|
+
|----------------------------|-------------------------------------------------------------------------------------|-----------------|
|
31
|
+
| batch_size | Event batch size to send to LM Logs. | 100 |
|
32
|
+
| message_key | Key that will be used by the plugin as the system key | "message" |
|
33
|
+
| lm_property | Key that will be used by LM to match resource based on property |"system.hostname"|
|
34
|
+
| keep_timestamp | If false, LM Logs will use the ingestion timestamp as the event timestamp | true |
|
35
|
+
| timestamp_is_key | If true, LM Logs will use a specified key as the event timestamp | false |
|
36
|
+
| timestamp_key | If timestamp_is_key is set, LM Logs will use this key in the event as the timestamp | "logtimestamp" |
|
37
|
+
|
38
|
+
See the [source code](lib/logstash/outputs/lmlogs.rb) for the full list of options
|
39
|
+
|
40
|
+
The syntax for `message_key` and `source_key` values are available in the [Logstash Event API Documentation](https://www.elastic.co/guide/en/logstash/current/event-api.html)
|
41
|
+
|
42
|
+
## Known issues
|
43
|
+
- Installation of the plugin fails on Logstash 6.2.1.
|
44
|
+
|
45
|
+
|
46
|
+
## Contributing
|
47
|
+
|
48
|
+
Bug reports and pull requests are welcome. This project is intended to
|
49
|
+
be a safe, welcoming space for collaboration.
|
50
|
+
|
51
|
+
## Development
|
52
|
+
|
53
|
+
We use docker to build the plugin. You can build it by running `docker-compose run jruby gem build logstash-output-lmlogs.gemspec `
|
@@ -0,0 +1,282 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require "logstash/outputs/base"
|
3
|
+
require "logstash/namespace"
|
4
|
+
require "logstash/json"
|
5
|
+
require 'uri'
|
6
|
+
require 'json'
|
7
|
+
require 'date'
|
8
|
+
require 'base64'
|
9
|
+
require 'openssl'
|
10
|
+
require 'manticore'
|
11
|
+
|
12
|
+
# An example output that does nothing.
|
13
|
+
class LogStash::Outputs::LMLogs < LogStash::Outputs::Base
|
14
|
+
class InvalidHTTPConfigError < StandardError; end
|
15
|
+
|
16
|
+
concurrency :shared
|
17
|
+
config_name "lmlogs"
|
18
|
+
|
19
|
+
# Event batch size to send to LM Logs. Increasing the batch size can increase throughput by reducing HTTP overhead
|
20
|
+
config :batch_size, :validate => :number, :default => 100
|
21
|
+
|
22
|
+
# Key that will be used by the plugin as the log message
|
23
|
+
config :message_key, :validate => :string, :default => "message"
|
24
|
+
|
25
|
+
# Key that will be used by the plugin as the system key
|
26
|
+
config :property_key, :validate => :string, :default => "host"
|
27
|
+
|
28
|
+
# Key that will be used by LM to match resource based on property
|
29
|
+
config :lm_property, :validate => :string, :default => "system.hostname"
|
30
|
+
|
31
|
+
# Keep logstash timestamp
|
32
|
+
config :keep_timestamp, :validate => :boolean, :default => true
|
33
|
+
|
34
|
+
# Use a configured message key for timestamp values
|
35
|
+
# Valid timestamp formats are ISO8601 strings or epoch in seconds, milliseconds or nanoseconds
|
36
|
+
config :timestamp_is_key, :validate => :boolean, :default => false
|
37
|
+
config :timestamp_key, :validate => :string, :default => "logtimestamp"
|
38
|
+
|
39
|
+
# Display debug logs
|
40
|
+
config :debug, :validate => :boolean, :default => false
|
41
|
+
|
42
|
+
# Timeout (in seconds) for the entire request
|
43
|
+
config :request_timeout, :validate => :number, :default => 60
|
44
|
+
|
45
|
+
# Timeout (in seconds) to wait for data on the socket. Default is `10s`
|
46
|
+
config :socket_timeout, :validate => :number, :default => 10
|
47
|
+
|
48
|
+
# Timeout (in seconds) to wait for a connection to be established. Default is `10s`
|
49
|
+
config :connect_timeout, :validate => :number, :default => 10
|
50
|
+
|
51
|
+
# Should redirects be followed? Defaults to `true`
|
52
|
+
config :follow_redirects, :validate => :boolean, :default => true
|
53
|
+
|
54
|
+
# Max number of concurrent connections. Defaults to `50`
|
55
|
+
config :pool_max, :validate => :number, :default => 50
|
56
|
+
|
57
|
+
# Max number of concurrent connections to a single host. Defaults to `25`
|
58
|
+
config :pool_max_per_route, :validate => :number, :default => 25
|
59
|
+
|
60
|
+
# Turn this on to enable HTTP keepalive support. We highly recommend setting `automatic_retries` to at least
|
61
|
+
# one with this to fix interactions with broken keepalive implementations.
|
62
|
+
config :keepalive, :validate => :boolean, :default => true
|
63
|
+
|
64
|
+
# How many times should the client retry a failing URL. We highly recommend NOT setting this value
|
65
|
+
# to zero if keepalive is enabled. Some servers incorrectly end keepalives early requiring a retry!
|
66
|
+
# Note: if `retry_non_idempotent` is set only GET, HEAD, PUT, DELETE, OPTIONS, and TRACE requests will be retried.
|
67
|
+
config :automatic_retries, :validate => :number, :default => 5
|
68
|
+
|
69
|
+
# If `automatic_retries` is enabled this will cause non-idempotent HTTP verbs (such as POST) to be retried.
|
70
|
+
config :retry_non_idempotent, :validate => :boolean, :default => true
|
71
|
+
|
72
|
+
# How long to wait before checking if the connection is stale before executing a request on a connection using keepalive.
|
73
|
+
# # You may want to set this lower, possibly to 0 if you get connection errors regularly
|
74
|
+
# Quoting the Apache commons docs (this client is based Apache Commmons):
|
75
|
+
# 'Defines period of inactivity in milliseconds after which persistent connections must be re-validated prior to being leased to the consumer. Non-positive value passed to this method disables connection validation. This check helps detect connections that have become stale (half-closed) while kept inactive in the pool.'
|
76
|
+
# See https://hc.apache.org/httpcomponents-client-ga/httpclient/apidocs/org/apache/http/impl/conn/PoolingHttpClientConnectionManager.html#setValidateAfterInactivity(int)[these docs for more info]
|
77
|
+
config :validate_after_inactivity, :validate => :number, :default => 200
|
78
|
+
|
79
|
+
# Enable cookie support. With this enabled the client will persist cookies
|
80
|
+
# across requests as a normal web browser would. Enabled by default
|
81
|
+
config :cookies, :validate => :boolean, :default => true
|
82
|
+
|
83
|
+
# If you'd like to use an HTTP proxy . This supports multiple configuration syntaxes:
|
84
|
+
#
|
85
|
+
# 1. Proxy host in form: `http://proxy.org:1234`
|
86
|
+
# 2. Proxy host in form: `{host => "proxy.org", port => 80, scheme => 'http', user => 'username@host', password => 'password'}`
|
87
|
+
# 3. Proxy host in form: `{url => 'http://proxy.org:1234', user => 'username@host', password => 'password'}`
|
88
|
+
config :proxy
|
89
|
+
|
90
|
+
# LM Portal Name
|
91
|
+
config :portal_name, :validate => :string, :required => true
|
92
|
+
|
93
|
+
# Username to use for HTTP auth.
|
94
|
+
config :access_id, :validate => :string, :required => true
|
95
|
+
|
96
|
+
# Password to use for HTTP auth
|
97
|
+
config :access_key, :validate => :password, :required => true
|
98
|
+
|
99
|
+
@@MAX_PAYLOAD_SIZE = 8*1024*1024
|
100
|
+
|
101
|
+
public
|
102
|
+
def register
|
103
|
+
@total = 0
|
104
|
+
@total_failed = 0
|
105
|
+
logger.info("Initialized LogicMonitor output plugin with configuration",
|
106
|
+
:host => @host)
|
107
|
+
|
108
|
+
end # def register
|
109
|
+
|
110
|
+
def client_config
|
111
|
+
c = {
|
112
|
+
connect_timeout: @connect_timeout,
|
113
|
+
socket_timeout: @socket_timeout,
|
114
|
+
request_timeout: @request_timeout,
|
115
|
+
follow_redirects: @follow_redirects,
|
116
|
+
automatic_retries: @automatic_retries,
|
117
|
+
retry_non_idempotent: @retry_non_idempotent,
|
118
|
+
check_connection_timeout: @validate_after_inactivity,
|
119
|
+
pool_max: @pool_max,
|
120
|
+
pool_max_per_route: @pool_max_per_route,
|
121
|
+
cookies: @cookies,
|
122
|
+
keepalive: @keepalive
|
123
|
+
}
|
124
|
+
|
125
|
+
if @proxy
|
126
|
+
# Symbolize keys if necessary
|
127
|
+
c[:proxy] = @proxy.is_a?(Hash) ?
|
128
|
+
@proxy.reduce({}) {|memo,(k,v)| memo[k.to_sym] = v; memo} :
|
129
|
+
@proxy
|
130
|
+
end
|
131
|
+
|
132
|
+
if @access_id
|
133
|
+
if !@access_key || !@access_key.value
|
134
|
+
raise ::LogStash::ConfigurationError, "access_id '#{@access_id}' specified without access_key!"
|
135
|
+
end
|
136
|
+
|
137
|
+
# Symbolize keys if necessary
|
138
|
+
c[:auth] = {
|
139
|
+
:access_id => @access_id,
|
140
|
+
:access_key => @access_key.value,
|
141
|
+
:eager => true
|
142
|
+
}
|
143
|
+
end
|
144
|
+
end
|
145
|
+
|
146
|
+
private
|
147
|
+
def make_client
|
148
|
+
puts client_config
|
149
|
+
Manticore::Client.new(client_config)
|
150
|
+
end
|
151
|
+
|
152
|
+
public
|
153
|
+
def client
|
154
|
+
@client ||= make_client
|
155
|
+
end
|
156
|
+
|
157
|
+
public
|
158
|
+
def close
|
159
|
+
@client.close
|
160
|
+
end
|
161
|
+
|
162
|
+
|
163
|
+
def generate_auth_string(body)
|
164
|
+
timestamp = DateTime.now.strftime('%Q')
|
165
|
+
hash_this = "POST#{timestamp}#{body}/log/ingest"
|
166
|
+
sign_this = OpenSSL::HMAC.hexdigest(
|
167
|
+
OpenSSL::Digest.new('sha256'),
|
168
|
+
"#{@access_key.value}",
|
169
|
+
hash_this
|
170
|
+
)
|
171
|
+
signature = Base64.strict_encode64(sign_this)
|
172
|
+
"LMv1 #{@access_id}:#{signature}:#{timestamp}"
|
173
|
+
end
|
174
|
+
|
175
|
+
def send_batch(events)
|
176
|
+
url = "https://" + @portal_name + ".logicmonitor.com/rest/log/ingest"
|
177
|
+
body = events.to_json
|
178
|
+
auth_string = generate_auth_string(body)
|
179
|
+
request = client.post(url, {
|
180
|
+
:body => body,
|
181
|
+
:headers => {
|
182
|
+
"Content-Type" => "application/json",
|
183
|
+
"User-Agent" => "LM Logs Logstash Plugin",
|
184
|
+
"Authorization" => "#{auth_string}"
|
185
|
+
}
|
186
|
+
})
|
187
|
+
|
188
|
+
request.on_success do |response|
|
189
|
+
if response.code == 202
|
190
|
+
@total += events.length
|
191
|
+
@logger.debug("Successfully sent ",
|
192
|
+
:response_code => response.code,
|
193
|
+
:batch_size => events.length,
|
194
|
+
:total_sent => @total,
|
195
|
+
:time => Time::now.utc)
|
196
|
+
elsif response.code == 207
|
197
|
+
log_failure(
|
198
|
+
"207 HTTP code - some of the events successfully parsed, some not. ",
|
199
|
+
:response_code => response.code,
|
200
|
+
:url => url,
|
201
|
+
:response_body => response.body,
|
202
|
+
:total_failed => @total_failed)
|
203
|
+
else
|
204
|
+
@total_failed += 1
|
205
|
+
log_failure(
|
206
|
+
"Encountered non-202/207 HTTP code #{response.code}",
|
207
|
+
:response_code => response.code,
|
208
|
+
:url => url,
|
209
|
+
:response_body => response.body,
|
210
|
+
:total_failed => @total_failed)
|
211
|
+
end
|
212
|
+
end
|
213
|
+
|
214
|
+
request.on_failure do |exception|
|
215
|
+
@total_failed += 1
|
216
|
+
log_failure("Could not access URL",
|
217
|
+
:url => url,
|
218
|
+
:method => @http_method,
|
219
|
+
:body => body,
|
220
|
+
:message => exception.message,
|
221
|
+
:class => exception.class.name,
|
222
|
+
:backtrace => exception.backtrace,
|
223
|
+
:total_failed => @total_failed
|
224
|
+
)
|
225
|
+
end
|
226
|
+
|
227
|
+
@logger.debug("Sending LM Logs",
|
228
|
+
:total => @total,
|
229
|
+
:time => Time::now.utc)
|
230
|
+
request.call
|
231
|
+
|
232
|
+
rescue Exception => e
|
233
|
+
@logger.error("[Exception=] #{e.message} #{e.backtrace}")
|
234
|
+
end
|
235
|
+
|
236
|
+
|
237
|
+
public
|
238
|
+
def multi_receive(events)
|
239
|
+
puts @@MAX_PAYLOAD_SIZE
|
240
|
+
if debug
|
241
|
+
puts events.to_json
|
242
|
+
end
|
243
|
+
|
244
|
+
events.each_slice(@batch_size) do |chunk|
|
245
|
+
documents = []
|
246
|
+
chunk.each do |event|
|
247
|
+
lmlogs_event = {
|
248
|
+
message: event.get(@message_key).to_s
|
249
|
+
}
|
250
|
+
|
251
|
+
lmlogs_event["_lm.resourceId"] = {}
|
252
|
+
lmlogs_event["_lm.resourceId"]["#{@lm_property}"] = event.get(@property_key.to_s)
|
253
|
+
|
254
|
+
if @keep_timestamp
|
255
|
+
lmlogs_event["timestamp"] = event.get("@timestamp")
|
256
|
+
end
|
257
|
+
|
258
|
+
if @timestamp_is_key
|
259
|
+
lmlogs_event["timestamp"] = event.get(@timestamp_key.to_s)
|
260
|
+
end
|
261
|
+
|
262
|
+
documents = isValidPayloadSize(documents,lmlogs_event,@@MAX_PAYLOAD_SIZE)
|
263
|
+
|
264
|
+
end
|
265
|
+
send_batch(documents)
|
266
|
+
end
|
267
|
+
end
|
268
|
+
|
269
|
+
def log_failure(message, opts)
|
270
|
+
@logger.error("[HTTP Output Failure] #{message}", opts)
|
271
|
+
end
|
272
|
+
|
273
|
+
def isValidPayloadSize(documents,lmlogs_event,max_payload_size)
|
274
|
+
if (documents.to_json.bytesize + lmlogs_event.to_json.bytesize) > max_payload_size
|
275
|
+
send_batch(documents)
|
276
|
+
documents = []
|
277
|
+
|
278
|
+
end
|
279
|
+
documents.push(lmlogs_event)
|
280
|
+
return documents
|
281
|
+
end
|
282
|
+
end # class LogStash::Outputs::LMLogs
|
@@ -0,0 +1,28 @@
|
|
1
|
+
Gem::Specification.new do |s|
|
2
|
+
s.name = 'logstash-output-lmlogs'
|
3
|
+
s.version = '0.1.0'
|
4
|
+
s.licenses = ['Apache License (2.0)']
|
5
|
+
s.summary = "Logstash output plugin for LM Logs"
|
6
|
+
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
7
|
+
s.authors = ["LogicMonitor"]
|
8
|
+
s.email = "support@logicmonitor.com"
|
9
|
+
s.homepage = "https://www.logicmonitor.com"
|
10
|
+
s.require_paths = ["lib"]
|
11
|
+
|
12
|
+
# Files
|
13
|
+
s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','Gemfile']
|
14
|
+
# Tests
|
15
|
+
s.test_files = s.files.grep(%r{^(test|spec|features)/})
|
16
|
+
|
17
|
+
# Special flag to let us know this is actually a logstash plugin
|
18
|
+
s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
|
19
|
+
|
20
|
+
# Gem dependencies
|
21
|
+
#
|
22
|
+
|
23
|
+
s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
|
24
|
+
s.add_runtime_dependency "logstash-codec-plain"
|
25
|
+
s.add_runtime_dependency 'manticore', '>= 0.5.2', '< 1.0.0'
|
26
|
+
|
27
|
+
s.add_development_dependency 'logstash-devutils'
|
28
|
+
end
|
@@ -0,0 +1,88 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require "logstash/devutils/rspec/spec_helper"
|
3
|
+
require "logstash/outputs/lmlogs"
|
4
|
+
require "logstash/event"
|
5
|
+
|
6
|
+
describe LogStash::Outputs::LMLogs do
|
7
|
+
let(:sample_event) { LogStash::Event.new("message" => "hello this is log") }
|
8
|
+
let(:client) { @lmlogs.client }
|
9
|
+
|
10
|
+
before do
|
11
|
+
@lmlogs = LogStash::Outputs::LMLogs.new(
|
12
|
+
"portal_name" => "localhost",
|
13
|
+
"access_id" => "access_id",
|
14
|
+
"access_key" => "access_key",
|
15
|
+
"batch_size" => 3,
|
16
|
+
"lm_property" => "test.property"
|
17
|
+
)
|
18
|
+
@lmlogs.register
|
19
|
+
allow(@lmlogs).to receive(:client).and_return(client)
|
20
|
+
allow(client).to receive(:post).and_call_original
|
21
|
+
end
|
22
|
+
|
23
|
+
before do
|
24
|
+
allow(@lmlogs).to receive(:client).and_return(client)
|
25
|
+
end
|
26
|
+
|
27
|
+
it "Forwards an event" do
|
28
|
+
expect(client).to receive(:post).once.and_call_original
|
29
|
+
@lmlogs.multi_receive([sample_event])
|
30
|
+
end
|
31
|
+
|
32
|
+
it "Batches multiple events and extracts metadata" do
|
33
|
+
event1 = LogStash::Event.new("message" => "hello this is log 1", "host" => "host1")
|
34
|
+
event2 = LogStash::Event.new("message" => "hello this is log 2", "host" => "host2")
|
35
|
+
event3 = LogStash::Event.new("message" => "hello this is log 3", "host" => "host3")
|
36
|
+
expect(client).to receive(:post).once.with("https://localhost.logicmonitor.com/rest/log/ingest",hash_including(:body => LogStash::Json.dump(
|
37
|
+
[{"message" => "hello this is log 1", "_lm.resourceId" => {"test.property" => "host1"}, "timestamp" => event1.timestamp.to_s},
|
38
|
+
{"message" => "hello this is log 2", "_lm.resourceId" => {"test.property" => "host2"}, "timestamp" => event2.timestamp.to_s},
|
39
|
+
{"message" => "hello this is log 3", "_lm.resourceId" => {"test.property" => "host3"}, "timestamp" => event3.timestamp.to_s}
|
40
|
+
]
|
41
|
+
))).and_call_original
|
42
|
+
@lmlogs.multi_receive([event1, event2, event3])
|
43
|
+
end
|
44
|
+
|
45
|
+
it "Batches data of size batch_size" do
|
46
|
+
expect(client).to receive(:post).exactly(2).times.and_call_original
|
47
|
+
@lmlogs.multi_receive([sample_event, sample_event, sample_event, sample_event, sample_event])
|
48
|
+
end
|
49
|
+
|
50
|
+
it "max payload exceeded test" do
|
51
|
+
|
52
|
+
document = [{"message" => "hello this is log 1", "_lm.resourceId" => {"test.property" => "host1"}, "timestamp" => "2021-03-22T04:28:55.907121106Z"},
|
53
|
+
{"message" => "hello this is log 2", "_lm.resourceId" => {"test.property" => "host2"}, "timestamp" => "2021-03-22T04:28:55.907321106Z"},
|
54
|
+
{"message" => "hello this is log 3", "_lm.resourceId" => {"test.property" => "host3"}, "timestamp" => "2021-03-22T04:28:55.907421106Z"}
|
55
|
+
]
|
56
|
+
|
57
|
+
lm_logs_event = {"message" => "hello this is log 4", "_lm.resourceId" => {"test.property" => "host3"}, "timestamp" => "2021-03-22T04:28:55.909421106Z"}
|
58
|
+
document_expected = [{"message" => "hello this is log 4", "_lm.resourceId" => {"test.property" => "host3"}, "timestamp" => "2021-03-22T04:28:55.909421106Z"}]
|
59
|
+
expect(client).to receive(:post).once.with("https://localhost.logicmonitor.com/rest/log/ingest",hash_including(:body => LogStash::Json.dump(
|
60
|
+
[{"message" => "hello this is log 1", "_lm.resourceId" => {"test.property" => "host1"}, "timestamp" => "2021-03-22T04:28:55.907121106Z"},
|
61
|
+
{"message" => "hello this is log 2", "_lm.resourceId" => {"test.property" => "host2"}, "timestamp" => "2021-03-22T04:28:55.907321106Z"},
|
62
|
+
{"message" => "hello this is log 3", "_lm.resourceId" => {"test.property" => "host3"}, "timestamp" => "2021-03-22T04:28:55.907421106Z"}
|
63
|
+
]
|
64
|
+
))).and_call_original
|
65
|
+
|
66
|
+
document_result = @lmlogs.isValidPayloadSize(document,lm_logs_event,document.to_json.bytesize)
|
67
|
+
expect(document_result).to eq(document_expected)
|
68
|
+
end
|
69
|
+
it "max payload in limit" do
|
70
|
+
|
71
|
+
document = [{"message" => "hello this is log 1", "_lm.resourceId" => {"test.property" => "host1"}, "timestamp" => "2021-03-22T04:28:55.907121106Z"},
|
72
|
+
{"message" => "hello this is log 2", "_lm.resourceId" => {"test.property" => "host2"}, "timestamp" => "2021-03-22T04:28:55.907321106Z"},
|
73
|
+
{"message" => "hello this is log 3", "_lm.resourceId" => {"test.property" => "host3"}, "timestamp" => "2021-03-22T04:28:55.907421106Z"}
|
74
|
+
]
|
75
|
+
|
76
|
+
lm_logs_event = {"message" => "hello this is log 4", "_lm.resourceId" => {"test.property" => "host3"}, "timestamp" => "2021-03-22T04:28:55.909421106Z"}
|
77
|
+
|
78
|
+
document_expected = [{"message" => "hello this is log 1", "_lm.resourceId" => {"test.property" => "host1"}, "timestamp" => "2021-03-22T04:28:55.907121106Z"},
|
79
|
+
{"message" => "hello this is log 2", "_lm.resourceId" => {"test.property" => "host2"}, "timestamp" => "2021-03-22T04:28:55.907321106Z"},
|
80
|
+
{"message" => "hello this is log 3", "_lm.resourceId" => {"test.property" => "host3"}, "timestamp" => "2021-03-22T04:28:55.907421106Z"},
|
81
|
+
{"message" => "hello this is log 4", "_lm.resourceId" => {"test.property" => "host3"}, "timestamp" => "2021-03-22T04:28:55.909421106Z"}]
|
82
|
+
expect(client).to receive(:post).exactly(0).times.and_call_original
|
83
|
+
|
84
|
+
document_result = @lmlogs.isValidPayloadSize(document,lm_logs_event,document.to_json.bytesize + lm_logs_event.to_json.bytesize)
|
85
|
+
expect(document_result).to eq(document_expected)
|
86
|
+
end
|
87
|
+
|
88
|
+
end
|
metadata
ADDED
@@ -0,0 +1,121 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: logstash-output-lmlogs
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- LogicMonitor
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2021-04-01 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
requirement: !ruby/object:Gem::Requirement
|
15
|
+
requirements:
|
16
|
+
- - ">="
|
17
|
+
- !ruby/object:Gem::Version
|
18
|
+
version: '1.60'
|
19
|
+
- - "<="
|
20
|
+
- !ruby/object:Gem::Version
|
21
|
+
version: '2.99'
|
22
|
+
name: logstash-core-plugin-api
|
23
|
+
prerelease: false
|
24
|
+
type: :runtime
|
25
|
+
version_requirements: !ruby/object:Gem::Requirement
|
26
|
+
requirements:
|
27
|
+
- - ">="
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: '1.60'
|
30
|
+
- - "<="
|
31
|
+
- !ruby/object:Gem::Version
|
32
|
+
version: '2.99'
|
33
|
+
- !ruby/object:Gem::Dependency
|
34
|
+
requirement: !ruby/object:Gem::Requirement
|
35
|
+
requirements:
|
36
|
+
- - ">="
|
37
|
+
- !ruby/object:Gem::Version
|
38
|
+
version: '0'
|
39
|
+
name: logstash-codec-plain
|
40
|
+
prerelease: false
|
41
|
+
type: :runtime
|
42
|
+
version_requirements: !ruby/object:Gem::Requirement
|
43
|
+
requirements:
|
44
|
+
- - ">="
|
45
|
+
- !ruby/object:Gem::Version
|
46
|
+
version: '0'
|
47
|
+
- !ruby/object:Gem::Dependency
|
48
|
+
requirement: !ruby/object:Gem::Requirement
|
49
|
+
requirements:
|
50
|
+
- - ">="
|
51
|
+
- !ruby/object:Gem::Version
|
52
|
+
version: 0.5.2
|
53
|
+
- - "<"
|
54
|
+
- !ruby/object:Gem::Version
|
55
|
+
version: 1.0.0
|
56
|
+
name: manticore
|
57
|
+
prerelease: false
|
58
|
+
type: :runtime
|
59
|
+
version_requirements: !ruby/object:Gem::Requirement
|
60
|
+
requirements:
|
61
|
+
- - ">="
|
62
|
+
- !ruby/object:Gem::Version
|
63
|
+
version: 0.5.2
|
64
|
+
- - "<"
|
65
|
+
- !ruby/object:Gem::Version
|
66
|
+
version: 1.0.0
|
67
|
+
- !ruby/object:Gem::Dependency
|
68
|
+
requirement: !ruby/object:Gem::Requirement
|
69
|
+
requirements:
|
70
|
+
- - ">="
|
71
|
+
- !ruby/object:Gem::Version
|
72
|
+
version: '0'
|
73
|
+
name: logstash-devutils
|
74
|
+
prerelease: false
|
75
|
+
type: :development
|
76
|
+
version_requirements: !ruby/object:Gem::Requirement
|
77
|
+
requirements:
|
78
|
+
- - ">="
|
79
|
+
- !ruby/object:Gem::Version
|
80
|
+
version: '0'
|
81
|
+
description: This gem is a Logstash plugin required to be installed on top of the
|
82
|
+
Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
|
83
|
+
gem is not a stand-alone program
|
84
|
+
email: support@logicmonitor.com
|
85
|
+
executables: []
|
86
|
+
extensions: []
|
87
|
+
extra_rdoc_files: []
|
88
|
+
files:
|
89
|
+
- CHANGELOG.md
|
90
|
+
- Gemfile
|
91
|
+
- README.md
|
92
|
+
- lib/logstash/outputs/lmlogs.rb
|
93
|
+
- logstash-output-lmlogs.gemspec
|
94
|
+
- spec/outputs/lmlogs_spec.rb
|
95
|
+
homepage: https://www.logicmonitor.com
|
96
|
+
licenses:
|
97
|
+
- Apache License (2.0)
|
98
|
+
metadata:
|
99
|
+
logstash_plugin: 'true'
|
100
|
+
logstash_group: output
|
101
|
+
post_install_message:
|
102
|
+
rdoc_options: []
|
103
|
+
require_paths:
|
104
|
+
- lib
|
105
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
106
|
+
requirements:
|
107
|
+
- - ">="
|
108
|
+
- !ruby/object:Gem::Version
|
109
|
+
version: '0'
|
110
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
111
|
+
requirements:
|
112
|
+
- - ">="
|
113
|
+
- !ruby/object:Gem::Version
|
114
|
+
version: '0'
|
115
|
+
requirements: []
|
116
|
+
rubygems_version: 3.1.6
|
117
|
+
signing_key:
|
118
|
+
specification_version: 4
|
119
|
+
summary: Logstash output plugin for LM Logs
|
120
|
+
test_files:
|
121
|
+
- spec/outputs/lmlogs_spec.rb
|