fluent-plugin-lm-logs 1.2.4 → 1.2.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +4 -2
- data/lib/fluent/plugin/out_lm.rb +192 -182
- data/lib/fluent/plugin/version.rb +1 -1
- metadata +3 -6
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 377dc56ec3f62bb91303431bbba701b179e5c5540feb4af72cfd91c27ac7b930
|
4
|
+
data.tar.gz: 80317f28d2cbe4a3315d715a408212c9d047874416e6e9b0fad84719fe6e4c76
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 5fd55fb16669b87f4118e3a3617042974829ebb765cf6396909c18573840661415740cb448621e0ed1c36896882a3dc6b43a9ae93b163fdce849cd39fd47b181
|
7
|
+
data.tar.gz: 83fdf0fe55a63007ba22f7bd9c1d447dc984cd2f741454c9746edef91a902a3020a68c04eb5b05df3ab9c2920079b27434d73e9c8b61a5ef195757c6866f1525
|
data/README.md
CHANGED
@@ -23,8 +23,9 @@ Create a custom `fluent.conf` or edit the existing one to specify which logs sho
|
|
23
23
|
resource_mapping {"<event_key>": "<lm_property>"}
|
24
24
|
company_name <lm_company_name>
|
25
25
|
company_domain <lm_company_domain>
|
26
|
-
|
26
|
+
access_id <lm_access_id>
|
27
27
|
access_key <lm_access_key>
|
28
|
+
resource_type <resource_type>
|
28
29
|
<buffer>
|
29
30
|
@type memory
|
30
31
|
flush_interval 1s
|
@@ -68,7 +69,8 @@ See the [LogicMonitor Helm repository](https://github.com/logicmonitor/k8s-helm-
|
|
68
69
|
| `resource_mapping` | The mapping that defines the source of the log event to the LM resource. In this case, the `<event_key>` in the incoming event is mapped to the value of `<lm_property>`.|
|
69
70
|
| `access_id` | LM API Token access ID. |
|
70
71
|
| `access_key` | LM API Token access key. |
|
71
|
-
| `
|
72
|
+
| `resource_type` | If a Resource Type is explicitly specified, that value will be statically applied to all ingested logs. If set to `##predef.externalResourceType##`, the Resource Type will be assigned dynamically based on the log context or configuration. If left blank, the Resource Type field will remain unset in the ingested logs. |
|
73
|
+
| `bearer_token` | LM API Bearer Token. Either specify `access_id` and `access_key` both or `bearer_token`. If all specified, LMv1 token(`access_id` and `access_key`) will be used for authentication with Logicmonitor. |
|
72
74
|
| `flush_interval` | Defines the time in seconds to wait before sending batches of logs to LogicMonitor. Default is `60s`. |
|
73
75
|
| `debug` | When `true`, logs more information to the fluentd console. |
|
74
76
|
| `force_encoding` | Specify charset when logs contains invalid utf-8 characters. |
|
data/lib/fluent/plugin/out_lm.rb
CHANGED
@@ -16,244 +16,254 @@ require_relative "version"
|
|
16
16
|
|
17
17
|
|
18
18
|
module Fluent
|
19
|
-
|
20
|
-
Fluent::Plugin
|
19
|
+
module Plugin
|
20
|
+
class LmOutput < Fluent::Plugin::Output
|
21
|
+
Fluent::Plugin.register_output('lm', self)
|
21
22
|
|
22
|
-
|
23
|
-
|
24
|
-
|
23
|
+
RESOURCE_MAPPING_KEY = "_lm.resourceId".freeze
|
24
|
+
DEVICELESS_KEY_SERVICE = "resource.service.name".freeze
|
25
|
+
DEVICELESS_KEY_NAMESPACE = "resource.service.namespace".freeze
|
25
26
|
|
26
|
-
|
27
|
+
# config_param defines a parameter. You can refer a parameter via @path instance variable
|
27
28
|
|
28
|
-
|
29
|
+
config_param :access_id, :string, :default => nil
|
29
30
|
|
30
|
-
|
31
|
+
config_param :access_key, :string, :default => nil, secret: true
|
31
32
|
|
32
|
-
|
33
|
+
config_param :company_name, :string, :default => "company_name"
|
33
34
|
|
34
|
-
|
35
|
+
config_param :resource_mapping, :hash, :default => {"host": "system.hostname", "hostname": "system.hostname"}
|
35
36
|
|
36
|
-
|
37
|
+
config_param :debug, :bool, :default => false
|
37
38
|
|
38
|
-
|
39
|
+
config_param :include_metadata, :bool, :default => false
|
39
40
|
|
40
|
-
|
41
|
+
config_param :force_encoding, :string, :default => ""
|
41
42
|
|
42
|
-
|
43
|
+
config_param :compression, :string, :default => ""
|
43
44
|
|
44
|
-
|
45
|
+
config_param :log_source, :string, :default => "lm-logs-fluentd"
|
45
46
|
|
46
|
-
|
47
|
+
config_param :version_id, :string, :default => "version_id"
|
47
48
|
|
48
|
-
|
49
|
+
config_param :device_less_logs, :bool, :default => false
|
49
50
|
|
50
|
-
|
51
|
+
config_param :http_proxy, :string, :default => nil
|
51
52
|
|
52
|
-
|
53
|
+
config_param :company_domain , :string, :default => "logicmonitor.com"
|
53
54
|
|
54
|
-
|
55
|
-
|
55
|
+
config_param :resource_type, :string, :default => ""
|
56
|
+
# Use bearer token for auth.
|
57
|
+
config_param :bearer_token, :string, :default => nil, secret: true
|
56
58
|
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
59
|
+
# This method is called before starting.
|
60
|
+
# 'conf' is a Hash that includes configuration parameters.
|
61
|
+
# If the configuration is invalid, raise Fluent::ConfigError.
|
62
|
+
def configure(conf)
|
63
|
+
super
|
64
|
+
end
|
63
65
|
|
64
|
-
|
65
|
-
|
66
|
-
|
66
|
+
def multi_workers_ready?
|
67
|
+
true
|
68
|
+
end
|
67
69
|
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
70
|
+
# This method is called when starting.
|
71
|
+
# Open sockets or files here.
|
72
|
+
def start
|
73
|
+
super
|
74
|
+
configure_auth
|
75
|
+
proxy_uri = :ENV
|
76
|
+
if @http_proxy
|
77
|
+
proxy_uri = URI.parse(http_proxy)
|
78
|
+
elsif ENV['HTTP_PROXY'] || ENV['http_proxy']
|
79
|
+
log.info("Using HTTP proxy defined in environment variable")
|
80
|
+
end
|
81
|
+
@http_client = Net::HTTP::Persistent.new name: "fluent-plugin-lm-logs", proxy: proxy_uri
|
82
|
+
@http_client.override_headers["Content-Type"] = "application/json"
|
83
|
+
@http_client.override_headers["User-Agent"] = log_source + "/" + LmLogsFluentPlugin::VERSION
|
84
|
+
@url = "https://#{@company_name}.#{@company_domain}/rest/log/ingest"
|
85
|
+
@uri = URI.parse(@url)
|
78
86
|
end
|
79
|
-
@http_client = Net::HTTP::Persistent.new name: "fluent-plugin-lm-logs", proxy: proxy_uri
|
80
|
-
@http_client.override_headers["Content-Type"] = "application/json"
|
81
|
-
@http_client.override_headers["User-Agent"] = log_source + "/" + LmLogsFluentPlugin::VERSION
|
82
|
-
@url = "https://#{@company_name}.#{@company_domain}/rest/log/ingest"
|
83
|
-
@uri = URI.parse(@url)
|
84
|
-
end
|
85
87
|
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
88
|
+
def configure_auth
|
89
|
+
@use_bearer_instead_of_lmv1 = false
|
90
|
+
if is_blank(@access_id) || is_blank(@access_key)
|
91
|
+
log.info "Access Id or access key blank / null. Using bearer token for authentication."
|
92
|
+
@use_bearer_instead_of_lmv1 = true
|
93
|
+
end
|
94
|
+
if @use_bearer_instead_of_lmv1 && is_blank(@bearer_token)
|
95
|
+
log.error "Bearer token not specified. Either access_id and access_key both or bearer_token must be specified for authentication with Logicmonitor."
|
96
|
+
raise ArgumentError, 'No valid authentication specified. Either access_id and access_key both or bearer_token must be specified for authentication with Logicmonitor.'
|
97
|
+
end
|
91
98
|
end
|
92
|
-
|
93
|
-
|
94
|
-
|
99
|
+
# This method is called when shutting down.
|
100
|
+
# Shutdown the thread and close sockets or files here.
|
101
|
+
def shutdown
|
102
|
+
super
|
103
|
+
@http_client.shutdown
|
95
104
|
end
|
96
|
-
end
|
97
|
-
# This method is called when shutting down.
|
98
|
-
# Shutdown the thread and close sockets or files here.
|
99
|
-
def shutdown
|
100
|
-
super
|
101
|
-
@http_client.shutdown
|
102
|
-
end
|
103
105
|
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
106
|
+
# This method is called when an event reaches to Fluentd.
|
107
|
+
# Convert the event to a raw string.
|
108
|
+
def format(tag, time, record)
|
109
|
+
[tag, time, record].to_msgpack
|
110
|
+
end
|
109
111
|
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
112
|
+
# This method is called every flush interval. Write the buffer chunk
|
113
|
+
# to files or databases here.
|
114
|
+
# 'chunk' is a buffer chunk that includes multiple formatted
|
115
|
+
# events. You can use 'data = chunk.read' to get all events and
|
116
|
+
# 'chunk.open {|io| ... }' to get IO objects.
|
117
|
+
#
|
118
|
+
# NOTE! This method is called by internal thread, not Fluentd's main thread. So IO wait doesn't affect other plugins.
|
119
|
+
def write(chunk)
|
120
|
+
events = []
|
121
|
+
chunk.msgpack_each do |(tag, time, record)|
|
122
|
+
event = process_record(tag,time,record)
|
123
|
+
if event != nil
|
124
|
+
events.push(event)
|
125
|
+
end
|
123
126
|
end
|
127
|
+
send_batch(events)
|
124
128
|
end
|
125
|
-
send_batch(events)
|
126
|
-
end
|
127
129
|
|
128
|
-
|
129
|
-
|
130
|
-
lm_event = {}
|
131
|
-
|
132
|
-
if @include_metadata
|
133
|
-
lm_event = get_metadata(record)
|
130
|
+
def formatted_to_msgpack_binary?
|
131
|
+
true
|
134
132
|
end
|
135
133
|
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
134
|
+
def process_record(tag, time, record)
|
135
|
+
resource_map = {}
|
136
|
+
lm_event = {}
|
137
|
+
|
138
|
+
if @include_metadata
|
139
|
+
lm_event = get_metadata(record)
|
140
|
+
end
|
141
|
+
|
142
|
+
if !@device_less_logs
|
143
|
+
# With devices
|
144
|
+
if record[RESOURCE_MAPPING_KEY] == nil
|
145
|
+
@resource_mapping.each do |key, value|
|
146
|
+
k = value
|
147
|
+
nestedVal = record
|
148
|
+
key.to_s.split('.').each { |x| nestedVal = nestedVal[x] }
|
149
|
+
if nestedVal != nil
|
150
|
+
resource_map[k] = nestedVal
|
151
|
+
end
|
145
152
|
end
|
146
|
-
|
147
|
-
|
153
|
+
lm_event[RESOURCE_MAPPING_KEY] = resource_map
|
154
|
+
else
|
155
|
+
lm_event[RESOURCE_MAPPING_KEY] = record[RESOURCE_MAPPING_KEY]
|
156
|
+
end
|
148
157
|
else
|
149
|
-
|
158
|
+
# Device less
|
159
|
+
if record[DEVICELESS_KEY_SERVICE]==nil
|
160
|
+
log.error "When device_less_logs is set \'true\', record must have \'service\'. Ignoring this event #{lm_event}."
|
161
|
+
return nil
|
162
|
+
else
|
163
|
+
lm_event[DEVICELESS_KEY_SERVICE] = encode_if_necessary(record[DEVICELESS_KEY_SERVICE])
|
164
|
+
if record[DEVICELESS_KEY_NAMESPACE]!=nil
|
165
|
+
lm_event[DEVICELESS_KEY_NAMESPACE] = encode_if_necessary(record[DEVICELESS_KEY_NAMESPACE])
|
166
|
+
end
|
167
|
+
end
|
150
168
|
end
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
log.error "When device_less_logs is set \'true\', record must have \'service\'. Ignoring this event #{lm_event}."
|
155
|
-
return nil
|
169
|
+
|
170
|
+
if record["timestamp"] != nil
|
171
|
+
lm_event["timestamp"] = record["timestamp"]
|
156
172
|
else
|
157
|
-
lm_event[
|
158
|
-
|
159
|
-
|
160
|
-
|
173
|
+
lm_event["timestamp"] = Time.at(time).utc.to_datetime.rfc3339
|
174
|
+
end
|
175
|
+
lm_event["message"] = encode_if_necessary(record["message"])
|
176
|
+
|
177
|
+
if !is_blank(@resource_type)
|
178
|
+
lm_event['_resource.type'] = resource_type
|
161
179
|
end
|
162
|
-
end
|
163
180
|
|
164
|
-
|
165
|
-
lm_event["timestamp"] = record["timestamp"]
|
166
|
-
else
|
167
|
-
lm_event["timestamp"] = Time.at(time).utc.to_datetime.rfc3339
|
181
|
+
return lm_event
|
168
182
|
end
|
169
|
-
lm_event["_resource.type"] = "Fluentd"
|
170
|
-
lm_event["message"] = encode_if_necessary(record["message"])
|
171
183
|
|
172
|
-
|
173
|
-
|
184
|
+
def get_metadata(record)
|
185
|
+
#if encoding is not defined we will skip going through each key val
|
186
|
+
#and return the whole record for performance reasons in case of a bulky record.
|
187
|
+
if @force_encoding == ""
|
188
|
+
return record
|
189
|
+
else
|
190
|
+
lm_event = {}
|
191
|
+
record.each do |key, value|
|
192
|
+
lm_event["#{key}"] = get_encoded_string(value)
|
193
|
+
end
|
194
|
+
return lm_event
|
195
|
+
end
|
196
|
+
end
|
174
197
|
|
175
|
-
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
else
|
181
|
-
lm_event = {}
|
182
|
-
record.each do |key, value|
|
183
|
-
lm_event["#{key}"] = get_encoded_string(value)
|
198
|
+
def encode_if_necessary(str)
|
199
|
+
if @force_encoding != ""
|
200
|
+
return get_encoded_string(str)
|
201
|
+
else
|
202
|
+
return str
|
184
203
|
end
|
185
|
-
return lm_event
|
186
204
|
end
|
187
|
-
end
|
188
205
|
|
189
|
-
|
190
|
-
|
191
|
-
return get_encoded_string(str)
|
192
|
-
else
|
193
|
-
return str
|
206
|
+
def get_encoded_string(str)
|
207
|
+
return str.force_encoding(@force_encoding).encode("UTF-8")
|
194
208
|
end
|
195
|
-
end
|
196
209
|
|
197
|
-
|
198
|
-
|
199
|
-
end
|
210
|
+
def send_batch(events)
|
211
|
+
body = events.to_json
|
200
212
|
|
201
|
-
|
202
|
-
|
213
|
+
if @debug
|
214
|
+
log.info "Sending #{events.length} events to logic monitor at #{@url}"
|
215
|
+
log.info "Request json #{body}"
|
216
|
+
end
|
203
217
|
|
204
|
-
|
205
|
-
|
206
|
-
log.info "Request json #{body}"
|
207
|
-
end
|
218
|
+
request = Net::HTTP::Post.new(@uri.request_uri)
|
219
|
+
request['authorization'] = generate_token(events)
|
208
220
|
|
209
|
-
|
210
|
-
|
221
|
+
if @compression == "gzip"
|
222
|
+
request['Content-Encoding'] = "gzip"
|
223
|
+
gzip = Zlib::GzipWriter.new(StringIO.new)
|
224
|
+
gzip << body
|
225
|
+
request.body = gzip.close.string
|
226
|
+
else
|
227
|
+
request.body = body
|
228
|
+
end
|
211
229
|
|
212
|
-
|
213
|
-
|
214
|
-
|
215
|
-
|
216
|
-
request.body = gzip.close.string
|
217
|
-
else
|
218
|
-
request.body = body
|
219
|
-
end
|
230
|
+
if @debug
|
231
|
+
log.info "Sending the below request headers to logicmonitor:"
|
232
|
+
request.each_header {|key,value| log.info "#{key} = #{value}" }
|
233
|
+
end
|
220
234
|
|
221
|
-
|
222
|
-
|
223
|
-
|
235
|
+
resp = @http_client.request @uri, request
|
236
|
+
if @debug || resp.kind_of?(Net::HTTPMultiStatus) || !resp.kind_of?(Net::HTTPSuccess)
|
237
|
+
log.info "Status code:#{resp.code} Request Id:#{resp.header['x-request-id']} message:#{resp.body}"
|
238
|
+
end
|
224
239
|
end
|
225
240
|
|
226
|
-
resp = @http_client.request @uri, request
|
227
|
-
if @debug || resp.kind_of?(Net::HTTPMultiStatus) || !resp.kind_of?(Net::HTTPSuccess)
|
228
|
-
log.info "Status code:#{resp.code} Request Id:#{resp.header['x-request-id']} message:#{resp.body}"
|
229
|
-
end
|
230
|
-
end
|
231
241
|
|
242
|
+
def generate_token(events)
|
232
243
|
|
233
|
-
|
234
|
-
|
235
|
-
|
236
|
-
|
237
|
-
|
238
|
-
|
239
|
-
|
240
|
-
|
241
|
-
|
242
|
-
|
243
|
-
|
244
|
-
|
245
|
-
|
246
|
-
return "LMv1 #{@access_id}:#{signature}:#{timestamp}"
|
244
|
+
if @use_bearer_instead_of_lmv1
|
245
|
+
return "Bearer #{@bearer_token}"
|
246
|
+
else
|
247
|
+
timestamp = DateTime.now.strftime('%Q')
|
248
|
+
signature = Base64.strict_encode64(
|
249
|
+
OpenSSL::HMAC.hexdigest(
|
250
|
+
OpenSSL::Digest.new('sha256'),
|
251
|
+
@access_key,
|
252
|
+
"POST#{timestamp}#{events.to_json}/log/ingest"
|
253
|
+
)
|
254
|
+
)
|
255
|
+
return "LMv1 #{@access_id}:#{signature}:#{timestamp}"
|
256
|
+
end
|
247
257
|
end
|
248
|
-
end
|
249
258
|
|
250
|
-
|
251
|
-
|
252
|
-
|
253
|
-
|
254
|
-
|
259
|
+
def is_blank(str)
|
260
|
+
if str.nil? || str.to_s.strip.empty?
|
261
|
+
return true
|
262
|
+
else
|
263
|
+
return false
|
264
|
+
end
|
255
265
|
end
|
256
|
-
end
|
257
266
|
|
267
|
+
end
|
258
268
|
end
|
259
269
|
end
|
metadata
CHANGED
@@ -1,14 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-lm-logs
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.2.
|
4
|
+
version: 1.2.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- LogicMonitor
|
8
|
-
autorequire:
|
9
8
|
bindir: bin
|
10
9
|
cert_chain: []
|
11
|
-
date:
|
10
|
+
date: 1980-01-02 00:00:00.000000000 Z
|
12
11
|
dependencies:
|
13
12
|
- !ruby/object:Gem::Dependency
|
14
13
|
name: fluentd
|
@@ -65,7 +64,6 @@ licenses:
|
|
65
64
|
metadata:
|
66
65
|
source_code_uri: https://github.com/logicmonitor/lm-logs-fluentd
|
67
66
|
documentation_uri: https://www.rubydoc.info/gems/lm-logs-fluentd
|
68
|
-
post_install_message:
|
69
67
|
rdoc_options: []
|
70
68
|
require_paths:
|
71
69
|
- lib
|
@@ -80,8 +78,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
80
78
|
- !ruby/object:Gem::Version
|
81
79
|
version: '0'
|
82
80
|
requirements: []
|
83
|
-
rubygems_version: 3.
|
84
|
-
signing_key:
|
81
|
+
rubygems_version: 3.6.7
|
85
82
|
specification_version: 4
|
86
83
|
summary: LogicMonitor logs fluentd output plugin
|
87
84
|
test_files: []
|