fluent-plugin-lm-logs 1.2.4 → 1.2.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 745bc2d1d5288affa3b239860d5ee6b00c01df4e718fe21ebab62eec9ef2a211
4
- data.tar.gz: 54b54ed790d7d0c56af9eea61b24024e33755157396fbf71f2ae6bbe700f3e14
3
+ metadata.gz: 377dc56ec3f62bb91303431bbba701b179e5c5540feb4af72cfd91c27ac7b930
4
+ data.tar.gz: 80317f28d2cbe4a3315d715a408212c9d047874416e6e9b0fad84719fe6e4c76
5
5
  SHA512:
6
- metadata.gz: 84f747c7449f169e0283ca639cf7c9460700a93d247b4a52ca633002ffa522f66b12a3ca0d1576c88fe3b65161cd4d26a8a204b893c7b4032141d51196763203
7
- data.tar.gz: cbe53ee7f70aa06e220e99585b4ff5e5a1e826c3d30a264f67bf2675994faeb08d8c78476283456316c37ea8e6da8cae70294b377a920d7eb433257cecb75d0d
6
+ metadata.gz: 5fd55fb16669b87f4118e3a3617042974829ebb765cf6396909c18573840661415740cb448621e0ed1c36896882a3dc6b43a9ae93b163fdce849cd39fd47b181
7
+ data.tar.gz: 83fdf0fe55a63007ba22f7bd9c1d447dc984cd2f741454c9746edef91a902a3020a68c04eb5b05df3ab9c2920079b27434d73e9c8b61a5ef195757c6866f1525
data/README.md CHANGED
@@ -23,8 +23,9 @@ Create a custom `fluent.conf` or edit the existing one to specify which logs sho
23
23
  resource_mapping {"<event_key>": "<lm_property>"}
24
24
  company_name <lm_company_name>
25
25
  company_domain <lm_company_domain>
26
- access_id <lm_access_id>
26
+ access_id <lm_access_id>
27
27
  access_key <lm_access_key>
28
+ resource_type <resource_type>
28
29
  <buffer>
29
30
  @type memory
30
31
  flush_interval 1s
@@ -68,7 +69,8 @@ See the [LogicMonitor Helm repository](https://github.com/logicmonitor/k8s-helm-
68
69
  | `resource_mapping` | The mapping that defines the source of the log event to the LM resource. In this case, the `<event_key>` in the incoming event is mapped to the value of `<lm_property>`.|
69
70
  | `access_id` | LM API Token access ID. |
70
71
  | `access_key` | LM API Token access key. |
71
- | `bearer_token` | LM API Bearer Token. Either specify `access_id` and `access_key` both or `bearer_token`. If all specified, LMv1 token(`access_id` and `access_key`) will be used for authentication with Logicmonitor. |
72
+ | `resource_type` | If a Resource Type is explicitly specified, that value will be statically applied to all ingested logs. If set to `##predef.externalResourceType##`, the Resource Type will be assigned dynamically based on the log context or configuration. If left blank, the Resource Type field will remain unset in the ingested logs. |
73
+ | `bearer_token` | LM API Bearer Token. Either specify `access_id` and `access_key` both or `bearer_token`. If all specified, LMv1 token(`access_id` and `access_key`) will be used for authentication with Logicmonitor. |
72
74
  | `flush_interval` | Defines the time in seconds to wait before sending batches of logs to LogicMonitor. Default is `60s`. |
73
75
  | `debug` | When `true`, logs more information to the fluentd console. |
74
76
  | `force_encoding` | Specify charset when logs contains invalid utf-8 characters. |
@@ -16,244 +16,254 @@ require_relative "version"
16
16
 
17
17
 
18
18
  module Fluent
19
- class LmOutput < BufferedOutput
20
- Fluent::Plugin.register_output('lm', self)
19
+ module Plugin
20
+ class LmOutput < Fluent::Plugin::Output
21
+ Fluent::Plugin.register_output('lm', self)
21
22
 
22
- RESOURCE_MAPPING_KEY = "_lm.resourceId".freeze
23
- DEVICELESS_KEY_SERVICE = "resource.service.name".freeze
24
- DEVICELESS_KEY_NAMESPACE = "resource.service.namespace".freeze
23
+ RESOURCE_MAPPING_KEY = "_lm.resourceId".freeze
24
+ DEVICELESS_KEY_SERVICE = "resource.service.name".freeze
25
+ DEVICELESS_KEY_NAMESPACE = "resource.service.namespace".freeze
25
26
 
26
- # config_param defines a parameter. You can refer a parameter via @path instance variable
27
+ # config_param defines a parameter. You can refer a parameter via @path instance variable
27
28
 
28
- config_param :access_id, :string, :default => nil
29
+ config_param :access_id, :string, :default => nil
29
30
 
30
- config_param :access_key, :string, :default => nil, secret: true
31
+ config_param :access_key, :string, :default => nil, secret: true
31
32
 
32
- config_param :company_name, :string, :default => "company_name"
33
+ config_param :company_name, :string, :default => "company_name"
33
34
 
34
- config_param :resource_mapping, :hash, :default => {"host": "system.hostname", "hostname": "system.hostname"}
35
+ config_param :resource_mapping, :hash, :default => {"host": "system.hostname", "hostname": "system.hostname"}
35
36
 
36
- config_param :debug, :bool, :default => false
37
+ config_param :debug, :bool, :default => false
37
38
 
38
- config_param :include_metadata, :bool, :default => false
39
+ config_param :include_metadata, :bool, :default => false
39
40
 
40
- config_param :force_encoding, :string, :default => ""
41
+ config_param :force_encoding, :string, :default => ""
41
42
 
42
- config_param :compression, :string, :default => ""
43
+ config_param :compression, :string, :default => ""
43
44
 
44
- config_param :log_source, :string, :default => "lm-logs-fluentd"
45
+ config_param :log_source, :string, :default => "lm-logs-fluentd"
45
46
 
46
- config_param :version_id, :string, :default => "version_id"
47
+ config_param :version_id, :string, :default => "version_id"
47
48
 
48
- config_param :device_less_logs, :bool, :default => false
49
+ config_param :device_less_logs, :bool, :default => false
49
50
 
50
- config_param :http_proxy, :string, :default => nil
51
+ config_param :http_proxy, :string, :default => nil
51
52
 
52
- config_param :company_domain , :string, :default => "logicmonitor.com"
53
+ config_param :company_domain , :string, :default => "logicmonitor.com"
53
54
 
54
- # Use bearer token for auth.
55
- config_param :bearer_token, :string, :default => nil, secret: true
55
+ config_param :resource_type, :string, :default => ""
56
+ # Use bearer token for auth.
57
+ config_param :bearer_token, :string, :default => nil, secret: true
56
58
 
57
- # This method is called before starting.
58
- # 'conf' is a Hash that includes configuration parameters.
59
- # If the configuration is invalid, raise Fluent::ConfigError.
60
- def configure(conf)
61
- super
62
- end
59
+ # This method is called before starting.
60
+ # 'conf' is a Hash that includes configuration parameters.
61
+ # If the configuration is invalid, raise Fluent::ConfigError.
62
+ def configure(conf)
63
+ super
64
+ end
63
65
 
64
- def multi_workers_ready?
65
- true
66
- end
66
+ def multi_workers_ready?
67
+ true
68
+ end
67
69
 
68
- # This method is called when starting.
69
- # Open sockets or files here.
70
- def start
71
- super
72
- configure_auth
73
- proxy_uri = :ENV
74
- if @http_proxy
75
- proxy_uri = URI.parse(http_proxy)
76
- elsif ENV['HTTP_PROXY'] || ENV['http_proxy']
77
- log.info("Using HTTP proxy defined in environment variable")
70
+ # This method is called when starting.
71
+ # Open sockets or files here.
72
+ def start
73
+ super
74
+ configure_auth
75
+ proxy_uri = :ENV
76
+ if @http_proxy
77
+ proxy_uri = URI.parse(http_proxy)
78
+ elsif ENV['HTTP_PROXY'] || ENV['http_proxy']
79
+ log.info("Using HTTP proxy defined in environment variable")
80
+ end
81
+ @http_client = Net::HTTP::Persistent.new name: "fluent-plugin-lm-logs", proxy: proxy_uri
82
+ @http_client.override_headers["Content-Type"] = "application/json"
83
+ @http_client.override_headers["User-Agent"] = log_source + "/" + LmLogsFluentPlugin::VERSION
84
+ @url = "https://#{@company_name}.#{@company_domain}/rest/log/ingest"
85
+ @uri = URI.parse(@url)
78
86
  end
79
- @http_client = Net::HTTP::Persistent.new name: "fluent-plugin-lm-logs", proxy: proxy_uri
80
- @http_client.override_headers["Content-Type"] = "application/json"
81
- @http_client.override_headers["User-Agent"] = log_source + "/" + LmLogsFluentPlugin::VERSION
82
- @url = "https://#{@company_name}.#{@company_domain}/rest/log/ingest"
83
- @uri = URI.parse(@url)
84
- end
85
87
 
86
- def configure_auth
87
- @use_bearer_instead_of_lmv1 = false
88
- if is_blank(@access_id) || is_blank(@access_key)
89
- log.info "Access Id or access key blank / null. Using bearer token for authentication."
90
- @use_bearer_instead_of_lmv1 = true
88
+ def configure_auth
89
+ @use_bearer_instead_of_lmv1 = false
90
+ if is_blank(@access_id) || is_blank(@access_key)
91
+ log.info "Access Id or access key blank / null. Using bearer token for authentication."
92
+ @use_bearer_instead_of_lmv1 = true
93
+ end
94
+ if @use_bearer_instead_of_lmv1 && is_blank(@bearer_token)
95
+ log.error "Bearer token not specified. Either access_id and access_key both or bearer_token must be specified for authentication with Logicmonitor."
96
+ raise ArgumentError, 'No valid authentication specified. Either access_id and access_key both or bearer_token must be specified for authentication with Logicmonitor.'
97
+ end
91
98
  end
92
- if @use_bearer_instead_of_lmv1 && is_blank(@bearer_token)
93
- log.error "Bearer token not specified. Either access_id and access_key both or bearer_token must be specified for authentication with Logicmonitor."
94
- raise ArgumentError, 'No valid authentication specified. Either access_id and access_key both or bearer_token must be specified for authentication with Logicmonitor.'
99
+ # This method is called when shutting down.
100
+ # Shutdown the thread and close sockets or files here.
101
+ def shutdown
102
+ super
103
+ @http_client.shutdown
95
104
  end
96
- end
97
- # This method is called when shutting down.
98
- # Shutdown the thread and close sockets or files here.
99
- def shutdown
100
- super
101
- @http_client.shutdown
102
- end
103
105
 
104
- # This method is called when an event reaches to Fluentd.
105
- # Convert the event to a raw string.
106
- def format(tag, time, record)
107
- [tag, time, record].to_msgpack
108
- end
106
+ # This method is called when an event reaches to Fluentd.
107
+ # Convert the event to a raw string.
108
+ def format(tag, time, record)
109
+ [tag, time, record].to_msgpack
110
+ end
109
111
 
110
- # This method is called every flush interval. Write the buffer chunk
111
- # to files or databases here.
112
- # 'chunk' is a buffer chunk that includes multiple formatted
113
- # events. You can use 'data = chunk.read' to get all events and
114
- # 'chunk.open {|io| ... }' to get IO objects.
115
- #
116
- # NOTE! This method is called by internal thread, not Fluentd's main thread. So IO wait doesn't affect other plugins.
117
- def write(chunk)
118
- events = []
119
- chunk.msgpack_each do |(tag, time, record)|
120
- event = process_record(tag,time,record)
121
- if event != nil
122
- events.push(event)
112
+ # This method is called every flush interval. Write the buffer chunk
113
+ # to files or databases here.
114
+ # 'chunk' is a buffer chunk that includes multiple formatted
115
+ # events. You can use 'data = chunk.read' to get all events and
116
+ # 'chunk.open {|io| ... }' to get IO objects.
117
+ #
118
+ # NOTE! This method is called by internal thread, not Fluentd's main thread. So IO wait doesn't affect other plugins.
119
+ def write(chunk)
120
+ events = []
121
+ chunk.msgpack_each do |(tag, time, record)|
122
+ event = process_record(tag,time,record)
123
+ if event != nil
124
+ events.push(event)
125
+ end
123
126
  end
127
+ send_batch(events)
124
128
  end
125
- send_batch(events)
126
- end
127
129
 
128
- def process_record(tag, time, record)
129
- resource_map = {}
130
- lm_event = {}
131
-
132
- if @include_metadata
133
- lm_event = get_metadata(record)
130
+ def formatted_to_msgpack_binary?
131
+ true
134
132
  end
135
133
 
136
- if !@device_less_logs
137
- # With devices
138
- if record[RESOURCE_MAPPING_KEY] == nil
139
- @resource_mapping.each do |key, value|
140
- k = value
141
- nestedVal = record
142
- key.to_s.split('.').each { |x| nestedVal = nestedVal[x] }
143
- if nestedVal != nil
144
- resource_map[k] = nestedVal
134
+ def process_record(tag, time, record)
135
+ resource_map = {}
136
+ lm_event = {}
137
+
138
+ if @include_metadata
139
+ lm_event = get_metadata(record)
140
+ end
141
+
142
+ if !@device_less_logs
143
+ # With devices
144
+ if record[RESOURCE_MAPPING_KEY] == nil
145
+ @resource_mapping.each do |key, value|
146
+ k = value
147
+ nestedVal = record
148
+ key.to_s.split('.').each { |x| nestedVal = nestedVal[x] }
149
+ if nestedVal != nil
150
+ resource_map[k] = nestedVal
151
+ end
145
152
  end
146
- end
147
- lm_event[RESOURCE_MAPPING_KEY] = resource_map
153
+ lm_event[RESOURCE_MAPPING_KEY] = resource_map
154
+ else
155
+ lm_event[RESOURCE_MAPPING_KEY] = record[RESOURCE_MAPPING_KEY]
156
+ end
148
157
  else
149
- lm_event[RESOURCE_MAPPING_KEY] = record[RESOURCE_MAPPING_KEY]
158
+ # Device less
159
+ if record[DEVICELESS_KEY_SERVICE]==nil
160
+ log.error "When device_less_logs is set \'true\', record must have \'service\'. Ignoring this event #{lm_event}."
161
+ return nil
162
+ else
163
+ lm_event[DEVICELESS_KEY_SERVICE] = encode_if_necessary(record[DEVICELESS_KEY_SERVICE])
164
+ if record[DEVICELESS_KEY_NAMESPACE]!=nil
165
+ lm_event[DEVICELESS_KEY_NAMESPACE] = encode_if_necessary(record[DEVICELESS_KEY_NAMESPACE])
166
+ end
167
+ end
150
168
  end
151
- else
152
- # Device less
153
- if record[DEVICELESS_KEY_SERVICE]==nil
154
- log.error "When device_less_logs is set \'true\', record must have \'service\'. Ignoring this event #{lm_event}."
155
- return nil
169
+
170
+ if record["timestamp"] != nil
171
+ lm_event["timestamp"] = record["timestamp"]
156
172
  else
157
- lm_event[DEVICELESS_KEY_SERVICE] = encode_if_necessary(record[DEVICELESS_KEY_SERVICE])
158
- if record[DEVICELESS_KEY_NAMESPACE]!=nil
159
- lm_event[DEVICELESS_KEY_NAMESPACE] = encode_if_necessary(record[DEVICELESS_KEY_NAMESPACE])
160
- end
173
+ lm_event["timestamp"] = Time.at(time).utc.to_datetime.rfc3339
174
+ end
175
+ lm_event["message"] = encode_if_necessary(record["message"])
176
+
177
+ if !is_blank(@resource_type)
178
+ lm_event['_resource.type'] = resource_type
161
179
  end
162
- end
163
180
 
164
- if record["timestamp"] != nil
165
- lm_event["timestamp"] = record["timestamp"]
166
- else
167
- lm_event["timestamp"] = Time.at(time).utc.to_datetime.rfc3339
181
+ return lm_event
168
182
  end
169
- lm_event["_resource.type"] = "Fluentd"
170
- lm_event["message"] = encode_if_necessary(record["message"])
171
183
 
172
- return lm_event
173
- end
184
+ def get_metadata(record)
185
+ #if encoding is not defined we will skip going through each key val
186
+ #and return the whole record for performance reasons in case of a bulky record.
187
+ if @force_encoding == ""
188
+ return record
189
+ else
190
+ lm_event = {}
191
+ record.each do |key, value|
192
+ lm_event["#{key}"] = get_encoded_string(value)
193
+ end
194
+ return lm_event
195
+ end
196
+ end
174
197
 
175
- def get_metadata(record)
176
- #if encoding is not defined we will skip going through each key val
177
- #and return the whole record for performance reasons in case of a bulky record.
178
- if @force_encoding == ""
179
- return record
180
- else
181
- lm_event = {}
182
- record.each do |key, value|
183
- lm_event["#{key}"] = get_encoded_string(value)
198
+ def encode_if_necessary(str)
199
+ if @force_encoding != ""
200
+ return get_encoded_string(str)
201
+ else
202
+ return str
184
203
  end
185
- return lm_event
186
204
  end
187
- end
188
205
 
189
- def encode_if_necessary(str)
190
- if @force_encoding != ""
191
- return get_encoded_string(str)
192
- else
193
- return str
206
+ def get_encoded_string(str)
207
+ return str.force_encoding(@force_encoding).encode("UTF-8")
194
208
  end
195
- end
196
209
 
197
- def get_encoded_string(str)
198
- return str.force_encoding(@force_encoding).encode("UTF-8")
199
- end
210
+ def send_batch(events)
211
+ body = events.to_json
200
212
 
201
- def send_batch(events)
202
- body = events.to_json
213
+ if @debug
214
+ log.info "Sending #{events.length} events to logic monitor at #{@url}"
215
+ log.info "Request json #{body}"
216
+ end
203
217
 
204
- if @debug
205
- log.info "Sending #{events.length} events to logic monitor at #{@url}"
206
- log.info "Request json #{body}"
207
- end
218
+ request = Net::HTTP::Post.new(@uri.request_uri)
219
+ request['authorization'] = generate_token(events)
208
220
 
209
- request = Net::HTTP::Post.new(@uri.request_uri)
210
- request['authorization'] = generate_token(events)
221
+ if @compression == "gzip"
222
+ request['Content-Encoding'] = "gzip"
223
+ gzip = Zlib::GzipWriter.new(StringIO.new)
224
+ gzip << body
225
+ request.body = gzip.close.string
226
+ else
227
+ request.body = body
228
+ end
211
229
 
212
- if @compression == "gzip"
213
- request['Content-Encoding'] = "gzip"
214
- gzip = Zlib::GzipWriter.new(StringIO.new)
215
- gzip << body
216
- request.body = gzip.close.string
217
- else
218
- request.body = body
219
- end
230
+ if @debug
231
+ log.info "Sending the below request headers to logicmonitor:"
232
+ request.each_header {|key,value| log.info "#{key} = #{value}" }
233
+ end
220
234
 
221
- if @debug
222
- log.info "Sending the below request headers to logicmonitor:"
223
- request.each_header {|key,value| log.info "#{key} = #{value}" }
235
+ resp = @http_client.request @uri, request
236
+ if @debug || resp.kind_of?(Net::HTTPMultiStatus) || !resp.kind_of?(Net::HTTPSuccess)
237
+ log.info "Status code:#{resp.code} Request Id:#{resp.header['x-request-id']} message:#{resp.body}"
238
+ end
224
239
  end
225
240
 
226
- resp = @http_client.request @uri, request
227
- if @debug || resp.kind_of?(Net::HTTPMultiStatus) || !resp.kind_of?(Net::HTTPSuccess)
228
- log.info "Status code:#{resp.code} Request Id:#{resp.header['x-request-id']} message:#{resp.body}"
229
- end
230
- end
231
241
 
242
+ def generate_token(events)
232
243
 
233
- def generate_token(events)
234
-
235
- if @use_bearer_instead_of_lmv1
236
- return "Bearer #{@bearer_token}"
237
- else
238
- timestamp = DateTime.now.strftime('%Q')
239
- signature = Base64.strict_encode64(
240
- OpenSSL::HMAC.hexdigest(
241
- OpenSSL::Digest.new('sha256'),
242
- @access_key,
243
- "POST#{timestamp}#{events.to_json}/log/ingest"
244
- )
245
- )
246
- return "LMv1 #{@access_id}:#{signature}:#{timestamp}"
244
+ if @use_bearer_instead_of_lmv1
245
+ return "Bearer #{@bearer_token}"
246
+ else
247
+ timestamp = DateTime.now.strftime('%Q')
248
+ signature = Base64.strict_encode64(
249
+ OpenSSL::HMAC.hexdigest(
250
+ OpenSSL::Digest.new('sha256'),
251
+ @access_key,
252
+ "POST#{timestamp}#{events.to_json}/log/ingest"
253
+ )
254
+ )
255
+ return "LMv1 #{@access_id}:#{signature}:#{timestamp}"
256
+ end
247
257
  end
248
- end
249
258
 
250
- def is_blank(str)
251
- if str.nil? || str.to_s.strip.empty?
252
- return true
253
- else
254
- return false
259
+ def is_blank(str)
260
+ if str.nil? || str.to_s.strip.empty?
261
+ return true
262
+ else
263
+ return false
264
+ end
255
265
  end
256
- end
257
266
 
267
+ end
258
268
  end
259
269
  end
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module LmLogsFluentPlugin
4
- VERSION = '1.2.4'
4
+ VERSION = '1.2.6'
5
5
  end
metadata CHANGED
@@ -1,14 +1,13 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-lm-logs
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.2.4
4
+ version: 1.2.6
5
5
  platform: ruby
6
6
  authors:
7
7
  - LogicMonitor
8
- autorequire:
9
8
  bindir: bin
10
9
  cert_chain: []
11
- date: 2024-12-24 00:00:00.000000000 Z
10
+ date: 1980-01-02 00:00:00.000000000 Z
12
11
  dependencies:
13
12
  - !ruby/object:Gem::Dependency
14
13
  name: fluentd
@@ -65,7 +64,6 @@ licenses:
65
64
  metadata:
66
65
  source_code_uri: https://github.com/logicmonitor/lm-logs-fluentd
67
66
  documentation_uri: https://www.rubydoc.info/gems/lm-logs-fluentd
68
- post_install_message:
69
67
  rdoc_options: []
70
68
  require_paths:
71
69
  - lib
@@ -80,8 +78,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
80
78
  - !ruby/object:Gem::Version
81
79
  version: '0'
82
80
  requirements: []
83
- rubygems_version: 3.5.22
84
- signing_key:
81
+ rubygems_version: 3.6.7
85
82
  specification_version: 4
86
83
  summary: LogicMonitor logs fluentd output plugin
87
84
  test_files: []