fluent-plugin-lm-logs 1.2.5 → 1.2.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 6acf353e0cbb1783eb22109eb555acad23540bade16fbca72a24bf96a030b906
4
- data.tar.gz: f87a8c226c61db34ee0dd2b5b5fe2e77ab7e1f2f51bf9b9ac2e5e378dbacb714
3
+ metadata.gz: 377dc56ec3f62bb91303431bbba701b179e5c5540feb4af72cfd91c27ac7b930
4
+ data.tar.gz: 80317f28d2cbe4a3315d715a408212c9d047874416e6e9b0fad84719fe6e4c76
5
5
  SHA512:
6
- metadata.gz: 916a3d78c9c0ddaed1b9ed4fb943c61f9132cfffe97782339c0d7afa9d38cde0949251946264b167d5deb9f69752711f4d8a21ac57fb26b60eeafe3d1481d2bf
7
- data.tar.gz: cd7f8a0388cf9f81c79638313bddeb4d01e69f3f072b803cd11d39f7b3dc5df7a9f8165fd225fc0cd8a938689ab6d1f92304746a97ea6a136dd55354876696e4
6
+ metadata.gz: 5fd55fb16669b87f4118e3a3617042974829ebb765cf6396909c18573840661415740cb448621e0ed1c36896882a3dc6b43a9ae93b163fdce849cd39fd47b181
7
+ data.tar.gz: 83fdf0fe55a63007ba22f7bd9c1d447dc984cd2f741454c9746edef91a902a3020a68c04eb5b05df3ab9c2920079b27434d73e9c8b61a5ef195757c6866f1525
data/README.md CHANGED
@@ -23,8 +23,9 @@ Create a custom `fluent.conf` or edit the existing one to specify which logs sho
23
23
  resource_mapping {"<event_key>": "<lm_property>"}
24
24
  company_name <lm_company_name>
25
25
  company_domain <lm_company_domain>
26
- access_id <lm_access_id>
26
+ access_id <lm_access_id>
27
27
  access_key <lm_access_key>
28
+ resource_type <resource_type>
28
29
  <buffer>
29
30
  @type memory
30
31
  flush_interval 1s
@@ -68,7 +69,8 @@ See the [LogicMonitor Helm repository](https://github.com/logicmonitor/k8s-helm-
68
69
  | `resource_mapping` | The mapping that defines the source of the log event to the LM resource. In this case, the `<event_key>` in the incoming event is mapped to the value of `<lm_property>`.|
69
70
  | `access_id` | LM API Token access ID. |
70
71
  | `access_key` | LM API Token access key. |
71
- | `bearer_token` | LM API Bearer Token. Either specify `access_id` and `access_key` both or `bearer_token`. If all specified, LMv1 token(`access_id` and `access_key`) will be used for authentication with Logicmonitor. |
72
+ | `resource_type` | If a Resource Type is explicitly specified, that value will be statically applied to all ingested logs. If set to `##predef.externalResourceType##`, the Resource Type will be assigned dynamically based on the log context or configuration. If left blank, the Resource Type field will remain unset in the ingested logs. |
73
+ | `bearer_token` | LM API Bearer Token. Either specify `access_id` and `access_key` both or `bearer_token`. If all specified, LMv1 token(`access_id` and `access_key`) will be used for authentication with Logicmonitor. |
72
74
  | `flush_interval` | Defines the time in seconds to wait before sending batches of logs to LogicMonitor. Default is `60s`. |
73
75
  | `debug` | When `true`, logs more information to the fluentd console. |
74
76
  | `force_encoding` | Specify charset when logs contains invalid utf-8 characters. |
@@ -21,7 +21,7 @@ Gem::Specification.new do |spec|
21
21
  spec.metadata["source_code_uri"] = "https://github.com/logicmonitor/lm-logs-fluentd"
22
22
  spec.metadata["documentation_uri"] = "https://www.rubydoc.info/gems/lm-logs-fluentd"
23
23
 
24
- spec.files = [".gitignore", "Gemfile", "LICENSE", "README.md", "Rakefile", "fluent-plugin-lm-logs.gemspec", "lib/fluent/plugin/version.rb", "lib/fluent/plugin/out_lm.rb", "lib/fluent/plugin/environment_detector.rb"]
24
+ spec.files = [".gitignore", "Gemfile", "LICENSE", "README.md", "Rakefile", "fluent-plugin-lm-logs.gemspec", "lib/fluent/plugin/version.rb", "lib/fluent/plugin/out_lm.rb"]
25
25
  spec.require_paths = ["lib"]
26
26
  spec.required_ruby_version = '>= 2.0.0'
27
27
 
@@ -10,293 +10,260 @@ require 'net/http'
10
10
  require 'net/http/persistent'
11
11
  require 'net/https'
12
12
  require('zlib')
13
- require_relative 'environment_detector'
14
13
 
15
14
  require_relative "version"
16
15
 
17
16
 
18
17
 
19
18
  module Fluent
20
- class LmOutput < BufferedOutput
21
- Fluent::Plugin.register_output('lm', self)
19
+ module Plugin
20
+ class LmOutput < Fluent::Plugin::Output
21
+ Fluent::Plugin.register_output('lm', self)
22
22
 
23
- RESOURCE_MAPPING_KEY = "_lm.resourceId".freeze
24
- DEVICELESS_KEY_SERVICE = "resource.service.name".freeze
25
- DEVICELESS_KEY_NAMESPACE = "resource.service.namespace".freeze
23
+ RESOURCE_MAPPING_KEY = "_lm.resourceId".freeze
24
+ DEVICELESS_KEY_SERVICE = "resource.service.name".freeze
25
+ DEVICELESS_KEY_NAMESPACE = "resource.service.namespace".freeze
26
26
 
27
- # config_param defines a parameter. You can refer a parameter via @path instance variable
27
+ # config_param defines a parameter. You can refer a parameter via @path instance variable
28
28
 
29
- config_param :access_id, :string, :default => nil
29
+ config_param :access_id, :string, :default => nil
30
30
 
31
- config_param :access_key, :string, :default => nil, secret: true
31
+ config_param :access_key, :string, :default => nil, secret: true
32
32
 
33
- config_param :company_name, :string, :default => "company_name"
33
+ config_param :company_name, :string, :default => "company_name"
34
34
 
35
- config_param :resource_mapping, :hash, :default => {"host": "system.hostname", "hostname": "system.hostname"}
35
+ config_param :resource_mapping, :hash, :default => {"host": "system.hostname", "hostname": "system.hostname"}
36
36
 
37
- config_param :debug, :bool, :default => false
37
+ config_param :debug, :bool, :default => false
38
38
 
39
- config_param :include_metadata, :bool, :default => false
39
+ config_param :include_metadata, :bool, :default => false
40
40
 
41
- config_param :force_encoding, :string, :default => ""
41
+ config_param :force_encoding, :string, :default => ""
42
42
 
43
- config_param :compression, :string, :default => ""
43
+ config_param :compression, :string, :default => ""
44
44
 
45
- config_param :log_source, :string, :default => "lm-logs-fluentd"
45
+ config_param :log_source, :string, :default => "lm-logs-fluentd"
46
46
 
47
- config_param :version_id, :string, :default => "version_id"
47
+ config_param :version_id, :string, :default => "version_id"
48
48
 
49
- config_param :device_less_logs, :bool, :default => false
49
+ config_param :device_less_logs, :bool, :default => false
50
50
 
51
- config_param :http_proxy, :string, :default => nil
51
+ config_param :http_proxy, :string, :default => nil
52
52
 
53
- config_param :company_domain , :string, :default => "logicmonitor.com"
53
+ config_param :company_domain , :string, :default => "logicmonitor.com"
54
54
 
55
- config_param :resource_type, :string, :default => ""
55
+ config_param :resource_type, :string, :default => ""
56
+ # Use bearer token for auth.
57
+ config_param :bearer_token, :string, :default => nil, secret: true
56
58
 
57
- # Use bearer token for auth.
58
- config_param :bearer_token, :string, :default => nil, secret: true
59
-
60
- # This method is called before starting.
61
- # 'conf' is a Hash that includes configuration parameters.
62
- # If the configuration is invalid, raise Fluent::ConfigError.
63
- def configure(conf)
64
- super
65
- end
59
+ # This method is called before starting.
60
+ # 'conf' is a Hash that includes configuration parameters.
61
+ # If the configuration is invalid, raise Fluent::ConfigError.
62
+ def configure(conf)
63
+ super
64
+ end
66
65
 
67
- def multi_workers_ready?
68
- true
69
- end
66
+ def multi_workers_ready?
67
+ true
68
+ end
70
69
 
71
- # This method is called when starting.
72
- # Open sockets or files here.
73
- def start
74
- super
75
- configure_auth
76
- proxy_uri = :ENV
77
- if @http_proxy
78
- proxy_uri = URI.parse(http_proxy)
79
- elsif ENV['HTTP_PROXY'] || ENV['http_proxy']
80
- log.info("Using HTTP proxy defined in environment variable")
70
+ # This method is called when starting.
71
+ # Open sockets or files here.
72
+ def start
73
+ super
74
+ configure_auth
75
+ proxy_uri = :ENV
76
+ if @http_proxy
77
+ proxy_uri = URI.parse(http_proxy)
78
+ elsif ENV['HTTP_PROXY'] || ENV['http_proxy']
79
+ log.info("Using HTTP proxy defined in environment variable")
80
+ end
81
+ @http_client = Net::HTTP::Persistent.new name: "fluent-plugin-lm-logs", proxy: proxy_uri
82
+ @http_client.override_headers["Content-Type"] = "application/json"
83
+ @http_client.override_headers["User-Agent"] = log_source + "/" + LmLogsFluentPlugin::VERSION
84
+ @url = "https://#{@company_name}.#{@company_domain}/rest/log/ingest"
85
+ @uri = URI.parse(@url)
81
86
  end
82
- @http_client = Net::HTTP::Persistent.new name: "fluent-plugin-lm-logs", proxy: proxy_uri
83
- @http_client.override_headers["Content-Type"] = "application/json"
84
- @http_client.override_headers["User-Agent"] = log_source + "/" + LmLogsFluentPlugin::VERSION
85
- @url = "https://#{@company_name}.#{@company_domain}/rest/log/ingest"
86
- @uri = URI.parse(@url)
87
- @detector = EnvironmentDetector.new
88
- @environment_info = @detector.detect
89
- @local_env_str = format_environment(@environment_info)
90
-
91
- log.info("Environment detected: #{@environment_info}")
92
- end
93
87
 
94
- def configure_auth
95
- @use_bearer_instead_of_lmv1 = false
96
- if is_blank(@access_id) || is_blank(@access_key)
97
- log.info "Access Id or access key blank / null. Using bearer token for authentication."
98
- @use_bearer_instead_of_lmv1 = true
88
+ def configure_auth
89
+ @use_bearer_instead_of_lmv1 = false
90
+ if is_blank(@access_id) || is_blank(@access_key)
91
+ log.info "Access Id or access key blank / null. Using bearer token for authentication."
92
+ @use_bearer_instead_of_lmv1 = true
93
+ end
94
+ if @use_bearer_instead_of_lmv1 && is_blank(@bearer_token)
95
+ log.error "Bearer token not specified. Either access_id and access_key both or bearer_token must be specified for authentication with Logicmonitor."
96
+ raise ArgumentError, 'No valid authentication specified. Either access_id and access_key both or bearer_token must be specified for authentication with Logicmonitor.'
97
+ end
99
98
  end
100
- if @use_bearer_instead_of_lmv1 && is_blank(@bearer_token)
101
- log.error "Bearer token not specified. Either access_id and access_key both or bearer_token must be specified for authentication with Logicmonitor."
102
- raise ArgumentError, 'No valid authentication specified. Either access_id and access_key both or bearer_token must be specified for authentication with Logicmonitor.'
99
+ # This method is called when shutting down.
100
+ # Shutdown the thread and close sockets or files here.
101
+ def shutdown
102
+ super
103
+ @http_client.shutdown
103
104
  end
104
- end
105
- # This method is called when shutting down.
106
- # Shutdown the thread and close sockets or files here.
107
- def shutdown
108
- super
109
- @http_client.shutdown
110
- end
111
105
 
112
- # This method is called when an event reaches to Fluentd.
113
- # Convert the event to a raw string.
114
- def format(tag, time, record)
115
- [tag, time, record].to_msgpack
116
- end
106
+ # This method is called when an event reaches to Fluentd.
107
+ # Convert the event to a raw string.
108
+ def format(tag, time, record)
109
+ [tag, time, record].to_msgpack
110
+ end
117
111
 
118
- # This method is called every flush interval. Write the buffer chunk
119
- # to files or databases here.
120
- # 'chunk' is a buffer chunk that includes multiple formatted
121
- # events. You can use 'data = chunk.read' to get all events and
122
- # 'chunk.open {|io| ... }' to get IO objects.
123
- #
124
- # NOTE! This method is called by internal thread, not Fluentd's main thread. So IO wait doesn't affect other plugins.
125
- def write(chunk)
126
- events = []
127
- chunk.msgpack_each do |(tag, time, record)|
128
- event = process_record(tag,time,record)
129
- if event != nil
130
- events.push(event)
112
+ # This method is called every flush interval. Write the buffer chunk
113
+ # to files or databases here.
114
+ # 'chunk' is a buffer chunk that includes multiple formatted
115
+ # events. You can use 'data = chunk.read' to get all events and
116
+ # 'chunk.open {|io| ... }' to get IO objects.
117
+ #
118
+ # NOTE! This method is called by internal thread, not Fluentd's main thread. So IO wait doesn't affect other plugins.
119
+ def write(chunk)
120
+ events = []
121
+ chunk.msgpack_each do |(tag, time, record)|
122
+ event = process_record(tag,time,record)
123
+ if event != nil
124
+ events.push(event)
125
+ end
131
126
  end
127
+ send_batch(events)
132
128
  end
133
- send_batch(events)
134
- end
135
129
 
136
- def process_record(tag, time, record)
137
- resource_map = {}
138
- lm_event = {}
139
-
140
- if @include_metadata
141
- lm_event = get_metadata(record)
130
+ def formatted_to_msgpack_binary?
131
+ true
142
132
  end
143
133
 
144
- if !@device_less_logs
145
- # With devices
146
- if record[RESOURCE_MAPPING_KEY] == nil
147
- @resource_mapping.each do |key, value|
148
- k = value
149
- nestedVal = record
150
- key.to_s.split('.').each { |x| nestedVal = nestedVal[x] }
151
- if nestedVal != nil
152
- resource_map[k] = nestedVal
134
+ def process_record(tag, time, record)
135
+ resource_map = {}
136
+ lm_event = {}
137
+
138
+ if @include_metadata
139
+ lm_event = get_metadata(record)
140
+ end
141
+
142
+ if !@device_less_logs
143
+ # With devices
144
+ if record[RESOURCE_MAPPING_KEY] == nil
145
+ @resource_mapping.each do |key, value|
146
+ k = value
147
+ nestedVal = record
148
+ key.to_s.split('.').each { |x| nestedVal = nestedVal[x] }
149
+ if nestedVal != nil
150
+ resource_map[k] = nestedVal
151
+ end
153
152
  end
154
- end
155
- lm_event[RESOURCE_MAPPING_KEY] = resource_map
153
+ lm_event[RESOURCE_MAPPING_KEY] = resource_map
154
+ else
155
+ lm_event[RESOURCE_MAPPING_KEY] = record[RESOURCE_MAPPING_KEY]
156
+ end
156
157
  else
157
- lm_event[RESOURCE_MAPPING_KEY] = record[RESOURCE_MAPPING_KEY]
158
+ # Device less
159
+ if record[DEVICELESS_KEY_SERVICE]==nil
160
+ log.error "When device_less_logs is set \'true\', record must have \'service\'. Ignoring this event #{lm_event}."
161
+ return nil
162
+ else
163
+ lm_event[DEVICELESS_KEY_SERVICE] = encode_if_necessary(record[DEVICELESS_KEY_SERVICE])
164
+ if record[DEVICELESS_KEY_NAMESPACE]!=nil
165
+ lm_event[DEVICELESS_KEY_NAMESPACE] = encode_if_necessary(record[DEVICELESS_KEY_NAMESPACE])
166
+ end
167
+ end
158
168
  end
159
- else
160
- # Device less
161
- if record[DEVICELESS_KEY_SERVICE]==nil
162
- log.error "When device_less_logs is set \'true\', record must have \'service\'. Ignoring this event #{lm_event}."
163
- return nil
169
+
170
+ if record["timestamp"] != nil
171
+ lm_event["timestamp"] = record["timestamp"]
164
172
  else
165
- lm_event[DEVICELESS_KEY_SERVICE] = encode_if_necessary(record[DEVICELESS_KEY_SERVICE])
166
- if record[DEVICELESS_KEY_NAMESPACE]!=nil
167
- lm_event[DEVICELESS_KEY_NAMESPACE] = encode_if_necessary(record[DEVICELESS_KEY_NAMESPACE])
168
- end
173
+ lm_event["timestamp"] = Time.at(time).utc.to_datetime.rfc3339
169
174
  end
170
- end
175
+ lm_event["message"] = encode_if_necessary(record["message"])
171
176
 
172
- if record["timestamp"] != nil
173
- lm_event["timestamp"] = record["timestamp"]
174
- else
175
- lm_event["timestamp"] = Time.at(time).utc.to_datetime.rfc3339
176
- end
177
- lm_event["message"] = encode_if_necessary(record["message"])
177
+ if !is_blank(@resource_type)
178
+ lm_event['_resource.type'] = resource_type
179
+ end
178
180
 
179
- resource_type = @resource_type || @detector.infer_resource_type(record, tag)
180
- if resource_type.nil? || resource_type.strip.empty? || resource_type == 'Unknown'
181
- resource_type = @local_env_str
181
+ return lm_event
182
182
  end
183
183
 
184
- lm_event['_resource.type'] = resource_type
185
-
186
- return lm_event
187
- end
188
-
189
- def get_metadata(record)
190
- #if encoding is not defined we will skip going through each key val
191
- #and return the whole record for performance reasons in case of a bulky record.
192
- if @force_encoding == ""
193
- return record
194
- else
195
- lm_event = {}
196
- record.each do |key, value|
197
- lm_event["#{key}"] = get_encoded_string(value)
184
+ def get_metadata(record)
185
+ #if encoding is not defined we will skip going through each key val
186
+ #and return the whole record for performance reasons in case of a bulky record.
187
+ if @force_encoding == ""
188
+ return record
189
+ else
190
+ lm_event = {}
191
+ record.each do |key, value|
192
+ lm_event["#{key}"] = get_encoded_string(value)
193
+ end
194
+ return lm_event
198
195
  end
199
- return lm_event
200
196
  end
201
- end
202
197
 
203
- def encode_if_necessary(str)
204
- if @force_encoding != ""
205
- return get_encoded_string(str)
206
- else
207
- return str
198
+ def encode_if_necessary(str)
199
+ if @force_encoding != ""
200
+ return get_encoded_string(str)
201
+ else
202
+ return str
203
+ end
208
204
  end
209
- end
210
205
 
211
- def get_encoded_string(str)
212
- return str.force_encoding(@force_encoding).encode("UTF-8")
213
- end
206
+ def get_encoded_string(str)
207
+ return str.force_encoding(@force_encoding).encode("UTF-8")
208
+ end
214
209
 
215
- def send_batch(events)
216
- body = events.to_json
210
+ def send_batch(events)
211
+ body = events.to_json
217
212
 
218
- if @debug
219
- log.info "Sending #{events.length} events to logic monitor at #{@url}"
220
- log.info "Request json #{body}"
221
- end
213
+ if @debug
214
+ log.info "Sending #{events.length} events to logic monitor at #{@url}"
215
+ log.info "Request json #{body}"
216
+ end
222
217
 
223
- request = Net::HTTP::Post.new(@uri.request_uri)
224
- request['authorization'] = generate_token(events)
218
+ request = Net::HTTP::Post.new(@uri.request_uri)
219
+ request['authorization'] = generate_token(events)
225
220
 
226
- if @compression == "gzip"
227
- request['Content-Encoding'] = "gzip"
228
- gzip = Zlib::GzipWriter.new(StringIO.new)
229
- gzip << body
230
- request.body = gzip.close.string
231
- else
232
- request.body = body
233
- end
221
+ if @compression == "gzip"
222
+ request['Content-Encoding'] = "gzip"
223
+ gzip = Zlib::GzipWriter.new(StringIO.new)
224
+ gzip << body
225
+ request.body = gzip.close.string
226
+ else
227
+ request.body = body
228
+ end
234
229
 
235
- if @debug
236
- log.info "Sending the below request headers to logicmonitor:"
237
- request.each_header {|key,value| log.info "#{key} = #{value}" }
238
- end
230
+ if @debug
231
+ log.info "Sending the below request headers to logicmonitor:"
232
+ request.each_header {|key,value| log.info "#{key} = #{value}" }
233
+ end
239
234
 
240
- resp = @http_client.request @uri, request
241
- if @debug || resp.kind_of?(Net::HTTPMultiStatus) || !resp.kind_of?(Net::HTTPSuccess)
242
- log.info "Status code:#{resp.code} Request Id:#{resp.header['x-request-id']} message:#{resp.body}"
235
+ resp = @http_client.request @uri, request
236
+ if @debug || resp.kind_of?(Net::HTTPMultiStatus) || !resp.kind_of?(Net::HTTPSuccess)
237
+ log.info "Status code:#{resp.code} Request Id:#{resp.header['x-request-id']} message:#{resp.body}"
238
+ end
243
239
  end
244
- end
245
240
 
246
241
 
247
- def generate_token(events)
248
-
249
- if @use_bearer_instead_of_lmv1
250
- return "Bearer #{@bearer_token}"
251
- else
252
- timestamp = DateTime.now.strftime('%Q')
253
- signature = Base64.strict_encode64(
254
- OpenSSL::HMAC.hexdigest(
255
- OpenSSL::Digest.new('sha256'),
256
- @access_key,
257
- "POST#{timestamp}#{events.to_json}/log/ingest"
258
- )
259
- )
260
- return "LMv1 #{@access_id}:#{signature}:#{timestamp}"
261
- end
262
- end
242
+ def generate_token(events)
263
243
 
264
- def is_blank(str)
265
- if str.nil? || str.to_s.strip.empty?
266
- return true
267
- else
268
- return false
244
+ if @use_bearer_instead_of_lmv1
245
+ return "Bearer #{@bearer_token}"
246
+ else
247
+ timestamp = DateTime.now.strftime('%Q')
248
+ signature = Base64.strict_encode64(
249
+ OpenSSL::HMAC.hexdigest(
250
+ OpenSSL::Digest.new('sha256'),
251
+ @access_key,
252
+ "POST#{timestamp}#{events.to_json}/log/ingest"
253
+ )
254
+ )
255
+ return "LMv1 #{@access_id}:#{signature}:#{timestamp}"
256
+ end
269
257
  end
270
- end
271
258
 
272
- def format_environment(env_info)
273
- runtime = env_info[:runtime]
274
- provider = env_info[:provider] if env_info.key?(:provider)
275
-
276
- case runtime
277
- when 'kubernetes'
278
- 'Kubernetes/Node'
279
- when 'docker'
280
- 'Docker/Host'
281
- when 'vm'
282
- case provider&.downcase
283
- when 'azure'
284
- 'Azure/VirtualMachine'
285
- when 'aws'
286
- 'AWS/EC2'
287
- when 'gcp'
288
- 'GCP/ComputeEngine'
259
+ def is_blank(str)
260
+ if str.nil? || str.to_s.strip.empty?
261
+ return true
289
262
  else
290
- 'Unknown/VirtualMachine'
263
+ return false
291
264
  end
292
- when 'physical'
293
- os = env_info[:os] || 'UnknownOS'
294
- product = env_info[:product] || 'UnknownHardware'
295
- "#{os} / #{product}"
296
- else
297
- 'UnknownEnvironment'
298
265
  end
299
- end
300
266
 
267
+ end
301
268
  end
302
269
  end
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module LmLogsFluentPlugin
4
- VERSION = '1.2.5'
4
+ VERSION = '1.2.6'
5
5
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-lm-logs
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.2.5
4
+ version: 1.2.6
5
5
  platform: ruby
6
6
  authors:
7
7
  - LogicMonitor
@@ -56,7 +56,6 @@ files:
56
56
  - README.md
57
57
  - Rakefile
58
58
  - fluent-plugin-lm-logs.gemspec
59
- - lib/fluent/plugin/environment_detector.rb
60
59
  - lib/fluent/plugin/out_lm.rb
61
60
  - lib/fluent/plugin/version.rb
62
61
  homepage: https://www.logicmonitor.com
@@ -1,201 +0,0 @@
1
- require 'net/http'
2
- require 'timeout'
3
- require 'json'
4
-
5
- class EnvironmentDetector
6
- METADATA_TIMEOUT = 1
7
-
8
- def detect
9
- if running_in_kubernetes?
10
- { runtime: 'kubernetes' }.merge(detect_node_info)
11
- elsif running_in_docker?
12
- { runtime: 'docker' }.merge(detect_node_info)
13
- else
14
- detect_host_environment
15
- end
16
- end
17
-
18
- def format_environment(env_info)
19
- runtime = env_info[:runtime]
20
- provider = env_info[:provider] if env_info.key?(:provider)
21
-
22
- case runtime
23
- when 'kubernetes'
24
- 'Kubernetes/Node'
25
- when 'docker'
26
- 'Docker/Host'
27
- when 'vm'
28
- case provider&.downcase
29
- when 'azure'
30
- 'Azure/VirtualMachine'
31
- when 'aws'
32
- 'AWS/EC2'
33
- when 'gcp'
34
- 'GCP/ComputeEngine'
35
- else
36
- 'Unknown/VirtualMachine'
37
- end
38
- when 'physical'
39
- os = env_info[:os] || 'UnknownOS'
40
- product = env_info[:product] || 'UnknownHardware'
41
- "#{os} / #{product}"
42
- else
43
- 'UnknownEnvironment'
44
- end
45
- end
46
-
47
- def infer_resource_type(record, tag = nil)
48
- return record['resource_type'] if record['resource_type']
49
-
50
- host = (record['host'] || record['hostname'] || '').to_s
51
- msg = (record['message'] || '').to_s
52
- program = (record['syslog_program'] || '').to_s
53
- tags = record['tags'] || []
54
- tag_down = tag&.downcase || ''
55
-
56
- host_down = host.downcase
57
- msg_down = msg.downcase
58
- program_down = program.downcase
59
- # From tag pattern (case-insensitive)
60
- return 'WindowsServer' if tag_down.include?('windows')
61
- return 'LinuxServer' if tag_down.include?('linux')
62
- return 'Kubernetes/Node' if tag_down.include?('k8s') || tag_down.include?('kubernetes')
63
- return 'Docker/Host' if tag_down.include?('docker')
64
-
65
- # Structured metadata
66
- return 'Kubernetes/Node' if record.key?('kubernetes')
67
- return 'Docker/Host' if record.key?('container_id') || record.dig('docker', 'container_id')
68
- return 'AWS/VirtualMachine' if host_down.start_with?('ip-') || msg_down.include?('amazon')
69
- return 'GCP/VirtualMachine' if host_down.include?('.c.') || host_down.include?('gcp')
70
- return 'Azure/VirtualMachine' if host_down.include?('cloudapp.net') || msg_down.include?('azure')
71
- return 'VMware/VirtualMachine' if msg_down.include?('vmware') || host_down.include?('vmware')
72
-
73
- return 'WindowsServer' if record.key?('EventID') || record.key?('ProviderName') || record.key?('Computer')
74
- return 'LinuxServer' if record.key?('syslog_facility') || program_down != ''
75
-
76
- return 'Firewall' if program_down.downcase.include?('firewalld') || msg_down.downcase.include?('iptables') || msg_down.include?('blocked by policy')
77
- return 'ACMEServer' if host_down.include?('acme') || msg_down.include?('ACME-Request') || tags.include?('acme')
78
- return 'WebServer' if msg_down.include?('nginx') || msg_down.include?('apache')
79
- return 'DatabaseServer' if msg_down.include?('mysql') || msg_down.include?('postgres') || msg_down.include?('oracle')
80
-
81
- 'Unknown'
82
- end
83
-
84
-
85
- private
86
-
87
- def running_in_kubernetes?
88
- ENV.key?('KUBERNETES_SERVICE_HOST') || ENV.key?('KUBERNETES_PORT')
89
- end
90
-
91
- def running_in_docker?
92
- return true if ENV['container'] == 'docker'
93
- cgroup = File.read('/proc/1/cgroup') rescue ''
94
- return true if cgroup.include?('docker') || cgroup.include?('containerd')
95
- File.exist?('/.dockerenv')
96
- end
97
-
98
- def detect_host_environment
99
- provider_info = detect_cloud_provider
100
- return { runtime: 'vm', provider: provider_info[:provider], details: provider_info[:details] } if provider_info
101
-
102
- os = detect_os
103
- product = detect_product_info
104
-
105
- if product.downcase.include?('xen hvm domu') && os.downcase.include?('amazon')
106
- return { runtime: 'vm', provider: 'aws', details: { os: os, product: product } }
107
- end
108
-
109
- { runtime: 'physical', os: os, product: product }
110
- end
111
-
112
- def detect_node_info
113
- { node_os: detect_os, node_product: detect_product_info }
114
- end
115
-
116
- def detect_cloud_provider
117
- azure_metadata || aws_metadata || gcp_metadata
118
- end
119
-
120
- def azure_metadata
121
- url = 'http://169.254.169.254/metadata/instance?api-version=2021-02-01'
122
- headers = { 'Metadata' => 'true' }
123
- response = fetch_metadata(url, headers)
124
- return unless response
125
- json = JSON.parse(response) rescue {}
126
- {
127
- provider: 'azure',
128
- details: {
129
- vm_id: json.dig('compute', 'vmId'),
130
- location: json.dig('compute', 'location'),
131
- name: json.dig('compute', 'name'),
132
- vm_size: json.dig('compute', 'vmSize')
133
- }
134
- }
135
- end
136
-
137
- def aws_metadata
138
- url = 'http://169.254.169.254/latest/meta-data/instance-id'
139
- response = fetch_metadata(url)
140
- return unless response
141
- { provider: 'aws', details: { instance_id: response.strip } }
142
- end
143
-
144
- def gcp_metadata
145
- url = 'http://169.254.169.254/computeMetadata/v1/instance/id'
146
- headers = { 'Metadata-Flavor' => 'Google' }
147
- response = fetch_metadata(url, headers)
148
- return unless response
149
- { provider: 'gcp', details: { instance_id: response.strip } }
150
- end
151
-
152
- def fetch_metadata(url, headers = {}, timeout_sec = METADATA_TIMEOUT)
153
- uri = URI(url)
154
- Timeout.timeout(timeout_sec) do
155
- req = Net::HTTP::Get.new(uri)
156
- headers.each { |k, v| req[k] = v }
157
- res = Net::HTTP.start(uri.host, uri.port, open_timeout: timeout_sec, read_timeout: timeout_sec) { |http| http.request(req) }
158
- return res.body if res.is_a?(Net::HTTPSuccess)
159
- end
160
- rescue Timeout::Error, SocketError, Errno::ECONNREFUSED, Errno::EHOSTUNREACH, EOFError
161
- nil
162
- end
163
-
164
- def detect_os
165
- if File.exist?('/etc/os-release')
166
- os_info = {}
167
- File.foreach('/etc/os-release') do |line|
168
- key, value = line.strip.split('=', 2)
169
- os_info[key] = value&.gsub('"', '')
170
- end
171
- "#{os_info['NAME']} #{os_info['VERSION']}"
172
- elsif RUBY_PLATFORM.include?('darwin')
173
- product_name = `sw_vers -productName`.strip
174
- product_version = `sw_vers -productVersion`.strip
175
- "#{product_name} #{product_version}"
176
- else
177
- `uname -a`.strip
178
- end
179
- rescue
180
- 'unknown'
181
- end
182
-
183
- def detect_product_info
184
- if File.exist?('/sys/class/dmi/id/sys_vendor') && File.exist?('/sys/class/dmi/id/product_name')
185
- vendor = read_file('/sys/class/dmi/id/sys_vendor')
186
- product = read_file('/sys/class/dmi/id/product_name')
187
- "#{vendor} #{product}".strip
188
- elsif RUBY_PLATFORM.include?('darwin')
189
- model = `system_profiler SPHardwareDataType | awk '/Model Identifier/ { print $3 }'`.strip
190
- model.empty? ? 'Mac' : model
191
- else
192
- 'unknown'
193
- end
194
- rescue
195
- 'unknown'
196
- end
197
-
198
- def read_file(path)
199
- File.read(path).strip if File.exist?(path)
200
- end
201
- end