fluent-plugin-lm-logs 1.2.6 → 1.2.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 377dc56ec3f62bb91303431bbba701b179e5c5540feb4af72cfd91c27ac7b930
-  data.tar.gz: 80317f28d2cbe4a3315d715a408212c9d047874416e6e9b0fad84719fe6e4c76
+  metadata.gz: 8458c35bd163ce3c82f8c7e71e9d83450e1063bee51f6f497f3a6714c7269fc1
+  data.tar.gz: 9ebae5b64a71e61b335f957ec61e3ceb27d766e5e4af1a634247845f082d5503
 SHA512:
-  metadata.gz: 5fd55fb16669b87f4118e3a3617042974829ebb765cf6396909c18573840661415740cb448621e0ed1c36896882a3dc6b43a9ae93b163fdce849cd39fd47b181
-  data.tar.gz: 83fdf0fe55a63007ba22f7bd9c1d447dc984cd2f741454c9746edef91a902a3020a68c04eb5b05df3ab9c2920079b27434d73e9c8b61a5ef195757c6866f1525
+  metadata.gz: e85a136c8d024082707e7daf0109de9f026ec2916a370c12e8a3b58b0fe7725e09601550fa2bd2d6a366c5e4fa63843589010b87f9642164962e26e24d174c65
+  data.tar.gz: 4b2e8b6b36c4baa54443068c9af875db9d93e57231bebcb48b0e419bd9fb68704c7da0dd9cd54901916cbf33f820f27f08e8ba82193056335e70495445539c01

data/README.md

@@ -34,6 +34,34 @@ Create a custom `fluent.conf` or edit the existing one to specify which logs sho
     debug false
 </match>
 ```
+### Dynamic resource type
+
+If you want to use a dynamic resource type, you can leave the `resource_type` field empty. The plugin will then automatically assign the resource type based on either of below:
+* If user assigns source-specific tags as below:
+```
+Tag windows.server1.logs
+Tag linux.vm02.logs
+```
+* In the fluentd conf file:
+```
+<filter **>
+  @type record_transformer
+  enable_ruby true
+  <record>
+    resource_type ${record["resource_type"] || "Unknown"}
+  </record>
+</filter>
+```
+* If the remote agent includes a host field (many do), we can use heuristics:
+```
+
+host = record['host'] || record['hostname'] || ''
+return 'AWS/VirtualMachine' if host.start_with?('ip-')
+return 'GCP/VirtualMachine' if host.include?('.c.') || host.include?('gcp')
+return 'WindowsServer' if host.include?('win')
+return 'LinuxServer' if host.include?('linux')
+'Unknown'
+```
 
 ### Request example
 
@@ -69,8 +97,8 @@ See the [LogicMonitor Helm repository](https://github.com/logicmonitor/k8s-helm-
 | `resource_mapping` | The mapping that defines the source of the log event to the LM resource. In this case, the `<event_key>` in the incoming event is mapped to the value of `<lm_property>`.|
 | `access_id` | LM API Token access ID. |
 | `access_key` | LM API Token access key. |
-| `resource_type` | If a Resource Type is explicitly specified, that value will be statically applied to all ingested logs. If set to `##predef.externalResourceType##`, the Resource Type will be assigned dynamically based on the log context or configuration. If left blank, the Resource Type field will remain unset in the ingested logs. |
-| `bearer_token` | LM API Bearer Token. Either specify `access_id` and `access_key` both or `bearer_token`. If all specified, LMv1 token(`access_id` and `access_key`) will be used for authentication with Logicmonitor. |
+| `resource_type` | If a Resource Type is specified, it will be statically applied to all ingested logs. If left blank, a dynamic Resource Type will be assigned. |
+| `bearer_token` | LM API Bearer Token. Either specify `access_id` and `access_key` both or `bearer_token`. If all specified, LMv1 token(`access_id` and `access_key`) will be used for authentication with Logicmonitor.   |
 | `flush_interval` | Defines the time in seconds to wait before sending batches of logs to LogicMonitor. Default is `60s`. |
 | `debug` | When `true`, logs more information to the fluentd console. |
 | `force_encoding` | Specify charset when logs contains invalid utf-8 characters. |

data/fluent-plugin-lm-logs.gemspec

@@ -21,7 +21,7 @@ Gem::Specification.new do |spec|
   spec.metadata["source_code_uri"]    = "https://github.com/logicmonitor/lm-logs-fluentd"
   spec.metadata["documentation_uri"]  = "https://www.rubydoc.info/gems/lm-logs-fluentd"
 
-  spec.files         = [".gitignore", "Gemfile", "LICENSE", "README.md", "Rakefile", "fluent-plugin-lm-logs.gemspec", "lib/fluent/plugin/version.rb", "lib/fluent/plugin/out_lm.rb"]
+  spec.files         = [".gitignore", "Gemfile", "LICENSE", "README.md", "Rakefile", "fluent-plugin-lm-logs.gemspec", "lib/fluent/plugin/version.rb", "lib/fluent/plugin/out_lm.rb", "lib/fluent/plugin/environment_detector.rb"]
   spec.require_paths = ["lib"]
   spec.required_ruby_version = '>= 2.0.0'
 

data/lib/fluent/plugin/environment_detector.rb

@@ -0,0 +1,201 @@
+require 'net/http'
+require 'timeout'
+require 'json'
+
+class EnvironmentDetector
+  METADATA_TIMEOUT = 1
+
+  def detect
+    if running_in_kubernetes?
+      { runtime: 'kubernetes' }.merge(detect_node_info)
+    elsif running_in_docker?
+      { runtime: 'docker' }.merge(detect_node_info)
+    else
+      detect_host_environment
+    end
+  end
+
+  def format_environment(env_info)
+    runtime = env_info[:runtime]
+    provider = env_info[:provider] if env_info.key?(:provider)
+
+    case runtime
+    when 'kubernetes'
+      'Kubernetes/Node'
+    when 'docker'
+      'Docker/Host'
+    when 'vm'
+      case provider&.downcase
+      when 'azure'
+        'Azure/VirtualMachine'
+      when 'aws'
+        'AWS/EC2'
+      when 'gcp'
+        'GCP/ComputeEngine'
+      else
+        'Unknown/VirtualMachine'
+      end
+    when 'physical'
+      os = env_info[:os] || 'UnknownOS'
+      product = env_info[:product] || 'UnknownHardware'
+      "#{os} / #{product}"
+    else
+      'UnknownEnvironment'
+    end
+  end
+
+  def infer_resource_type(record, tag = nil)
+    return record['resource_type'] if record['resource_type']
+
+    host = (record['host'] || record['hostname'] || '').to_s
+    msg = (record['message'] || '').to_s
+    program = (record['syslog_program'] || '').to_s
+    tags = record['tags'] || []
+    tag_down = tag&.downcase || ''
+
+    host_down = host.downcase
+    msg_down = msg.downcase
+    program_down = program.downcase
+    # From tag pattern (case-insensitive)
+    return 'WindowsServer' if tag_down.include?('windows')
+    return 'LinuxServer' if tag_down.include?('linux')
+    return 'Kubernetes/Node' if tag_down.include?('k8s') || tag_down.include?('kubernetes')
+    return 'Docker/Host' if tag_down.include?('docker')
+
+    # Structured metadata
+    return 'Kubernetes/Node' if record.key?('kubernetes')
+    return 'Docker/Host' if record.key?('container_id') || record.dig('docker', 'container_id')
+    return 'AWS/VirtualMachine' if host_down.start_with?('ip-') || msg_down.include?('amazon')
+    return 'GCP/VirtualMachine' if host_down.include?('.c.') || host_down.include?('gcp')
+    return 'Azure/VirtualMachine' if host_down.include?('cloudapp.net') || msg_down.include?('azure')
+    return 'VMware/VirtualMachine' if msg_down.include?('vmware') || host_down.include?('vmware')
+
+    return 'WindowsServer' if record.key?('EventID') || record.key?('ProviderName') || record.key?('Computer')
+    return 'LinuxServer' if record.key?('syslog_facility') || program_down != ''
+
+    return 'Firewall' if program_down.downcase.include?('firewalld') || msg_down.downcase.include?('iptables') || msg_down.include?('blocked by policy')
+    return 'ACMEServer' if host_down.include?('acme') || msg_down.include?('ACME-Request') || tags.include?('acme')
+    return 'WebServer' if msg_down.include?('nginx') || msg_down.include?('apache')
+    return 'DatabaseServer' if msg_down.include?('mysql') || msg_down.include?('postgres') || msg_down.include?('oracle')
+
+    'Unknown'
+  end
+
+
+  private
+
+  def running_in_kubernetes?
+    ENV.key?('KUBERNETES_SERVICE_HOST') || ENV.key?('KUBERNETES_PORT')
+  end
+
+  def running_in_docker?
+    return true if ENV['container'] == 'docker'
+    cgroup = File.read('/proc/1/cgroup') rescue ''
+    return true if cgroup.include?('docker') || cgroup.include?('containerd')
+    File.exist?('/.dockerenv')
+  end
+
+  def detect_host_environment
+    provider_info = detect_cloud_provider
+    return { runtime: 'vm', provider: provider_info[:provider], details: provider_info[:details] } if provider_info
+
+    os = detect_os
+    product = detect_product_info
+
+    if product.downcase.include?('xen hvm domu') && os.downcase.include?('amazon')
+      return { runtime: 'vm', provider: 'aws', details: { os: os, product: product } }
+    end
+
+    { runtime: 'physical', os: os, product: product }
+  end
+
+  def detect_node_info
+    { node_os: detect_os, node_product: detect_product_info }
+  end
+
+  def detect_cloud_provider
+    azure_metadata || aws_metadata || gcp_metadata
+  end
+
+  def azure_metadata
+    url = 'http://169.254.169.254/metadata/instance?api-version=2021-02-01'
+    headers = { 'Metadata' => 'true' }
+    response = fetch_metadata(url, headers)
+    return unless response
+    json = JSON.parse(response) rescue {}
+    {
+      provider: 'azure',
+      details: {
+        vm_id: json.dig('compute', 'vmId'),
+        location: json.dig('compute', 'location'),
+        name: json.dig('compute', 'name'),
+        vm_size: json.dig('compute', 'vmSize')
+      }
+    }
+  end
+
+  def aws_metadata
+    url = 'http://169.254.169.254/latest/meta-data/instance-id'
+    response = fetch_metadata(url)
+    return unless response
+    { provider: 'aws', details: { instance_id: response.strip } }
+  end
+
+  def gcp_metadata
+    url = 'http://169.254.169.254/computeMetadata/v1/instance/id'
+    headers = { 'Metadata-Flavor' => 'Google' }
+    response = fetch_metadata(url, headers)
+    return unless response
+    { provider: 'gcp', details: { instance_id: response.strip } }
+  end
+
+  def fetch_metadata(url, headers = {}, timeout_sec = METADATA_TIMEOUT)
+    uri = URI(url)
+    Timeout.timeout(timeout_sec) do
+      req = Net::HTTP::Get.new(uri)
+      headers.each { |k, v| req[k] = v }
+      res = Net::HTTP.start(uri.host, uri.port, open_timeout: timeout_sec, read_timeout: timeout_sec) { |http| http.request(req) }
+      return res.body if res.is_a?(Net::HTTPSuccess)
+    end
+  rescue Timeout::Error, SocketError, Errno::ECONNREFUSED, Errno::EHOSTUNREACH, EOFError
+    nil
+  end
+
+  def detect_os
+    if File.exist?('/etc/os-release')
+      os_info = {}
+      File.foreach('/etc/os-release') do |line|
+        key, value = line.strip.split('=', 2)
+        os_info[key] = value&.gsub('"', '')
+      end
+      "#{os_info['NAME']} #{os_info['VERSION']}"
+    elsif RUBY_PLATFORM.include?('darwin')
+      product_name = `sw_vers -productName`.strip
+      product_version = `sw_vers -productVersion`.strip
+      "#{product_name} #{product_version}"
+    else
+      `uname -a`.strip
+    end
+  rescue
+    'unknown'
+  end
+
+  def detect_product_info
+    if File.exist?('/sys/class/dmi/id/sys_vendor') && File.exist?('/sys/class/dmi/id/product_name')
+      vendor = read_file('/sys/class/dmi/id/sys_vendor')
+      product = read_file('/sys/class/dmi/id/product_name')
+      "#{vendor} #{product}".strip
+    elsif RUBY_PLATFORM.include?('darwin')
+      model = `system_profiler SPHardwareDataType | awk '/Model Identifier/ { print $3 }'`.strip
+      model.empty? ? 'Mac' : model
+    else
+      'unknown'
+    end
+  rescue
+    'unknown'
+  end
+
+  def read_file(path)
+    File.read(path).strip if File.exist?(path)
+  end
+end

data/lib/fluent/plugin/out_lm.rb

@@ -10,6 +10,7 @@ require 'net/http'
 require 'net/http/persistent'
 require 'net/https'
 require('zlib')
+require_relative 'environment_detector'
 
 require_relative "version"
 
@@ -83,6 +84,11 @@ module Fluent
         @http_client.override_headers["User-Agent"] = log_source + "/" + LmLogsFluentPlugin::VERSION
         @url = "https://#{@company_name}.#{@company_domain}/rest/log/ingest"
         @uri = URI.parse(@url)
+        @detector = EnvironmentDetector.new
+        @environment_info = @detector.detect
+        @local_env_str = format_environment(@environment_info)
+
+        log.info("Environment detected: #{@environment_info}")
       end
 
       def configure_auth
@@ -174,10 +180,13 @@ module Fluent
         end
         lm_event["message"] = encode_if_necessary(record["message"])
 
-        if !is_blank(@resource_type)
-          lm_event['_resource.type'] = resource_type
+        resource_type = @resource_type || @detector.infer_resource_type(record, tag)
+        if resource_type.nil? || resource_type.strip.empty? || resource_type == 'Unknown'
+          resource_type = @local_env_str
         end
 
+        lm_event['_resource.type'] = resource_type
+
         return lm_event
       end
 
@@ -264,6 +273,34 @@ module Fluent
         end
       end
 
+      def format_environment(env_info)
+        runtime = env_info[:runtime]
+        provider = env_info[:provider] if env_info.key?(:provider)
+
+        case runtime
+        when 'kubernetes'
+          'Kubernetes/Node'
+        when 'docker'
+          'Docker/Host'
+        when 'vm'
+          case provider&.downcase
+          when 'azure'
+            'Azure/VirtualMachine'
+          when 'aws'
+            'AWS/EC2'
+          when 'gcp'
+            'GCP/ComputeEngine'
+          else
+            'Unknown/VirtualMachine'
+          end
+        when 'physical'
+          os = env_info[:os] || 'UnknownOS'
+          product = env_info[:product] || 'UnknownHardware'
+          "#{os} / #{product}"
+        else
+          'UnknownEnvironment'
+        end
+      end
     end
   end
 end

data/lib/fluent/plugin/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module LmLogsFluentPlugin
-    VERSION = '1.2.6'
+    VERSION = '1.2.7'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-lm-logs
 version: !ruby/object:Gem::Version
-  version: 1.2.6
+  version: 1.2.7
 platform: ruby
 authors:
 - LogicMonitor
@@ -56,6 +56,7 @@ files:
 - README.md
 - Rakefile
 - fluent-plugin-lm-logs.gemspec
+- lib/fluent/plugin/environment_detector.rb
 - lib/fluent/plugin/out_lm.rb
 - lib/fluent/plugin/version.rb
 homepage: https://www.logicmonitor.com