fluent-plugin-lm-logs 1.2.4 → 1.2.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 745bc2d1d5288affa3b239860d5ee6b00c01df4e718fe21ebab62eec9ef2a211
-  data.tar.gz: 54b54ed790d7d0c56af9eea61b24024e33755157396fbf71f2ae6bbe700f3e14
+  metadata.gz: 377dc56ec3f62bb91303431bbba701b179e5c5540feb4af72cfd91c27ac7b930
+  data.tar.gz: 80317f28d2cbe4a3315d715a408212c9d047874416e6e9b0fad84719fe6e4c76
 SHA512:
-  metadata.gz: 84f747c7449f169e0283ca639cf7c9460700a93d247b4a52ca633002ffa522f66b12a3ca0d1576c88fe3b65161cd4d26a8a204b893c7b4032141d51196763203
-  data.tar.gz: cbe53ee7f70aa06e220e99585b4ff5e5a1e826c3d30a264f67bf2675994faeb08d8c78476283456316c37ea8e6da8cae70294b377a920d7eb433257cecb75d0d
+  metadata.gz: 5fd55fb16669b87f4118e3a3617042974829ebb765cf6396909c18573840661415740cb448621e0ed1c36896882a3dc6b43a9ae93b163fdce849cd39fd47b181
+  data.tar.gz: 83fdf0fe55a63007ba22f7bd9c1d447dc984cd2f741454c9746edef91a902a3020a68c04eb5b05df3ab9c2920079b27434d73e9c8b61a5ef195757c6866f1525

data/README.md

@@ -23,8 +23,9 @@ Create a custom `fluent.conf` or edit the existing one to specify which logs sho
     resource_mapping {"<event_key>": "<lm_property>"}
     company_name <lm_company_name>
     company_domain <lm_company_domain>
-	access_id <lm_access_id>
+    access_id <lm_access_id>
     access_key <lm_access_key>
+    resource_type <resource_type>
       <buffer>
         @type memory
         flush_interval 1s
@@ -68,7 +69,8 @@ See the [LogicMonitor Helm repository](https://github.com/logicmonitor/k8s-helm-
 | `resource_mapping` | The mapping that defines the source of the log event to the LM resource. In this case, the `<event_key>` in the incoming event is mapped to the value of `<lm_property>`.|
 | `access_id` | LM API Token access ID. |
 | `access_key` | LM API Token access key. |
-| `bearer_token` | LM API Bearer Token. Either specify `access_id` and `access_key` both or `bearer_token`. If all specified, LMv1 token(`access_id` and `access_key`) will be used for authentication with Logicmonitor.   |
+| `resource_type` | If a Resource Type is explicitly specified, that value will be statically applied to all ingested logs. If set to `##predef.externalResourceType##`, the Resource Type will be assigned dynamically based on the log context or configuration. If left blank, the Resource Type field will remain unset in the ingested logs. |
+| `bearer_token` | LM API Bearer Token. Either specify `access_id` and `access_key` both or `bearer_token`. If all specified, LMv1 token(`access_id` and `access_key`) will be used for authentication with Logicmonitor. |
 | `flush_interval` | Defines the time in seconds to wait before sending batches of logs to LogicMonitor. Default is `60s`. |
 | `debug` | When `true`, logs more information to the fluentd console. |
 | `force_encoding` | Specify charset when logs contains invalid utf-8 characters. |

data/lib/fluent/plugin/out_lm.rb

@@ -16,244 +16,254 @@ require_relative "version"
 
 
 module Fluent
-  class LmOutput < BufferedOutput
-    Fluent::Plugin.register_output('lm', self)
+  module Plugin
+    class LmOutput < Fluent::Plugin::Output
+      Fluent::Plugin.register_output('lm', self)
 
-    RESOURCE_MAPPING_KEY      = "_lm.resourceId".freeze
-    DEVICELESS_KEY_SERVICE    = "resource.service.name".freeze
-    DEVICELESS_KEY_NAMESPACE  = "resource.service.namespace".freeze
+      RESOURCE_MAPPING_KEY      = "_lm.resourceId".freeze
+      DEVICELESS_KEY_SERVICE    = "resource.service.name".freeze
+      DEVICELESS_KEY_NAMESPACE  = "resource.service.namespace".freeze
 
-    # config_param defines a parameter. You can refer a parameter via @path instance variable
+      # config_param defines a parameter. You can refer a parameter via @path instance variable
 
-    config_param :access_id,  :string, :default => nil
+      config_param :access_id,  :string, :default => nil
 
-    config_param :access_key,  :string, :default => nil, secret: true
+      config_param :access_key,  :string, :default => nil, secret: true
 
-    config_param :company_name,  :string, :default => "company_name"
+      config_param :company_name,  :string, :default => "company_name"
 
-    config_param :resource_mapping,  :hash, :default => {"host": "system.hostname", "hostname": "system.hostname"}
+      config_param :resource_mapping,  :hash, :default => {"host": "system.hostname", "hostname": "system.hostname"}
 
-    config_param :debug,  :bool, :default => false
+      config_param :debug,  :bool, :default => false
 
-    config_param :include_metadata,  :bool, :default => false
+      config_param :include_metadata,  :bool, :default => false
 
-    config_param :force_encoding,  :string, :default => ""
+      config_param :force_encoding,  :string, :default => ""
 
-    config_param :compression,  :string, :default => ""
+      config_param :compression,  :string, :default => ""
 
-    config_param :log_source,  :string, :default => "lm-logs-fluentd"
+      config_param :log_source,  :string, :default => "lm-logs-fluentd"
 
-    config_param :version_id,  :string, :default => "version_id"
+      config_param :version_id,  :string, :default => "version_id"
 
-    config_param :device_less_logs,  :bool, :default => false
+      config_param :device_less_logs,  :bool, :default => false
 
-    config_param :http_proxy,   :string, :default => nil
+      config_param :http_proxy,   :string, :default => nil
 
-    config_param :company_domain ,  :string, :default => "logicmonitor.com"
+      config_param :company_domain ,  :string, :default => "logicmonitor.com"
 
-    # Use bearer token for auth.
-    config_param :bearer_token, :string, :default => nil, secret: true
+      config_param :resource_type,  :string, :default => ""
+      # Use bearer token for auth.
+      config_param :bearer_token, :string, :default => nil, secret: true
 
-    # This method is called before starting.
-    # 'conf' is a Hash that includes configuration parameters.
-    # If the configuration is invalid, raise Fluent::ConfigError.
-    def configure(conf)
-      super
-    end
+      # This method is called before starting.
+      # 'conf' is a Hash that includes configuration parameters.
+      # If the configuration is invalid, raise Fluent::ConfigError.
+      def configure(conf)
+        super
+      end
 
-    def multi_workers_ready?
-      true
-    end
+      def multi_workers_ready?
+        true
+      end
 
-    # This method is called when starting.
-    # Open sockets or files here.
-    def start
-      super
-      configure_auth
-      proxy_uri = :ENV
-      if @http_proxy
-        proxy_uri = URI.parse(http_proxy)
-      elsif ENV['HTTP_PROXY'] || ENV['http_proxy']
-        log.info("Using HTTP proxy defined in environment variable")
+      # This method is called when starting.
+      # Open sockets or files here.
+      def start
+        super
+        configure_auth
+        proxy_uri = :ENV
+        if @http_proxy
+          proxy_uri = URI.parse(http_proxy)
+        elsif ENV['HTTP_PROXY'] || ENV['http_proxy']
+          log.info("Using HTTP proxy defined in environment variable")
+        end
+        @http_client = Net::HTTP::Persistent.new name: "fluent-plugin-lm-logs", proxy: proxy_uri
+        @http_client.override_headers["Content-Type"] = "application/json"
+        @http_client.override_headers["User-Agent"] = log_source + "/" + LmLogsFluentPlugin::VERSION
+        @url = "https://#{@company_name}.#{@company_domain}/rest/log/ingest"
+        @uri = URI.parse(@url)
       end
-      @http_client = Net::HTTP::Persistent.new name: "fluent-plugin-lm-logs", proxy: proxy_uri
-      @http_client.override_headers["Content-Type"] = "application/json"
-      @http_client.override_headers["User-Agent"] = log_source + "/" + LmLogsFluentPlugin::VERSION
-      @url = "https://#{@company_name}.#{@company_domain}/rest/log/ingest"
-      @uri = URI.parse(@url)
-    end
 
-    def configure_auth
-      @use_bearer_instead_of_lmv1 = false
-      if is_blank(@access_id) || is_blank(@access_key)
-        log.info "Access Id or access key blank / null. Using bearer token for authentication."
-        @use_bearer_instead_of_lmv1 = true
+      def configure_auth
+        @use_bearer_instead_of_lmv1 = false
+        if is_blank(@access_id) || is_blank(@access_key)
+          log.info "Access Id or access key blank / null. Using bearer token for authentication."
+          @use_bearer_instead_of_lmv1 = true
+        end
+        if @use_bearer_instead_of_lmv1 && is_blank(@bearer_token)
+          log.error "Bearer token not specified. Either access_id and access_key both or bearer_token must be specified for authentication with Logicmonitor."
+          raise ArgumentError, 'No valid authentication specified. Either access_id and access_key both or bearer_token must be specified for authentication with Logicmonitor.'
+        end
       end
-      if @use_bearer_instead_of_lmv1 && is_blank(@bearer_token)
-        log.error "Bearer token not specified. Either access_id and access_key both or bearer_token must be specified for authentication with Logicmonitor."
-        raise ArgumentError, 'No valid authentication specified. Either access_id and access_key both or bearer_token must be specified for authentication with Logicmonitor.'
+      # This method is called when shutting down.
+      # Shutdown the thread and close sockets or files here.
+      def shutdown
+        super
+        @http_client.shutdown
       end
-    end
-    # This method is called when shutting down.
-    # Shutdown the thread and close sockets or files here.
-    def shutdown
-      super
-      @http_client.shutdown
-    end
 
-    # This method is called when an event reaches to Fluentd.
-    # Convert the event to a raw string.
-    def format(tag, time, record)
-      [tag, time, record].to_msgpack
-    end
+      # This method is called when an event reaches to Fluentd.
+      # Convert the event to a raw string.
+      def format(tag, time, record)
+        [tag, time, record].to_msgpack
+      end
 
-    # This method is called every flush interval. Write the buffer chunk
-    # to files or databases here.
-    # 'chunk' is a buffer chunk that includes multiple formatted
-    # events. You can use 'data = chunk.read' to get all events and
-    # 'chunk.open {|io| ... }' to get IO objects.
-    #
-    # NOTE! This method is called by internal thread, not Fluentd's main thread. So IO wait doesn't affect other plugins.
-    def write(chunk)
-      events = []
-      chunk.msgpack_each do |(tag, time, record)|
-        event = process_record(tag,time,record)
-        if event != nil
-          events.push(event)
+      # This method is called every flush interval. Write the buffer chunk
+      # to files or databases here.
+      # 'chunk' is a buffer chunk that includes multiple formatted
+      # events. You can use 'data = chunk.read' to get all events and
+      # 'chunk.open {|io| ... }' to get IO objects.
+      #
+      # NOTE! This method is called by internal thread, not Fluentd's main thread. So IO wait doesn't affect other plugins.
+      def write(chunk)
+        events = []
+          chunk.msgpack_each do |(tag, time, record)|
+          event = process_record(tag,time,record)
+          if event != nil
+            events.push(event)
+          end
         end
+        send_batch(events)
       end
-      send_batch(events)
-    end
 
-    def process_record(tag, time, record)
-      resource_map = {}
-      lm_event = {}
-
-      if @include_metadata
-        lm_event = get_metadata(record)
+      def formatted_to_msgpack_binary?
+        true
       end
 
-      if !@device_less_logs
-        # With devices
-        if record[RESOURCE_MAPPING_KEY] == nil
-            @resource_mapping.each do |key, value|
-              k = value
-              nestedVal = record
-              key.to_s.split('.').each { |x| nestedVal = nestedVal[x] }
-              if nestedVal != nil
-                resource_map[k] = nestedVal
+      def process_record(tag, time, record)
+        resource_map = {}
+        lm_event = {}
+
+        if @include_metadata
+          lm_event = get_metadata(record)
+        end
+
+        if !@device_less_logs
+          # With devices
+          if record[RESOURCE_MAPPING_KEY] == nil
+              @resource_mapping.each do |key, value|
+                k = value
+                nestedVal = record
+                key.to_s.split('.').each { |x| nestedVal = nestedVal[x] }
+                if nestedVal != nil
+                  resource_map[k] = nestedVal
+                end
               end
-            end
-          lm_event[RESOURCE_MAPPING_KEY] = resource_map
+            lm_event[RESOURCE_MAPPING_KEY] = resource_map
+          else
+            lm_event[RESOURCE_MAPPING_KEY] = record[RESOURCE_MAPPING_KEY]
+          end
         else
-          lm_event[RESOURCE_MAPPING_KEY] = record[RESOURCE_MAPPING_KEY]
+          # Device less
+          if record[DEVICELESS_KEY_SERVICE]==nil
+            log.error "When device_less_logs is set \'true\', record must have \'service\'. Ignoring this event #{lm_event}."
+            return nil
+          else
+            lm_event[DEVICELESS_KEY_SERVICE] = encode_if_necessary(record[DEVICELESS_KEY_SERVICE])
+            if record[DEVICELESS_KEY_NAMESPACE]!=nil
+              lm_event[DEVICELESS_KEY_NAMESPACE] = encode_if_necessary(record[DEVICELESS_KEY_NAMESPACE])
+            end
+          end
         end
-      else
-        # Device less
-        if record[DEVICELESS_KEY_SERVICE]==nil
-          log.error "When device_less_logs is set \'true\', record must have \'service\'. Ignoring this event #{lm_event}."
-          return nil
+
+        if record["timestamp"] != nil
+          lm_event["timestamp"] = record["timestamp"]
         else
-          lm_event[DEVICELESS_KEY_SERVICE] = encode_if_necessary(record[DEVICELESS_KEY_SERVICE])
-          if record[DEVICELESS_KEY_NAMESPACE]!=nil
-            lm_event[DEVICELESS_KEY_NAMESPACE] = encode_if_necessary(record[DEVICELESS_KEY_NAMESPACE])
-          end
+          lm_event["timestamp"] = Time.at(time).utc.to_datetime.rfc3339
+        end
+        lm_event["message"] = encode_if_necessary(record["message"])
+
+        if !is_blank(@resource_type)
+          lm_event['_resource.type'] = resource_type
         end
-      end
 
-      if record["timestamp"] != nil
-        lm_event["timestamp"] = record["timestamp"]
-      else
-        lm_event["timestamp"] = Time.at(time).utc.to_datetime.rfc3339
+        return lm_event
       end
-      lm_event["_resource.type"] = "Fluentd"
-      lm_event["message"] = encode_if_necessary(record["message"])
 
-      return lm_event
-    end
+      def get_metadata(record)
+        #if encoding is not defined we will skip going through each key val
+        #and return the whole record for performance reasons in case of a bulky record.
+        if @force_encoding == ""
+          return record
+        else
+          lm_event = {}
+          record.each do |key, value|
+            lm_event["#{key}"] = get_encoded_string(value)
+          end
+          return lm_event
+        end
+      end
 
-    def get_metadata(record)
-      #if encoding is not defined we will skip going through each key val
-      #and return the whole record for performance reasons in case of a bulky record.
-      if @force_encoding == ""
-        return record
-      else
-        lm_event = {}
-        record.each do |key, value|
-          lm_event["#{key}"] = get_encoded_string(value)
+      def encode_if_necessary(str)
+        if @force_encoding != ""
+          return get_encoded_string(str)
+        else
+          return str
         end
-        return lm_event
       end
-    end
 
-    def encode_if_necessary(str)
-      if @force_encoding != ""
-        return get_encoded_string(str)
-      else
-        return str
+      def get_encoded_string(str)
+        return str.force_encoding(@force_encoding).encode("UTF-8")
       end
-    end
 
-    def get_encoded_string(str)
-      return str.force_encoding(@force_encoding).encode("UTF-8")
-    end
+      def send_batch(events)
+        body = events.to_json
 
-    def send_batch(events)
-      body = events.to_json
+        if @debug
+          log.info "Sending #{events.length} events to logic monitor at #{@url}"
+          log.info "Request json #{body}"
+        end
 
-      if @debug
-        log.info "Sending #{events.length} events to logic monitor at #{@url}"
-        log.info "Request json #{body}"
-      end
+        request = Net::HTTP::Post.new(@uri.request_uri)
+        request['authorization'] = generate_token(events)
 
-      request = Net::HTTP::Post.new(@uri.request_uri)
-      request['authorization'] = generate_token(events)
+        if @compression == "gzip"
+          request['Content-Encoding'] = "gzip"
+          gzip = Zlib::GzipWriter.new(StringIO.new)
+          gzip << body
+          request.body = gzip.close.string
+        else
+          request.body = body
+        end
 
-      if @compression == "gzip"
-        request['Content-Encoding'] = "gzip"
-        gzip = Zlib::GzipWriter.new(StringIO.new)
-        gzip << body
-        request.body = gzip.close.string
-      else
-        request.body = body
-      end
+        if @debug
+          log.info "Sending the below request headers to logicmonitor:"
+          request.each_header {|key,value| log.info "#{key} = #{value}" }
+        end
 
-      if @debug
-        log.info "Sending the below request headers to logicmonitor:"
-        request.each_header {|key,value| log.info "#{key} = #{value}" }
+        resp = @http_client.request @uri, request
+        if @debug || resp.kind_of?(Net::HTTPMultiStatus) || !resp.kind_of?(Net::HTTPSuccess)
+          log.info "Status code:#{resp.code} Request Id:#{resp.header['x-request-id']} message:#{resp.body}"
+        end
       end
 
-      resp = @http_client.request @uri, request
-      if @debug || resp.kind_of?(Net::HTTPMultiStatus) || !resp.kind_of?(Net::HTTPSuccess)
-        log.info "Status code:#{resp.code} Request Id:#{resp.header['x-request-id']} message:#{resp.body}"
-      end
-    end
 
+      def generate_token(events)
 
-    def generate_token(events)
-
-      if @use_bearer_instead_of_lmv1
-        return "Bearer #{@bearer_token}"
-      else
-        timestamp = DateTime.now.strftime('%Q')
-        signature = Base64.strict_encode64(
-            OpenSSL::HMAC.hexdigest(
-                OpenSSL::Digest.new('sha256'),
-                @access_key,
-                "POST#{timestamp}#{events.to_json}/log/ingest"
-            )
-        )
-        return "LMv1 #{@access_id}:#{signature}:#{timestamp}"
+        if @use_bearer_instead_of_lmv1
+          return "Bearer #{@bearer_token}"
+        else
+          timestamp = DateTime.now.strftime('%Q')
+          signature = Base64.strict_encode64(
+              OpenSSL::HMAC.hexdigest(
+                  OpenSSL::Digest.new('sha256'),
+                  @access_key,
+                  "POST#{timestamp}#{events.to_json}/log/ingest"
+              )
+          )
+          return "LMv1 #{@access_id}:#{signature}:#{timestamp}"
+        end
       end
-    end
 
-    def is_blank(str)
-      if str.nil? || str.to_s.strip.empty?
-        return true
-      else
-        return false
+      def is_blank(str)
+        if str.nil? || str.to_s.strip.empty?
+          return true
+        else
+          return false
+        end
       end
-    end
 
+    end
   end
 end

data/lib/fluent/plugin/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module LmLogsFluentPlugin
-    VERSION = '1.2.4'
+    VERSION = '1.2.6'
 end

metadata

@@ -1,14 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-lm-logs
 version: !ruby/object:Gem::Version
-  version: 1.2.4
+  version: 1.2.6
 platform: ruby
 authors:
 - LogicMonitor
-autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-12-24 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -65,7 +64,6 @@ licenses:
 metadata:
   source_code_uri: https://github.com/logicmonitor/lm-logs-fluentd
   documentation_uri: https://www.rubydoc.info/gems/lm-logs-fluentd
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -80,8 +78,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.22
-signing_key:
+rubygems_version: 3.6.7
 specification_version: 4
 summary: LogicMonitor logs fluentd output plugin
 test_files: []