fluent-plugin-lm-logs 0.0.12 → 1.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c930b2614b184eeba146418a22a10d732b8fd7aa57aabc8db45ec6d9e8608f0a
-  data.tar.gz: 6ced74d3ff2e35a3771b63d7f88572c2ebfd4e3dccd23e7a450a5bce6b7a22f7
+  metadata.gz: 1c2298de6fe81e9eb49052ee49d55180193d254fee912339b9c8f6fad6c7aea1
+  data.tar.gz: 0a64fdb982ca4d8768175db534198b942d96fdcb22519913c5c077b7201bb6ea
 SHA512:
-  metadata.gz: dd6ecf139c62cfba239000b45f134bd5c162ea773112438223fcbc20291ddaf97aebc3dec7aca2469639160f585baa2af628b94d445cee01dcb10d5bea7ffc31
-  data.tar.gz: e4e97f690c535f223b8c934bcdcf03cf3821f799f6018520ea8c2400eeb726b75de98d7eefa52ab80abf679ec85cc8941123596275f964755b4751372ffc1daf
+  metadata.gz: 95fe45b670b97925e672ecad76a724ed9eef24fd1e1685a131a4375ac7e662161aa5612ec34f3fc99e0f5817b24e7d640481b457bd466f2d2b3455f8e004e394
+  data.tar.gz: c69f47a5aad81cb1fbf778f14bc3238ee0271cb9530516a31120e46829aa7cf28336d509f9d6be2ddb155794ec9926896b2fc9738f524a30cf222f31013e322c

data/README.md

@@ -1,13 +1,13 @@
 
 [![Gem Version](https://badge.fury.io/rb/fluent-plugin-lm-logs.svg)](http://badge.fury.io/rb/fluent-plugin-lm-logs)
-# lm-logs-fluentd (beta)
+# lm-logs-fluentd
 This output plugin sends Fluentd records to the configured LogicMonitor account.
 
 ## Prerequisites
 
 Install the plugin:
-* With gem:       `gem install fluent-plugin-lm-logs`
-* For deb packages:       `td-agent-gem install fluent-plugin-lm-logs`
+* With gem (if td-agent/fluentd is installed along with native ruby):       `gem install fluent-plugin-lm-logs`
+* For native td-agent/fluentd plugin handling:       `td-agent-gem install fluent-plugin-lm-logs`
 
 Alternatively, you can add `out_lm.rb` to your Fluentd plugins directory.
 
@@ -68,4 +68,9 @@ See the [LogicMonitor Helm repository](https://github.com/logicmonitor/k8s-helm-
 | `access_key` | LM API Token access key. |
 | `flush_interval` | Defines the time in seconds to wait before sending batches of logs to LogicMonitor. Default is `60s`. |
 | `debug` | When `true`, logs more information to the fluentd console. |
-| `force_encoding` | Specify charset when logs contains invalid utf-8 characters. |
+| `force_encoding` | Specify charset when logs contains invalid utf-8 characters. |
+| `include_metadata` | When `true`, appends additional metadata to the log. default `false`.  |
+| `device_less_logs` | When `true`, do not map log with any resource. record must have `service` and `namespace` when `true`. default `false`. |
+| `metadata_to_exclude` | `String array` of keys of metadata to exclude. default `empty` |
+
+

data/Rakefile

@@ -1 +1,7 @@
 require "bundler/gem_tasks"
+require "rake/testtask"
+task default: "test"
+
+Rake::TestTask.new do |task|
+ task.pattern = "test/plugin/*.rb"
+end

data/fluent-plugin-lm-logs.gemspec

@@ -5,7 +5,7 @@
 
 Gem::Specification.new do |spec|
   spec.name                           = "fluent-plugin-lm-logs"
-  spec.version                        = '0.0.12'
+  spec.version                        = '1.0.2'
   spec.authors                        = ["LogicMonitor"]
   spec.email                          = "rubygems@logicmonitor.com"
   spec.summary                        = "LogicMonitor logs fluentd output plugin"

data/lib/fluent/plugin/out_lm.rb

@@ -26,11 +26,21 @@ module Fluent
     config_param :resource_mapping,  :hash, :default => {"host": "system.hostname", "hostname": "system.hostname"}
 
     config_param :debug,  :bool, :default => false
+
+    config_param :include_metadata,  :bool, :default => false
 		
     config_param :force_encoding,  :string, :default => ""
 
     config_param :compression,  :string, :default => ""
 
+    config_param :log_source,  :string, :default => "lm-logs-fluentd"
+
+    config_param :version_id,  :string, :default => "version_id"
+
+    config_param :device_less_logs,  :bool, :default => false
+
+    config_param :metadata_to_exclude,  :array, default: [], value_type: :string
+  
     # This method is called before starting.
     # 'conf' is a Hash that includes configuration parameters.
     # If the configuration is invalid, raise Fluent::ConfigError.
@@ -67,7 +77,9 @@ module Fluent
       events = []
       chunk.msgpack_each do |(tag, time, record)|
         event = process_record(tag,time,record)
-        events.push(event)
+        if event != nil
+          events.push(event)
+        end
       end
       send_batch(events)
     end
@@ -75,24 +87,20 @@ module Fluent
     def process_record(tag, time, record)
       resource_map = {}
       lm_event = {}
-      lm_event["message"] = record["message"]
-      
-      if @force_encoding != ""
-        lm_event["message"] = lm_event["message"].force_encoding(@force_encoding).encode("UTF-8")
-      end
-
-      if record["_lm.resourceId"] == nil
-          @resource_mapping.each do |key, value|
-            k = value
-            nestedVal = record
-            key.to_s.split('.').each { |x| nestedVal = nestedVal[x] }
-            if nestedVal != nil
-              resource_map[k] = nestedVal
+      if !@device_less_logs
+        if record["_lm.resourceId"] == nil
+            @resource_mapping.each do |key, value|
+              k = value
+              nestedVal = record
+              key.to_s.split('.').each { |x| nestedVal = nestedVal[x] }
+              if nestedVal != nil
+                resource_map[k] = nestedVal
+              end
             end
-          end
-        lm_event["_lm.resourceId"] = resource_map
-      else
-        lm_event["_lm.resourceId"] = record["_lm.resourceId"]
+          lm_event["_lm.resourceId"] = resource_map
+        else
+          lm_event["_lm.resourceId"] = record["_lm.resourceId"]
+        end
       end
 
       if record["timestamp"] != nil
@@ -101,6 +109,34 @@ module Fluent
         lm_event["timestamp"] = Time.at(time).utc.to_datetime.rfc3339
       end
 
+      if @include_metadata
+        record.each do |key, value|
+          case key
+          when *@metadata_to_exclude.concat(["timestamp","_lm.resourceId","log","message"])
+          else    
+              lm_event["#{key}"] = value
+  
+              if @force_encoding != ""
+                  lm_event["#{key}"] = lm_event["#{key}"].force_encoding(@force_encoding).encode("UTF-8")
+              end
+          end
+        end
+      end
+      lm_event["message"] = record["message"]
+    
+      if @force_encoding != ""
+        lm_event["message"] = lm_event["message"].force_encoding(@force_encoding).encode("UTF-8")
+      end
+
+      if @device_less_logs
+        if record["service"]==nil || record["namespace"] == nil
+          log.error "When device_less_logs is set \'true\', record must have \'service\' and \'namespace\'. Ignoring this event #{lm_event}."
+          return nil
+        else
+          lm_event["service"] = record["service"]
+          lm_event["namespace"] = record["namespace"] 
+        end
+      end
       return lm_event
     end
 
@@ -120,6 +156,7 @@ module Fluent
       request = Net::HTTP::Post.new(uri.request_uri)
       request['authorization'] = generate_token(events)
       request['Content-type'] = "application/json"
+      request['User-Agent'] = log_source + "/" + version_id
 
       if @compression == "gzip"
         request['Content-Encoding'] = "gzip"
@@ -130,6 +167,11 @@ module Fluent
         request.body = body
       end
 
+      if @debug
+        log.info "Sending the below request headers to logicmonitor:"
+        request.each_header {|key,value| log.info "#{key} = #{value}" }
+      end
+
       resp = http.request(request)
       if @debug || (!resp.kind_of? Net::HTTPSuccess)
         log.info "Status code:#{resp.code} Request Id:#{resp.header['x-request-id']}"
@@ -149,4 +191,4 @@ module Fluent
       "LMv1 #{@access_id}:#{signature}:#{timestamp}"
     end
   end
-end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-lm-logs
 version: !ruby/object:Gem::Version
-  version: 0.0.12
+  version: 1.0.2
 platform: ruby
 authors:
 - LogicMonitor
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-11-25 00:00:00.000000000 Z
+date: 2022-03-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -65,7 +65,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
+rubygems_version: 3.3.7
 signing_key: 
 specification_version: 4
 summary: LogicMonitor logs fluentd output plugin