fluent-plugin-lm-logs 1.0.1 → 1.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6ac69aa367c60b885d377a7c818fbfcb70c8af34d86315fee0d2b18778351a92
-  data.tar.gz: b3cfdfbe4362b7f8d71a79d5aa34e92cc2b29c1a63b18a00b7aaa0cbe09cb14a
+  metadata.gz: 1c2298de6fe81e9eb49052ee49d55180193d254fee912339b9c8f6fad6c7aea1
+  data.tar.gz: 0a64fdb982ca4d8768175db534198b942d96fdcb22519913c5c077b7201bb6ea
 SHA512:
-  metadata.gz: aae1c2c44c9cc03c7256147322eb34d14eb551ea115a5befde0aa56903afb9a98e21d5670f0c253ac6fc7ef9a0be65ebd827ccc0581bda097597de77a1a5861f
-  data.tar.gz: fb6d25f6b71ad3c30084840e145ec1e7c4d98d036773762babb3b9d18c8136f5f384f85f065c3758972448f861f4e5617a5d57a97efcb5e0d77f30c8a945b446
+  metadata.gz: 95fe45b670b97925e672ecad76a724ed9eef24fd1e1685a131a4375ac7e662161aa5612ec34f3fc99e0f5817b24e7d640481b457bd466f2d2b3455f8e004e394
+  data.tar.gz: c69f47a5aad81cb1fbf778f14bc3238ee0271cb9530516a31120e46829aa7cf28336d509f9d6be2ddb155794ec9926896b2fc9738f524a30cf222f31013e322c

data/README.md

@@ -6,8 +6,8 @@ This output plugin sends Fluentd records to the configured LogicMonitor account.
 ## Prerequisites
 
 Install the plugin:
-* With gem:       `gem install fluent-plugin-lm-logs`
-* For deb packages:       `td-agent-gem install fluent-plugin-lm-logs`
+* With gem (if td-agent/fluentd is installed along with native ruby):       `gem install fluent-plugin-lm-logs`
+* For native td-agent/fluentd plugin handling:       `td-agent-gem install fluent-plugin-lm-logs`
 
 Alternatively, you can add `out_lm.rb` to your Fluentd plugins directory.
 
@@ -69,3 +69,8 @@ See the [LogicMonitor Helm repository](https://github.com/logicmonitor/k8s-helm-
 | `flush_interval` | Defines the time in seconds to wait before sending batches of logs to LogicMonitor. Default is `60s`. |
 | `debug` | When `true`, logs more information to the fluentd console. |
 | `force_encoding` | Specify charset when logs contains invalid utf-8 characters. |
+| `include_metadata` | When `true`, appends additional metadata to the log. default `false`.  |
+| `device_less_logs` | When `true`, do not map log with any resource. record must have `service` and `namespace` when `true`. default `false`. |
+| `metadata_to_exclude` | `String array` of keys of metadata to exclude. default `empty` |
+
+

data/Rakefile

@@ -1 +1,7 @@
 require "bundler/gem_tasks"
+require "rake/testtask"
+task default: "test"
+
+Rake::TestTask.new do |task|
+ task.pattern = "test/plugin/*.rb"
+end

data/fluent-plugin-lm-logs.gemspec

@@ -5,7 +5,7 @@
 
 Gem::Specification.new do |spec|
   spec.name                           = "fluent-plugin-lm-logs"
-  spec.version                        = '1.0.1'
+  spec.version                        = '1.0.2'
   spec.authors                        = ["LogicMonitor"]
   spec.email                          = "rubygems@logicmonitor.com"
   spec.summary                        = "LogicMonitor logs fluentd output plugin"

data/lib/fluent/plugin/out_lm.rb

@@ -26,6 +26,8 @@ module Fluent
     config_param :resource_mapping,  :hash, :default => {"host": "system.hostname", "hostname": "system.hostname"}
 
     config_param :debug,  :bool, :default => false
+
+    config_param :include_metadata,  :bool, :default => false
 		
     config_param :force_encoding,  :string, :default => ""
 
@@ -34,7 +36,11 @@ module Fluent
     config_param :log_source,  :string, :default => "lm-logs-fluentd"
 
     config_param :version_id,  :string, :default => "version_id"
-    
+
+    config_param :device_less_logs,  :bool, :default => false
+
+    config_param :metadata_to_exclude,  :array, default: [], value_type: :string
+  
     # This method is called before starting.
     # 'conf' is a Hash that includes configuration parameters.
     # If the configuration is invalid, raise Fluent::ConfigError.
@@ -71,7 +77,9 @@ module Fluent
       events = []
       chunk.msgpack_each do |(tag, time, record)|
         event = process_record(tag,time,record)
-        events.push(event)
+        if event != nil
+          events.push(event)
+        end
       end
       send_batch(events)
     end
@@ -79,24 +87,20 @@ module Fluent
     def process_record(tag, time, record)
       resource_map = {}
       lm_event = {}
-      lm_event["message"] = record["message"]
-      
-      if @force_encoding != ""
-        lm_event["message"] = lm_event["message"].force_encoding(@force_encoding).encode("UTF-8")
-      end
-
-      if record["_lm.resourceId"] == nil
-          @resource_mapping.each do |key, value|
-            k = value
-            nestedVal = record
-            key.to_s.split('.').each { |x| nestedVal = nestedVal[x] }
-            if nestedVal != nil
-              resource_map[k] = nestedVal
+      if !@device_less_logs
+        if record["_lm.resourceId"] == nil
+            @resource_mapping.each do |key, value|
+              k = value
+              nestedVal = record
+              key.to_s.split('.').each { |x| nestedVal = nestedVal[x] }
+              if nestedVal != nil
+                resource_map[k] = nestedVal
+              end
             end
-          end
-        lm_event["_lm.resourceId"] = resource_map
-      else
-        lm_event["_lm.resourceId"] = record["_lm.resourceId"]
+          lm_event["_lm.resourceId"] = resource_map
+        else
+          lm_event["_lm.resourceId"] = record["_lm.resourceId"]
+        end
       end
 
       if record["timestamp"] != nil
@@ -105,6 +109,34 @@ module Fluent
         lm_event["timestamp"] = Time.at(time).utc.to_datetime.rfc3339
       end
 
+      if @include_metadata
+        record.each do |key, value|
+          case key
+          when *@metadata_to_exclude.concat(["timestamp","_lm.resourceId","log","message"])
+          else    
+              lm_event["#{key}"] = value
+  
+              if @force_encoding != ""
+                  lm_event["#{key}"] = lm_event["#{key}"].force_encoding(@force_encoding).encode("UTF-8")
+              end
+          end
+        end
+      end
+      lm_event["message"] = record["message"]
+    
+      if @force_encoding != ""
+        lm_event["message"] = lm_event["message"].force_encoding(@force_encoding).encode("UTF-8")
+      end
+
+      if @device_less_logs
+        if record["service"]==nil || record["namespace"] == nil
+          log.error "When device_less_logs is set \'true\', record must have \'service\' and \'namespace\'. Ignoring this event #{lm_event}."
+          return nil
+        else
+          lm_event["service"] = record["service"]
+          lm_event["namespace"] = record["namespace"] 
+        end
+      end
       return lm_event
     end
 
@@ -159,4 +191,4 @@ module Fluent
       "LMv1 #{@access_id}:#{signature}:#{timestamp}"
     end
   end
-end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-lm-logs
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 1.0.2
 platform: ruby
 authors:
 - LogicMonitor
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-04-15 00:00:00.000000000 Z
+date: 2022-03-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -65,7 +65,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.15
+rubygems_version: 3.3.7
 signing_key: 
 specification_version: 4
 summary: LogicMonitor logs fluentd output plugin