fluent-plugin-lm-logs 1.0.1 → 1.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6ac69aa367c60b885d377a7c818fbfcb70c8af34d86315fee0d2b18778351a92
-  data.tar.gz: b3cfdfbe4362b7f8d71a79d5aa34e92cc2b29c1a63b18a00b7aaa0cbe09cb14a
+  metadata.gz: 74ceac6a53b86efe8123cd6ec143d4839de6043815ebb10e763e2b3019122396
+  data.tar.gz: f72441fc5646c2cdc6f602f066fca505e1bb713371a90bdbf019ea6cf07c80b0
 SHA512:
-  metadata.gz: aae1c2c44c9cc03c7256147322eb34d14eb551ea115a5befde0aa56903afb9a98e21d5670f0c253ac6fc7ef9a0be65ebd827ccc0581bda097597de77a1a5861f
-  data.tar.gz: fb6d25f6b71ad3c30084840e145ec1e7c4d98d036773762babb3b9d18c8136f5f384f85f065c3758972448f861f4e5617a5d57a97efcb5e0d77f30c8a945b446
+  metadata.gz: ee9e805c8fe449abca81d6fdfff713130359c7b3cb5c246a48fc37e8f6a6a93ab25c2203c1aa5ecfb441ca0a3e7223373ebad31b2a5314660eba5ab5bb31ed0c
+  data.tar.gz: a910b4872388fade3cf044e7f2c9e2562a23d5730a2faffc4f4daa9f942220a44c02fa3daead5e078e662eaddb8ecb27c034f3bb190324267b0b4a04e953cd87

data/README.md

@@ -6,8 +6,8 @@ This output plugin sends Fluentd records to the configured LogicMonitor account.
 ## Prerequisites
 
 Install the plugin:
-* With gem:       `gem install fluent-plugin-lm-logs`
-* For deb packages:       `td-agent-gem install fluent-plugin-lm-logs`
+* With gem (if td-agent/fluentd is installed along with native ruby):       `gem install fluent-plugin-lm-logs`
+* For native td-agent/fluentd plugin handling:       `td-agent-gem install fluent-plugin-lm-logs`
 
 Alternatively, you can add `out_lm.rb` to your Fluentd plugins directory.
 
@@ -69,3 +69,8 @@ See the [LogicMonitor Helm repository](https://github.com/logicmonitor/k8s-helm-
 | `flush_interval` | Defines the time in seconds to wait before sending batches of logs to LogicMonitor. Default is `60s`. |
 | `debug` | When `true`, logs more information to the fluentd console. |
 | `force_encoding` | Specify charset when logs contains invalid utf-8 characters. |
+| `include_metadata` | When `true`, appends additional metadata to the log. default `false`.  |
+| `device_less_logs` | When `true`, do not map log with any resource. record must have `service` when `true`. default `false`. |
+
+
+

data/Rakefile

@@ -1 +1,7 @@
 require "bundler/gem_tasks"
+require "rake/testtask"
+task default: "test"
+
+Rake::TestTask.new do |task|
+ task.pattern = "test/plugin/*.rb"
+end

data/fluent-plugin-lm-logs.gemspec

@@ -5,7 +5,7 @@
 
 Gem::Specification.new do |spec|
   spec.name                           = "fluent-plugin-lm-logs"
-  spec.version                        = '1.0.1'
+  spec.version                        = '1.0.4'
   spec.authors                        = ["LogicMonitor"]
   spec.email                          = "rubygems@logicmonitor.com"
   spec.summary                        = "LogicMonitor logs fluentd output plugin"

data/lib/fluent/plugin/out_lm.rb

@@ -15,6 +15,10 @@ module Fluent
   class LmOutput < BufferedOutput
     Fluent::Plugin.register_output('lm', self)
 
+    RESOURCE_MAPPING_KEY      = "_lm.resourceId".freeze
+    DEVICELESS_KEY_SERVICE    = "resource.service.name".freeze
+    DEVICELESS_KEY_NAMESPACE  = "resource.service.namespace".freeze
+
     # config_param defines a parameter. You can refer a parameter via @path instance variable
 
     config_param :access_id,  :string, :default => "access_id"
@@ -26,6 +30,8 @@ module Fluent
     config_param :resource_mapping,  :hash, :default => {"host": "system.hostname", "hostname": "system.hostname"}
 
     config_param :debug,  :bool, :default => false
+
+    config_param :include_metadata,  :bool, :default => false
 		
     config_param :force_encoding,  :string, :default => ""
 
@@ -34,7 +40,9 @@ module Fluent
     config_param :log_source,  :string, :default => "lm-logs-fluentd"
 
     config_param :version_id,  :string, :default => "version_id"
-    
+
+    config_param :device_less_logs,  :bool, :default => false
+  
     # This method is called before starting.
     # 'conf' is a Hash that includes configuration parameters.
     # If the configuration is invalid, raise Fluent::ConfigError.
@@ -71,7 +79,9 @@ module Fluent
       events = []
       chunk.msgpack_each do |(tag, time, record)|
         event = process_record(tag,time,record)
-        events.push(event)
+        if event != nil
+          events.push(event)
+        end
       end
       send_batch(events)
     end
@@ -79,24 +89,37 @@ module Fluent
     def process_record(tag, time, record)
       resource_map = {}
       lm_event = {}
-      lm_event["message"] = record["message"]
-      
-      if @force_encoding != ""
-        lm_event["message"] = lm_event["message"].force_encoding(@force_encoding).encode("UTF-8")
+
+      if @include_metadata
+        lm_event = get_metadata(record)
       end
 
-      if record["_lm.resourceId"] == nil
-          @resource_mapping.each do |key, value|
-            k = value
-            nestedVal = record
-            key.to_s.split('.').each { |x| nestedVal = nestedVal[x] }
-            if nestedVal != nil
-              resource_map[k] = nestedVal
+      if !@device_less_logs
+        # With devices
+        if record[RESOURCE_MAPPING_KEY] == nil
+            @resource_mapping.each do |key, value|
+              k = value
+              nestedVal = record
+              key.to_s.split('.').each { |x| nestedVal = nestedVal[x] }
+              if nestedVal != nil
+                resource_map[k] = nestedVal
+              end
             end
-          end
-        lm_event["_lm.resourceId"] = resource_map
+          lm_event[RESOURCE_MAPPING_KEY] = resource_map
+        else
+          lm_event[RESOURCE_MAPPING_KEY] = record[RESOURCE_MAPPING_KEY]
+        end
       else
-        lm_event["_lm.resourceId"] = record["_lm.resourceId"]
+        # Device less
+        if record[DEVICELESS_KEY_SERVICE]==nil
+          log.error "When device_less_logs is set \'true\', record must have \'service\'. Ignoring this event #{lm_event}."
+          return nil
+        else
+          lm_event[DEVICELESS_KEY_SERVICE] = encode_if_necessary(record[DEVICELESS_KEY_SERVICE])
+          if record[DEVICELESS_KEY_NAMESPACE]!=nil
+            lm_event[DEVICELESS_KEY_NAMESPACE] = encode_if_necessary(record[DEVICELESS_KEY_NAMESPACE]) 
+          end
+        end
       end
 
       if record["timestamp"] != nil
@@ -105,9 +128,37 @@ module Fluent
         lm_event["timestamp"] = Time.at(time).utc.to_datetime.rfc3339
       end
 
+      lm_event["message"] = encode_if_necessary(record["message"])
+
       return lm_event
     end
 
+    def get_metadata(record)
+      #if encoding is not defined we will skip going through each key val 
+      #and return the whole record for performance reasons in case of a bulky record.
+      if @force_encoding == "" 
+        return record
+      else
+        lm_event = {}
+        record.each do |key, value| 
+          lm_event["#{key}"] = get_encoded_string(value)
+        end
+        return lm_event
+      end
+    end
+
+    def encode_if_necessary(str)
+      if @force_encoding != ""
+        return get_encoded_string(str)
+      else
+        return str
+      end
+    end
+
+    def get_encoded_string(str)
+      return str.force_encoding(@force_encoding).encode("UTF-8")
+    end
+
     def send_batch(events)
       url = "https://#{@company_name}.logicmonitor.com/rest/log/ingest"
       body = events.to_json
@@ -159,4 +210,4 @@ module Fluent
       "LMv1 #{@access_id}:#{signature}:#{timestamp}"
     end
   end
-end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-lm-logs
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 1.0.4
 platform: ruby
 authors:
 - LogicMonitor
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-04-15 00:00:00.000000000 Z
+date: 2022-05-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -65,7 +65,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.15
+rubygems_version: 3.3.7
 signing_key: 
 specification_version: 4
 summary: LogicMonitor logs fluentd output plugin