fluent-plugin-lm-logs 1.0.0 → 1.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a22de1423e4cac56363b99b31828c5add123ac4f843d4aeae4112aab02c4b6b4
-  data.tar.gz: e35849d3c43ef4f83fdeecae137b2d4e38f5c69ec52a7bf147191ac46773e61c
+  metadata.gz: fbacc1e0e4eb5389c11d4b23c28eda6100919da6d7de0abbda9edf13dffbf682
+  data.tar.gz: 3bffdba784febf493f69ba0be9c1745f53d7aebb8f66ef92bf23b252fb91d2bb
 SHA512:
-  metadata.gz: f34bab74bbf201ca0af3ff2de35307560f5bb79a37b9ea549b5627f421c7214ea2a5ab7304efe6ddc54f774af161fa594aed8a6c4ce75b9dfb38f2eb2a852788
-  data.tar.gz: 14341d2be42d2d7530cc964178f514a530adb3c8d3679ff60ab6cdef653f83eaf38f70b68bb1578a7393f7bd6172c63f9185623eb7acf450f4db343ca2f0abfb
+  metadata.gz: 34362b69e402d439ee415089540a5f3d1532bb5f44c997bca79437e57a64286579b3e9c8349ebb44095bf86e976e2b197ddd9f7f20a32600e2c164685310b62e
+  data.tar.gz: '0631937f8f545e675735e207d6e6137748ef0bc1445cdab27c1b0179f0513daa271fbee4bcbe9000aa8fea1df155e5f208f82fdb6002a6e58f133f09c2616e57'

data/README.md

@@ -6,8 +6,8 @@ This output plugin sends Fluentd records to the configured LogicMonitor account.
 ## Prerequisites
 
 Install the plugin:
-* With gem:       `gem install fluent-plugin-lm-logs`
-* For deb packages:       `td-agent-gem install fluent-plugin-lm-logs`
+* With gem (if td-agent/fluentd is installed along with native ruby):       `gem install fluent-plugin-lm-logs`
+* For native td-agent/fluentd plugin handling:       `td-agent-gem install fluent-plugin-lm-logs`
 
 Alternatively, you can add `out_lm.rb` to your Fluentd plugins directory.
 
@@ -69,3 +69,8 @@ See the [LogicMonitor Helm repository](https://github.com/logicmonitor/k8s-helm-
 | `flush_interval` | Defines the time in seconds to wait before sending batches of logs to LogicMonitor. Default is `60s`. |
 | `debug` | When `true`, logs more information to the fluentd console. |
 | `force_encoding` | Specify charset when logs contains invalid utf-8 characters. |
+| `include_metadata` | When `true`, appends additional metadata to the log. default `false`.  |
+| `device_less_logs` | When `true`, do not map log with any resource. record must have `service` when `true`. default `false`. |
+
+
+

data/Rakefile

@@ -1 +1,7 @@
 require "bundler/gem_tasks"
+require "rake/testtask"
+task default: "test"
+
+Rake::TestTask.new do |task|
+ task.pattern = "test/plugin/*.rb"
+end

data/fluent-plugin-lm-logs.gemspec

@@ -5,7 +5,7 @@
 
 Gem::Specification.new do |spec|
   spec.name                           = "fluent-plugin-lm-logs"
-  spec.version                        = '1.0.0'
+  spec.version                        = '1.0.3'
   spec.authors                        = ["LogicMonitor"]
   spec.email                          = "rubygems@logicmonitor.com"
   spec.summary                        = "LogicMonitor logs fluentd output plugin"

data/lib/fluent/plugin/out_lm.rb

@@ -26,6 +26,8 @@ module Fluent
     config_param :resource_mapping,  :hash, :default => {"host": "system.hostname", "hostname": "system.hostname"}
 
     config_param :debug,  :bool, :default => false
+
+    config_param :include_metadata,  :bool, :default => false
 		
     config_param :force_encoding,  :string, :default => ""
 
@@ -34,7 +36,9 @@ module Fluent
     config_param :log_source,  :string, :default => "lm-logs-fluentd"
 
     config_param :version_id,  :string, :default => "version_id"
-    
+
+    config_param :device_less_logs,  :bool, :default => false
+  
     # This method is called before starting.
     # 'conf' is a Hash that includes configuration parameters.
     # If the configuration is invalid, raise Fluent::ConfigError.
@@ -71,7 +75,9 @@ module Fluent
       events = []
       chunk.msgpack_each do |(tag, time, record)|
         event = process_record(tag,time,record)
-        events.push(event)
+        if event != nil
+          events.push(event)
+        end
       end
       send_batch(events)
     end
@@ -79,24 +85,32 @@ module Fluent
     def process_record(tag, time, record)
       resource_map = {}
       lm_event = {}
-      lm_event["message"] = record["message"]
-      
-      if @force_encoding != ""
-        lm_event["message"] = lm_event["message"].force_encoding(@force_encoding).encode("UTF-8")
+
+      if @include_metadata
+        lm_event = get_metadata(record)
       end
 
-      if record["_lm.resourceId"] == nil
-          @resource_mapping.each do |key, value|
-            k = value
-            nestedVal = record
-            key.to_s.split('.').each { |x| nestedVal = nestedVal[x] }
-            if nestedVal != nil
-              resource_map[k] = nestedVal
+      if !@device_less_logs
+        if record["_lm.resourceId"] == nil
+            @resource_mapping.each do |key, value|
+              k = value
+              nestedVal = record
+              key.to_s.split('.').each { |x| nestedVal = nestedVal[x] }
+              if nestedVal != nil
+                resource_map[k] = nestedVal
+              end
             end
-          end
-        lm_event["_lm.resourceId"] = resource_map
+          lm_event["_lm.resourceId"] = resource_map
+        else
+          lm_event["_lm.resourceId"] = record["_lm.resourceId"]
+        end
       else
-        lm_event["_lm.resourceId"] = record["_lm.resourceId"]
+        if record["service"]==nil
+          log.error "When device_less_logs is set \'true\', record must have \'service\'. Ignoring this event #{lm_event}."
+          return nil
+        else
+          lm_event["service"] = encode_if_necessary(record["service"]) 
+        end  
       end
 
       if record["timestamp"] != nil
@@ -105,9 +119,37 @@ module Fluent
         lm_event["timestamp"] = Time.at(time).utc.to_datetime.rfc3339
       end
 
+      lm_event["message"] = encode_if_necessary(record["message"])
+
       return lm_event
     end
 
+    def get_metadata(record)
+      #if encoding is not defined we will skip going through each key val 
+      #and return the whole record for performance reasons in case of a bulky record.
+      if @force_encoding == "" 
+        return record
+      else
+        lm_event = {}
+        record.each do |key, value| 
+          lm_event["#{key}"] = get_encoded_string(value)
+        end
+        return lm_event
+      end
+    end
+
+    def encode_if_necessary(str)
+      if @force_encoding != ""
+        return get_encoded_string(str)
+      else
+        return str
+      end
+    end
+
+    def get_encoded_string(str)
+      return str.force_encoding(@force_encoding).encode("UTF-8")
+    end
+
     def send_batch(events)
       url = "https://#{@company_name}.logicmonitor.com/rest/log/ingest"
       body = events.to_json
@@ -124,7 +166,7 @@ module Fluent
       request = Net::HTTP::Post.new(uri.request_uri)
       request['authorization'] = generate_token(events)
       request['Content-type'] = "application/json"
-      request['User-Agent'] = log_source+version_id
+      request['User-Agent'] = log_source + "/" + version_id
 
       if @compression == "gzip"
         request['Content-Encoding'] = "gzip"
@@ -135,6 +177,11 @@ module Fluent
         request.body = body
       end
 
+      if @debug
+        log.info "Sending the below request headers to logicmonitor:"
+        request.each_header {|key,value| log.info "#{key} = #{value}" }
+      end
+
       resp = http.request(request)
       if @debug || (!resp.kind_of? Net::HTTPSuccess)
         log.info "Status code:#{resp.code} Request Id:#{resp.header['x-request-id']}"
@@ -154,4 +201,4 @@ module Fluent
       "LMv1 #{@access_id}:#{signature}:#{timestamp}"
     end
   end
-end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-lm-logs
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.0.3
 platform: ruby
 authors:
 - LogicMonitor
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-03-31 00:00:00.000000000 Z
+date: 2022-03-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -65,7 +65,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.3
+rubygems_version: 3.3.7
 signing_key: 
 specification_version: 4
 summary: LogicMonitor logs fluentd output plugin