fluent-plugin-lm-logs 0.0.8 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 321e02b24adb9528c90c7cdddd17ebdbc193dd3abcd0a66cdb6b7eaa1394b78e
-  data.tar.gz: c0ab73b0026f74fde357f5cdb38d1c2c0618b06a2e2389b5ba246d5eee7eb3ea
+  metadata.gz: a22de1423e4cac56363b99b31828c5add123ac4f843d4aeae4112aab02c4b6b4
+  data.tar.gz: e35849d3c43ef4f83fdeecae137b2d4e38f5c69ec52a7bf147191ac46773e61c
 SHA512:
-  metadata.gz: 0c863145fc51cc9473113c469d81acf6e96a770481b83665b9ed11e15e13fc2c259d6bad983092620813de982ede3f9df74389fdf3e2635f4298d3a1d807c5b4
-  data.tar.gz: b87af1e8f8469d7b365aad2aa670d526f022132b8e7a630caef0e342c7c979db8e47a53e6e2bc79deb07c394c15e1391b1ec40e85f7f33e96f9f3c62dd4e70e9
+  metadata.gz: f34bab74bbf201ca0af3ff2de35307560f5bb79a37b9ea549b5627f421c7214ea2a5ab7304efe6ddc54f774af161fa594aed8a6c4ce75b9dfb38f2eb2a852788
+  data.tar.gz: 14341d2be42d2d7530cc964178f514a530adb3c8d3679ff60ab6cdef653f83eaf38f70b68bb1578a7393f7bd6172c63f9185623eb7acf450f4db343ca2f0abfb

data/README.md

@@ -1,6 +1,6 @@
 
 [![Gem Version](https://badge.fury.io/rb/fluent-plugin-lm-logs.svg)](http://badge.fury.io/rb/fluent-plugin-lm-logs)
-# lm-logs-fluentd (beta)
+# lm-logs-fluentd
 This output plugin sends Fluentd records to the configured LogicMonitor account.
 
 ## Prerequisites
@@ -20,10 +20,10 @@ Create a custom `fluent.conf` or edit the existing one to specify which logs sho
 # send them to LogicMonitor
 <match lm.**>
     @type lm
-    company_name <account_name>
     resource_mapping {"<event_key>": "<lm_property>"}
-    access_id <your_lm_access_id>
-    access_key <your_lm_access_key>
+    company_name <lm_company_name>
+	access_id <lm_access_id>
+    access_key <lm_access_key>
       <buffer>
         @type memory
         flush_interval 1s
@@ -39,7 +39,7 @@ Sending:
 
 `curl -X POST -d 'json={"message":"hello LogicMonitor from fluentd", "event_key":"lm_property_value"}' http://localhost:8888/lm.test`
 
-Returns the event:
+Produces this event:
 ```
 {
     "message": "hello LogicMonitor from fluentd"
@@ -49,9 +49,8 @@ Returns the event:
 **Note:** Make sure that logs have a message field. Requests sent without a message will not be accepted. 
 
 ### Kubernetes
-Logic Monitor collects k8s logs automatically via FluentD daemonset, It requires a plugin for this. We recommend using [fluent-plugin-kubernetes_metadata_filter](https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter) to collect  Kubernetes metadata.
-
-See the [example](https://github.com/logicmonitor/lm-logs-fluentd/tree/master/Examples/k8s) to forward k8s logs to Logic Monitor.
+The Kubernetes configuration for LM Logs is deployed as a Helm chart.
+See the [LogicMonitor Helm repository](https://github.com/logicmonitor/k8s-helm-charts/tree/master/lm-logs).
 
 ### Resource mapping examples
 
@@ -68,4 +67,5 @@ See the [example](https://github.com/logicmonitor/lm-logs-fluentd/tree/master/Ex
 | `access_id` | LM API Token access ID. |
 | `access_key` | LM API Token access key. |
 | `flush_interval` | Defines the time in seconds to wait before sending batches of logs to LogicMonitor. Default is `60s`. |
-| `debug` | When `true`, logs more information to the fluentd console. |
+| `debug` | When `true`, logs more information to the fluentd console. |
+| `force_encoding` | Specify charset when logs contains invalid utf-8 characters. |

data/fluent-plugin-lm-logs.gemspec

@@ -1,14 +1,14 @@
 # Unless explicitly stated otherwise all files in this repository are licensed
 # under the Apache License Version 2.0.
-# This product includes software developed at Logic Monitor (https://www.logicmonitor.com).
-# Copyright 2020 Logic Monitor, Inc.
+# This product includes software developed at LogicMonitor (https://www.logicmonitor.com).
+# Copyright 2020 LogicMonitor, Inc.
 
 Gem::Specification.new do |spec|
   spec.name                           = "fluent-plugin-lm-logs"
-  spec.version                        = '0.0.8'
-  spec.authors                        = ["Logic Monitor"]
+  spec.version                        = '1.0.0'
+  spec.authors                        = ["LogicMonitor"]
   spec.email                          = "rubygems@logicmonitor.com"
-  spec.summary                        = "Logic Monitor logs fluentd output plugin"
+  spec.summary                        = "LogicMonitor logs fluentd output plugin"
   spec.description                    = "This output plugin sends fluentd records to the configured LogicMonitor account."
   spec.homepage                       = "https://www.logicmonitor.com"
   spec.license                        = "Apache-2.0"

data/lib/fluent/plugin/out_lm.rb

@@ -5,9 +5,10 @@ require 'uri'
 require 'digest'
 require 'json'
 require 'openssl'
-require "base64"
+require 'base64'
 require 'net/http'
 require 'net/https'
+require('zlib')
 
 
 module Fluent
@@ -25,7 +26,15 @@ module Fluent
     config_param :resource_mapping,  :hash, :default => {"host": "system.hostname", "hostname": "system.hostname"}
 
     config_param :debug,  :bool, :default => false
+		
+    config_param :force_encoding,  :string, :default => ""
 
+    config_param :compression,  :string, :default => ""
+
+    config_param :log_source,  :string, :default => "lm-logs-fluentd"
+
+    config_param :version_id,  :string, :default => "version_id"
+    
     # This method is called before starting.
     # 'conf' is a Hash that includes configuration parameters.
     # If the configuration is invalid, raise Fluent::ConfigError.
@@ -71,6 +80,11 @@ module Fluent
       resource_map = {}
       lm_event = {}
       lm_event["message"] = record["message"]
+      
+      if @force_encoding != ""
+        lm_event["message"] = lm_event["message"].force_encoding(@force_encoding).encode("UTF-8")
+      end
+
       if record["_lm.resourceId"] == nil
           @resource_mapping.each do |key, value|
             k = value
@@ -84,8 +98,13 @@ module Fluent
       else
         lm_event["_lm.resourceId"] = record["_lm.resourceId"]
       end
-  
-      lm_event["timestamp"] = Time.at(time).utc.to_datetime.rfc3339
+
+      if record["timestamp"] != nil
+        lm_event["timestamp"] = record["timestamp"]
+      else
+        lm_event["timestamp"] = Time.at(time).utc.to_datetime.rfc3339
+      end
+
       return lm_event
     end
 
@@ -93,10 +112,9 @@ module Fluent
       url = "https://#{@company_name}.logicmonitor.com/rest/log/ingest"
       body = events.to_json
       uri = URI.parse(url)
-
-      log.info "Sending #{events.length} events to logic monitor at #{url}"
       
       if @debug
+        log.info "Sending #{events.length} events to logic monitor at #{url}"
         log.info "Request json #{body}"
       end
 
@@ -105,13 +123,21 @@ module Fluent
 
       request = Net::HTTP::Post.new(uri.request_uri)
       request['authorization'] = generate_token(events)
-      request['Content-type'] = "application/json"  
-      request.body = body
+      request['Content-type'] = "application/json"
+      request['User-Agent'] = log_source+version_id
+
+      if @compression == "gzip"
+        request['Content-Encoding'] = "gzip"
+        gzip = Zlib::GzipWriter.new(StringIO.new)
+        gzip << body
+        request.body = gzip.close.string
+      else
+        request.body = body
+      end
 
       resp = http.request(request)
       if @debug || (!resp.kind_of? Net::HTTPSuccess)
-        log.info "Status code #{resp.code}"
-        log.info "Request Id #{resp.header['x-request-id']}"
+        log.info "Status code:#{resp.code} Request Id:#{resp.header['x-request-id']}"
       end
     end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-lm-logs
 version: !ruby/object:Gem::Version
-  version: 0.0.8
+  version: 1.0.0
 platform: ruby
 authors:
-- Logic Monitor
+- LogicMonitor
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-09-16 00:00:00.000000000 Z
+date: 2021-03-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -65,8 +65,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.2.3
 signing_key: 
 specification_version: 4
-summary: Logic Monitor logs fluentd output plugin
+summary: LogicMonitor logs fluentd output plugin
 test_files: []