logstash-output-lmlogs 1.2.1 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3236f68981e8e3fb68a9335240874b9f8fb93b59372893425992f0c9c86f2895
-  data.tar.gz: 850265953883226fd2c82b3ad00acf894667aaca86396af454af6d9d59411acc
+  metadata.gz: 46b4784b47b720c638e2f6cb53ace887c3ff3bd821cc27ca675fc3bf6b5b3325
+  data.tar.gz: 2dd4c277e4b769d4580ea1474d00f058d2f08f1d718da6e715c51ab16f722679
 SHA512:
-  metadata.gz: 86a1e6be32d0c8f976371dd9441aa282c980a7886e4940b5e298ae28406c5f9e1934b751264a40010e7b94a307076248274c350ae87e51ce33d1b5cea1a1b57a
-  data.tar.gz: 01ca2ae8f258141614eef6312fe18c394b7602bd9e88e86c57d6f810471ee204889da918da4704c999d2942ef368925ba675df9a6bc2fb4169e842beb4fe7455
+  metadata.gz: 118161034fbd6b10b946449a75ccabc6fcce733006824711bdbcde2fff864f65ac249bd1de2f78600e975e71f47740881e946c4f2335a82956e6a8b130f44af5
+  data.tar.gz: f28a74d295e3f14bca4bfed4389c5dd32cfaaecb8f5b550168f2542494b0876c50b95d42661d101879284f9a7f532a74551b7fa1989abf83289b046f210e33de

data/CHANGELOG.md

@@ -1,2 +1,18 @@
 ## 1.0.0
 - First version of the plugin
+## 1.0.1
+- Proxy support
+## 1.0.2
+- Update debug logs
+## 1.1.0
+- Disable mfa for gem push
+## 1.2.0
+- Add metadata by default to every log
+## 1.2.1
+- Fix ensuring metadata exists before deleting the original while forwarding to lm-logs
+## 1.3.0
+- Add bearer token support for authentication with Logicmonitor
+## 1.3.1
+- Fix user agent populated in headers
+## 2.0.0
+- Dont send event metadata by default. Add config include_metadata_keys for including custom metadata

data/README.md

@@ -1,6 +1,6 @@
 [![Gem Version](https://badge.fury.io/rb/logstash-output-lmlogs.svg)](https://badge.fury.io/rb/logstash-output-lmlogs)
 
-This plugin sends Logstash events to the [Logicmonitor Logs](https://www.logicmonitor.com) 
+This plugin sends Logstash events to the [Logicmonitor Logs](https://www.logicmonitor.com)
 
 # Getting started
 
@@ -20,6 +20,7 @@ output {
     }
 }
 ```
+You would need either `access_id` and `access_id` both or `bearer_token` for authentication with Logicmonitor.
 
 
 
@@ -27,27 +28,28 @@ output {
 
 | Option | Description| Default |
 | --- | --- | --- |
-| batch_size | Event batch size to send to LM Logs.| 100 | 
+| batch_size | Event batch size to send to LM Logs.| 100 |
 | message_key | Key that will be used by the plugin as the system key | "message" |
 | lm_property | Key that will be used by LM to match resource based on property | "system.hostname" |
 | keep_timestamp | If false, LM Logs will use the ingestion timestamp as the event timestamp | true |
 | timestamp_is_key  | If true, LM Logs will use a specified key as the event timestamp | false |
 | timestamp_key | If timestamp_is_key is set, LM Logs will use this key in the event as the timestamp | "logtimestamp"  |
-| include_metadata  | If false, the metadata fields will not be sent to LM Logs  | true |
+| include_metadata  | If true, all metadata fields will be sent to LM Logs  | false |
+| include_metadata_keys  | Array of json keys for which plugin looks for these keys and adds as event meatadata. A dot "." can be used to add nested subjson. If config `include_metadata` is set to true, all metadata will be sent regardless of this config. | [] |
 
 See the [source code](lib/logstash/outputs/lmlogs.rb) for the full list of options
 
 The syntax for `message_key` and `source_key` values are available in the [Logstash Event API Documentation](https://www.elastic.co/guide/en/logstash/current/event-api.html)
 
-## Known issues 
+## Known issues
  - Installation of the plugin fails on Logstash 6.2.1.
- 
- 
+
+
  ## Contributing
- 
+
  Bug reports and pull requests are welcome. This project is intended to
  be a safe, welcoming space for collaboration.
- 
+
  ## Development
- 
+
 We use docker to build the plugin. You can build it by running  `docker-compose run jruby gem build logstash-output-lmlogs.gemspec `

data/lib/logstash/outputs/lmlogs.rb

@@ -8,6 +8,7 @@ require 'date'
 require 'base64'
 require 'openssl'
 require 'manticore'
+require_relative "version"
 
 # An example output that does nothing.
 class LogStash::Outputs::LMLogs < LogStash::Outputs::Base
@@ -30,7 +31,7 @@ class LogStash::Outputs::LMLogs < LogStash::Outputs::Base
 
   # Keep logstash timestamp
   config :keep_timestamp, :validate => :boolean, :default => true
-  
+
   # Use a configured message key for timestamp values
   # Valid timestamp formats are ISO8601 strings or epoch in seconds, milliseconds or nanoseconds
   config :timestamp_is_key, :validate => :boolean, :default => false
@@ -91,13 +92,19 @@ class LogStash::Outputs::LMLogs < LogStash::Outputs::Base
   config :portal_name, :validate => :string, :required => true
 
   # Username to use for HTTP auth.
-  config :access_id, :validate => :string, :required => true
+  config :access_id, :validate => :string, :required => false, :default => nil
 
   # Include/Exclude metadata from sending to LM Logs
-  config :include_metadata, :validate => :boolean, :default => true
+  config :include_metadata, :validate => :boolean, :default => false
 
   # Password to use for HTTP auth
-  config :access_key, :validate => :password, :required => true
+  config :access_key, :validate => :password, :required => false, :default => nil
+
+  # Use bearer token instead of access key/id for authentication.
+  config :bearer_token, :validate => :password, :required => false, :default => nil
+
+  # json keys for which plugin looks for these keys and adds as event meatadata. A dot "." can be used to add nested subjson.
+  config :include_metadata_keys, :validate => :array, :required => false, :default => []
 
   @@MAX_PAYLOAD_SIZE = 8*1024*1024
 
@@ -110,8 +117,16 @@ class LogStash::Outputs::LMLogs < LogStash::Outputs::Base
     @total_failed = 0
     logger.info("Initialized LogicMonitor output plugin with configuration",
                 :host => @host)
-    logger.info("Max Payload Size: ", 
+    logger.info("Max Payload Size: ",
                 :size => @@MAX_PAYLOAD_SIZE)
+    configure_auth
+
+    @final_metadata_keys = Hash.new
+    if @include_metadata_keys.any?
+      include_metadata_keys.each do | nested_key |
+        @final_metadata_keys[nested_key] = nested_key.to_s.split('.')
+      end
+    end
 
   end # def register
 
@@ -137,19 +152,7 @@ class LogStash::Outputs::LMLogs < LogStash::Outputs::Base
                       @proxy
     end
 
-    if @access_id
-      if !@access_key || !@access_key.value
-        raise ::LogStash::ConfigurationError, "access_id '#{@access_id}' specified without access_key!"
-      end
-
-      # Symbolize keys if necessary
-      # c[:auth] = {
-      #     :user => @access_id,
-      #     :password => @access_key.value,
-      #     :eager => true
-      # }
-    end
-	  log_debug("manticore client config: ", :client => c)              
+    log_debug("manticore client config: ", :client => c)
     return c
   end
 
@@ -168,21 +171,35 @@ class LogStash::Outputs::LMLogs < LogStash::Outputs::Base
     @client.close
   end
 
-
+  def configure_auth
+    @use_bearer_instead_of_lmv1 = false
+    if @access_id == nil || @access_key.value == nil
+      @logger.info "Access Id or access key null. Using bearer token for authentication."
+      @use_bearer_instead_of_lmv1 = true
+    end
+    if @use_bearer_instead_of_lmv1 && @bearer_token.value == nil
+      @logger.error "Bearer token not specified. Either access_id and access_key both or bearer_token must be specified for authentication with Logicmonitor."
+      raise LogStash::ConfigurationError, 'No valid authentication specified. Either access_id and access_key both or bearer_token must be specified for authentication with Logicmonitor.'
+    end
+  end
   def generate_auth_string(body)
-    timestamp = DateTime.now.strftime('%Q')
-    hash_this = "POST#{timestamp}#{body}/log/ingest"
-    sign_this = OpenSSL::HMAC.hexdigest(
-                  OpenSSL::Digest.new('sha256'),
-                  "#{@access_key.value}",
-                  hash_this
-                )
-    signature = Base64.strict_encode64(sign_this)
-    "LMv1 #{@access_id}:#{signature}:#{timestamp}"
+    if @use_bearer_instead_of_lmv1
+      return "Bearer #{@bearer_token.value}"
+    else
+      timestamp = DateTime.now.strftime('%Q')
+      hash_this = "POST#{timestamp}#{body}/log/ingest"
+      sign_this = OpenSSL::HMAC.hexdigest(
+                    OpenSSL::Digest.new('sha256'),
+                    "#{@access_key.value}",
+                    hash_this
+                  )
+      signature = Base64.strict_encode64(sign_this)
+      return "LMv1 #{@access_id}:#{signature}:#{timestamp}"
+    end
   end
 
   def send_batch(events)
-    log_debug("Started sending logs to LM: ", 
+    log_debug("Started sending logs to LM: ",
                   :time => Time::now.utc)
     url = "https://" + @portal_name + ".logicmonitor.com/rest/log/ingest"
     body = events.to_json
@@ -191,7 +208,7 @@ class LogStash::Outputs::LMLogs < LogStash::Outputs::Base
         :body => body,
         :headers => {
                 "Content-Type" => "application/json",
-                "User-Agent" => "LM Logs Logstash Plugin",
+                "User-Agent" => "lm-logs-logstash/" + LmLogsLogstashPlugin::VERSION,
                 "Authorization" => "#{auth_string}"
         }
     })
@@ -249,7 +266,7 @@ class LogStash::Outputs::LMLogs < LogStash::Outputs::Base
     elsif debug
       @logger.debug(message, *opts)
     end
-  end 
+  end
 
   public
   def multi_receive(events)
@@ -260,34 +277,48 @@ class LogStash::Outputs::LMLogs < LogStash::Outputs::Base
     events.each_slice(@batch_size) do |chunk|
       documents = []
       chunk.each do |event|
-        event_json = JSON.parse(event.to_json)
-        lmlogs_event = {}
-
-        if @include_metadata
-         lmlogs_event = event_json
-         lmlogs_event.delete("@timestamp")  # remove redundant timestamp field
-         if lmlogs_event.dig("event", "original") != nil
-            lmlogs_event["event"].delete("original") # remove redundant log field
-         end 
-        end
 
-        lmlogs_event["message"] = event.get(@message_key).to_s
-        lmlogs_event["_lm.resourceId"] = {}
-        lmlogs_event["_lm.resourceId"]["#{@lm_property}"] = event.get(@property_key.to_s)
+        documents = isValidPayloadSize(documents, processEvent(event), @@MAX_PAYLOAD_SIZE)
+      end
+      send_batch(documents)
+    end
+  end
 
-        if @keep_timestamp
-          lmlogs_event["timestamp"] = event.get("@timestamp")
-        end
-        
-        if @timestamp_is_key
-          lmlogs_event["timestamp"] = event.get(@timestamp_key.to_s)
-        end
 
-        documents = isValidPayloadSize(documents,lmlogs_event,@@MAX_PAYLOAD_SIZE)
+  def processEvent(event)
+    event_json = JSON.parse(event.to_json)
+    lmlogs_event = {}
 
+    if @include_metadata
+      lmlogs_event = event_json
+      lmlogs_event.delete("@timestamp")  # remove redundant timestamp field
+      if lmlogs_event.dig("event", "original") != nil
+        lmlogs_event["event"].delete("original") # remove redundant log field
+      end
+    elsif @final_metadata_keys
+      @final_metadata_keys.each do | key, value |
+        nestedVal = event_json
+        value.each { |x| nestedVal = nestedVal[x] }
+        if nestedVal != nil
+          lmlogs_event[key] = nestedVal
+        end
       end
-      send_batch(documents)
     end
+
+    lmlogs_event["message"] = event.get(@message_key).to_s
+    lmlogs_event["_lm.resourceId"] = {}
+    lmlogs_event["_lm.resourceId"]["#{@lm_property}"] = event.get(@property_key.to_s)
+
+    if @keep_timestamp
+      lmlogs_event["timestamp"] = event.get("@timestamp")
+    end
+
+    if @timestamp_is_key
+      lmlogs_event["timestamp"] = event.get(@timestamp_key.to_s)
+    end
+
+    return lmlogs_event
+
   end
 
   def log_failure(message, opts)
@@ -295,10 +326,10 @@ class LogStash::Outputs::LMLogs < LogStash::Outputs::Base
   end
 
   def isValidPayloadSize(documents,lmlogs_event,max_payload_size)
-    if (documents.to_json.bytesize + lmlogs_event.to_json.bytesize) >  max_payload_size 
+    if (documents.to_json.bytesize + lmlogs_event.to_json.bytesize) >  max_payload_size
           send_batch(documents)
           documents = []
-          
+
     end
     documents.push(lmlogs_event)
     return documents

data/lib/logstash/outputs/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module LmLogsLogstashPlugin
+  VERSION = '2.0.0'
+end

data/logstash-output-lmlogs.gemspec

@@ -1,6 +1,14 @@
+
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+require "logstash/outputs/version.rb"
+
+
+
 Gem::Specification.new do |s|
   s.name = 'logstash-output-lmlogs'
-  s.version         = '1.2.1'
+  s.version         = LmLogsLogstashPlugin::VERSION
   s.licenses = ['Apache-2.0']
   s.summary = "Logstash output plugin for LM Logs"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -25,4 +33,5 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'manticore', '>= 0.5.2', '< 1.0.0'
 
   s.add_development_dependency 'logstash-devutils'
+  s.add_development_dependency 'hashdiff', '>= 1.0.0'
 end

data/spec/outputs/auth_spec.rb

@@ -0,0 +1,60 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/lmlogs"
+require "logstash/event"
+
+describe LogStash::Outputs::LMLogs do
+
+    let(:sample_lm_logs_event){{"message" => "hello this is log 1", "_lm.resourceId" => {"test.property" => "host1"}, "timestamp" => "2021-03-22T04:28:55.907121106Z"}}
+
+    def create_output_plugin_with_conf(conf)
+        return LogStash::Outputs::LMLogs.new(conf)
+    end 
+
+    it "with no auth specified" do
+        puts "auth test"
+        plugin = create_output_plugin_with_conf({
+            "portal_name" => "localhost"
+        })
+        expect { plugin.configure_auth() }.to raise_error(LogStash::ConfigurationError)
+    end
+
+    it "access_key id is specified with no bearer" do
+        puts "auth test"
+        plugin = create_output_plugin_with_conf({
+            "portal_name" => "localhost",
+            "access_id" => "abcd",
+            "access_key" => "abcd"
+        })
+        plugin.configure_auth()
+        auth_string  = plugin.generate_auth_string([sample_lm_logs_event])
+
+        expect(auth_string).to start_with("LMv1 abcd:")
+    end
+
+    it "when access id /key not specified but bearer specified" do
+        plugin = create_output_plugin_with_conf({
+            "portal_name" => "localhost",
+            "access_id" => "abcd",
+            "bearer_token" => "abcd"
+        })
+        plugin.configure_auth()
+        auth_string  = plugin.generate_auth_string([sample_lm_logs_event])
+
+        expect(auth_string).to eq("Bearer abcd")
+    end
+
+    it "when access id /key bearer all specified, use lmv1" do
+        puts "auth test"
+        plugin = create_output_plugin_with_conf({
+            "portal_name" => "localhost",
+            "access_id" => "abcd",
+            "access_key" => "abcd",
+            "bearer_token" => "abcd"
+        })
+        plugin.configure_auth()
+        auth_string  = plugin.generate_auth_string([sample_lm_logs_event])
+
+        expect(auth_string).to start_with("LMv1 abcd:")
+    end
+end

data/spec/outputs/metadata_spec.rb

@@ -0,0 +1,86 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/lmlogs"
+require "logstash/event"
+require "hashdiff"
+
+
+describe LogStash::Outputs::LMLogs do
+
+
+    let(:logstash_event) {LogStash::Event.new("message" => "hello this is log 1",
+    "host" => "host1",
+    "nested1" => {"nested2" => {"nested3" => "value"},
+                  "nested2a" => {"nested3a" => {"nested4" => "valueA"}},
+                  "nested2b" => {"nested3b" => "value"}
+                      },
+    "nested1_" => "value",
+    "nested" => {"nested2" => {"nested3" => "value",
+                                "nested3b" => "value"},
+                  "nested_ignored" => "somevalue"
+                }
+    )}
+    let(:sample_lm_logs_event){{"message" => "hello this is log 1", "_lm.resourceId" => {"test.property" => "host1"}, "timestamp" => "2021-03-22T04:28:55.907121106Z"}}
+    let(:include_metadata_keys) {["host", "nested1.nested2.nested3", "nested1.nested2a", "nested.nested2" ]}
+
+    def create_output_plugin_with_conf(conf)
+        return LogStash::Outputs::LMLogs.new(conf)
+    end
+
+    def check_same_hash(h1,h2)
+
+    end
+
+    it "default behaviour" do
+      puts "default behaviour"
+      plugin = create_output_plugin_with_conf({
+          "portal_name" => "localhost",
+          "access_id" => "abcd",
+          "access_key" => "abcd",
+          "lm_property" => "system.hostname",
+          "property_key" => "host"
+      })
+      constructed_event = plugin.processEvent(logstash_event)
+      expected_event = {
+        "message" => "hello this is log 1",
+        "timestamp" => logstash_event.timestamp,
+        "_lm.resourceId" => {"system.hostname" => "host1"},
+
+      }
+      puts " actual : #{constructed_event} \n expected : #{expected_event}"
+
+      expect(Hashdiff.diff(constructed_event,expected_event)).to eq([])
+    end
+
+    it "with include_metadata set to true" do
+      puts "with include_metadata set to true"
+      plugin = create_output_plugin_with_conf({
+        "portal_name" => "localhost",
+        "access_id" => "abcd",
+        "access_key" => "abcd",
+        "lm_property" => "system.hostname",
+        "property_key" => "host",
+        "include_metadata" => true
+      })
+      constructed_event = plugin.processEvent(logstash_event)
+      expected_event = {
+        "message" => "hello this is log 1",
+        "timestamp" => logstash_event.timestamp,
+        "@version" => "1",
+        "_lm.resourceId" => {"system.hostname" => "host1"},
+        "host" => "host1",
+        "nested1" => {"nested2" => {"nested3" => "value"},
+                      "nested2a" => {"nested3a" => {"nested4" => "valueA"}},
+                      "nested2b" => {"nested3b" => "value"}
+                      },
+        "nested1_" => "value",
+        "nested" => {"nested2" => {"nested3" => "value",
+                                "nested3b" => "value"},
+                    "nested_ignored" => "somevalue"
+                  }
+      }
+      puts " actual : #{constructed_event} \n expected : #{expected_event}"
+      puts " hash diff : #{Hashdiff.diff(constructed_event,expected_event)}"
+      expect(Hashdiff.diff(constructed_event,expected_event)).to eq([])
+    end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-lmlogs
 version: !ruby/object:Gem::Version
-  version: 1.2.1
+  version: 2.0.0
 platform: ruby
 authors:
 - LogicMonitor
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-12-23 00:00:00.000000000 Z
+date: 2023-11-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -78,6 +78,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  name: hashdiff
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.0
 description: This gem is a Logstash plugin required to be installed on top of the
   Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
   gem is not a stand-alone program
@@ -90,8 +104,11 @@ files:
 - Gemfile
 - README.md
 - lib/logstash/outputs/lmlogs.rb
+- lib/logstash/outputs/version.rb
 - logstash-output-lmlogs.gemspec
+- spec/outputs/auth_spec.rb
 - spec/outputs/lmlogs_spec.rb
+- spec/outputs/metadata_spec.rb
 homepage: https://www.logicmonitor.com
 licenses:
 - Apache-2.0
@@ -119,4 +136,6 @@ signing_key:
 specification_version: 4
 summary: Logstash output plugin for LM Logs
 test_files:
+- spec/outputs/auth_spec.rb
 - spec/outputs/lmlogs_spec.rb
+- spec/outputs/metadata_spec.rb