logstash-output-lmlogs 1.3.0 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 870bd26e9fabeb697cb8dd8f66eeb7f243b4be3fcf51c0d3babb81260a08b3d9
-  data.tar.gz: 548fff67d397ef9a6c51a2bebb0c4f44d2b3decd57faa801bd9ac2e1048edad2
+  metadata.gz: 46b4784b47b720c638e2f6cb53ace887c3ff3bd821cc27ca675fc3bf6b5b3325
+  data.tar.gz: 2dd4c277e4b769d4580ea1474d00f058d2f08f1d718da6e715c51ab16f722679
 SHA512:
-  metadata.gz: 504cdc43d302a8d504a38627e3b914e8514331ae889dbcfba7fa3f0da6f010aeb09aa19437f5d32e73e100db4d0f839f9c182eb4135b78cfb1d538a6ea0f4314
-  data.tar.gz: 78a8b8b391c4887bfa5a06b12d044d03c9d3b9f2cfdc9231ae5a9d0a70ac28d824e59ce493cf8a4f3d2fc7675ecd0a7ce8b7d71c60fa8fc57664a7937793ad07
+  metadata.gz: 118161034fbd6b10b946449a75ccabc6fcce733006824711bdbcde2fff864f65ac249bd1de2f78600e975e71f47740881e946c4f2335a82956e6a8b130f44af5
+  data.tar.gz: f28a74d295e3f14bca4bfed4389c5dd32cfaaecb8f5b550168f2542494b0876c50b95d42661d101879284f9a7f532a74551b7fa1989abf83289b046f210e33de

data/CHANGELOG.md

@@ -7,8 +7,12 @@
 ## 1.1.0
 - Disable mfa for gem push
 ## 1.2.0
-- Add metadata by default to every log 
+- Add metadata by default to every log
 ## 1.2.1
 - Fix ensuring metadata exists before deleting the original while forwarding to lm-logs
 ## 1.3.0
-- Add bearer token support for authentication with Logicmonitor
+- Add bearer token support for authentication with Logicmonitor
+## 1.3.1
+- Fix user agent populated in headers
+## 2.0.0
+- Dont send event metadata by default. Add config include_metadata_keys for including custom metadata

data/README.md

@@ -1,6 +1,6 @@
 [![Gem Version](https://badge.fury.io/rb/logstash-output-lmlogs.svg)](https://badge.fury.io/rb/logstash-output-lmlogs)
 
-This plugin sends Logstash events to the [Logicmonitor Logs](https://www.logicmonitor.com) 
+This plugin sends Logstash events to the [Logicmonitor Logs](https://www.logicmonitor.com)
 
 # Getting started
 
@@ -20,6 +20,7 @@ output {
     }
 }
 ```
+You would need either `access_id` and `access_id` both or `bearer_token` for authentication with Logicmonitor.
 
 
 
@@ -27,27 +28,28 @@ output {
 
 | Option | Description| Default |
 | --- | --- | --- |
-| batch_size | Event batch size to send to LM Logs.| 100 | 
+| batch_size | Event batch size to send to LM Logs.| 100 |
 | message_key | Key that will be used by the plugin as the system key | "message" |
 | lm_property | Key that will be used by LM to match resource based on property | "system.hostname" |
 | keep_timestamp | If false, LM Logs will use the ingestion timestamp as the event timestamp | true |
 | timestamp_is_key  | If true, LM Logs will use a specified key as the event timestamp | false |
 | timestamp_key | If timestamp_is_key is set, LM Logs will use this key in the event as the timestamp | "logtimestamp"  |
-| include_metadata  | If false, the metadata fields will not be sent to LM Logs  | true |
+| include_metadata  | If true, all metadata fields will be sent to LM Logs  | false |
+| include_metadata_keys  | Array of json keys for which plugin looks for these keys and adds as event meatadata. A dot "." can be used to add nested subjson. If config `include_metadata` is set to true, all metadata will be sent regardless of this config. | [] |
 
 See the [source code](lib/logstash/outputs/lmlogs.rb) for the full list of options
 
 The syntax for `message_key` and `source_key` values are available in the [Logstash Event API Documentation](https://www.elastic.co/guide/en/logstash/current/event-api.html)
 
-## Known issues 
+## Known issues
  - Installation of the plugin fails on Logstash 6.2.1.
- 
- 
+
+
  ## Contributing
- 
+
  Bug reports and pull requests are welcome. This project is intended to
  be a safe, welcoming space for collaboration.
- 
+
  ## Development
- 
+
 We use docker to build the plugin. You can build it by running  `docker-compose run jruby gem build logstash-output-lmlogs.gemspec `

data/lib/logstash/outputs/lmlogs.rb

@@ -8,6 +8,7 @@ require 'date'
 require 'base64'
 require 'openssl'
 require 'manticore'
+require_relative "version"
 
 # An example output that does nothing.
 class LogStash::Outputs::LMLogs < LogStash::Outputs::Base
@@ -94,7 +95,7 @@ class LogStash::Outputs::LMLogs < LogStash::Outputs::Base
   config :access_id, :validate => :string, :required => false, :default => nil
 
   # Include/Exclude metadata from sending to LM Logs
-  config :include_metadata, :validate => :boolean, :default => true
+  config :include_metadata, :validate => :boolean, :default => false
 
   # Password to use for HTTP auth
   config :access_key, :validate => :password, :required => false, :default => nil
@@ -102,6 +103,9 @@ class LogStash::Outputs::LMLogs < LogStash::Outputs::Base
   # Use bearer token instead of access key/id for authentication.
   config :bearer_token, :validate => :password, :required => false, :default => nil
 
+  # json keys for which plugin looks for these keys and adds as event meatadata. A dot "." can be used to add nested subjson.
+  config :include_metadata_keys, :validate => :array, :required => false, :default => []
+
   @@MAX_PAYLOAD_SIZE = 8*1024*1024
 
   # For developer debugging.
@@ -116,6 +120,14 @@ class LogStash::Outputs::LMLogs < LogStash::Outputs::Base
     logger.info("Max Payload Size: ",
                 :size => @@MAX_PAYLOAD_SIZE)
     configure_auth
+
+    @final_metadata_keys = Hash.new
+    if @include_metadata_keys.any?
+      include_metadata_keys.each do | nested_key |
+        @final_metadata_keys[nested_key] = nested_key.to_s.split('.')
+      end
+    end
+
   end # def register
 
   def client_config
@@ -164,12 +176,12 @@ class LogStash::Outputs::LMLogs < LogStash::Outputs::Base
     if @access_id == nil || @access_key.value == nil
       @logger.info "Access Id or access key null. Using bearer token for authentication."
       @use_bearer_instead_of_lmv1 = true
-    end  
-    if @use_bearer_instead_of_lmv1 && @bearer_token.value == nil 
+    end
+    if @use_bearer_instead_of_lmv1 && @bearer_token.value == nil
       @logger.error "Bearer token not specified. Either access_id and access_key both or bearer_token must be specified for authentication with Logicmonitor."
       raise LogStash::ConfigurationError, 'No valid authentication specified. Either access_id and access_key both or bearer_token must be specified for authentication with Logicmonitor.'
     end
-  end  
+  end
   def generate_auth_string(body)
     if @use_bearer_instead_of_lmv1
       return "Bearer #{@bearer_token.value}"
@@ -196,7 +208,7 @@ class LogStash::Outputs::LMLogs < LogStash::Outputs::Base
         :body => body,
         :headers => {
                 "Content-Type" => "application/json",
-                "User-Agent" => "LM Logs Logstash Plugin",
+                "User-Agent" => "lm-logs-logstash/" + LmLogsLogstashPlugin::VERSION,
                 "Authorization" => "#{auth_string}"
         }
     })
@@ -265,34 +277,48 @@ class LogStash::Outputs::LMLogs < LogStash::Outputs::Base
     events.each_slice(@batch_size) do |chunk|
       documents = []
       chunk.each do |event|
-        event_json = JSON.parse(event.to_json)
-        lmlogs_event = {}
-
-        if @include_metadata
-         lmlogs_event = event_json
-         lmlogs_event.delete("@timestamp")  # remove redundant timestamp field
-         if lmlogs_event.dig("event", "original") != nil
-            lmlogs_event["event"].delete("original") # remove redundant log field
-         end
-        end
 
-        lmlogs_event["message"] = event.get(@message_key).to_s
-        lmlogs_event["_lm.resourceId"] = {}
-        lmlogs_event["_lm.resourceId"]["#{@lm_property}"] = event.get(@property_key.to_s)
+        documents = isValidPayloadSize(documents, processEvent(event), @@MAX_PAYLOAD_SIZE)
+      end
+      send_batch(documents)
+    end
+  end
 
-        if @keep_timestamp
-          lmlogs_event["timestamp"] = event.get("@timestamp")
-        end
 
-        if @timestamp_is_key
-          lmlogs_event["timestamp"] = event.get(@timestamp_key.to_s)
+  def processEvent(event)
+    event_json = JSON.parse(event.to_json)
+    lmlogs_event = {}
+
+    if @include_metadata
+      lmlogs_event = event_json
+      lmlogs_event.delete("@timestamp")  # remove redundant timestamp field
+      if lmlogs_event.dig("event", "original") != nil
+        lmlogs_event["event"].delete("original") # remove redundant log field
+      end
+    elsif @final_metadata_keys
+      @final_metadata_keys.each do | key, value |
+        nestedVal = event_json
+        value.each { |x| nestedVal = nestedVal[x] }
+        if nestedVal != nil
+          lmlogs_event[key] = nestedVal
         end
+      end
+    end
 
-        documents = isValidPayloadSize(documents,lmlogs_event,@@MAX_PAYLOAD_SIZE)
+    lmlogs_event["message"] = event.get(@message_key).to_s
+    lmlogs_event["_lm.resourceId"] = {}
+    lmlogs_event["_lm.resourceId"]["#{@lm_property}"] = event.get(@property_key.to_s)
 
-      end
-      send_batch(documents)
+    if @keep_timestamp
+      lmlogs_event["timestamp"] = event.get("@timestamp")
+    end
+
+    if @timestamp_is_key
+      lmlogs_event["timestamp"] = event.get(@timestamp_key.to_s)
     end
+
+    return lmlogs_event
+
   end
 
   def log_failure(message, opts)

data/lib/logstash/outputs/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module LmLogsLogstashPlugin
+  VERSION = '2.0.0'
+end

data/logstash-output-lmlogs.gemspec

@@ -1,6 +1,14 @@
+
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+require "logstash/outputs/version.rb"
+
+
+
 Gem::Specification.new do |s|
   s.name = 'logstash-output-lmlogs'
-  s.version         = '1.3.0'
+  s.version         = LmLogsLogstashPlugin::VERSION
   s.licenses = ['Apache-2.0']
   s.summary = "Logstash output plugin for LM Logs"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -25,4 +33,5 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'manticore', '>= 0.5.2', '< 1.0.0'
 
   s.add_development_dependency 'logstash-devutils'
+  s.add_development_dependency 'hashdiff', '>= 1.0.0'
 end

data/spec/outputs/metadata_spec.rb

@@ -0,0 +1,86 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/lmlogs"
+require "logstash/event"
+require "hashdiff"
+
+
+describe LogStash::Outputs::LMLogs do
+
+
+    let(:logstash_event) {LogStash::Event.new("message" => "hello this is log 1",
+    "host" => "host1",
+    "nested1" => {"nested2" => {"nested3" => "value"},
+                  "nested2a" => {"nested3a" => {"nested4" => "valueA"}},
+                  "nested2b" => {"nested3b" => "value"}
+                      },
+    "nested1_" => "value",
+    "nested" => {"nested2" => {"nested3" => "value",
+                                "nested3b" => "value"},
+                  "nested_ignored" => "somevalue"
+                }
+    )}
+    let(:sample_lm_logs_event){{"message" => "hello this is log 1", "_lm.resourceId" => {"test.property" => "host1"}, "timestamp" => "2021-03-22T04:28:55.907121106Z"}}
+    let(:include_metadata_keys) {["host", "nested1.nested2.nested3", "nested1.nested2a", "nested.nested2" ]}
+
+    def create_output_plugin_with_conf(conf)
+        return LogStash::Outputs::LMLogs.new(conf)
+    end
+
+    def check_same_hash(h1,h2)
+
+    end
+
+    it "default behaviour" do
+      puts "default behaviour"
+      plugin = create_output_plugin_with_conf({
+          "portal_name" => "localhost",
+          "access_id" => "abcd",
+          "access_key" => "abcd",
+          "lm_property" => "system.hostname",
+          "property_key" => "host"
+      })
+      constructed_event = plugin.processEvent(logstash_event)
+      expected_event = {
+        "message" => "hello this is log 1",
+        "timestamp" => logstash_event.timestamp,
+        "_lm.resourceId" => {"system.hostname" => "host1"},
+
+      }
+      puts " actual : #{constructed_event} \n expected : #{expected_event}"
+
+      expect(Hashdiff.diff(constructed_event,expected_event)).to eq([])
+    end
+
+    it "with include_metadata set to true" do
+      puts "with include_metadata set to true"
+      plugin = create_output_plugin_with_conf({
+        "portal_name" => "localhost",
+        "access_id" => "abcd",
+        "access_key" => "abcd",
+        "lm_property" => "system.hostname",
+        "property_key" => "host",
+        "include_metadata" => true
+      })
+      constructed_event = plugin.processEvent(logstash_event)
+      expected_event = {
+        "message" => "hello this is log 1",
+        "timestamp" => logstash_event.timestamp,
+        "@version" => "1",
+        "_lm.resourceId" => {"system.hostname" => "host1"},
+        "host" => "host1",
+        "nested1" => {"nested2" => {"nested3" => "value"},
+                      "nested2a" => {"nested3a" => {"nested4" => "valueA"}},
+                      "nested2b" => {"nested3b" => "value"}
+                      },
+        "nested1_" => "value",
+        "nested" => {"nested2" => {"nested3" => "value",
+                                "nested3b" => "value"},
+                    "nested_ignored" => "somevalue"
+                  }
+      }
+      puts " actual : #{constructed_event} \n expected : #{expected_event}"
+      puts " hash diff : #{Hashdiff.diff(constructed_event,expected_event)}"
+      expect(Hashdiff.diff(constructed_event,expected_event)).to eq([])
+    end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-lmlogs
 version: !ruby/object:Gem::Version
-  version: 1.3.0
+  version: 2.0.0
 platform: ruby
 authors:
 - LogicMonitor
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-07-06 00:00:00.000000000 Z
+date: 2023-11-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -78,6 +78,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  name: hashdiff
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.0
 description: This gem is a Logstash plugin required to be installed on top of the
   Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
   gem is not a stand-alone program
@@ -90,9 +104,11 @@ files:
 - Gemfile
 - README.md
 - lib/logstash/outputs/lmlogs.rb
+- lib/logstash/outputs/version.rb
 - logstash-output-lmlogs.gemspec
 - spec/outputs/auth_spec.rb
 - spec/outputs/lmlogs_spec.rb
+- spec/outputs/metadata_spec.rb
 homepage: https://www.logicmonitor.com
 licenses:
 - Apache-2.0
@@ -122,3 +138,4 @@ summary: Logstash output plugin for LM Logs
 test_files:
 - spec/outputs/auth_spec.rb
 - spec/outputs/lmlogs_spec.rb
+- spec/outputs/metadata_spec.rb