logstash-output-lmlogs 1.0.1 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d6d83c13647807e66508b976a76aa5481dc4412c8631d0184f2a4cd793384a3e
-  data.tar.gz: b2d5a69f85702525e83095cc9cab180ab194402873fe18317e597cf2f591331d
+  metadata.gz: 6420014cba38216a8de9dea6094e645b80e56a0751a954db2ce76fc6e57e5b72
+  data.tar.gz: 06113b6ebb5cc2ca01017a7a801c48ac990e57c15dad4d6ad45aef811c8bf962
 SHA512:
-  metadata.gz: e7f30119cf9877d14b7362eb3c0fe6dbec7ed8fb95de34596ff1311ced6fc511286f982680ad3d50c632640fc4d7ec2e53d54548fd3081155872d63eb1d25936
-  data.tar.gz: 693702b1e37126660b117de0b34a8758579bf6fa20e26479b55b7e2022090d64497041c7ddb9563d169297f41aa82ce142d6b7e13a97558defdc0d9e8f3300c4
+  metadata.gz: 4b2ed08cbd38d9fa41b715372384fde2872e9a98552352b4494b13185e62c5fd1df1de26a13a7b4c81951ae8c91d6fb2cad1677c152e17c260a73ca1fbd5ef2c
+  data.tar.gz: d42834b440dd45429b4cd0705487c6ed23c68176b560b3af1b74d315bff204ac4608080e452e7404437a636578f58a0b4984bf5124e0e1758d2368aca0456ab9

data/README.md

@@ -33,6 +33,7 @@ output {
 | keep_timestamp | If false, LM Logs will use the ingestion timestamp as the event timestamp | true |
 | timestamp_is_key  | If true, LM Logs will use a specified key as the event timestamp | false |
 | timestamp_key | If timestamp_is_key is set, LM Logs will use this key in the event as the timestamp | "logtimestamp"  |
+| include_metadata  | If false, the metadata fields will not be sent to LM Logs  | true |
 
 See the [source code](lib/logstash/outputs/lmlogs.rb) for the full list of options
 

data/lib/logstash/outputs/lmlogs.rb

@@ -93,17 +93,25 @@ class LogStash::Outputs::LMLogs < LogStash::Outputs::Base
   # Username to use for HTTP auth.
   config :access_id, :validate => :string, :required => true
 
+  # Include/Exclude metadata from sending to LM Logs
+  config :include_metadata, :validate => :boolean, :default => true
+
   # Password to use for HTTP auth
   config :access_key, :validate => :password, :required => true
 
   @@MAX_PAYLOAD_SIZE = 8*1024*1024
 
+  # For developer debugging.
+  @@CONSOLE_LOGS = false
+
   public
   def register
     @total = 0
     @total_failed = 0
     logger.info("Initialized LogicMonitor output plugin with configuration",
                 :host => @host)
+    logger.info("Max Payload Size: ", 
+                :size => @@MAX_PAYLOAD_SIZE)
 
   end # def register
 
@@ -141,7 +149,7 @@ class LogStash::Outputs::LMLogs < LogStash::Outputs::Base
       #     :eager => true
       # }
     end
-    @logger.debug("manticore client config ", :client => c)                
+	  log_debug("manticore client config: ", :client => c)              
     return c
   end
 
@@ -174,6 +182,8 @@ class LogStash::Outputs::LMLogs < LogStash::Outputs::Base
   end
 
   def send_batch(events)
+    log_debug("Started sending logs to LM: ", 
+                  :time => Time::now.utc)
     url = "https://" + @portal_name + ".logicmonitor.com/rest/log/ingest"
     body = events.to_json
     auth_string = generate_auth_string(body)
@@ -189,7 +199,7 @@ class LogStash::Outputs::LMLogs < LogStash::Outputs::Base
     request.on_success do |response|
       if response.code == 202
         @total += events.length
-        @logger.debug("Successfully sent ",
+        log_debug("Successfully sent ",
                       :response_code => response.code,
                       :batch_size => events.length,
                       :total_sent => @total,
@@ -214,10 +224,9 @@ class LogStash::Outputs::LMLogs < LogStash::Outputs::Base
 
     request.on_failure do |exception|
       @total_failed += 1
-      log_failure("Could not access URL",
+      log_failure("The request failed. ",
                   :url => url,
                   :method => @http_method,
-                  :body => body,
                   :message => exception.message,
                   :class => exception.class.name,
                   :backtrace => exception.backtrace,
@@ -225,7 +234,7 @@ class LogStash::Outputs::LMLogs < LogStash::Outputs::Base
       )
     end
 
-    @logger.debug("Sending LM Logs",
+    log_debug("Completed sending logs to LM",
                   :total => @total,
                   :time => Time::now.utc)
     request.call
@@ -234,21 +243,33 @@ class LogStash::Outputs::LMLogs < LogStash::Outputs::Base
     @logger.error("[Exception=] #{e.message} #{e.backtrace}")
   end
 
+  def log_debug(message, *opts)
+    if @@CONSOLE_LOGS
+      puts "[#{DateTime::now}] [logstash.outputs.lmlogs] [DEBUG] #{message} #{opts.to_s}"
+    elsif debug
+      @logger.debug(message, *opts)
+    end
+  end 
 
   public
   def multi_receive(events)
-    puts @@MAX_PAYLOAD_SIZE
-    if debug
-      puts events.to_json
-    end
+    if events.length() > 0
+      log_debug(events.to_json)
+	  end
 
     events.each_slice(@batch_size) do |chunk|
       documents = []
       chunk.each do |event|
-        lmlogs_event = {
-          message: event.get(@message_key).to_s
-        }
+        event_json = JSON.parse(event.to_json)
+        lmlogs_event = {}
+
+        if @include_metadata
+         lmlogs_event = event_json
+         lmlogs_event.delete("@timestamp")  # remove redundant timestamp field
+         lmlogs_event["event"].delete("original") # remove redundant log field
+        end
 
+        lmlogs_event["message"] = event.get(@message_key).to_s
         lmlogs_event["_lm.resourceId"] = {}
         lmlogs_event["_lm.resourceId"]["#{@lm_property}"] = event.get(@property_key.to_s)
 

data/logstash-output-lmlogs.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-output-lmlogs'
-  s.version         = '1.0.1'
-  s.licenses = ['Apache License (2.0)']
+  s.version         = '1.2.0'
+  s.licenses = ['Apache-2.0']
   s.summary = "Logstash output plugin for LM Logs"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
   s.authors = ["LogicMonitor"]
@@ -15,7 +15,7 @@ Gem::Specification.new do |s|
   s.test_files = s.files.grep(%r{^(test|spec|features)/})
 
   # Special flag to let us know this is actually a logstash plugin
-  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" , "rubygems_mfa_required" => "false"}
 
   # Gem dependencies
   #

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-lmlogs
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 1.2.0
 platform: ruby
 authors:
 - LogicMonitor
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-06-08 00:00:00.000000000 Z
+date: 2022-12-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -94,10 +94,11 @@ files:
 - spec/outputs/lmlogs_spec.rb
 homepage: https://www.logicmonitor.com
 licenses:
-- Apache License (2.0)
+- Apache-2.0
 metadata:
   logstash_plugin: 'true'
   logstash_group: output
+  rubygems_mfa_required: 'false'
 post_install_message:
 rdoc_options: []
 require_paths:
@@ -113,7 +114,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.29
+rubygems_version: 3.3.25
 signing_key:
 specification_version: 4
 summary: Logstash output plugin for LM Logs