logstash-output-lmlogs 1.0.0 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7a6870b79a3aa4828273152c3955c0bd9ab4268336497c5f2a149b3194775842
-  data.tar.gz: 20a754e8964d1ddb7ee0e6f272b6a72035603071aef7e13863713ab0acc936d3
+  metadata.gz: d0215e007f329ca2c6538f6ec2d65932fe43d87c1396772e03c98ca132bc932e
+  data.tar.gz: 288bf0b9b00ab06c19a02d4f540daed77d50a797fcaaa4c96912b2ca1cc59140
 SHA512:
-  metadata.gz: de15b190857a5c4343189c4e3cac70fda17b44672140dc31afd19b77ce6298ac205685957e922c0c687fc4cf913be3d11c93be6706d795be83362f1c801d840f
-  data.tar.gz: 6b972a9c27bd57f27e2503ba32fdca11afe36b72a45e3395d5e2a68b6efaca72fe557bc23e5900bb1a1e92196adb3a28ddb67a55f0db6e471df1d81773fb0a9a
+  metadata.gz: 7703c1da2e96bb1fbf1f55e72c04f8d3c22c0fe164f07496d70535b7eb358f67064b0021c3857b551e46419a6a1a62f5a26a88efed83c4b3135be1a48da943be
+  data.tar.gz: a49bbfa5e88c5dc18d72d7d37f5141da160ee020802a2812680b5433c343cab5a70d11a5af94f2ecde638455efc836d8b47aca798411dba648b369322b8cb25d

data/CHANGELOG.md

@@ -1,2 +1,2 @@
-## 0.1.0
+## 1.0.0
 - First version of the plugin

data/README.md

@@ -1,4 +1,4 @@
-[![Build Status](https://travis-ci.org/unomaly/logstash-output-unomaly.svg?branch=master)](https://travis-ci.org/unomaly/logstash-output-unomaly)
+[![Gem Version](https://badge.fury.io/rb/logstash-output-lmlogs.svg)](https://badge.fury.io/rb/logstash-output-lmlogs)
 
 This plugin sends Logstash events to the [Logicmonitor Logs](https://www.logicmonitor.com) 
 
@@ -21,19 +21,18 @@ output {
 }
 ```
 
-```
 
 
-# Important options
+## Important options
 
-| Option                     | Description                                                                         | Default         |
-|----------------------------|-------------------------------------------------------------------------------------|-----------------|
-| batch_size                 | Event batch size to send to LM Logs.                                                | 100             | 
-| message_key                | Key that will be used by the plugin as the system key                               | "message"       |
-| lm_property                | Key that will be used by LM to match resource based on property                     |"system.hostname"|
-| keep_timestamp             | If false, LM Logs will use the ingestion timestamp as the event timestamp           | true            |
-| timestamp_is_key           | If true, LM Logs will use a specified key as the event timestamp                    | false           |
-| timestamp_key              | If timestamp_is_key is set, LM Logs will use this key in the event as the timestamp | "logtimestamp"  |
+| Option | Description| Default |
+| --- | --- | --- |
+| batch_size | Event batch size to send to LM Logs.| 100 | 
+| message_key | Key that will be used by the plugin as the system key | "message" |
+| lm_property | Key that will be used by LM to match resource based on property | "system.hostname" |
+| keep_timestamp | If false, LM Logs will use the ingestion timestamp as the event timestamp | true |
+| timestamp_is_key  | If true, LM Logs will use a specified key as the event timestamp | false |
+| timestamp_key | If timestamp_is_key is set, LM Logs will use this key in the event as the timestamp | "logtimestamp"  |
 
 See the [source code](lib/logstash/outputs/lmlogs.rb) for the full list of options
 

data/lib/logstash/outputs/lmlogs.rb

@@ -98,12 +98,17 @@ class LogStash::Outputs::LMLogs < LogStash::Outputs::Base
 
   @@MAX_PAYLOAD_SIZE = 8*1024*1024
 
+  # For developer debugging.
+  @@CONSOLE_LOGS = false
+
   public
   def register
     @total = 0
     @total_failed = 0
     logger.info("Initialized LogicMonitor output plugin with configuration",
                 :host => @host)
+    logger.info("Max Payload Size: ", 
+                :size => @@MAX_PAYLOAD_SIZE)
 
   end # def register
 
@@ -135,17 +140,18 @@ class LogStash::Outputs::LMLogs < LogStash::Outputs::Base
       end
 
       # Symbolize keys if necessary
-      c[:auth] = {
-          :access_id => @access_id,
-          :access_key => @access_key.value,
-          :eager => true
-      }
+      # c[:auth] = {
+      #     :user => @access_id,
+      #     :password => @access_key.value,
+      #     :eager => true
+      # }
     end
+	  log_debug("manticore client config: ", :client => c)              
+    return c
   end
 
   private
   def make_client
-    puts client_config
     Manticore::Client.new(client_config)
   end
 
@@ -173,6 +179,8 @@ class LogStash::Outputs::LMLogs < LogStash::Outputs::Base
   end
 
   def send_batch(events)
+    log_debug("Started sending logs to LM: ", 
+                  :time => Time::now.utc)
     url = "https://" + @portal_name + ".logicmonitor.com/rest/log/ingest"
     body = events.to_json
     auth_string = generate_auth_string(body)
@@ -188,7 +196,7 @@ class LogStash::Outputs::LMLogs < LogStash::Outputs::Base
     request.on_success do |response|
       if response.code == 202
         @total += events.length
-        @logger.debug("Successfully sent ",
+        log_debug("Successfully sent ",
                       :response_code => response.code,
                       :batch_size => events.length,
                       :total_sent => @total,
@@ -213,10 +221,9 @@ class LogStash::Outputs::LMLogs < LogStash::Outputs::Base
 
     request.on_failure do |exception|
       @total_failed += 1
-      log_failure("Could not access URL",
+      log_failure("The request failed. ",
                   :url => url,
                   :method => @http_method,
-                  :body => body,
                   :message => exception.message,
                   :class => exception.class.name,
                   :backtrace => exception.backtrace,
@@ -224,7 +231,7 @@ class LogStash::Outputs::LMLogs < LogStash::Outputs::Base
       )
     end
 
-    @logger.debug("Sending LM Logs",
+    log_debug("Completed sending logs to LM",
                   :total => @total,
                   :time => Time::now.utc)
     request.call
@@ -233,20 +240,29 @@ class LogStash::Outputs::LMLogs < LogStash::Outputs::Base
     @logger.error("[Exception=] #{e.message} #{e.backtrace}")
   end
 
+  def log_debug(message, *opts)
+    if @@CONSOLE_LOGS
+      puts "[#{DateTime::now}] [logstash.outputs.lmlogs] [DEBUG] #{message} #{opts.to_s}"
+    elsif debug
+      @logger.debug(message, *opts)
+    end
+  end 
 
   public
   def multi_receive(events)
-    puts @@MAX_PAYLOAD_SIZE
-    if debug
-      puts events.to_json
-    end
+    if events.length() > 0
+      log_debug(events.to_json)
+	  end
 
     events.each_slice(@batch_size) do |chunk|
       documents = []
       chunk.each do |event|
-        lmlogs_event = {
-          message: event.get(@message_key).to_s
-        }
+        event_json = JSON.parse(event.to_json)
+        lmlogs_event = event_json
+        lmlogs_event.delete("@timestamp")  # remove redundant timestamp field
+        lmlogs_event["event"].delete("original") # remove redundant log field
+
+        lmlogs_event["message"] = event.get(@message_key).to_s
 
         lmlogs_event["_lm.resourceId"] = {}
         lmlogs_event["_lm.resourceId"]["#{@lm_property}"] = event.get(@property_key.to_s)

data/logstash-output-lmlogs.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-output-lmlogs'
-  s.version         = '1.0.0'
-  s.licenses = ['Apache License (2.0)']
+  s.version         = '1.1.0'
+  s.licenses = ['Apache-2.0']
   s.summary = "Logstash output plugin for LM Logs"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
   s.authors = ["LogicMonitor"]
@@ -15,7 +15,7 @@ Gem::Specification.new do |s|
   s.test_files = s.files.grep(%r{^(test|spec|features)/})
 
   # Special flag to let us know this is actually a logstash plugin
-  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" , "rubygems_mfa_required" => "false"}
 
   # Gem dependencies
   #

data/spec/outputs/lmlogs_spec.rb

@@ -6,6 +6,7 @@ require "logstash/event"
 describe LogStash::Outputs::LMLogs do
   let(:sample_event) { LogStash::Event.new("message" => "hello this is log") }
   let(:client) { @lmlogs.client }
+  let(:sample_lm_logs_event){{"message" => "hello this is log 1", "_lm.resourceId" => {"test.property" => "host1"}, "timestamp" => "2021-03-22T04:28:55.907121106Z"}}
 
   before do
     @lmlogs = LogStash::Outputs::LMLogs.new(
@@ -47,20 +48,14 @@ describe LogStash::Outputs::LMLogs do
     @lmlogs.multi_receive([sample_event, sample_event, sample_event, sample_event, sample_event])
   end
 
-  it "max payload exceeded test" do
+  it "max payload exceeded" do
 
-    document = [{"message" => "hello this is log 1", "_lm.resourceId" => {"test.property" => "host1"}, "timestamp" => "2021-03-22T04:28:55.907121106Z"},
-        {"message" => "hello this is log 2", "_lm.resourceId" => {"test.property" => "host2"}, "timestamp" => "2021-03-22T04:28:55.907321106Z"},
-        {"message" => "hello this is log 3", "_lm.resourceId" => {"test.property" => "host3"}, "timestamp" => "2021-03-22T04:28:55.907421106Z"}
-        ]
+    document = [sample_lm_logs_event,sample_lm_logs_event]
 
-    lm_logs_event = {"message" => "hello this is log 4", "_lm.resourceId" => {"test.property" => "host3"}, "timestamp" => "2021-03-22T04:28:55.909421106Z"}
-    document_expected = [{"message" => "hello this is log 4", "_lm.resourceId" => {"test.property" => "host3"}, "timestamp" => "2021-03-22T04:28:55.909421106Z"}]
+    lm_logs_event = {"message" => "hello this is log 3", "_lm.resourceId" => {"test.property" => "host3"}, "timestamp" => "2021-03-22T04:28:55.909421106Z"}
+    document_expected = [lm_logs_event]
     expect(client).to receive(:post).once.with("https://localhost.logicmonitor.com/rest/log/ingest",hash_including(:body => LogStash::Json.dump(
-        [{"message" => "hello this is log 1", "_lm.resourceId" => {"test.property" => "host1"}, "timestamp" => "2021-03-22T04:28:55.907121106Z"},
-        {"message" => "hello this is log 2", "_lm.resourceId" => {"test.property" => "host2"}, "timestamp" => "2021-03-22T04:28:55.907321106Z"},
-        {"message" => "hello this is log 3", "_lm.resourceId" => {"test.property" => "host3"}, "timestamp" => "2021-03-22T04:28:55.907421106Z"}
-        ]
+        [sample_lm_logs_event,sample_lm_logs_event]
         ))).and_call_original
 
     document_result = @lmlogs.isValidPayloadSize(document,lm_logs_event,document.to_json.bytesize)
@@ -68,17 +63,11 @@ describe LogStash::Outputs::LMLogs do
   end
   it "max payload in limit" do
 
-    document = [{"message" => "hello this is log 1", "_lm.resourceId" => {"test.property" => "host1"}, "timestamp" => "2021-03-22T04:28:55.907121106Z"},
-        {"message" => "hello this is log 2", "_lm.resourceId" => {"test.property" => "host2"}, "timestamp" => "2021-03-22T04:28:55.907321106Z"},
-        {"message" => "hello this is log 3", "_lm.resourceId" => {"test.property" => "host3"}, "timestamp" => "2021-03-22T04:28:55.907421106Z"}
-        ]
+    document = [sample_lm_logs_event]
 
-    lm_logs_event = {"message" => "hello this is log 4", "_lm.resourceId" => {"test.property" => "host3"}, "timestamp" => "2021-03-22T04:28:55.909421106Z"}
+    lm_logs_event = {"message" => "hello this is log 2", "_lm.resourceId" => {"test.property" => "host3"}, "timestamp" => "2021-03-22T04:28:55.909421106Z"}
     
-    document_expected = [{"message" => "hello this is log 1", "_lm.resourceId" => {"test.property" => "host1"}, "timestamp" => "2021-03-22T04:28:55.907121106Z"},
-        {"message" => "hello this is log 2", "_lm.resourceId" => {"test.property" => "host2"}, "timestamp" => "2021-03-22T04:28:55.907321106Z"},
-        {"message" => "hello this is log 3", "_lm.resourceId" => {"test.property" => "host3"}, "timestamp" => "2021-03-22T04:28:55.907421106Z"},
-        {"message" => "hello this is log 4", "_lm.resourceId" => {"test.property" => "host3"}, "timestamp" => "2021-03-22T04:28:55.909421106Z"}]
+    document_expected = [sample_lm_logs_event,lm_logs_event]
     expect(client).to receive(:post).exactly(0).times.and_call_original
 
     document_result = @lmlogs.isValidPayloadSize(document,lm_logs_event,document.to_json.bytesize + lm_logs_event.to_json.bytesize)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-lmlogs
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.1.0
 platform: ruby
 authors:
 - LogicMonitor
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-04-01 00:00:00.000000000 Z
+date: 2022-11-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -94,10 +94,11 @@ files:
 - spec/outputs/lmlogs_spec.rb
 homepage: https://www.logicmonitor.com
 licenses:
-- Apache License (2.0)
+- Apache-2.0
 metadata:
   logstash_plugin: 'true'
   logstash_group: output
+  rubygems_mfa_required: 'false'
 post_install_message:
 rdoc_options: []
 require_paths:
@@ -113,7 +114,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.6
+rubygems_version: 3.3.25
 signing_key:
 specification_version: 4
 summary: Logstash output plugin for LM Logs