logstash-output-lmlogs 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: a4781f24993d21cefddbbdfdca25641f5382a4db7243a2ed0f06a15e58fdb870
+  data.tar.gz: cfaca0ff01103adbb6df7c6cf693c6db7eb06f435adfb0dd62ac7baf57de4d74
+SHA512:
+  metadata.gz: b79cb9b9306d41b5d1faf705208a1e3cacfc227bfb1ba1acf7b3580ed1f517e89ff148e05ea6cf892651d8997279a1ffad3c96bac4459a4bdacbcfc3fa1d3476
+  data.tar.gz: fff0d6826fa34cf0b6867c6406f5ca99d5c6e1244adfa1293a8d09e2d2393338c4c70d411b0f929365d4eb57eac5113f9537b9bfee815bf34bc68ec216527f20

data/CHANGELOG.md

@@ -0,0 +1,2 @@
+## 0.1.0
+- First version of the plugin

data/Gemfile

@@ -0,0 +1,11 @@
+source 'https://rubygems.org'
+
+gemspec
+
+logstash_path = ENV["LOGSTASH_PATH"] || "../../logstash"
+use_logstash_source = ENV["LOGSTASH_SOURCE"] && ENV["LOGSTASH_SOURCE"].to_s == "1"
+
+if Dir.exist?(logstash_path) && use_logstash_source
+  gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
+  gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
+end

data/README.md

@@ -0,0 +1,53 @@
+[![Build Status](https://travis-ci.org/unomaly/logstash-output-unomaly.svg?branch=master)](https://travis-ci.org/unomaly/logstash-output-unomaly)
+
+This plugin sends Logstash events to the [Logicmonitor Logs](https://www.logicmonitor.com) 
+
+# Getting started
+
+## Installing through rubygems
+
+Run the following on your Logstash instance
+
+`logstash-plugin install logstash-output-lmlogs`
+
+## Minimal configuration
+```
+output {
+    lmlogs {
+        portal_name => "your company name"
+        access_id => "your lm access id"
+        access_key => "your access key"
+    }
+}
+```
+
+```
+
+
+# Important options
+
+| Option                     | Description                                                                         | Default         |
+|----------------------------|-------------------------------------------------------------------------------------|-----------------|
+| batch_size                 | Event batch size to send to LM Logs.                                                | 100             | 
+| message_key                | Key that will be used by the plugin as the system key                               | "message"       |
+| lm_property                | Key that will be used by LM to match resource based on property                     |"system.hostname"|
+| keep_timestamp             | If false, LM Logs will use the ingestion timestamp as the event timestamp           | true            |
+| timestamp_is_key           | If true, LM Logs will use a specified key as the event timestamp                    | false           |
+| timestamp_key              | If timestamp_is_key is set, LM Logs will use this key in the event as the timestamp | "logtimestamp"  |
+
+See the [source code](lib/logstash/outputs/lmlogs.rb) for the full list of options
+
+The syntax for `message_key` and `source_key` values are available in the [Logstash Event API Documentation](https://www.elastic.co/guide/en/logstash/current/event-api.html)
+
+## Known issues 
+ - Installation of the plugin fails on Logstash 6.2.1.
+ 
+ 
+ ## Contributing
+ 
+ Bug reports and pull requests are welcome. This project is intended to
+ be a safe, welcoming space for collaboration.
+ 
+ ## Development
+ 
+We use docker to build the plugin. You can build it by running  `docker-compose run jruby gem build logstash-output-lmlogs.gemspec `

data/lib/logstash/outputs/lmlogs.rb

@@ -0,0 +1,282 @@
+# encoding: utf-8
+require "logstash/outputs/base"
+require "logstash/namespace"
+require "logstash/json"
+require 'uri'
+require 'json'
+require 'date'
+require 'base64'
+require 'openssl'
+require 'manticore'
+
+# An example output that does nothing.
+class LogStash::Outputs::LMLogs < LogStash::Outputs::Base
+  class InvalidHTTPConfigError < StandardError; end
+
+  concurrency :shared
+  config_name "lmlogs"
+
+  # Event batch size to send to LM Logs. Increasing the batch size can increase throughput by reducing HTTP overhead
+  config :batch_size, :validate => :number, :default => 100
+
+  # Key that will be used by the plugin as the log message
+  config :message_key, :validate => :string, :default => "message"
+
+  # Key that will be used by the plugin as the system key
+  config :property_key, :validate => :string, :default => "host"
+
+  # Key that will be used by LM to match resource based on property
+  config :lm_property, :validate => :string, :default => "system.hostname"
+
+  # Keep logstash timestamp
+  config :keep_timestamp, :validate => :boolean, :default => true
+  
+  # Use a configured message key for timestamp values
+  # Valid timestamp formats are ISO8601 strings or epoch in seconds, milliseconds or nanoseconds
+  config :timestamp_is_key, :validate => :boolean, :default => false
+  config :timestamp_key, :validate => :string, :default => "logtimestamp"
+
+  # Display debug logs
+  config :debug, :validate => :boolean, :default => false
+
+  # Timeout (in seconds) for the entire request
+  config :request_timeout, :validate => :number, :default => 60
+
+  # Timeout (in seconds) to wait for data on the socket. Default is `10s`
+  config :socket_timeout, :validate => :number, :default => 10
+
+  # Timeout (in seconds) to wait for a connection to be established. Default is `10s`
+  config :connect_timeout, :validate => :number, :default => 10
+
+  # Should redirects be followed? Defaults to `true`
+  config :follow_redirects, :validate => :boolean, :default => true
+
+  # Max number of concurrent connections. Defaults to `50`
+  config :pool_max, :validate => :number, :default => 50
+
+  # Max number of concurrent connections to a single host. Defaults to `25`
+  config :pool_max_per_route, :validate => :number, :default => 25
+
+  # Turn this on to enable HTTP keepalive support. We highly recommend setting `automatic_retries` to at least
+  # one with this to fix interactions with broken keepalive implementations.
+  config :keepalive, :validate => :boolean, :default => true
+
+  # How many times should the client retry a failing URL. We highly recommend NOT setting this value
+  # to zero if keepalive is enabled. Some servers incorrectly end keepalives early requiring a retry!
+  # Note: if `retry_non_idempotent` is set only GET, HEAD, PUT, DELETE, OPTIONS, and TRACE requests will be retried.
+  config :automatic_retries, :validate => :number, :default => 5
+
+  # If `automatic_retries` is enabled this will cause non-idempotent HTTP verbs (such as POST) to be retried.
+  config :retry_non_idempotent, :validate => :boolean, :default => true
+
+  # How long to wait before checking if the connection is stale before executing a request on a connection using keepalive.
+  # # You may want to set this lower, possibly to 0 if you get connection errors regularly
+  # Quoting the Apache commons docs (this client is based Apache Commmons):
+  # 'Defines period of inactivity in milliseconds after which persistent connections must be re-validated prior to being leased to the consumer. Non-positive value passed to this method disables connection validation. This check helps detect connections that have become stale (half-closed) while kept inactive in the pool.'
+  # See https://hc.apache.org/httpcomponents-client-ga/httpclient/apidocs/org/apache/http/impl/conn/PoolingHttpClientConnectionManager.html#setValidateAfterInactivity(int)[these docs for more info]
+  config :validate_after_inactivity, :validate => :number, :default => 200
+
+  # Enable cookie support. With this enabled the client will persist cookies
+  # across requests as a normal web browser would. Enabled by default
+  config :cookies, :validate => :boolean, :default => true
+
+  # If you'd like to use an HTTP proxy . This supports multiple configuration syntaxes:
+  #
+  # 1. Proxy host in form: `http://proxy.org:1234`
+  # 2. Proxy host in form: `{host => "proxy.org", port => 80, scheme => 'http', user => 'username@host', password => 'password'}`
+  # 3. Proxy host in form: `{url =>  'http://proxy.org:1234', user => 'username@host', password => 'password'}`
+  config :proxy
+
+  # LM Portal Name
+  config :portal_name, :validate => :string, :required => true
+
+  # Username to use for HTTP auth.
+  config :access_id, :validate => :string, :required => true
+
+  # Password to use for HTTP auth
+  config :access_key, :validate => :password, :required => true
+
+  @@MAX_PAYLOAD_SIZE = 8*1024*1024
+
+  public
+  def register
+    @total = 0
+    @total_failed = 0
+    logger.info("Initialized LogicMonitor output plugin with configuration",
+                :host => @host)
+
+  end # def register
+
+  def client_config
+    c = {
+        connect_timeout: @connect_timeout,
+        socket_timeout: @socket_timeout,
+        request_timeout: @request_timeout,
+        follow_redirects: @follow_redirects,
+        automatic_retries: @automatic_retries,
+        retry_non_idempotent: @retry_non_idempotent,
+        check_connection_timeout: @validate_after_inactivity,
+        pool_max: @pool_max,
+        pool_max_per_route: @pool_max_per_route,
+        cookies: @cookies,
+        keepalive: @keepalive
+    }
+
+    if @proxy
+      # Symbolize keys if necessary
+      c[:proxy] = @proxy.is_a?(Hash) ?
+                      @proxy.reduce({}) {|memo,(k,v)| memo[k.to_sym] = v; memo} :
+                      @proxy
+    end
+
+    if @access_id
+      if !@access_key || !@access_key.value
+        raise ::LogStash::ConfigurationError, "access_id '#{@access_id}' specified without access_key!"
+      end
+
+      # Symbolize keys if necessary
+      c[:auth] = {
+          :access_id => @access_id,
+          :access_key => @access_key.value,
+          :eager => true
+      }
+    end
+  end
+
+  private
+  def make_client
+    puts client_config
+    Manticore::Client.new(client_config)
+  end
+
+  public
+  def client
+    @client ||= make_client
+  end
+
+  public
+  def close
+    @client.close
+  end
+
+
+  def generate_auth_string(body)
+    timestamp = DateTime.now.strftime('%Q')
+    hash_this = "POST#{timestamp}#{body}/log/ingest"
+    sign_this = OpenSSL::HMAC.hexdigest(
+                  OpenSSL::Digest.new('sha256'),
+                  "#{@access_key.value}",
+                  hash_this
+                )
+    signature = Base64.strict_encode64(sign_this)
+    "LMv1 #{@access_id}:#{signature}:#{timestamp}"
+  end
+
+  def send_batch(events)
+    url = "https://" + @portal_name + ".logicmonitor.com/rest/log/ingest"
+    body = events.to_json
+    auth_string = generate_auth_string(body)
+    request = client.post(url, {
+        :body => body,
+        :headers => {
+                "Content-Type" => "application/json",
+                "User-Agent" => "LM Logs Logstash Plugin",
+                "Authorization" => "#{auth_string}"
+        }
+    })
+
+    request.on_success do |response|
+      if response.code == 202
+        @total += events.length
+        @logger.debug("Successfully sent ",
+                      :response_code => response.code,
+                      :batch_size => events.length,
+                      :total_sent => @total,
+                      :time => Time::now.utc)
+      elsif response.code == 207
+        log_failure(
+          "207 HTTP code - some of the events successfully parsed, some not. ",
+          :response_code => response.code,
+          :url => url,
+          :response_body => response.body,
+          :total_failed => @total_failed)
+      else
+        @total_failed += 1
+        log_failure(
+            "Encountered non-202/207 HTTP code #{response.code}",
+            :response_code => response.code,
+            :url => url,
+            :response_body => response.body,
+            :total_failed => @total_failed)
+      end
+    end
+
+    request.on_failure do |exception|
+      @total_failed += 1
+      log_failure("Could not access URL",
+                  :url => url,
+                  :method => @http_method,
+                  :body => body,
+                  :message => exception.message,
+                  :class => exception.class.name,
+                  :backtrace => exception.backtrace,
+                  :total_failed => @total_failed
+      )
+    end
+
+    @logger.debug("Sending LM Logs",
+                  :total => @total,
+                  :time => Time::now.utc)
+    request.call
+
+  rescue Exception => e
+    @logger.error("[Exception=] #{e.message} #{e.backtrace}")
+  end
+
+
+  public
+  def multi_receive(events)
+    puts @@MAX_PAYLOAD_SIZE
+    if debug
+      puts events.to_json
+    end
+
+    events.each_slice(@batch_size) do |chunk|
+      documents = []
+      chunk.each do |event|
+        lmlogs_event = {
+          message: event.get(@message_key).to_s
+        }
+
+        lmlogs_event["_lm.resourceId"] = {}
+        lmlogs_event["_lm.resourceId"]["#{@lm_property}"] = event.get(@property_key.to_s)
+
+        if @keep_timestamp
+          lmlogs_event["timestamp"] = event.get("@timestamp")
+        end
+        
+        if @timestamp_is_key
+          lmlogs_event["timestamp"] = event.get(@timestamp_key.to_s)
+        end
+
+        documents = isValidPayloadSize(documents,lmlogs_event,@@MAX_PAYLOAD_SIZE)
+
+      end
+      send_batch(documents)
+    end
+  end
+
+  def log_failure(message, opts)
+    @logger.error("[HTTP Output Failure] #{message}", opts)
+  end
+
+  def isValidPayloadSize(documents,lmlogs_event,max_payload_size)
+    if (documents.to_json.bytesize + lmlogs_event.to_json.bytesize) >  max_payload_size 
+          send_batch(documents)
+          documents = []
+          
+    end
+    documents.push(lmlogs_event)
+    return documents
+  end
+end # class LogStash::Outputs::LMLogs

data/logstash-output-lmlogs.gemspec

@@ -0,0 +1,28 @@
+Gem::Specification.new do |s|
+  s.name = 'logstash-output-lmlogs'
+  s.version         = '0.1.0'
+  s.licenses = ['Apache License (2.0)']
+  s.summary = "Logstash output plugin for LM Logs"
+  s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
+  s.authors = ["LogicMonitor"]
+  s.email = "support@logicmonitor.com"
+  s.homepage = "https://www.logicmonitor.com"
+  s.require_paths = ["lib"]
+
+  # Files
+  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','Gemfile']
+   # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
+
+  # Gem dependencies
+  #
+
+  s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
+  s.add_runtime_dependency "logstash-codec-plain"
+  s.add_runtime_dependency 'manticore', '>= 0.5.2', '< 1.0.0'
+
+  s.add_development_dependency 'logstash-devutils'
+end

data/spec/outputs/lmlogs_spec.rb

@@ -0,0 +1,88 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/lmlogs"
+require "logstash/event"
+
+describe LogStash::Outputs::LMLogs do
+  let(:sample_event) { LogStash::Event.new("message" => "hello this is log") }
+  let(:client) { @lmlogs.client }
+
+  before do
+    @lmlogs = LogStash::Outputs::LMLogs.new(
+      "portal_name" => "localhost",
+      "access_id" => "access_id",
+      "access_key" => "access_key",
+      "batch_size" => 3,
+      "lm_property" => "test.property"
+      )
+    @lmlogs.register
+    allow(@lmlogs).to receive(:client).and_return(client)
+    allow(client).to receive(:post).and_call_original
+  end
+
+  before do
+    allow(@lmlogs).to receive(:client).and_return(client)
+  end
+
+  it "Forwards an event" do
+      expect(client).to receive(:post).once.and_call_original
+      @lmlogs.multi_receive([sample_event])
+  end
+
+  it "Batches multiple events and extracts metadata" do
+    event1 = LogStash::Event.new("message" => "hello this is log 1", "host" => "host1")
+    event2 = LogStash::Event.new("message" => "hello this is log 2", "host" => "host2")
+    event3 = LogStash::Event.new("message" => "hello this is log 3", "host" => "host3")
+    expect(client).to receive(:post).once.with("https://localhost.logicmonitor.com/rest/log/ingest",hash_including(:body => LogStash::Json.dump(
+        [{"message" => "hello this is log 1", "_lm.resourceId" => {"test.property" => "host1"}, "timestamp" => event1.timestamp.to_s},
+        {"message" => "hello this is log 2", "_lm.resourceId" => {"test.property" => "host2"}, "timestamp" => event2.timestamp.to_s},
+        {"message" => "hello this is log 3", "_lm.resourceId" => {"test.property" => "host3"}, "timestamp" => event3.timestamp.to_s}
+        ]
+        ))).and_call_original
+    @lmlogs.multi_receive([event1, event2, event3])
+  end
+
+  it "Batches data of size batch_size" do
+    expect(client).to receive(:post).exactly(2).times.and_call_original
+    @lmlogs.multi_receive([sample_event, sample_event, sample_event, sample_event, sample_event])
+  end
+
+  it "max payload exceeded test" do
+
+    document = [{"message" => "hello this is log 1", "_lm.resourceId" => {"test.property" => "host1"}, "timestamp" => "2021-03-22T04:28:55.907121106Z"},
+        {"message" => "hello this is log 2", "_lm.resourceId" => {"test.property" => "host2"}, "timestamp" => "2021-03-22T04:28:55.907321106Z"},
+        {"message" => "hello this is log 3", "_lm.resourceId" => {"test.property" => "host3"}, "timestamp" => "2021-03-22T04:28:55.907421106Z"}
+        ]
+
+    lm_logs_event = {"message" => "hello this is log 4", "_lm.resourceId" => {"test.property" => "host3"}, "timestamp" => "2021-03-22T04:28:55.909421106Z"}
+    document_expected = [{"message" => "hello this is log 4", "_lm.resourceId" => {"test.property" => "host3"}, "timestamp" => "2021-03-22T04:28:55.909421106Z"}]
+    expect(client).to receive(:post).once.with("https://localhost.logicmonitor.com/rest/log/ingest",hash_including(:body => LogStash::Json.dump(
+        [{"message" => "hello this is log 1", "_lm.resourceId" => {"test.property" => "host1"}, "timestamp" => "2021-03-22T04:28:55.907121106Z"},
+        {"message" => "hello this is log 2", "_lm.resourceId" => {"test.property" => "host2"}, "timestamp" => "2021-03-22T04:28:55.907321106Z"},
+        {"message" => "hello this is log 3", "_lm.resourceId" => {"test.property" => "host3"}, "timestamp" => "2021-03-22T04:28:55.907421106Z"}
+        ]
+        ))).and_call_original
+
+    document_result = @lmlogs.isValidPayloadSize(document,lm_logs_event,document.to_json.bytesize)
+    expect(document_result).to eq(document_expected)
+  end
+  it "max payload in limit" do
+
+    document = [{"message" => "hello this is log 1", "_lm.resourceId" => {"test.property" => "host1"}, "timestamp" => "2021-03-22T04:28:55.907121106Z"},
+        {"message" => "hello this is log 2", "_lm.resourceId" => {"test.property" => "host2"}, "timestamp" => "2021-03-22T04:28:55.907321106Z"},
+        {"message" => "hello this is log 3", "_lm.resourceId" => {"test.property" => "host3"}, "timestamp" => "2021-03-22T04:28:55.907421106Z"}
+        ]
+
+    lm_logs_event = {"message" => "hello this is log 4", "_lm.resourceId" => {"test.property" => "host3"}, "timestamp" => "2021-03-22T04:28:55.909421106Z"}
+    
+    document_expected = [{"message" => "hello this is log 1", "_lm.resourceId" => {"test.property" => "host1"}, "timestamp" => "2021-03-22T04:28:55.907121106Z"},
+        {"message" => "hello this is log 2", "_lm.resourceId" => {"test.property" => "host2"}, "timestamp" => "2021-03-22T04:28:55.907321106Z"},
+        {"message" => "hello this is log 3", "_lm.resourceId" => {"test.property" => "host3"}, "timestamp" => "2021-03-22T04:28:55.907421106Z"},
+        {"message" => "hello this is log 4", "_lm.resourceId" => {"test.property" => "host3"}, "timestamp" => "2021-03-22T04:28:55.909421106Z"}]
+    expect(client).to receive(:post).exactly(0).times.and_call_original
+
+    document_result = @lmlogs.isValidPayloadSize(document,lm_logs_event,document.to_json.bytesize + lm_logs_event.to_json.bytesize)
+    expect(document_result).to eq(document_expected)
+  end
+
+end

metadata

@@ -0,0 +1,121 @@
+--- !ruby/object:Gem::Specification
+name: logstash-output-lmlogs
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- LogicMonitor
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2021-04-01 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.60'
+    - - "<="
+      - !ruby/object:Gem::Version
+        version: '2.99'
+  name: logstash-core-plugin-api
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.60'
+    - - "<="
+      - !ruby/object:Gem::Version
+        version: '2.99'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-codec-plain
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.5.2
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  name: manticore
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.5.2
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-devutils
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: This gem is a Logstash plugin required to be installed on top of the
+  Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
+  gem is not a stand-alone program
+email: support@logicmonitor.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- Gemfile
+- README.md
+- lib/logstash/outputs/lmlogs.rb
+- logstash-output-lmlogs.gemspec
+- spec/outputs/lmlogs_spec.rb
+homepage: https://www.logicmonitor.com
+licenses:
+- Apache License (2.0)
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: output
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.6
+signing_key:
+specification_version: 4
+summary: Logstash output plugin for LM Logs
+test_files:
+- spec/outputs/lmlogs_spec.rb