logstash-output-loginsight 0.2.2 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d742a0d884f089a5d629ec11e8de53b0d6d3b2d1
-  data.tar.gz: 7420c34950d0ecd870dd87bf3e55f830987f3cc9
+  metadata.gz: 2b777be5a6adf27e1e8e453c5566aee137e7ae10
+  data.tar.gz: 61ce5259f75d3feab5210cf336e284a2df5bee79
 SHA512:
-  metadata.gz: 806d3a63f2ca1977b2c7521766857773e31847eac2a7b04afc45a8d2a887ed44a7da8c2d52a2590be595a686d4abbdcb81fc299b0259916537b9c0268c195237
-  data.tar.gz: 123af688b5c4d03da03421f7ece11e8fac31bc66a99672b4f87e3cc9aaed062c246c437ac3fb61efa7a69dcf12b8f96e1e06884c6fc9926db577dac975e5bb7a
+  metadata.gz: 27bdaaf99fcd06500230af83f5f298be78c29dda5bbef73aaf8265a14c5e3d9580bef8d27485ffb64ca1128dc4bdb1b387c14e27bd047fc21a9078b10fc5c832
+  data.tar.gz: b03dbc07bef187db1da924cb64e85eaae0966d7d172acdea17c93619f6727d6762bcd6388983829922d8a67bd9e77b06fce4c3bbc43d96a7c99eca28b2b37a4e

data/README.md

@@ -30,12 +30,12 @@ loginsight {
 
 | option | default | notes |
 | --- | --- | --- |
-| `host`  |       | required remote sserver to connect to |
+| `host`  |       | required remote server FQDN or IP |
 | `port`  | `9543`  | ingestion api port 9000 uses http |
 | `proto` | `https` | `https` or `http` |
 | `uuid`  | `id` or `0` | unique identifier for client |
-| `verify` | `True` | verify certificate chain and hostname for SSL connections |
-| `ca_file` |       | alternate certificate chain to trust |
+| `ssl_certificate_validation` | `True` | verify certificate chain and hostname for SSL connections |
+| `cacert` |       | alternate certificate chain to trust, PEM-formatted |
 
 ## Self-signed Certificate
 

data/lib/logstash/outputs/loginsight.rb

@@ -2,10 +2,7 @@
 # Copyright © 2017 VMware, Inc. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0
 
-require "logstash/outputs/base"
-require "logstash/namespace"
-require "stud/buffer"
-require "manticore"
+require "logstash/outputs/http"
 
 # This output plugin is used to send Events to a VMware vRealize Log Insight cluster,
 # preserving existing fields on Events as key=value fields. Timestamps are transmitted
@@ -13,69 +10,74 @@ require "manticore"
 
 # output { loginsight { host => ["10.11.12.13"] } }
 
-class LogStash::Outputs::Loginsight < LogStash::Outputs::Base
-  include Stud::Buffer
+class LogStash::Outputs::Loginsight < LogStash::Outputs::Http
 
   config_name "loginsight"
 
+
   config :host, :validate => :string, :required => true
   config :port, :validate => :number, :default => 9543
-  config :proto, :validate => :string, :default => "https"
+  config :proto, :validate => :string, :default => 'https'
   config :uuid, :validate => :string, :default => nil
-  config :verify, :validate => :boolean, :default => true
-  config :ca_file, :validate => :string, :default => nil
 
-  config :flush_size, :validate => :number, :default => 100
-  config :idle_flush_time, :validate => :number, :default => 1
+  config :verify, :default => true, :deprecated => 'Deprecated alias for "ssl_certificate_validation". Insecure. For self-signed certs, use openssl s_client to save server\'s certificate to a PEM-formatted file. Then pass the filename in "cacert" option.'
+  config :ca_file, :validate => :string, :default => nil, :deprecated => 'Deprecated alias for "cacert", specify path to PEM-formatted file.'
+
+  config :flush_size, :validate => :number, :default => 1, :obsolete => 'Has no effect. Events are sent without delay.'
+  config :idle_flush_time, :validate => :number, :default => 1, :obsolete => 'Has no effect. Events are sent without delay.'
 
   # Fields that will be renamed or dropped.
   config :adjusted_fields, :validate => :hash, :default => {
-    "hostname" => "host",  # unlikely to be present, preserve anyway
-    "host" => "hostname",  # desired change
-    "@version" => nil,  # drop
-    "@timestamp" => nil,  # drop, already mapped to "timestamp" in event_hash
-    "message" => nil,  # drop, already mapped to "text" in event_hash
-    "timestamp" => "timestamp_",  # Log Insight will refuse events with a "timestamp" field.
+      'hostname' => 'host',  # unlikely to be present, preserve anyway
+      'host' => 'hostname',  # desired change
+      '@version' => nil,  # drop
+      '@timestamp' => nil,  # drop, already mapped to "timestamp" in event_hash
+      'message' => nil,  # drop, already mapped to "text" in event_hash
+      'timestamp' => 'timestamp_',  # Log Insight will refuse events with a "timestamp" field.
   }
 
-  concurrency :single
+  config :url, :validate => :string, :default => nil, :deprecated => 'Use "host", "port", "proto" and "uuid" instead.'
+
+
+  # Remove configuration options from superclass that don't make sense for this plugin.
+  @config.delete('http_method')  # CFAPI is post-only
+  @config.delete('format')
+  @config.delete('message')
 
   public
   def register
+
+    if @cacert.nil?
+      @cacert = @ca_file
+    end
+
+    unless @verify.nil?
+      @ssl_certificate_validation = @verify
+    end
+
+    # Hard-wired options
+    @http_method = 'post'
+    @format = 'json'
+    @content_type = 'application/json'
+
     @uuid ||= ( @id or 0 )  # Default UUID
     @logger.debug("Starting up agent #{@uuid}")
-    @url = "#{@proto}://#{@host}:#{@port}/api/v1/events/ingest/#{@uuid}"
 
-    if  @proto == "https"
-      @client = Manticore::Client.new(headers: {"Content-Type" => "application/json"} , ssl:{ verify: @verify , ca_file:  @ca_file } )
-    else
-      @client = Manticore::Client.new(headers: {"Content-Type" => "application/json"} )
+    if @url.nil?
+      @url = "#{@proto}://#{@host}:#{@port}/api/v1/events/ingest/#{@uuid}"
     end
 
-    @logger.debug("Client", :client => @client)
+    super
 
-    buffer_initialize(
-      :max_items => @flush_size,
-      :max_interval => @idle_flush_time,
-      :logger => @logger
-    )
   end # def register
 
-  public
-  def receive(event)
-    @logger.debug("Event received", :event => event)
-    buffer_receive(event)
-  end # def receive
-
-  public
-  def flush(events, database, teardown = false)
-    @logger.debug? and @logger.debug("Flushing #{events.size} events - Teardown? #{teardown}")
-    
-    post(cfapi(events))
+  # override function from parent class, Http, removing other format modes
+  def event_body(event)
+    LogStash::Json.dump(cfapi([event]))
   end
 
   def timestamp_in_milliseconds(timestamp)
-    return (timestamp.to_f * 1000).to_i
+    (timestamp.to_f * 1000).to_i
   end
 
   # Frame the events in the hash-array structure required by Log Insight
@@ -86,39 +88,27 @@ class LogStash::Outputs::Loginsight < LogStash::Outputs::Base
     events.each do |event|
       # Create an outbound event; this can be serialized to json and sent
       event_hash = {
-        "timestamp" => timestamp_in_milliseconds(event.get("@timestamp")),
-        "text" => (event.get("message") or ""),
+        'timestamp' => timestamp_in_milliseconds(event.get('@timestamp')),
+        'text' => (event.get('message') or ''),
       }
 
       # Map fields from the event to the desired form
-      event_hash["fields"] = merge_hash(event.to_hash)
+      event_hash['fields'] = merge_hash(event.to_hash)
         .reject { |key,value| @adjusted_fields.has_key?(key) and @adjusted_fields[key] == nil }  # drop banned fields
         .map {|k,v| [ @adjusted_fields.has_key?(k) ? @adjusted_fields[k] : k,v] }  # rename fields
-        .map {|k,v| { "name" => (k), "content" => v } }  # Convert a hashmap {k=>v, k2=>v2} to a list [{name=>k, content=>v}, {name=>k2, content=>v2}]
+        .map {|k,v| { 'name' => (safefield(k)), 'content' => v } }  # Convert a hashmap {k=>v, k2=>v2} to a list [{name=>k, content=>v}, {name=>k2, content=>v2}]
 
         messages.push(event_hash)
     end # events.each do
 
-    return { "events" => messages }  # Framing required by CFAPI.
-  end # def flush
+    { 'events' => messages }  # Framing required by CFAPI.
+  end # def cfapi
 
   # Return a copy of the fieldname with non-alphanumeric characters removed.
   def safefield(fieldname)
-    fieldname.gsub(/[^a-zA-Z0-9\_]/, '')  # TODO: Correct pattern for a valid fieldname. Must deny leading numbers.
+    fieldname.gsub(/[^a-zA-Z0-9_]/, '')  # TODO: Correct pattern for a valid fieldname. Must deny leading numbers.
   end
 
-  def post(messages)
-    @logger.debug("post(body)", :messages => messages)
-    
-    body = LogStash::Json.dump(messages)
-    @logger.debug("json-dump", :body => body)
-    
-    @logger.debug("attempting connection", :url => @url)
-    response = @client.post!(@url, :body => body)
-    @logger.debug("result", :response => response)
-
-  end # def post
-
   # Recursively merge a nested dictionary into a flat dictionary with dotted keys.
   def merge_hash(hash, prelude = nil)
     hash.reduce({}) do |acc, kv|

data/logstash-output-loginsight.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name          = 'logstash-output-loginsight'
-  s.version       = '0.2.2'
+  s.version       = '0.3.0'
   s.licenses      = ['Apache-2.0']
   s.summary       = 'Output events to a Log Insight server. This uses the Ingestion API protocol.'
   s.description   = 'This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install logstash-output-loginsight. This gem is not a stand-alone program.'
@@ -25,5 +25,6 @@ Gem::Specification.new do |s|
   s.add_development_dependency "logstash-devutils", ">= 0"#, ">= 1.3.1"
   s.add_development_dependency "rspec", ">= 0"
   s.add_development_dependency "logstash-codec-plain", ">= 0"
+  s.add_development_dependency "logstash-output-http", ">= 0"
 
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-loginsight
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.3.0
 platform: ruby
 authors:
 - Alan Castonguay
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-08-31 00:00:00.000000000 Z
+date: 2017-09-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -106,6 +106,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-output-http
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install logstash-output-loginsight. This gem is not a stand-alone program.
 email: acastonguay@vmware.com
 executables: []