logstash-output-loginsight 0.2.2 → 0.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +3 -3
- data/lib/logstash/outputs/loginsight.rb +50 -60
- data/logstash-output-loginsight.gemspec +2 -1
- metadata +16 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 2b777be5a6adf27e1e8e453c5566aee137e7ae10
|
4
|
+
data.tar.gz: 61ce5259f75d3feab5210cf336e284a2df5bee79
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 27bdaaf99fcd06500230af83f5f298be78c29dda5bbef73aaf8265a14c5e3d9580bef8d27485ffb64ca1128dc4bdb1b387c14e27bd047fc21a9078b10fc5c832
|
7
|
+
data.tar.gz: b03dbc07bef187db1da924cb64e85eaae0966d7d172acdea17c93619f6727d6762bcd6388983829922d8a67bd9e77b06fce4c3bbc43d96a7c99eca28b2b37a4e
|
data/README.md
CHANGED
@@ -30,12 +30,12 @@ loginsight {
|
|
30
30
|
|
31
31
|
| option | default | notes |
|
32
32
|
| --- | --- | --- |
|
33
|
-
| `host` | | required remote
|
33
|
+
| `host` | | required remote server FQDN or IP |
|
34
34
|
| `port` | `9543` | ingestion api port 9000 uses http |
|
35
35
|
| `proto` | `https` | `https` or `http` |
|
36
36
|
| `uuid` | `id` or `0` | unique identifier for client |
|
37
|
-
| `
|
38
|
-
| `
|
37
|
+
| `ssl_certificate_validation` | `True` | verify certificate chain and hostname for SSL connections |
|
38
|
+
| `cacert` | | alternate certificate chain to trust, PEM-formatted |
|
39
39
|
|
40
40
|
## Self-signed Certificate
|
41
41
|
|
@@ -2,10 +2,7 @@
|
|
2
2
|
# Copyright © 2017 VMware, Inc. All Rights Reserved.
|
3
3
|
# SPDX-License-Identifier: Apache-2.0
|
4
4
|
|
5
|
-
require "logstash/outputs/
|
6
|
-
require "logstash/namespace"
|
7
|
-
require "stud/buffer"
|
8
|
-
require "manticore"
|
5
|
+
require "logstash/outputs/http"
|
9
6
|
|
10
7
|
# This output plugin is used to send Events to a VMware vRealize Log Insight cluster,
|
11
8
|
# preserving existing fields on Events as key=value fields. Timestamps are transmitted
|
@@ -13,69 +10,74 @@ require "manticore"
|
|
13
10
|
|
14
11
|
# output { loginsight { host => ["10.11.12.13"] } }
|
15
12
|
|
16
|
-
class LogStash::Outputs::Loginsight < LogStash::Outputs::
|
17
|
-
include Stud::Buffer
|
13
|
+
class LogStash::Outputs::Loginsight < LogStash::Outputs::Http
|
18
14
|
|
19
15
|
config_name "loginsight"
|
20
16
|
|
17
|
+
|
21
18
|
config :host, :validate => :string, :required => true
|
22
19
|
config :port, :validate => :number, :default => 9543
|
23
|
-
config :proto, :validate => :string, :default =>
|
20
|
+
config :proto, :validate => :string, :default => 'https'
|
24
21
|
config :uuid, :validate => :string, :default => nil
|
25
|
-
config :verify, :validate => :boolean, :default => true
|
26
|
-
config :ca_file, :validate => :string, :default => nil
|
27
22
|
|
28
|
-
config :
|
29
|
-
config :
|
23
|
+
config :verify, :default => true, :deprecated => 'Deprecated alias for "ssl_certificate_validation". Insecure. For self-signed certs, use openssl s_client to save server\'s certificate to a PEM-formatted file. Then pass the filename in "cacert" option.'
|
24
|
+
config :ca_file, :validate => :string, :default => nil, :deprecated => 'Deprecated alias for "cacert", specify path to PEM-formatted file.'
|
25
|
+
|
26
|
+
config :flush_size, :validate => :number, :default => 1, :obsolete => 'Has no effect. Events are sent without delay.'
|
27
|
+
config :idle_flush_time, :validate => :number, :default => 1, :obsolete => 'Has no effect. Events are sent without delay.'
|
30
28
|
|
31
29
|
# Fields that will be renamed or dropped.
|
32
30
|
config :adjusted_fields, :validate => :hash, :default => {
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
31
|
+
'hostname' => 'host', # unlikely to be present, preserve anyway
|
32
|
+
'host' => 'hostname', # desired change
|
33
|
+
'@version' => nil, # drop
|
34
|
+
'@timestamp' => nil, # drop, already mapped to "timestamp" in event_hash
|
35
|
+
'message' => nil, # drop, already mapped to "text" in event_hash
|
36
|
+
'timestamp' => 'timestamp_', # Log Insight will refuse events with a "timestamp" field.
|
39
37
|
}
|
40
38
|
|
41
|
-
|
39
|
+
config :url, :validate => :string, :default => nil, :deprecated => 'Use "host", "port", "proto" and "uuid" instead.'
|
40
|
+
|
41
|
+
|
42
|
+
# Remove configuration options from superclass that don't make sense for this plugin.
|
43
|
+
@config.delete('http_method') # CFAPI is post-only
|
44
|
+
@config.delete('format')
|
45
|
+
@config.delete('message')
|
42
46
|
|
43
47
|
public
|
44
48
|
def register
|
49
|
+
|
50
|
+
if @cacert.nil?
|
51
|
+
@cacert = @ca_file
|
52
|
+
end
|
53
|
+
|
54
|
+
unless @verify.nil?
|
55
|
+
@ssl_certificate_validation = @verify
|
56
|
+
end
|
57
|
+
|
58
|
+
# Hard-wired options
|
59
|
+
@http_method = 'post'
|
60
|
+
@format = 'json'
|
61
|
+
@content_type = 'application/json'
|
62
|
+
|
45
63
|
@uuid ||= ( @id or 0 ) # Default UUID
|
46
64
|
@logger.debug("Starting up agent #{@uuid}")
|
47
|
-
@url = "#{@proto}://#{@host}:#{@port}/api/v1/events/ingest/#{@uuid}"
|
48
65
|
|
49
|
-
if
|
50
|
-
@
|
51
|
-
else
|
52
|
-
@client = Manticore::Client.new(headers: {"Content-Type" => "application/json"} )
|
66
|
+
if @url.nil?
|
67
|
+
@url = "#{@proto}://#{@host}:#{@port}/api/v1/events/ingest/#{@uuid}"
|
53
68
|
end
|
54
69
|
|
55
|
-
|
70
|
+
super
|
56
71
|
|
57
|
-
buffer_initialize(
|
58
|
-
:max_items => @flush_size,
|
59
|
-
:max_interval => @idle_flush_time,
|
60
|
-
:logger => @logger
|
61
|
-
)
|
62
72
|
end # def register
|
63
73
|
|
64
|
-
|
65
|
-
def
|
66
|
-
|
67
|
-
buffer_receive(event)
|
68
|
-
end # def receive
|
69
|
-
|
70
|
-
public
|
71
|
-
def flush(events, database, teardown = false)
|
72
|
-
@logger.debug? and @logger.debug("Flushing #{events.size} events - Teardown? #{teardown}")
|
73
|
-
|
74
|
-
post(cfapi(events))
|
74
|
+
# override function from parent class, Http, removing other format modes
|
75
|
+
def event_body(event)
|
76
|
+
LogStash::Json.dump(cfapi([event]))
|
75
77
|
end
|
76
78
|
|
77
79
|
def timestamp_in_milliseconds(timestamp)
|
78
|
-
|
80
|
+
(timestamp.to_f * 1000).to_i
|
79
81
|
end
|
80
82
|
|
81
83
|
# Frame the events in the hash-array structure required by Log Insight
|
@@ -86,39 +88,27 @@ class LogStash::Outputs::Loginsight < LogStash::Outputs::Base
|
|
86
88
|
events.each do |event|
|
87
89
|
# Create an outbound event; this can be serialized to json and sent
|
88
90
|
event_hash = {
|
89
|
-
|
90
|
-
|
91
|
+
'timestamp' => timestamp_in_milliseconds(event.get('@timestamp')),
|
92
|
+
'text' => (event.get('message') or ''),
|
91
93
|
}
|
92
94
|
|
93
95
|
# Map fields from the event to the desired form
|
94
|
-
event_hash[
|
96
|
+
event_hash['fields'] = merge_hash(event.to_hash)
|
95
97
|
.reject { |key,value| @adjusted_fields.has_key?(key) and @adjusted_fields[key] == nil } # drop banned fields
|
96
98
|
.map {|k,v| [ @adjusted_fields.has_key?(k) ? @adjusted_fields[k] : k,v] } # rename fields
|
97
|
-
.map {|k,v| {
|
99
|
+
.map {|k,v| { 'name' => (safefield(k)), 'content' => v } } # Convert a hashmap {k=>v, k2=>v2} to a list [{name=>k, content=>v}, {name=>k2, content=>v2}]
|
98
100
|
|
99
101
|
messages.push(event_hash)
|
100
102
|
end # events.each do
|
101
103
|
|
102
|
-
|
103
|
-
end # def
|
104
|
+
{ 'events' => messages } # Framing required by CFAPI.
|
105
|
+
end # def cfapi
|
104
106
|
|
105
107
|
# Return a copy of the fieldname with non-alphanumeric characters removed.
|
106
108
|
def safefield(fieldname)
|
107
|
-
fieldname.gsub(/[^a-zA-Z0-
|
109
|
+
fieldname.gsub(/[^a-zA-Z0-9_]/, '') # TODO: Correct pattern for a valid fieldname. Must deny leading numbers.
|
108
110
|
end
|
109
111
|
|
110
|
-
def post(messages)
|
111
|
-
@logger.debug("post(body)", :messages => messages)
|
112
|
-
|
113
|
-
body = LogStash::Json.dump(messages)
|
114
|
-
@logger.debug("json-dump", :body => body)
|
115
|
-
|
116
|
-
@logger.debug("attempting connection", :url => @url)
|
117
|
-
response = @client.post!(@url, :body => body)
|
118
|
-
@logger.debug("result", :response => response)
|
119
|
-
|
120
|
-
end # def post
|
121
|
-
|
122
112
|
# Recursively merge a nested dictionary into a flat dictionary with dotted keys.
|
123
113
|
def merge_hash(hash, prelude = nil)
|
124
114
|
hash.reduce({}) do |acc, kv|
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'logstash-output-loginsight'
|
3
|
-
s.version = '0.
|
3
|
+
s.version = '0.3.0'
|
4
4
|
s.licenses = ['Apache-2.0']
|
5
5
|
s.summary = 'Output events to a Log Insight server. This uses the Ingestion API protocol.'
|
6
6
|
s.description = 'This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install logstash-output-loginsight. This gem is not a stand-alone program.'
|
@@ -25,5 +25,6 @@ Gem::Specification.new do |s|
|
|
25
25
|
s.add_development_dependency "logstash-devutils", ">= 0"#, ">= 1.3.1"
|
26
26
|
s.add_development_dependency "rspec", ">= 0"
|
27
27
|
s.add_development_dependency "logstash-codec-plain", ">= 0"
|
28
|
+
s.add_development_dependency "logstash-output-http", ">= 0"
|
28
29
|
|
29
30
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-output-loginsight
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Alan Castonguay
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-
|
11
|
+
date: 2017-09-14 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -106,6 +106,20 @@ dependencies:
|
|
106
106
|
- - ">="
|
107
107
|
- !ruby/object:Gem::Version
|
108
108
|
version: '0'
|
109
|
+
- !ruby/object:Gem::Dependency
|
110
|
+
requirement: !ruby/object:Gem::Requirement
|
111
|
+
requirements:
|
112
|
+
- - ">="
|
113
|
+
- !ruby/object:Gem::Version
|
114
|
+
version: '0'
|
115
|
+
name: logstash-output-http
|
116
|
+
prerelease: false
|
117
|
+
type: :development
|
118
|
+
version_requirements: !ruby/object:Gem::Requirement
|
119
|
+
requirements:
|
120
|
+
- - ">="
|
121
|
+
- !ruby/object:Gem::Version
|
122
|
+
version: '0'
|
109
123
|
description: This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install logstash-output-loginsight. This gem is not a stand-alone program.
|
110
124
|
email: acastonguay@vmware.com
|
111
125
|
executables: []
|