logstash-output-loginsight 0.2.2 → 0.3.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/CONTRIBUTORS +1 -1
- data/README.md +3 -3
- data/lib/logstash/outputs/loginsight.rb +50 -60
- data/logstash-output-loginsight.gemspec +5 -4
- metadata +34 -19
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: d5ed6267b9963df4a92fc07824580bbf58b0ae69df769d0a4bdbbe1b326b2359
|
4
|
+
data.tar.gz: d143fa9d194d17c8a673a78302e74c05e7df5fa225bd7a0dba6886055e7938e4
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: bf555927dd53f0f9231bc201fdb0c990fee68f903566faba42a680dcb79c2e1bffa6483db8c8aa37cb72f716afa0c275c19a4245fabc63002b6fa2edd5f57019
|
7
|
+
data.tar.gz: b117dc4e674ee7c16d19175fb0cdc0f71387585b4696b866cdd840d1fed127a7eea58c2aec3b1bffac2f20dc0e3ef9038d5dfe94f173c7c244faba94f1893dee
|
data/CONTRIBUTORS
CHANGED
@@ -2,7 +2,7 @@ The following is a list of people who have contributed ideas, code, bug
|
|
2
2
|
reports, or in general have helped logstash along its way.
|
3
3
|
|
4
4
|
Contributors:
|
5
|
-
*
|
5
|
+
* Ellie Ayla - ellieayla
|
6
6
|
|
7
7
|
Note: If you've sent us patches, bug reports, or otherwise contributed to
|
8
8
|
Logstash, and you aren't on the list above and want to be, please let us know
|
data/README.md
CHANGED
@@ -30,12 +30,12 @@ loginsight {
|
|
30
30
|
|
31
31
|
| option | default | notes |
|
32
32
|
| --- | --- | --- |
|
33
|
-
| `host` | | required remote
|
33
|
+
| `host` | | required remote server FQDN or IP |
|
34
34
|
| `port` | `9543` | ingestion api port 9000 uses http |
|
35
35
|
| `proto` | `https` | `https` or `http` |
|
36
36
|
| `uuid` | `id` or `0` | unique identifier for client |
|
37
|
-
| `
|
38
|
-
| `
|
37
|
+
| `ssl_certificate_validation` | `True` | verify certificate chain and hostname for SSL connections |
|
38
|
+
| `cacert` | | alternate certificate chain to trust, PEM-formatted |
|
39
39
|
|
40
40
|
## Self-signed Certificate
|
41
41
|
|
@@ -2,10 +2,7 @@
|
|
2
2
|
# Copyright © 2017 VMware, Inc. All Rights Reserved.
|
3
3
|
# SPDX-License-Identifier: Apache-2.0
|
4
4
|
|
5
|
-
require "logstash/outputs/
|
6
|
-
require "logstash/namespace"
|
7
|
-
require "stud/buffer"
|
8
|
-
require "manticore"
|
5
|
+
require "logstash/outputs/http"
|
9
6
|
|
10
7
|
# This output plugin is used to send Events to a VMware vRealize Log Insight cluster,
|
11
8
|
# preserving existing fields on Events as key=value fields. Timestamps are transmitted
|
@@ -13,69 +10,74 @@ require "manticore"
|
|
13
10
|
|
14
11
|
# output { loginsight { host => ["10.11.12.13"] } }
|
15
12
|
|
16
|
-
class LogStash::Outputs::Loginsight < LogStash::Outputs::
|
17
|
-
include Stud::Buffer
|
13
|
+
class LogStash::Outputs::Loginsight < LogStash::Outputs::Http
|
18
14
|
|
19
15
|
config_name "loginsight"
|
20
16
|
|
17
|
+
|
21
18
|
config :host, :validate => :string, :required => true
|
22
19
|
config :port, :validate => :number, :default => 9543
|
23
|
-
config :proto, :validate => :string, :default =>
|
20
|
+
config :proto, :validate => :string, :default => 'https'
|
24
21
|
config :uuid, :validate => :string, :default => nil
|
25
|
-
config :verify, :validate => :boolean, :default => true
|
26
|
-
config :ca_file, :validate => :string, :default => nil
|
27
22
|
|
28
|
-
config :
|
29
|
-
config :
|
23
|
+
config :verify, :default => true, :deprecated => 'Deprecated alias for "ssl_certificate_validation". Insecure. For self-signed certs, use openssl s_client to save server\'s certificate to a PEM-formatted file. Then pass the filename in "cacert" option.'
|
24
|
+
config :ca_file, :validate => :string, :default => nil, :deprecated => 'Deprecated alias for "cacert", specify path to PEM-formatted file.'
|
25
|
+
|
26
|
+
config :flush_size, :validate => :number, :default => 1, :obsolete => 'Has no effect. Events are sent without delay.'
|
27
|
+
config :idle_flush_time, :validate => :number, :default => 1, :obsolete => 'Has no effect. Events are sent without delay.'
|
30
28
|
|
31
29
|
# Fields that will be renamed or dropped.
|
32
30
|
config :adjusted_fields, :validate => :hash, :default => {
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
31
|
+
'hostname' => 'host', # unlikely to be present, preserve anyway
|
32
|
+
'host' => 'hostname', # desired change
|
33
|
+
'@version' => nil, # drop
|
34
|
+
'@timestamp' => nil, # drop, already mapped to "timestamp" in event_hash
|
35
|
+
'message' => nil, # drop, already mapped to "text" in event_hash
|
36
|
+
'timestamp' => 'timestamp_', # Log Insight will refuse events with a "timestamp" field.
|
39
37
|
}
|
40
38
|
|
41
|
-
|
39
|
+
config :url, :validate => :string, :default => nil, :deprecated => 'Use "host", "port", "proto" and "uuid" instead.'
|
40
|
+
|
41
|
+
|
42
|
+
# Remove configuration options from superclass that don't make sense for this plugin.
|
43
|
+
@config.delete('http_method') # CFAPI is post-only
|
44
|
+
@config.delete('format')
|
45
|
+
@config.delete('message')
|
42
46
|
|
43
47
|
public
|
44
48
|
def register
|
49
|
+
|
50
|
+
if @cacert.nil?
|
51
|
+
@cacert = @ca_file
|
52
|
+
end
|
53
|
+
|
54
|
+
unless @verify.nil?
|
55
|
+
@ssl_certificate_validation = @verify
|
56
|
+
end
|
57
|
+
|
58
|
+
# Hard-wired options
|
59
|
+
@http_method = 'post'
|
60
|
+
@format = 'json'
|
61
|
+
@content_type = 'application/json'
|
62
|
+
|
45
63
|
@uuid ||= ( @id or 0 ) # Default UUID
|
46
64
|
@logger.debug("Starting up agent #{@uuid}")
|
47
|
-
@url = "#{@proto}://#{@host}:#{@port}/api/v1/events/ingest/#{@uuid}"
|
48
65
|
|
49
|
-
if
|
50
|
-
@
|
51
|
-
else
|
52
|
-
@client = Manticore::Client.new(headers: {"Content-Type" => "application/json"} )
|
66
|
+
if @url.nil?
|
67
|
+
@url = "#{@proto}://#{@host}:#{@port}/api/v1/events/ingest/#{@uuid}"
|
53
68
|
end
|
54
69
|
|
55
|
-
|
70
|
+
super
|
56
71
|
|
57
|
-
buffer_initialize(
|
58
|
-
:max_items => @flush_size,
|
59
|
-
:max_interval => @idle_flush_time,
|
60
|
-
:logger => @logger
|
61
|
-
)
|
62
72
|
end # def register
|
63
73
|
|
64
|
-
|
65
|
-
def
|
66
|
-
|
67
|
-
buffer_receive(event)
|
68
|
-
end # def receive
|
69
|
-
|
70
|
-
public
|
71
|
-
def flush(events, database, teardown = false)
|
72
|
-
@logger.debug? and @logger.debug("Flushing #{events.size} events - Teardown? #{teardown}")
|
73
|
-
|
74
|
-
post(cfapi(events))
|
74
|
+
# override function from parent class, Http, removing other format modes
|
75
|
+
def event_body(event)
|
76
|
+
LogStash::Json.dump(cfapi([event]))
|
75
77
|
end
|
76
78
|
|
77
79
|
def timestamp_in_milliseconds(timestamp)
|
78
|
-
|
80
|
+
(timestamp.to_f * 1000).to_i
|
79
81
|
end
|
80
82
|
|
81
83
|
# Frame the events in the hash-array structure required by Log Insight
|
@@ -86,39 +88,27 @@ class LogStash::Outputs::Loginsight < LogStash::Outputs::Base
|
|
86
88
|
events.each do |event|
|
87
89
|
# Create an outbound event; this can be serialized to json and sent
|
88
90
|
event_hash = {
|
89
|
-
|
90
|
-
|
91
|
+
'timestamp' => timestamp_in_milliseconds(event.get('@timestamp')),
|
92
|
+
'text' => (event.get('message') or ''),
|
91
93
|
}
|
92
94
|
|
93
95
|
# Map fields from the event to the desired form
|
94
|
-
event_hash[
|
96
|
+
event_hash['fields'] = merge_hash(event.to_hash)
|
95
97
|
.reject { |key,value| @adjusted_fields.has_key?(key) and @adjusted_fields[key] == nil } # drop banned fields
|
96
98
|
.map {|k,v| [ @adjusted_fields.has_key?(k) ? @adjusted_fields[k] : k,v] } # rename fields
|
97
|
-
.map {|k,v| {
|
99
|
+
.map {|k,v| { 'name' => (safefield(k)), 'content' => v } } # Convert a hashmap {k=>v, k2=>v2} to a list [{name=>k, content=>v}, {name=>k2, content=>v2}]
|
98
100
|
|
99
101
|
messages.push(event_hash)
|
100
102
|
end # events.each do
|
101
103
|
|
102
|
-
|
103
|
-
end # def
|
104
|
+
{ 'events' => messages } # Framing required by CFAPI.
|
105
|
+
end # def cfapi
|
104
106
|
|
105
107
|
# Return a copy of the fieldname with non-alphanumeric characters removed.
|
106
108
|
def safefield(fieldname)
|
107
|
-
fieldname.gsub(/[^a-zA-Z0-
|
109
|
+
fieldname.gsub(/[^a-zA-Z0-9_]/, '') # TODO: Correct pattern for a valid fieldname. Must deny leading numbers.
|
108
110
|
end
|
109
111
|
|
110
|
-
def post(messages)
|
111
|
-
@logger.debug("post(body)", :messages => messages)
|
112
|
-
|
113
|
-
body = LogStash::Json.dump(messages)
|
114
|
-
@logger.debug("json-dump", :body => body)
|
115
|
-
|
116
|
-
@logger.debug("attempting connection", :url => @url)
|
117
|
-
response = @client.post!(@url, :body => body)
|
118
|
-
@logger.debug("result", :response => response)
|
119
|
-
|
120
|
-
end # def post
|
121
|
-
|
122
112
|
# Recursively merge a nested dictionary into a flat dictionary with dotted keys.
|
123
113
|
def merge_hash(hash, prelude = nil)
|
124
114
|
hash.reduce({}) do |acc, kv|
|
@@ -1,12 +1,12 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'logstash-output-loginsight'
|
3
|
-
s.version = '0.
|
3
|
+
s.version = '0.3.1'
|
4
4
|
s.licenses = ['Apache-2.0']
|
5
5
|
s.summary = 'Output events to a Log Insight server. This uses the Ingestion API protocol.'
|
6
6
|
s.description = 'This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install logstash-output-loginsight. This gem is not a stand-alone program.'
|
7
|
-
s.homepage = 'https://github.com/
|
8
|
-
s.authors = ['
|
9
|
-
s.email = '
|
7
|
+
s.homepage = 'https://github.com/ellieayla/logstash-output-loginsight'
|
8
|
+
s.authors = ['Ellie Ayla']
|
9
|
+
s.email = 'git@verselogic.net'
|
10
10
|
s.require_paths = ['lib']
|
11
11
|
|
12
12
|
# Files
|
@@ -25,5 +25,6 @@ Gem::Specification.new do |s|
|
|
25
25
|
s.add_development_dependency "logstash-devutils", ">= 0"#, ">= 1.3.1"
|
26
26
|
s.add_development_dependency "rspec", ">= 0"
|
27
27
|
s.add_development_dependency "logstash-codec-plain", ">= 0"
|
28
|
+
s.add_development_dependency "logstash-output-http", ">= 0"
|
28
29
|
|
29
30
|
end
|
metadata
CHANGED
@@ -1,16 +1,17 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-output-loginsight
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.3.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
|
-
-
|
7
|
+
- Ellie Ayla
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2023-06-20 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
|
+
name: logstash-core-plugin-api
|
14
15
|
requirement: !ruby/object:Gem::Requirement
|
15
16
|
requirements:
|
16
17
|
- - ">="
|
@@ -19,9 +20,8 @@ dependencies:
|
|
19
20
|
- - "<="
|
20
21
|
- !ruby/object:Gem::Version
|
21
22
|
version: '2.99'
|
22
|
-
name: logstash-core-plugin-api
|
23
|
-
prerelease: false
|
24
23
|
type: :runtime
|
24
|
+
prerelease: false
|
25
25
|
version_requirements: !ruby/object:Gem::Requirement
|
26
26
|
requirements:
|
27
27
|
- - ">="
|
@@ -31,6 +31,7 @@ dependencies:
|
|
31
31
|
- !ruby/object:Gem::Version
|
32
32
|
version: '2.99'
|
33
33
|
- !ruby/object:Gem::Dependency
|
34
|
+
name: manticore
|
34
35
|
requirement: !ruby/object:Gem::Requirement
|
35
36
|
requirements:
|
36
37
|
- - "~>"
|
@@ -39,9 +40,8 @@ dependencies:
|
|
39
40
|
- - ">="
|
40
41
|
- !ruby/object:Gem::Version
|
41
42
|
version: 0.6.0
|
42
|
-
name: manticore
|
43
|
-
prerelease: false
|
44
43
|
type: :runtime
|
44
|
+
prerelease: false
|
45
45
|
version_requirements: !ruby/object:Gem::Requirement
|
46
46
|
requirements:
|
47
47
|
- - "~>"
|
@@ -51,63 +51,79 @@ dependencies:
|
|
51
51
|
- !ruby/object:Gem::Version
|
52
52
|
version: 0.6.0
|
53
53
|
- !ruby/object:Gem::Dependency
|
54
|
+
name: logstash-core
|
54
55
|
requirement: !ruby/object:Gem::Requirement
|
55
56
|
requirements:
|
56
57
|
- - ">="
|
57
58
|
- !ruby/object:Gem::Version
|
58
59
|
version: '0'
|
59
|
-
name: logstash-core
|
60
|
-
prerelease: false
|
61
60
|
type: :runtime
|
61
|
+
prerelease: false
|
62
62
|
version_requirements: !ruby/object:Gem::Requirement
|
63
63
|
requirements:
|
64
64
|
- - ">="
|
65
65
|
- !ruby/object:Gem::Version
|
66
66
|
version: '0'
|
67
67
|
- !ruby/object:Gem::Dependency
|
68
|
+
name: logstash-devutils
|
68
69
|
requirement: !ruby/object:Gem::Requirement
|
69
70
|
requirements:
|
70
71
|
- - ">="
|
71
72
|
- !ruby/object:Gem::Version
|
72
73
|
version: '0'
|
73
|
-
name: logstash-devutils
|
74
|
-
prerelease: false
|
75
74
|
type: :development
|
75
|
+
prerelease: false
|
76
76
|
version_requirements: !ruby/object:Gem::Requirement
|
77
77
|
requirements:
|
78
78
|
- - ">="
|
79
79
|
- !ruby/object:Gem::Version
|
80
80
|
version: '0'
|
81
81
|
- !ruby/object:Gem::Dependency
|
82
|
+
name: rspec
|
82
83
|
requirement: !ruby/object:Gem::Requirement
|
83
84
|
requirements:
|
84
85
|
- - ">="
|
85
86
|
- !ruby/object:Gem::Version
|
86
87
|
version: '0'
|
87
|
-
name: rspec
|
88
|
-
prerelease: false
|
89
88
|
type: :development
|
89
|
+
prerelease: false
|
90
90
|
version_requirements: !ruby/object:Gem::Requirement
|
91
91
|
requirements:
|
92
92
|
- - ">="
|
93
93
|
- !ruby/object:Gem::Version
|
94
94
|
version: '0'
|
95
95
|
- !ruby/object:Gem::Dependency
|
96
|
+
name: logstash-codec-plain
|
96
97
|
requirement: !ruby/object:Gem::Requirement
|
97
98
|
requirements:
|
98
99
|
- - ">="
|
99
100
|
- !ruby/object:Gem::Version
|
100
101
|
version: '0'
|
101
|
-
|
102
|
+
type: :development
|
102
103
|
prerelease: false
|
104
|
+
version_requirements: !ruby/object:Gem::Requirement
|
105
|
+
requirements:
|
106
|
+
- - ">="
|
107
|
+
- !ruby/object:Gem::Version
|
108
|
+
version: '0'
|
109
|
+
- !ruby/object:Gem::Dependency
|
110
|
+
name: logstash-output-http
|
111
|
+
requirement: !ruby/object:Gem::Requirement
|
112
|
+
requirements:
|
113
|
+
- - ">="
|
114
|
+
- !ruby/object:Gem::Version
|
115
|
+
version: '0'
|
103
116
|
type: :development
|
117
|
+
prerelease: false
|
104
118
|
version_requirements: !ruby/object:Gem::Requirement
|
105
119
|
requirements:
|
106
120
|
- - ">="
|
107
121
|
- !ruby/object:Gem::Version
|
108
122
|
version: '0'
|
109
|
-
description: This gem is a Logstash plugin required to be installed on top of the
|
110
|
-
|
123
|
+
description: This gem is a Logstash plugin required to be installed on top of the
|
124
|
+
Logstash core pipeline using $LS_HOME/bin/logstash-plugin install logstash-output-loginsight.
|
125
|
+
This gem is not a stand-alone program.
|
126
|
+
email: git@verselogic.net
|
111
127
|
executables: []
|
112
128
|
extensions: []
|
113
129
|
extra_rdoc_files: []
|
@@ -120,7 +136,7 @@ files:
|
|
120
136
|
- lib/logstash/outputs/loginsight.rb
|
121
137
|
- logstash-output-loginsight.gemspec
|
122
138
|
- spec/outputs/loginsight_spec.rb
|
123
|
-
homepage: https://github.com/
|
139
|
+
homepage: https://github.com/ellieayla/logstash-output-loginsight
|
124
140
|
licenses:
|
125
141
|
- Apache-2.0
|
126
142
|
metadata:
|
@@ -141,8 +157,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
141
157
|
- !ruby/object:Gem::Version
|
142
158
|
version: '0'
|
143
159
|
requirements: []
|
144
|
-
|
145
|
-
rubygems_version: 2.6.8
|
160
|
+
rubygems_version: 3.3.7
|
146
161
|
signing_key:
|
147
162
|
specification_version: 4
|
148
163
|
summary: Output events to a Log Insight server. This uses the Ingestion API protocol.
|