logstash-filter-cloudfoundry 0.2.0 → 0.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/logstash/filters/cloudfoundry.rb +46 -15
- data/logstash-filter-cloudfoundry.gemspec +3 -5
- metadata +6 -6
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: bbd641a5d1f0798692f1b8906ed06f63afc2c633
|
4
|
+
data.tar.gz: ea84e455e4d94075cff2437494e375f865336249
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 92fe5abf5c2038c2b6104aa0a56d61438631d5a09781ae8643855aebb0794d7b098bfec730c08fa7cf928d802e770995b7632e7e271152999113fe6047d29fc3
|
7
|
+
data.tar.gz: 9bdbb110d18ee58e2783e8227dfc34d32a83972ce4e9a1d985e2a796cbc4f8cfe959af5206c7152c6d43943f37ff119b5bdf00ce587418d3bf691702f7c6c004
|
@@ -7,6 +7,7 @@ require "open-uri"
|
|
7
7
|
require "rufus/scheduler"
|
8
8
|
require "open3"
|
9
9
|
|
10
|
+
|
10
11
|
# The Cloud Foundry filter performs a lookup against a Cloud Foundry foundation to provide the following pieces of
|
11
12
|
# meta-data to an application log
|
12
13
|
# - Org name
|
@@ -24,19 +25,37 @@ require "open3"
|
|
24
25
|
# }
|
25
26
|
# }
|
26
27
|
#
|
27
|
-
#
|
28
|
+
# For this plugin to work you will need the CF CLI installed on your system. (https://github.com/cloudfoundry/cli)
|
29
|
+
#
|
30
|
+
# This filter only processes 1 event at a time so the use of this plugin can significantly slow down your pipeline's
|
28
31
|
# throughput if you have a high latency network. In the event of an outage (network or cloud foundry infrastructure), a
|
29
32
|
# retry flag can bet set to reduce the number of failed log-in and curl attempts to the Cloud Foundry endpoint. This
|
30
33
|
# will allow the pipeline to function without severely impacting throughput. Additionally adjusting the cache flush
|
31
34
|
# period and a cache items TTL will reduce slow down to your pipeline's throughput at the cost of additional resource
|
32
35
|
# consumption.
|
33
36
|
#
|
34
|
-
#
|
35
|
-
#
|
36
|
-
#
|
37
|
+
# To set up receiving logs from multiple foundations the user executing logstash will need write premissions to it's
|
38
|
+
# home directiory. To achieve connecting to multiple CF endpoints at once the CF_HOME variable needs to be a unique
|
39
|
+
# path for each initialization of the plugin. The configuration should look similar to the following:
|
40
|
+
#
|
41
|
+
# filter{
|
42
|
+
# if "zone1" in [tags]
|
43
|
+
# cloudfoundry{
|
44
|
+
# cf_api => "https://api.cf-domain1.com"
|
45
|
+
# ....
|
46
|
+
# }
|
47
|
+
# }
|
48
|
+
# if "zone2" in [tags]
|
49
|
+
# cloudfoundry{
|
50
|
+
# cf_api => "https://api.cf-domain2.com"
|
51
|
+
# ....
|
52
|
+
# }
|
53
|
+
# }
|
54
|
+
# }
|
55
|
+
#
|
56
|
+
#
|
37
57
|
|
38
58
|
class LogStash::Filters::CloudFoundry < LogStash::Filters::Base
|
39
|
-
#TODO: Multi-foundation suppport
|
40
59
|
#TODO: Pull "log-type" field from raw Cloud Foundry log
|
41
60
|
#TODO: Determine windows equivalent of a linux "timeout" command to improve pipeline throughput during an outage
|
42
61
|
|
@@ -57,10 +76,10 @@ class LogStash::Filters::CloudFoundry < LogStash::Filters::Base
|
|
57
76
|
# Any Cloud Foundry Space that a users account has access to
|
58
77
|
config :cf_space, :validate => :string
|
59
78
|
|
60
|
-
# Skip SSL validation while
|
79
|
+
# Skip SSL validation while logging into the endpoint
|
61
80
|
config :skip_ssl_validation, :validate => :boolean, :default => true
|
62
81
|
|
63
|
-
# How often scheduler is run to clean up cache
|
82
|
+
# How often the scheduler is run to clean up cache
|
64
83
|
config :cache_flush_time, :validate => :string, :default => "10m"
|
65
84
|
|
66
85
|
# A cache items time to live
|
@@ -69,16 +88,18 @@ class LogStash::Filters::CloudFoundry < LogStash::Filters::Base
|
|
69
88
|
# After a failed attempt to reach the Cloud Foundry endpoint, how long should the plugin wait before using the cf CLI again
|
70
89
|
config :cf_retry_cli_timeout, :validate => :number, :default => 0
|
71
90
|
|
72
|
-
# If the the Cloud Foundry API can not find GUID, cache it so plugin won't
|
91
|
+
# If the the Cloud Foundry API can not find GUID, cache it so plugin won't waste resouces continuously curling it
|
73
92
|
config :cache_invalid_guids, :validate => :boolean, :default => false
|
74
93
|
|
75
94
|
public
|
76
95
|
def register
|
77
96
|
|
97
|
+
#Check input
|
78
98
|
if @cf_api.empty? || @cf_user.empty? || @cf_password.empty? || @cf_org.empty? || @cf_space.empty?
|
79
99
|
raise "Required paramters where left blank."
|
80
100
|
end
|
81
|
-
|
101
|
+
|
102
|
+
#Set up cache and scheduler
|
82
103
|
@app_cache = Hash.new
|
83
104
|
@app_cache_mutex = Mutex.new
|
84
105
|
@scheduler = Rufus::Scheduler.new
|
@@ -92,13 +113,25 @@ class LogStash::Filters::CloudFoundry < LogStash::Filters::Base
|
|
92
113
|
end
|
93
114
|
end
|
94
115
|
|
116
|
+
#define CF_HOME path
|
117
|
+
@cf_path = "#{ENV['HOME']}/#{@cf_api.gsub(/[^0-9A-Za-z]/, '')}"
|
118
|
+
stdout, stderr, status = Open3.capture3("mkdir #{@cf_path}")
|
119
|
+
|
120
|
+
#Check folder creation status.
|
121
|
+
unless status.success?
|
122
|
+
unless stderr.include?("File exists")
|
123
|
+
raise "CF-Home-Folder-Creation: #{stderr}"
|
124
|
+
end
|
125
|
+
end
|
126
|
+
|
127
|
+
#Login to CF endpoint
|
95
128
|
login_query = cflogin
|
96
129
|
raise "CF-login-failed: #{login_query[:stdout]}" unless login_query[:status]
|
97
130
|
|
98
131
|
@cf_retry_cli_timeout > 0 ? @cf_retry_cli = true : @cf_retry_cli = false
|
99
132
|
@login_next = Time.now.to_i
|
100
133
|
@cf_logged_in = true
|
101
|
-
|
134
|
+
|
102
135
|
rescue Exception => e
|
103
136
|
|
104
137
|
@cf_retry_cli = false
|
@@ -181,9 +214,9 @@ class LogStash::Filters::CloudFoundry < LogStash::Filters::Base
|
|
181
214
|
if query[:stdout]['metadata'].nil?
|
182
215
|
|
183
216
|
if @cache_invalid_guids && !guid.blank?
|
184
|
-
|
217
|
+
@app_cache_mutex.synchronize { @app_cache[guid] = {"info" => {}, "expire_at" => Time.now.to_i + @cache_age_time} }
|
185
218
|
end
|
186
|
-
|
219
|
+
|
187
220
|
raise "CF-curl-inavlid: #{query[:stdout]}"
|
188
221
|
end
|
189
222
|
else
|
@@ -218,11 +251,9 @@ class LogStash::Filters::CloudFoundry < LogStash::Filters::Base
|
|
218
251
|
|
219
252
|
private
|
220
253
|
def cf(cmd)
|
221
|
-
|
222
|
-
stdout, stderr, status = Open3.capture3("cf #{cmd}")
|
254
|
+
stdout, stderr, status = Open3.capture3({"CF_HOME" => "#{@cf_home}"}, "cf #{cmd}")
|
223
255
|
command_output = { :stdout => valid_json?(stdout), :stderr => stderr, :status => status.success?}
|
224
256
|
command_output
|
225
|
-
|
226
257
|
end # def cf
|
227
258
|
|
228
259
|
private
|
@@ -1,12 +1,12 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'logstash-filter-cloudfoundry'
|
3
|
-
s.version = '0.
|
3
|
+
s.version = '0.3.0'
|
4
4
|
s.licenses = ['Apache License (2.0)']
|
5
5
|
s.summary = "Plugin used to assign meta-data to cloud foundry logs"
|
6
6
|
s.description = "This filter will add App, Space, and Org metadata to CloudFoundry logs. This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
|
7
7
|
s.authors = ["Tyler Stigliano"]
|
8
8
|
s.email = 'info@elastic.co'
|
9
|
-
s.homepage = "https://github.com/
|
9
|
+
s.homepage = "https://github.com/logstash-plugins/logstash-filter-example/"
|
10
10
|
s.require_paths = ["lib"]
|
11
11
|
|
12
12
|
# Files
|
@@ -18,7 +18,7 @@ Gem::Specification.new do |s|
|
|
18
18
|
s.metadata = { "logstash_plugin" => "true", "logstash_group" => "filter" }
|
19
19
|
|
20
20
|
# Gem dependencies
|
21
|
-
s.add_runtime_dependency "logstash-core", '>= 1.
|
21
|
+
s.add_runtime_dependency "logstash-core", '>= 1.4.0', '< 3.0.0'
|
22
22
|
#s.add_runtime_dependency "json"
|
23
23
|
#s.add_runtime_dependency "open-uri"
|
24
24
|
#s.add_runtime_dependency "rufus/scheduler"
|
@@ -26,5 +26,3 @@ Gem::Specification.new do |s|
|
|
26
26
|
|
27
27
|
s.add_development_dependency 'logstash-devutils'
|
28
28
|
end
|
29
|
-
|
30
|
-
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-filter-cloudfoundry
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Tyler Stigliano
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-09-
|
11
|
+
date: 2016-09-24 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: logstash-core
|
@@ -16,7 +16,7 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - ">="
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: 1.
|
19
|
+
version: 1.4.0
|
20
20
|
- - "<"
|
21
21
|
- !ruby/object:Gem::Version
|
22
22
|
version: 3.0.0
|
@@ -26,7 +26,7 @@ dependencies:
|
|
26
26
|
requirements:
|
27
27
|
- - ">="
|
28
28
|
- !ruby/object:Gem::Version
|
29
|
-
version: 1.
|
29
|
+
version: 1.4.0
|
30
30
|
- - "<"
|
31
31
|
- !ruby/object:Gem::Version
|
32
32
|
version: 3.0.0
|
@@ -67,7 +67,7 @@ files:
|
|
67
67
|
- lib/logstash/filters/cloudfoundry.rb
|
68
68
|
- logstash-filter-cloudfoundry.gemspec
|
69
69
|
- spec/filters/cloudfoundry_spec.rb
|
70
|
-
homepage: https://github.com/
|
70
|
+
homepage: https://github.com/logstash-plugins/logstash-filter-example/
|
71
71
|
licenses:
|
72
72
|
- Apache License (2.0)
|
73
73
|
metadata:
|
@@ -89,7 +89,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
89
89
|
version: '0'
|
90
90
|
requirements: []
|
91
91
|
rubyforge_project:
|
92
|
-
rubygems_version: 2.
|
92
|
+
rubygems_version: 2.5.1
|
93
93
|
signing_key:
|
94
94
|
specification_version: 4
|
95
95
|
summary: Plugin used to assign meta-data to cloud foundry logs
|