logstash-filter-cloudfoundry 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 165928e63355d42e3b36f00a8b7a4e6e978e0425
-  data.tar.gz: 637c3741d91e596dac3b14e916762458677c47bb
+  metadata.gz: bbd641a5d1f0798692f1b8906ed06f63afc2c633
+  data.tar.gz: ea84e455e4d94075cff2437494e375f865336249
 SHA512:
-  metadata.gz: 0e0c14a67cbfaf80ac9d18f4a190ef811fbb1bb98cc095625fe9f5eb8854982e597108feb3b655596a15abaa68906e4775c3ad455ae2d19ef33160c36dcdf656
-  data.tar.gz: e00c582b1ee0808015507954d41b26762e7a70814a3e36070896c8440f7e0f692cf88ec8e17aee90c5f07337eb93a6e5b0d1722c5021a5c0111b7bbffc16099f
+  metadata.gz: 92fe5abf5c2038c2b6104aa0a56d61438631d5a09781ae8643855aebb0794d7b098bfec730c08fa7cf928d802e770995b7632e7e271152999113fe6047d29fc3
+  data.tar.gz: 9bdbb110d18ee58e2783e8227dfc34d32a83972ce4e9a1d985e2a796cbc4f8cfe959af5206c7152c6d43943f37ff119b5bdf00ce587418d3bf691702f7c6c004

data/lib/logstash/filters/cloudfoundry.rb

@@ -7,6 +7,7 @@ require "open-uri"
 require "rufus/scheduler"
 require "open3"
 
+
 # The Cloud Foundry filter performs a lookup against a Cloud Foundry foundation to provide the following pieces of
 # meta-data to an application log
 #  - Org name
@@ -24,19 +25,37 @@ require "open3"
 #     }
 #   }
 #
-# This filter, only processes 1 event at a time, so the use of this plugin can significantly slow down your pipeline's
+# For this plugin to work you will need the CF CLI installed on your system. (https://github.com/cloudfoundry/cli)
+#
+# This filter only processes 1 event at a time so the use of this plugin can significantly slow down your pipeline's
 # throughput if you have a high latency network. In the event of an outage (network or cloud foundry infrastructure), a
 # retry flag can bet set to reduce the number of failed log-in and curl attempts to the Cloud Foundry endpoint. This
 # will allow the pipeline to function without severely impacting throughput. Additionally adjusting the cache flush
 # period and a cache items TTL will reduce slow down to your pipeline's throughput at the cost of additional resource
 # consumption.
 #
-# Currently, this filter can only handle logs from a single Cloud Foundry foundation. Due to a limitation with the
-# Cloud Foundry CLI (command line interface) only a single foundation can be logged into at a time. This causes race
-# conditions that this plugin currently won't handle gracefully.
+# To set up receiving logs from multiple foundations the user executing logstash will need write premissions to it's
+# home directiory. To achieve connecting to multiple CF endpoints at once the CF_HOME variable needs to be a unique
+# path for each initialization of the plugin. The configuration should look similar to the following: 
+#
+#   filter{
+#     if "zone1" in [tags]
+#         cloudfoundry{
+#             cf_api      => "https://api.cf-domain1.com"
+#             ....
+#         }
+#     }
+#     if "zone2" in [tags]
+#         cloudfoundry{
+#             cf_api      => "https://api.cf-domain2.com"
+#             ....
+#         }
+#     }
+#   }
+#
+#
 
 class LogStash::Filters::CloudFoundry < LogStash::Filters::Base
-  #TODO: Multi-foundation suppport
   #TODO: Pull "log-type" field from raw Cloud Foundry log
   #TODO: Determine windows equivalent of a linux "timeout" command to improve pipeline throughput during an outage
 
@@ -57,10 +76,10 @@ class LogStash::Filters::CloudFoundry < LogStash::Filters::Base
   # Any Cloud Foundry Space that a users account has access to
   config :cf_space,       		       :validate => :string
 
-  # Skip SSL validation while loging into the endpoint
+  # Skip SSL validation while logging into the endpoint
   config :skip_ssl_validation, 		   :validate => :boolean, :default => true
 
-  # How often scheduler is run to clean up cache
+  # How often the scheduler is run to clean up cache
   config :cache_flush_time,    		   :validate => :string,  :default => "10m"
 
   # A cache items time to live
@@ -69,16 +88,18 @@ class LogStash::Filters::CloudFoundry < LogStash::Filters::Base
   # After a failed attempt to reach the Cloud Foundry endpoint, how long should the plugin wait before using the cf CLI again
   config :cf_retry_cli_timeout,	      :validate => :number,  :default => 0
 
-  # If the the Cloud Foundry API can not find GUID, cache it so plugin won't want resouces continuously curling it 
+  # If the the Cloud Foundry API can not find GUID, cache it so plugin won't waste resouces continuously curling it 
   config :cache_invalid_guids, 		  :validate => :boolean, :default => false
   
   public
   def register
 
+    #Check input
     if @cf_api.empty? || @cf_user.empty? || @cf_password.empty? || @cf_org.empty? || @cf_space.empty?
       raise "Required paramters where left blank."
     end
-
+	
+    #Set up cache and scheduler
     @app_cache       = Hash.new
     @app_cache_mutex = Mutex.new
     @scheduler       = Rufus::Scheduler.new
@@ -92,13 +113,25 @@ class LogStash::Filters::CloudFoundry < LogStash::Filters::Base
       end
     end
 
+    #define CF_HOME path
+    @cf_path = "#{ENV['HOME']}/#{@cf_api.gsub(/[^0-9A-Za-z]/, '')}"	
+    stdout, stderr, status = Open3.capture3("mkdir #{@cf_path}")	
+	
+    #Check folder creation status. 
+    unless status.success?
+      unless stderr.include?("File exists")
+        raise "CF-Home-Folder-Creation: #{stderr}"
+      end
+    end
+
+    #Login to CF endpoint
     login_query = cflogin
     raise "CF-login-failed: #{login_query[:stdout]}" unless login_query[:status]
 
     @cf_retry_cli_timeout > 0 ? @cf_retry_cli = true : @cf_retry_cli = false
     @login_next = Time.now.to_i
     @cf_logged_in = true
-
+	
   rescue Exception => e
 
     @cf_retry_cli = false
@@ -181,9 +214,9 @@ class LogStash::Filters::CloudFoundry < LogStash::Filters::Base
       if query[:stdout]['metadata'].nil?
 		
 	if @cache_invalid_guids && !guid.blank?
-  	  @app_cache_mutex.synchronize { @app_cache[guid] = {"info" => {}, "expire_at" => Time.now.to_i + @cache_age_time} }
+	  @app_cache_mutex.synchronize { @app_cache[guid] = {"info" => {}, "expire_at" => Time.now.to_i + @cache_age_time} }
 	end
-	  
+	 
         raise "CF-curl-inavlid: #{query[:stdout]}"
       end
     else
@@ -218,11 +251,9 @@ class LogStash::Filters::CloudFoundry < LogStash::Filters::Base
 
   private
   def cf(cmd)
-
-    stdout, stderr, status = Open3.capture3("cf #{cmd}")
+    stdout, stderr, status = Open3.capture3({"CF_HOME" => "#{@cf_home}"}, "cf #{cmd}")	
     command_output = { :stdout => valid_json?(stdout), :stderr => stderr, :status => status.success?}
     command_output
-
   end # def cf
 
   private

data/logstash-filter-cloudfoundry.gemspec

@@ -1,12 +1,12 @@
 Gem::Specification.new do |s|
   s.name          = 'logstash-filter-cloudfoundry'
-  s.version       = '0.2.0'
+  s.version       = '0.3.0'
   s.licenses      = ['Apache License (2.0)']
   s.summary       = "Plugin used to assign meta-data to cloud foundry logs"
   s.description   = "This filter will add App, Space, and Org metadata to CloudFoundry logs. This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
   s.authors       = ["Tyler Stigliano"]
   s.email         = 'info@elastic.co'
-  s.homepage      = "https://github.com/ahahtyler/logstash-filter-cloudfoundry"
+  s.homepage      = "https://github.com/logstash-plugins/logstash-filter-example/"
   s.require_paths = ["lib"]
 
   # Files
@@ -18,7 +18,7 @@ Gem::Specification.new do |s|
   s.metadata = { "logstash_plugin" => "true", "logstash_group" => "filter" }
 
   # Gem dependencies
-  s.add_runtime_dependency "logstash-core", '>= 1.5.0', '< 3.0.0'
+  s.add_runtime_dependency "logstash-core", '>= 1.4.0', '< 3.0.0'
   #s.add_runtime_dependency "json"  
   #s.add_runtime_dependency "open-uri"
   #s.add_runtime_dependency "rufus/scheduler"
@@ -26,5 +26,3 @@ Gem::Specification.new do |s|
   
   s.add_development_dependency 'logstash-devutils'
 end
-
-

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-cloudfoundry
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Tyler Stigliano
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-09-20 00:00:00.000000000 Z
+date: 2016-09-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-core
@@ -16,7 +16,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.5.0
+        version: 1.4.0
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -26,7 +26,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.5.0
+        version: 1.4.0
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -67,7 +67,7 @@ files:
 - lib/logstash/filters/cloudfoundry.rb
 - logstash-filter-cloudfoundry.gemspec
 - spec/filters/cloudfoundry_spec.rb
-homepage: https://github.com/ahahtyler/logstash-filter-cloudfoundry
+homepage: https://github.com/logstash-plugins/logstash-filter-example/
 licenses:
 - Apache License (2.0)
 metadata:
@@ -89,7 +89,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.8
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: Plugin used to assign meta-data to cloud foundry logs