fluent-plugin-spectrum 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9bc88f1954fef9f516c4ef7c8f7fd134a6ed957d
-  data.tar.gz: de6e8bfdd3f67695df5de0f2f62fee25f9c15a91
+  metadata.gz: 4a907d9e3749f2604e633e9d2e7d208b2595eca8
+  data.tar.gz: 6bf4af623fa6d9aed08638ea59d474a694bf2e87
 SHA512:
-  metadata.gz: ec23275e1753d9d7bce921d6a8e3a6df5a48225dcde19a9fb5aaf6f7d1f3a1352a8b415676d7c131e1c54a4d657f12aaac1643ca9697adbed7de92df773cfb36
-  data.tar.gz: 54f6270b853b64f33290044b9d9a910dc7adb8d25aa3dacc2165ea91711a6ad643f79f7fb61d991aa3d575c2dbadf24a2c1727c0d43ae7b905fb76698fd745cd
+  metadata.gz: b6d14688b87fa043d58a4eaef3394507a82c64b0178a7d32f435a00d5d1c6ea429a1ed44853cd8c4af268d997ca7d3a0748b78e40d4dbf2e75fc020440783266
+  data.tar.gz: a48028226c73f7a715a43861ed31fe9eec1ecf2aac9ba7ec536a88cfa6ed39ba1b81b2314326fc1ea29021001115ba012266d206837b214fc2f3783407e91378

data/README.md

@@ -20,17 +20,19 @@ Or install it yourself as:
 ## Usage
 Add the following into your fluentd config.
 
-    <source>
-      type spectrum        # required, choosing the input plugin.
-      endpoint hostname    # required, FQDN of Spectrum EndPoint
-      username ws_user     # optional, interface to listen on, default 0 for all.
-      password ws_password # optional, interface to listen on, default 0 for all.
-      tag alert.spectrum   # optional, tag to assign to events, default is alert.spectrum 
-    </source>
-
-    <match alert.spectrum>
-      type stdout
-    </match>
+	<source>
+	 type spectrum # required, choosing the input plugin
+	 endpoint spectrumapi.corp.yourdomain.net # required, FQDN of spectrum endpoint
+	 user username # required, username for APIs
+	 pass password # required, password for APIs
+	 tag alert.spectrum # optional, tag to assign to events, default is alert.spectrum
+	 interval 60 # optional, interval in seconds for how often to poll, defaults to 300
+	 include_raw false # optional, include original object as key raw
+	</source>
+
+	<match alert.spectrum>
+	 type stdout
+	</match>
 
 Now startup fluentd
 
@@ -40,4 +42,8 @@ Send a test
     TBD
 
 ## To Do
-Things left to do, not in any particular order.
+Add retry login. On timeout/failure retry, how often, increasing delay? (how would that affect polling time, possible duplicates?)
+All flag to allow specifying spectrum attributes to get or get _ALL_
+Add flag to allow start date/time if users want to backfill data from a specific date. then start loop. 
+Add flag to disable loop, if users only wanted to backfill from datetime to now or specific end time. 
+change loop to allow multiple runs to stack on eachother to avoid missing data?

data/fluent-plugin-spectrum.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-spectrum"
-  gem.version       = "0.0.1"
+  gem.version       = "0.0.2"
   gem.date          = '2015-02-24'
   gem.authors       = ["Alex Pena"]
   gem.email         = ["alex_pena@intuit.com"]

data/lib/fluent/plugin/in_spectrum.rb

@@ -2,9 +2,19 @@ module Fluent
 # TODO:
 # Add a fields all or list option
 # error checking in every section  
+    # class for handling interval in loop
+  class TimerWatcher < Coolio::TimerWatcher
+    def initialize(interval, repeat, &callback)
+      @callback = callback
+      super(interval, repeat)
+    end # def initialize
+    def on_timer
+      @callback.call
+    end # def on_timer
+  end
+
   class SpectrumInput < Input
     Fluent::Plugin.register_input('spectrum', self)
-
     config_param :tag, :string, :default => "alert.spectrum"
     config_param :endpoint, :string, :default => "pleasechangeme.com" #fqdn of endpoint
     config_param :interval, :integer, :default => '300' #Default 5 minutes
@@ -13,6 +23,19 @@ module Fluent
     config_param :include_raw, :string, :default => "false" #Include original object as raw
     config_param :attributes, :string, :default => "ALL" # fields to include, ALL for... well, ALL.
 
+    # function to UTF8 encode
+    def to_utf8(str)
+      str = str.force_encoding('UTF-8')
+      return str if str.valid_encoding?
+      str.encode("UTF-8", 'binary', invalid: :replace, undef: :replace, replace: '')
+    end
+
+    def parseAttributes(alarmAttribute)
+      key = @spectrum_access_code[alarmAttribute['@id'].to_s].to_s
+      value = ((to_utf8(alarmAttribute['$'].to_s)).strip).gsub(/\r?\n/, " ")
+      return key,value
+    end
+
     def initialize
       require 'rest_client'
       require 'json'
@@ -92,24 +115,23 @@ module Fluent
       super
       @loop.stop
       @thread.join
+    rescue
+      $log.error "Spectrum :: unexpected error", :error=>$!.to_s
+      $log.error_backtrace
     end # def shutdown
 
     def run
       @loop.run
+    rescue
+        $log.error "Spectrum :: unexpected error", :error=>$!.to_s
+        $log.error_backtrace
     end # def run
 
     def input
       alertStartTime = Engine.now.to_i - @interval.to_i 
-      $log.info "Spectrum :: Polling for alerts for time period: #{alertStartTime.to_i} - #{Engine.now.to_i}"
+      $log.info "Spectrum :: Polling alerts for time period: #{alertStartTime.to_i} - #{Engine.now.to_i}"
 
-      # function to UTF8 encode
-      def to_utf8(str)
-        str = str.force_encoding('UTF-8')
-        return str if str.valid_encoding?
-        str.encode("UTF-8", 'binary', invalid: :replace, undef: :replace, replace: '')
-      end
-
-      # XML for spectrum post
+      # Format XML for spectrum post
       @xml = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>
       <rs:alarm-request throttlesize=\"10000\"
       xmlns:rs=\"http://www.ca.com/spectrum/restful/schema/request\"
@@ -132,51 +154,57 @@ module Fluent
       # Post to Spectrum and parse results  
       responsePost=spectrumEnd.post @xml,:content_type => 'application/xml',:accept => 'application/json'
       body = JSON.parse(responsePost.body)
-      # Only continue if we have alarms in results
+
+      # Processing for multiple alerts returned
       if body['ns1.alarm-response-list']['@total-alarms'].to_i > 1
-        $log.info "Spectrum :: returned #{body['ns1.alarm-response-list']['@total-alarms'].to_i} alarms"
-        # iterate through alarms in results
+        $log.info "Spectrum :: returned #{body['ns1.alarm-response-list']['@total-alarms'].to_i} alarms for period #{alertStartTime.to_i} - #{Engine.now.to_i}"
+        # iterate through each alarm
         body['ns1.alarm-response-list']['ns1.alarm-responses']['ns1.alarm'].each do |alarm|
+          # Create initial structure
           record_hash = Hash.new # temp hash to hold attributes of alarm
+          raw_array = Array.new # temp hash to hold attributes of alarm for raw
           record_hash['event_type'] = @tag.to_s
           record_hash['intermediary_source'] = @endpoint.to_s
-          # iterate though alarm attributes and add to temp hash  
+          # iterate though alarm attributes
           alarm['ns1.attribute'].each do |attribute|
-            record_hash[@spectrum_access_code[attribute['@id'].to_s].to_s] = to_utf8(attribute['$'].to_s)
+            key,value = parseAttributes(attribute)
+            record_hash[key] = value
+            if @include_raw.to_s == "true"
+              raw_array << { "#{key}" => "#{value}" }
+            end
           end
-          # include raw?
+          # append raw object
           if @include_raw.to_s == "true"  
-            record_hash['raw'] = alarm  
+            record_hash[:raw] = raw_array
           end
           Engine.emit(@tag, record_hash['CREATION_DATE'].to_i,record_hash)
         end
+      # Processing for single alarm returned  
       elsif body['ns1.alarm-response-list']['@total-alarms'].to_i == 1
-        $log.info "Spectrum :: returned #{body['ns1.alarm-response-list']['@total-alarms'].to_i} alarms"
+        $log.info "Spectrum :: returned #{body['ns1.alarm-response-list']['@total-alarms'].to_i} alarms for period #{alertStartTime.to_i} - #{Engine.now.to_i}"
+        # Create initial structure
         record_hash = Hash.new # temp hash to hold attributes of alarm
+        raw_array = Array.new # temp hash to hold attributes of alarm for raw
         record_hash['event_type'] = @tag.to_s
         record_hash['intermediary_source'] = @endpoint.to_s
         # iterate though alarm attributes and add to temp hash  
         body['ns1.alarm-response-list']['ns1.alarm-responses']['ns1.alarm']['ns1.attribute'].each do |attribute|
-          record_hash[@spectrum_access_code[attribute['@id'].to_s].to_s] = to_utf8(attribute['$'].to_s)
+          key,value = parseAttributes(attribute)
+          record_hash[key] = value
+          if @include_raw.to_s == "true"
+            raw_array << { "#{key}" => "#{value}" }
+          end
         end
-        # include raw?
+        # append raw object
         if @include_raw.to_s == "true"  
-          record_hash['raw'] = alarm  
+          record_hash[:raw] = raw_array
         end
         Engine.emit(@tag, record_hash['CREATION_DATE'].to_i,record_hash)
-      end  
-    end
+      # No alarms returned
+      else
+        $log.info "Spectrum :: returned #{body['ns1.alarm-response-list']['@total-alarms'].to_i} alarms for period #{alertStartTime.to_i} - #{Engine.now.to_i}"
+      end    
+    
+    end # def input
   end # class SpectrumInput
-
-  # class for handling interval in loop
-  class TimerWatcher < Coolio::TimerWatcher
-    def initialize(interval, repeat, &callback)
-      @callback = callback
-      super(interval, repeat)
-    end # def initialize
-    def on_timer
-      @callback.call
-    end # def on_timer
-  end
-
 end # module Fluent

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-spectrum
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - Alex Pena