fluent-plugin-spectrum 0.0.7 → 0.0.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: dce92432a6a0648acac71c3abcc728f1f7cb313b
-  data.tar.gz: 9e12765096932803e03d30e71c09a3cb78bf0718
+  metadata.gz: 76bb8bd0bc0343b655aa47cccd1f710b4aa17759
+  data.tar.gz: 436a59a9735ca316f71fa76fc3cc8cc275b636b6
 SHA512:
-  metadata.gz: 8e6419d7bcbcfe308e0a2d62b916f6e61b5cc1327569317a5d65be87a728d358042dfd316f8038a893e40fe1a686bd41a81611b1e192c1ff4b5d07c93871cb82
-  data.tar.gz: efabb69d231fb482311be547596ae17c5486d995837af219aa8cd6e7441145731148daed2903dce4d82d44fc266fa3fdf16a3cafea4a67a944a7c133eb7f81f1
+  metadata.gz: 73e8268a657d5a4526a246956cff5641ca31e201c263f9aeaf896632996f72e12a65142879ac11af562aeb5d9db31b82c78dd2fdfac55868ee616b788b03169f
+  data.tar.gz: 433e2d2cc00ecb0486a4408e3819a2bf443f04fedd9d2e14a6c13b08b456c32d07aeb61924356425eb74f2db3e2fcf75b65011ae42050cd4999ba859e11200de

data/.gitignore

@@ -14,4 +14,5 @@ rdoc
 spec/reports
 test/tmp
 test/version_tmp
-tmp
+tmp
+.DS_Store

data/README.md

@@ -25,6 +25,9 @@ Or install it yourself as:
     $ gem install fluent-plugin-spectrum
 
 ## Usage
+
+### Input plugin
+
 Add the following into your fluentd config.
 
 Simple:
@@ -32,8 +35,8 @@ Simple:
     <source>
       type spectrum
       endpoint spectrum.yourdomain.com 	# required, FQDN of endpoint
-      user username  # required
-      pass password  # required
+      username username  # required
+      password password  # required
       interval 60    # optional, interval in seconds, defaults to 300
     </source>
     <match alert.spectrum>
@@ -45,11 +48,17 @@ Advanced:
     <source>
       type spectrum
       endpoint spectrum.yourdomain.com 	# required, FQDN of endpoint
-      user username                     # required
-      pass password                     # required
+      username username                     # required
+      password password                     # required
       interval 60                       # optional, interval in seconds, defaults to 300
-      state_file /tmp/spectrum_state    # optional, file to keep state    
+      state_type file                   # optional, set the type for store state (file or redis)
+      state_file /tmp/spectrum_state  # optional, file to keep state or file to get redis configure (need state_type setup)
+      state_tag  spectrum             # optional, tag for store state(need state_type, state_file setup)
+      tag alert.spectrum              # optional, add your own tag for tha alert
+      attributes  ALARM_ID,CREATION_DATE,HOSTNAME   # optional, select attributes that you want to poll in alerts, default value is __ALL__
     </source>
+
+    
     # using rename_key to map to new keynames
     <match alert.spectrum>
       type rename_key
@@ -92,6 +101,73 @@ Verify:
 		Output:
 		2015-03-05 15:04:00 -0800 alert.spectrum: {"event_type":"alert.spectrum","intermediary_source":"spectrumapi001.corp.yourdomain.net","ALARM_ID":"54f8e0e0-e706-12c2-0165-005056a07ac5","CREATION_DATE":"1425596640","SEVERITY":"3","ALARM_TITLE":"LOGMATCH TRAPSEND CRIT","HOSTNAME":"yourhost001.corp.yourdomain.net","IP_ADDRESS":"10.10.0.14","ORIGINATING_EVENT_ATTR":"A SEC logmatch trapsend CRIT Your Alert Message here","MODEL_STRING":"Host_Device","ACKNOWLEDGED":"false","ALARM_STATUS":"","OCCURRENCES":"1","TROUBLE_SHOOTER":"","USER_CLEARABLE":"true","TROUBLE_TICKET_ID":"","PERSISTENT":"true","GC_NAME":"Your_Global_Collection"}
 
+
+### Output plugin
+
+Add the following into your fluentd config.
+
+```
+#set source,  here use kafka as example
+<source>
+  type   kafka
+  host   localhost  #<broker host>
+  port   9092 # <broker port: default=9092>
+  topics argos-parser #<listening topics(separate with comma',')>
+  format json #<input text type (text|json|ltsv|msgpack)>
+</source>
+
+# once match specific tag, use spectrum output plugin
+<match argos-parser>
+  type spectrum
+  endpoint spectrum.yourdomain.com  # required, FQDN of endpoint 
+  user username # required
+  pass password # required
+  interval 10 # interval in seconds
+  model_mh your_model_handler # required, you need to create model in spectrum first
+  event_type_id your_event_type_id # required, you need to set event type in spectrum first 
+  spectrum_key event_type # key in alert to check if alert is from spectrum
+  spectrum_value alert.raw.spectrum # value to match is its from spectrum
+  alarm_ID_key source_event_id  # key in the alert that associate with alarm_ID for calling spectrum PUT alarms api 
+  
+  # For 3rd party alerts
+  # Create new events in Spectrum
+  # Set these parameters according to varbind keys of your event type in spectrum 
+  # and the keynames in your original event (which you want to push to spectrum)
+  <event_rename_rules>
+    #key_varbind origin_event_keyname 
+    2 source_hostname
+    100  creation_time 
+    101  criticality
+    102  source_ip 
+    103  alert_description   
+    104  application_name  
+    105  business_unit_l2  
+    106  business_unit_l3  
+    107  business_unit_l4  
+    108  cmdb_ci_sys_id
+  </event_rename_rules>  
+
+
+  # Update existing alarms from Spectrum
+  # set these parameters according to keys of alarm in spectrum that you want to update
+  # and the the keynames in your original event 
+  <alarm_rename_rules>
+    #key_spectrum_alarm origin_event_keyname 
+    0xffff00f6 application_name 
+    0xffff00f7 business_unit_l2
+    0xffff00f8 business_unit_l3
+    0xffff00f9 business_unit_l4
+    0xffff00fa cmdb_ci_sysid
+  </alarm_rename_rules>
+
+</match>
+```
+
+Now startup fluentd
+
+    $ sudo fluentd -c fluent.conf &
+
+
 ## To Do
 * All flag to allow specifying spectrum attributes to get or get _ALL_
 * Add flag to allow start date/time if users want to backfill data from a specific date. then start loop. 

data/fluent-plugin-spectrum.gemspec

@@ -3,10 +3,10 @@ $:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-spectrum"
-  gem.version       = "0.0.7"
-  gem.date          = '2015-04-06'
-  gem.authors       = ["Alex Pena"]
-  gem.email         = ["pena.alex@gmail.com"]
+  gem.version       = "0.0.8" 
+  gem.date          = '2015-05-07'
+  gem.authors       = ["Alex Pena","Ling Zhang"]
+  gem.email         = ["pena.alex@gmail.com", "zhangling.ice@gmail.com"]
   gem.summary       = %q{Fluentd plugin for managing monitoring alerts from CA Spectrum}
   gem.description   = %q{Fluentd input/output plugin for managing monitoring alerts from CA Spectrum. Input supports polling CA Spectrum APIs. Output currently only supports updating events retrieved from Spectrum.}
   gem.homepage      = 'https://github.com/Bigel0w/fluent-plugin-spectrum'
@@ -26,4 +26,5 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency "fluentd", '~> 0.12'
   gem.add_runtime_dependency "json", '~> 1.8'
   gem.add_runtime_dependency "rest-client", '~> 1.8'
+  gem.add_runtime_dependency "highwatermark", '~> 0.1'
 end

data/lib/fluent/plugin/in_spectrum.rb

@@ -8,11 +8,18 @@ module Fluent
     config_param  :endpoint,      :string,  :default => nil
     config_param  :username,      :string,  :default => nil
     config_param  :password,      :string,  :default => nil
+    
+    config_param  :state_tag,     :string,  :default => "spectrum"
+    config_param  :state_type,    :string,  :default => "memory"
     config_param  :state_file,    :string,  :default => nil
-    config_param  :attributes,    :string,  :default => "ALL"
+    config_param  :redis_host,    :string,  :default => nil
+    config_param  :redis_port,    :string,  :default => nil
+
+    config_param  :attributes,    :string,  :default => "__ALL__"
     config_param  :interval,      :integer, :default => INTERVAL_MIN
     config_param  :select_limit,  :integer, :default => 10000
     config_param  :include_raw,   :bool,    :default => "false"
+    
 
     # Classes
     class TimerWatcher < Coolio::TimerWatcher
@@ -29,45 +36,6 @@ module Fluent
       end # def on_timer
     end
 
-    class StateStore
-      def initialize(path)
-        require 'yaml'
-        @path = path
-        if File.exists?(@path)
-          @data = YAML.load_file(@path)
-          if @data == false || @data == []
-            @data = {}
-          elsif !@data.is_a?(Hash)
-            raise "Spectrum :: ConfigError state_file on #{@path.inspect} is invalid"
-          end
-        else
-          @data = {}
-        end
-      end
-
-      def last_records
-        @data['last_records'] ||= {}
-      end
-
-      def update!
-        File.open(@path, 'w') {|f|
-          f.write YAML.dump(@data)
-        }
-      end
-    end
-
-    class MemoryStateStore
-      def initialize
-        @data = {}
-      end
-      
-      def last_records
-        @data['last_records'] ||= {}
-      end
-      
-      def update!
-      end
-    end
 
     # function to UTF8 encode
     def to_utf8(str)
@@ -85,6 +53,8 @@ module Fluent
     def initialize
       require 'rest-client'
       require 'json'
+      require 'highwatermark'
+      require 'yaml'
       super
     end # def initialize
 
@@ -101,11 +71,21 @@ module Fluent
         raise ConfigError, "Spectrum :: ConfigError 'interval' must be #{INTERVAL_MIN} or over."
       end
       # Warn about optional state file
-      unless @state_file
-        $log.warn "Spectrum :: 'state_file PATH' parameter is not set to a valid source."
-        $log.warn "Spectrum :: this parameter is highly recommended to save the last known good timestamp to resume event consuming"
+      unless @state_type == "file" || @state_type =="redis"
+        $log.warn "Spectrum :: 'state_type' is not set to file or redis"
+        $log.warn "Spectrum :: state file or redis are recommended to save the last known good timestamp to resume event consuming"
       end
-      # map of Spectrum attribute codes to names
+
+      @highwatermark_parameters={
+        "state_tag" => @state_tag,     
+        "state_type" => @state_type,
+        "state_file" => @state_file,
+        "redis_host" => @redis_host,
+        "redis_port" => @redis_port      
+      }
+      $log.info "highwatermark_parameters: #{@highwatermark_parameters}"
+
+      # default setting for @spectrum_access_code
       @spectrum_access_code={
         "0x11f9c" => "ALARM_ID",
         "0x11f4e" => "CREATION_DATE",
@@ -123,37 +103,63 @@ module Fluent
         "0x12022" => "TROUBLE_TICKET_ID",
         "0x12942" => "PERSISTENT",
         "0x12adb" => "GC_NAME",
-        "0x57f0105" => "Custom_Project",
-        #{}"0x11f51" => "CLEARED_BY_USER_NAME",
-        #{}"0x11f52" => "EVENT_ID_LIST",
-        #{}"0x11f53" => "MODEL_HANDLE",
-        #{}"0x11f54" => "PRIMARY_ALARM",
-        #{}"0x11fc4" => "ALARM_SOURCE",
-        #{}"0x11fc6" => "TROUBLE_SHOOTER_MH",
-        #{}"0x12a6c" => "TROUBLE_SHOOTER_EMAIL",
-        #{}"0x1290d" => "IMPACT_SEVERITY",
-        #{}"0x1290e" => "IMPACT_SCOPE",
-        #{}"0x1298a" => "IMPACT_TYPE_LIST",
-        #{}"0x12948" => "DIAGNOSIS_LOG",
-        #{}"0x129aa" => "MODEL_ID",
-        #{}"0x129ab" => "MODEL_TYPE_ID",
-        #{}"0x129af" => "CLEAR_DATE",
-        #{}"0x12a04" => "SYMPTOM_LIST_ATTR",
-        #{}"0x12a6f" => "EVENT_SYMPTOM_LIST_ATTR",
-        #{}"0x12a05" => "CAUSE_LIST_ATTR",
-        #{}"0x12a06" => "SYMPTOM_COUNT_ATTR",
-        #{}"0x12a70" => "EVENT_SYMPTOM_COUNT_ATTR",
-        #{}"0x12a07" => "CAUSE_COUNT_ATTR",
-        #{}"0x12a63" => "WEB_CONTEXT_URL",
-        #{}"0x12a6b" => "COMBINED_IMPACT_TYPE_LIST",
-        #{}"0x11f50" => "CAUSE_CODE",
-        #{}"0x10009" => "SECURITY_STRING"
+        "0x57f0105" => "CUSTOM_PROJECT",
+        "0x11f51" => "CLEARED_BY_USER_NAME",
+        "0x11f52" => "EVENT_ID_LIST",
+        "0x11f53" => "MODEL_HANDLE",
+        "0x11f54" => "PRIMARY_ALARM",
+        "0x11fc4" => "ALARM_SOURCE",
+        "0x11fc6" => "TROUBLE_SHOOTER_MH",
+        "0x12a6c" => "TROUBLE_SHOOTER_EMAIL",
+        "0x1290d" => "IMPACT_SEVERITY",
+        "0x1290e" => "IMPACT_SCOPE",
+        "0x1298a" => "IMPACT_TYPE_LIST",
+        "0x12948" => "DIAGNOSIS_LOG",
+        "0x129aa" => "MODEL_ID",
+        "0x129ab" => "MODEL_TYPE_ID",
+        "0x129af" => "CLEAR_DATE",
+        "0x12a04" => "SYMPTOM_LIST_ATTR",
+        "0x12a6f" => "EVENT_SYMPTOM_LIST_ATTR",
+        "0x12a05" => "CAUSE_LIST_ATTR",
+        "0x12a06" => "SYMPTOM_COUNT_ATTR",
+        "0x12a70" => "EVENT_SYMPTOM_COUNT_ATTR",
+        "0x12a07" => "CAUSE_COUNT_ATTR",
+        "0x12a63" => "WEB_CONTEXT_URL",
+        "0x12a6b" => "COMBINED_IMPACT_TYPE_LIST",
+        "0x11f50" => "CAUSE_CODE",
+        "0x10009" => "SECURITY_STRING"
       }
+
       # Create XML chunk for attributes we care about
       @attr_of_interest=""
-      @spectrum_access_code.each do |key, array|
-        @attr_of_interest += " <rs:requested-attribute id=\"#{key}\"/>"
-      end
+      if(@attributes.upcase === "__ALL__")
+        $log.info "all attributes"
+        @spectrum_access_code.each do |key, value|
+          $log.info "key: #{key},  value: #{value}"
+          @attr_of_interest += " <rs:requested-attribute id=\"#{key}\"/>"
+        end
+      else
+        $log.info "selected attributes"
+        @attributes.split(",").each {|attr|         
+          key=""
+          value=""
+          # if it's hex code
+          if @spectrum_access_code.has_key?(attr.strip)
+            key = attr.strip
+            value = @spectrum_access_code.fetch(key)
+          # if it's the name
+          elsif @spectrum_access_code.has_value?(attr.strip.upcase)
+            value = attr.strip.upcase
+            key = @spectrum_access_code.key(value)
+          # if it's invalid input, not the hex code or name in the map
+          else 
+            raise ConfigError, "Spectrum :: ConfigError attribute '#{attr}' is not in the hash map"
+          end
+          $log.info "key: #{key},  value: #{value}"
+          @attr_of_interest += " <rs:requested-attribute id=\"#{key}\"/>"
+        }
+      end      
+
       # URL Resource
       def resource
         @url = 'http://' + @endpoint.to_s + '/spectrum/restful/alarms'
@@ -163,7 +169,7 @@ module Fluent
 
     def start
       @stop_flag = false
-      @state_store = @state_file.nil? ? MemoryStateStore.new : StateStore.new(@state_file)
+      @highwatermark = Highwatermark::HighWaterMark.new(@highwatermark_parameters)
       @loop = Coolio::Loop.new
       @loop.attach(TimerWatcher.new(@interval, true, &method(:on_timer)))
       @thread = Thread.new(&method(:run))
@@ -186,10 +192,12 @@ module Fluent
     def on_timer
       if not @stop_flag
         pollingStart = Engine.now.to_i
-        if @state_store.last_records.has_key?("spectrum") 
-          alertStartTime = @state_store.last_records['spectrum']
+        if @highwatermark.last_records(@state_tag)
+          alertStartTime = @highwatermark.last_records(@state_tag)
+          $log.info "got hwm form state file: #{alertStartTime.to_i}"
         else
           alertStartTime = (pollingStart.to_i - @interval.to_i)
+          $log.info "no hwm, got new alert start time: #{alertStartTime.to_i}"
         end
         pollingEnd = ''
         pollingDuration = ''
@@ -218,7 +226,6 @@ module Fluent
           res=resource.post @xml,:content_type => 'application/xml',:accept => 'application/json'
           body = JSON.parse(res.body)
           pollingEnd = Time.parse(res.headers[:date]).to_i
-          @state_store.last_records['spectrum'] = pollingEnd
           pollingDuration = Engine.now.to_i - pollingStart
         end  
 
@@ -273,7 +280,7 @@ module Fluent
         else
           $log.info "Spectrum :: returned #{body['ns1.alarm-response-list']['@total-alarms'].to_i} alarms for period < #{alertStartTime.to_i} took #{pollingDuration.to_i} seconds, ended at #{pollingEnd}"
         end
-        @state_store.update!  
+        @highwatermark.update_records(pollingEnd,@state_tag)
       end
     end # def input
   end # class SpectrumInput

data/lib/fluent/plugin/out_spectrum.rb

@@ -1,136 +1,86 @@
 module Fluent
-  class SpectrumOut < Output
+  class SpectrumOut< Output
     # First, register the plugin. NAME is the name of this plugin
     # and identifies the plugin in the configuration file.
     Fluent::Plugin.register_output('spectrum', self)
     
-    config_param :tag, :string, default:'alert.spectrum.out' 
-    #config_param :tag, :string, :default => "alert.spectrum"
     config_param :endpoint, :string, :default => "pleasechangeme.com" #fqdn of endpoint
-    config_param :interval, :integer, :default => '300' #Default 5 minutes
     config_param :user, :string, :default => "username"
     config_param :pass, :string, :default => "password"
-    config_param :include_raw, :string, :default => "false" #Include original object as raw
-    config_param :attributes, :string, :default => "ALL" # fields to include, ALL for... well, ALL.
-
-    def parseAttributes(alarmAttribute)
-      key = @spectrum_access_code[alarmAttribute['@id'].to_s].to_s
-      value = ((to_utf8(alarmAttribute['$'].to_s)).strip).gsub(/\r?\n/, " ")
-      return key,value
-    end
+    config_param :interval, :integer, :default => '300' #Default 5 minutes
+    config_param :model_mh, :string, :default => "model_mh"
+    config_param :event_type_id, :string, :default => "event_type_id"
+    config_param :spectrum_key, :string, :default => "event_type" # key in the alert to check if alert is from spectrum
+    config_param :spectrum_value, :string, :default => "alert.raw.spectrum"# value to match is its from spectrum
+    config_param :alarm_ID_key, :string, :default => "source_event_id" # key in the alert that associate with alarm_ID for calling spectrum PUT alarms api 
+    config_param :debug, :bool, :default => false
 
     def initialize
       require 'rest-client'
       require 'json'
-      require 'pp'
-      require 'cgi'
+      require 'cgi' # verify we need --yes, we need it, to_utf8 could not used to create valid url and xml
       super
     end # def initialize
 
+    # function to UTF8 encode
+    def to_utf8(str)
+      str = str.force_encoding('UTF-8')
+      return str if str.valid_encoding?
+      str.encode("UTF-8", 'binary', invalid: :replace, undef: :replace, replace: '')
+    end
+
+    def parse_rename_rule rule
+      if rule.match /^([^\s]+)\s+(.+)$/
+        return $~.captures
+      end
+    end
 
     # This method is called before starting.
     def configure(conf)
-      super 
-      # Read property file and create a hash
-      @rename_rules = []
-      conf_rename_rules = conf.keys.select { |k| k =~ /^rename_rule(\d+)$/ }
-      conf_rename_rules.sort_by { |r| r.sub('rename_rule', '').to_i }.each do |r|
-        key_regexp, new_key = parse_rename_rule conf[r]
+      super
+      # Read configuration for event_rename_rules and create a hash
+      @event_rename_rules = []
+      conf.elements.select { |element| element.name == 'event_rename_rules' }.each { |element|
+        element.each_pair { |key_varbind, origin_event_keyname|
+          element.has_key?(key_varbind) # to suppress unread configuration warning
+          @event_rename_rules << { key_varbind: key_varbind, origin_event_keyname: origin_event_keyname }
+          $log.info "Added event_rename_rules: #{@event_rename_rules.last}"
+        }
+      }
 
-        if key_regexp.nil? || new_key.nil?
-          raise Fluent::ConfigError, "Failed to parse: #{r} #{conf[r]}"
-        end
 
-        if @rename_rules.map { |r| r[:key_regexp] }.include? /#{key_regexp}/
-          raise Fluent::ConfigError, "Duplicated rules for key #{key_regexp}: #{@rename_rules}"
-        end
+      # Read configuration for alarm_rename_rules and create a hash
+      @alarm_rename_rules = []
+      conf.elements.select { |element| element.name == 'alarm_rename_rules' }.each { |element|
+        element.each_pair { |key_spectrum_alarm, origin_event_keyname|
+          element.has_key?(key_spectrum_alarm) # to suppress unread configuration warning
+          @alarm_rename_rules << { key_spectrum_alarm: key_spectrum_alarm, origin_event_keyname: origin_event_keyname }
+          $log.info "Added alarm_rename_rules: #{@alarm_rename_rules.last}"
+        }
+      }
 
-        #@rename_rules << { key_regexp: /#{key_regexp}/, new_key: new_key }
-        @rename_rules << { key_regexp: key_regexp, new_key: new_key }
-        $log.info "Added rename key rule: #{r} #{@rename_rules.last}"
+      
+      # Setup URL Resource
+      @alarms_url = 'http://' + @endpoint.to_s + '/spectrum/restful/alarms/'
+      @events_url = 'http://' + @endpoint.to_s + '/spectrum/restful/events'
+      def events_resource
+        RestClient::Resource.new(@events_url, :user => @user, :password => @pass, :open_timeout => 5, :timeout => (@interval * 3))
       end
 
-      raise Fluent::ConfigError, "No rename rules are given" if @rename_rules.empty?
-      @conf = conf
-      # map of Spectrum attribute codes to names
-      @spectrum_access_code={
-        "0x11f9c" => "ALARM_ID",
-        "0x11f4e" => "CREATION_DATE",
-        "0x11f56" => "SEVERITY",
-        "0x12b4c" => "ALARM_TITLE",
-        "0x1006e" => "HOSTNAME",
-        "0x12d7f" => "IP_ADDRESS",
-        "0x1296e" => "ORIGINATING_EVENT_ATTR",
-        "0x10000" => "MODEL_STRING",  
-        "0x11f4d" => "ACKNOWLEDGED",
-        "0x11f4f" => "ALARM_STATUS",
-        "0x11fc5" => "OCCURRENCES",
-        "0x11f57" => "TROUBLE_SHOOTER",
-        "0x11f9b" => "USER_CLEARABLE",
-        "0x12022" => "TROUBLE_TICKET_ID",
-        "0x12942" => "PERSISTENT",
-        "0x12adb" => "GC_NAME",
-        "0x57f0105" => "Custom_Project",
-        "0x11f4d" => "ACKNOWLEDGED",
-        "0xffff00ed" => "application_name",
-        "0xffff00f1" => "business_unit_l1",
-        "0xffff00f2" => "business_unit_l2",
-        "0xffff00f3" => "business_unit_l3",
-        "0xffff00f4" => "business_unit_l4",
-        "0xffff00f0" => "cmdb_ci_sysid",
-
-        #{}"0x11f51" => "CLEARED_BY_USER_NAME",
-        #{}"0x11f52" => "EVENT_ID_LIST",
-        #{}"0x11f53" => "MODEL_HANDLE",
-        #{}"0x11f54" => "PRIMARY_ALARM",
-        #{}"0x11fc4" => "ALARM_SOURCE",
-        #{}"0x11fc6" => "TROUBLE_SHOOTER_MH",
-        #{}"0x12a6c" => "TROUBLE_SHOOTER_EMAIL",
-        #{}"0x1290d" => "IMPACT_SEVERITY",
-        #{}"0x1290e" => "IMPACT_SCOPE",
-        #{}"0x1298a" => "IMPACT_TYPE_LIST",
-        #{}"0x12948" => "DIAGNOSIS_LOG",
-        #{}"0x129aa" => "MODEL_ID",
-        #{}"0x129ab" => "MODEL_TYPE_ID",
-        #{}"0x129af" => "CLEAR_DATE",
-        #{}"0x12a04" => "SYMPTOM_LIST_ATTR",
-        #{}"0x12a6f" => "EVENT_SYMPTOM_LIST_ATTR",
-        #{}"0x12a05" => "CAUSE_LIST_ATTR",
-        #{}"0x12a06" => "SYMPTOM_COUNT_ATTR",
-        #{}"0x12a70" => "EVENT_SYMPTOM_COUNT_ATTR",
-        #{}"0x12a07" => "CAUSE_COUNT_ATTR",
-        #{}"0x12a63" => "WEB_CONTEXT_URL",
-        #{}"0x12a6b" => "COMBINED_IMPACT_TYPE_LIST",
-        #{}"0x11f50" => "CAUSE_CODE",
-        #{}"0x10009" => "SECURITY_STRING"
-      }
-      # Create XML chunk for attributes we care about
-      @attr_of_interest=""
-      @spectrum_access_code.each do |key, array|
-        @attr_of_interest += " <rs:requested-attribute id=\"#{key}\"/>"
+      def alarms_resource
+        RestClient::Resource.new(@alarms_url, :user => @user, :password => @pass, :open_timeout => 5, :timeout => (@interval * 3))
       end
 
 
-      # Setup URL Resource
-      	@url = 'http://' + @endpoint.to_s + '/spectrum/restful/alarms/'
-   end
-
-  def parse_rename_rule rule
-    if rule.match /^([^\s]+)\s+(.+)$/
-      return $~.captures
-    end
-  end
-
+    end # end of def configure
     # This method is called when starting.
     def start
       super
     end
-
     # This method is called when shutting down.
     def shutdown
       super
     end
-
     # This method is called when an event reaches Fluentd.
     # 'es' is a Fluent::EventStream object that includes multiple events.
     # You can use 'es.each {|time,record["event"]| ... }' to retrieve events.
@@ -141,53 +91,105 @@ module Fluent
     def emit(tag, es, chain)
       chain.next
       es.each {|time,record|
-        $stderr.puts "OK!"
-	## Check if the incoming event already has an event id (alarm id) and a corresponding tag of spectrum 
-	if (record["event"].has_key?("source_event_id") && record["event"].has_key?("event_type"))  
-		# If the value on event_type is spectrum, then it means that ti is already from spectrum and needs an update		
-		if (record["event"].has_value?("alert.spectrum"))
-
-			# Create an empty hash
-			alertUpdateHash=Hash.new
-
-			# Parse thro the array hash that contains name value pairs for hash mapping and add new records to a new hash
-			@rename_rules.each { |rule| 
-				pp rule[:new_key]
-				alertUpdateHash[rule[:key_regexp]]=record["event"][rule[:new_key]]
-			}
-			# construct the url and iterate to construct the uri
-			@urlrest = @url + record["event"]["source_event_id"]
-       			alertUpdateHash.each do |attr, val| 
-				if (val.nil? || val.empty?)
-					next
-				else
-					if (@urlrest =~ /#{record["event"]["source_event_id"]}\s*$/)
-						@urlrest = @urlrest + "?attr=" + attr + "&val=" + CGI.escape(val.to_s)
-					else
-						@urlrest = @urlrest + "&attr=" + attr + "&val=" + CGI.escape(val.to_s)
-					end
-				end
-			end	
-			
-			$log.info "Rest url " + @urlrest
-       			#RestClient::Resource.new(@user,@pass)
-			begin		
-				responsePostAffEnt=RestClient::Resource.new(@urlrest,@user,@pass).put(@urlrest,:content_type => 'application/json')
-			rescue Exception => e 
-				$log.error "Error in restful put call."
-				log.error e.backtrace.inspect
-				$log.error responsePostAffEnt
-			end
-		
-		else
-
-			# For now just throw to stdout
-			$log.info record["event"]
-
-		end		
-
-	end
-      }
-    end
-  end
-end
+        
+        if (record["event"].has_key?(@alarm_ID_key) && record["event"].has_key?(@spectrum_key) ) 
+          ######native spectrum alert ########################
+          ######PUT alarm to update enriched fields
+          if (record["event"][@spectrum_key] == @spectrum_value) 
+            $log.info "The alert is from spectrum"
+            # Create an empty hash
+            alertUpdateHash=Hash.new
+            # Parse thro the array hash that contains name value pairs for hash mapping and add new records to a new hash
+            @alarm_rename_rules.each { |rule| 
+              puts rule[:origin_event_keyname]
+              alertUpdateHash[rule[:key_spectrum_alarm]]=record["event"][rule[:origin_event_keyname]]
+            }
+            # construct the alarms PUT uri for update triggerd alarm withe enriched fields
+            @alarms_urlrest = @alarms_url + record["event"][@alarm_ID_key]
+            @attr_count = 0
+            alertUpdateHash.each do |attr, val| 
+              if (val.nil? || val.empty?)
+                next
+              else
+                if (@attr_count == 0)
+                  @alarms_urlrest = @alarms_urlrest + "?attr=" + attr + "&val=" + CGI.escape(val.to_s)
+                  # @alarms_urlrest = @alarms_urlrest + "?attr=" + attr + "&val=" + to_utf8(val.to_s)
+                  @attr_count +=1
+                else
+                  @alarms_urlrest = @alarms_urlrest + "&attr=" + attr + "&val=" + CGI.escape(val.to_s)
+                  # @alarms_urlrest = @alarms_urlrest + "&attr=" + attr + "&val=" + to_utf8(val.to_s)
+                  @attr_count +=1
+                end
+              end
+            end
+            $log.info "Rest url for PUT alarms: " + @alarms_urlrest            
+            
+            begin 
+              # alarmPutRes = alarms_resource.put @alarms_urlrest,:content_type => 'application/json'
+              alarmPutRes = RestClient::Resource.new(@alarms_urlrest,@user,@pass).put(@alarms_urlrest,:content_type => 'application/json')
+              $log.info alarmPutRes 
+            end
+
+          ######3rd party alert #######################
+          ######Post an event and then trigger an alarm ######   
+          else
+            $log.info "The alert is from 3rd party"           
+            # Create an empty hash
+            alertNewHash=Hash.new
+            # Parse thro the array hash that contains name value pairs for hash mapping and add new records to a new hash
+            @event_rename_rules.each { |rule| 
+              if(debug)
+                $log.info rule[:key_varbind]+": "+ rule[:origin_event_keyname]
+              end
+              alertNewHash[rule[:key_varbind]]=record["event"][rule[:origin_event_keyname]]
+            }
+            # construct the xml
+            @post_event_xml ="<?xml version=\"1.0\" encoding=\"UTF-8\"?>
+            <rs:event-request throttlesize=\"10\"
+              xmlns:rs=\"http://www.ca.com/spectrum/restful/schema/request\"
+              xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"
+              xsi:schemaLocation=\"http://www.ca.com/spectrum/restful/schema/request ../../../xsd/Request.xsd\">
+              <rs:event>
+                <rs:target-models>
+                 <rs:model mh= \"#{model_mh}\" />
+                </rs:target-models>
+             
+               <!-- event ID -->
+                <rs:event-type id=\"#{event_type_id}\"/>
+             
+                <!-- attributes/varbinds -->"
+            alertNewHash.each do |attr, val| 
+              if (val.nil? || val.empty?)
+                @post_event_xml += "\n <rs:varbind id=\""+ attr + "\"></rs:varbind>"
+              else
+                @post_event_xml += "\n <rs:varbind id=\""+ attr + "\">"+ CGI.escapeHTML(val) +"</rs:varbind>"
+              end
+            end
+            @post_event_xml += "
+                    </rs:event>
+                  </rs:event-request>"
+            @triggered_event_id = ''
+            if(debug)
+              $log.info "Rest url for post events: " + @events_url               
+              $log.info "xml: " +@post_event_xml 
+            end   
+            begin        
+              # eventPostRes = RestClient::Resource.new(@events_url,@user,@pass).post(@post_event_xml,:content_type => 'application/xml')
+              eventPostRes = events_resource.post @post_event_xml,:content_type => 'application/xml',:accept => 'application/json'
+              $log.info eventPostRes
+              eventPostResBody = JSON.parse(eventPostRes.body)
+              @triggered_event_id = eventPostResBody['ns1.event-response-list']['ns1.event-response']['@id']
+              # $log.info "event id is: " + @triggered_event_id
+            end
+
+          end #end of checking alerts is from 3rd party or spectrum
+          
+        else # if don't have @alarm_ID_key and @spectrum_key
+          $log.info "The alert don't have '#{@alarm_ID_key}' and '#{@spectrum_key}' "
+          $log.info record["event"]
+        end 
+
+      } # end of loop for each record
+    end  #end of emit
+  end #end of class SpectrumOtherOut 
+end #end of module

data/sample/spectrum.access.code.conf.sample

@@ -0,0 +1,44 @@
+# map of Spectrum attribute codes to names
+spectrum_access_code:{
+    "0x11f9c" => "ALARM_ID",
+    "0x11f4e" => "CREATION_DATE",
+    "0x11f56" => "SEVERITY",
+    "0x12b4c" => "ALARM_TITLE",
+    "0x1006e" => "HOSTNAME",
+    "0x12d7f" => "IP_ADDRESS",
+    "0x1296e" => "ORIGINATING_EVENT_ATTR",
+    "0x10000" => "MODEL_STRING",  
+    "0x11f4d" => "ACKNOWLEDGED",
+    "0x11f4f" => "ALARM_STATUS",
+    "0x11fc5" => "OCCURRENCES",
+    "0x11f57" => "TROUBLE_SHOOTER",
+    "0x11f9b" => "USER_CLEARABLE",
+    "0x12022" => "TROUBLE_TICKET_ID",
+    "0x12942" => "PERSISTENT",
+    "0x12adb" => "GC_NAME",
+    "0x57f0105" => "CUSTOM_PROJECT",
+    "0x11f51" => "CLEARED_BY_USER_NAME",
+    "0x11f52" => "EVENT_ID_LIST",
+    "0x11f53" => "MODEL_HANDLE",
+    "0x11f54" => "PRIMARY_ALARM",
+    "0x11fc4" => "ALARM_SOURCE",
+    "0x11fc6" => "TROUBLE_SHOOTER_MH",
+    "0x12a6c" => "TROUBLE_SHOOTER_EMAIL",
+    "0x1290d" => "IMPACT_SEVERITY",
+    "0x1290e" => "IMPACT_SCOPE",
+    "0x1298a" => "IMPACT_TYPE_LIST",
+    "0x12948" => "DIAGNOSIS_LOG",
+    "0x129aa" => "MODEL_ID",
+    "0x129ab" => "MODEL_TYPE_ID",
+    "0x129af" => "CLEAR_DATE",
+    "0x12a04" => "SYMPTOM_LIST_ATTR",
+    "0x12a6f" => "EVENT_SYMPTOM_LIST_ATTR",
+    "0x12a05" => "CAUSE_LIST_ATTR",
+    "0x12a06" => "SYMPTOM_COUNT_ATTR",
+    "0x12a70" => "EVENT_SYMPTOM_COUNT_ATTR",
+    "0x12a07" => "CAUSE_COUNT_ATTR",
+    "0x12a63" => "WEB_CONTEXT_URL",
+    "0x12a6b" => "COMBINED_IMPACT_TYPE_LIST",
+    "0x11f50" => "CAUSE_CODE",
+    "0x10009" => "SECURITY_STRING"
+}

metadata

@@ -1,14 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-spectrum
 version: !ruby/object:Gem::Version
-  version: 0.0.7
+  version: 0.0.8
 platform: ruby
 authors:
 - Alex Pena
+- Ling Zhang
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-04-06 00:00:00.000000000 Z
+date: 2015-05-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -108,11 +109,26 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.8'
+- !ruby/object:Gem::Dependency
+  name: highwatermark
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
 description: Fluentd input/output plugin for managing monitoring alerts from CA Spectrum.
   Input supports polling CA Spectrum APIs. Output currently only supports updating
   events retrieved from Spectrum.
 email:
 - pena.alex@gmail.com
+- zhangling.ice@gmail.com
 executables: []
 extensions: []
 extra_rdoc_files: []
@@ -125,6 +141,7 @@ files:
 - fluent-plugin-spectrum.gemspec
 - lib/fluent/plugin/in_spectrum.rb
 - lib/fluent/plugin/out_spectrum.rb
+- sample/spectrum.access.code.conf.sample
 - sample/spectrum.conf.advanced_sample
 - sample/spectrum.conf.sample
 - test/helper.rb