fluent-plugin-spectrum 0.0.7 → 0.0.8

Sign up to get free protection for your applications and to get access to all the features.
Files changed (8) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +2 -1
  3. data/README.md +81 -5
  4. data/fluent-plugin-spectrum.gemspec +5 -4
  5. data/lib/fluent/plugin/in_spectrum.rb +84 -77
  6. data/lib/fluent/plugin/out_spectrum.rb +151 -149
  7. data/sample/spectrum.access.code.conf.sample +44 -0
  8. metadata +19 -2
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: dce92432a6a0648acac71c3abcc728f1f7cb313b
4
- data.tar.gz: 9e12765096932803e03d30e71c09a3cb78bf0718
3
+ metadata.gz: 76bb8bd0bc0343b655aa47cccd1f710b4aa17759
4
+ data.tar.gz: 436a59a9735ca316f71fa76fc3cc8cc275b636b6
5
5
  SHA512:
6
- metadata.gz: 8e6419d7bcbcfe308e0a2d62b916f6e61b5cc1327569317a5d65be87a728d358042dfd316f8038a893e40fe1a686bd41a81611b1e192c1ff4b5d07c93871cb82
7
- data.tar.gz: efabb69d231fb482311be547596ae17c5486d995837af219aa8cd6e7441145731148daed2903dce4d82d44fc266fa3fdf16a3cafea4a67a944a7c133eb7f81f1
6
+ metadata.gz: 73e8268a657d5a4526a246956cff5641ca31e201c263f9aeaf896632996f72e12a65142879ac11af562aeb5d9db31b82c78dd2fdfac55868ee616b788b03169f
7
+ data.tar.gz: 433e2d2cc00ecb0486a4408e3819a2bf443f04fedd9d2e14a6c13b08b456c32d07aeb61924356425eb74f2db3e2fcf75b65011ae42050cd4999ba859e11200de
data/.gitignore CHANGED
@@ -14,4 +14,5 @@ rdoc
14
14
  spec/reports
15
15
  test/tmp
16
16
  test/version_tmp
17
- tmp
17
+ tmp
18
+ .DS_Store
data/README.md CHANGED
@@ -25,6 +25,9 @@ Or install it yourself as:
25
25
  $ gem install fluent-plugin-spectrum
26
26
 
27
27
  ## Usage
28
+
29
+ ### Input plugin
30
+
28
31
  Add the following into your fluentd config.
29
32
 
30
33
  Simple:
@@ -32,8 +35,8 @@ Simple:
32
35
  <source>
33
36
  type spectrum
34
37
  endpoint spectrum.yourdomain.com # required, FQDN of endpoint
35
- user username # required
36
- pass password # required
38
+ username username # required
39
+ password password # required
37
40
  interval 60 # optional, interval in seconds, defaults to 300
38
41
  </source>
39
42
  <match alert.spectrum>
@@ -45,11 +48,17 @@ Advanced:
45
48
  <source>
46
49
  type spectrum
47
50
  endpoint spectrum.yourdomain.com # required, FQDN of endpoint
48
- user username # required
49
- pass password # required
51
+ username username # required
52
+ password password # required
50
53
  interval 60 # optional, interval in seconds, defaults to 300
51
- state_file /tmp/spectrum_state # optional, file to keep state
54
+ state_type file # optional, set the type for store state (file or redis)
55
+ state_file /tmp/spectrum_state # optional, file to keep state or file to get redis configure (need state_type setup)
56
+ state_tag spectrum # optional, tag for store state(need state_type, state_file setup)
57
+ tag alert.spectrum # optional, add your own tag for tha alert
58
+ attributes ALARM_ID,CREATION_DATE,HOSTNAME # optional, select attributes that you want to poll in alerts, default value is __ALL__
52
59
  </source>
60
+
61
+
53
62
  # using rename_key to map to new keynames
54
63
  <match alert.spectrum>
55
64
  type rename_key
@@ -92,6 +101,73 @@ Verify:
92
101
  Output:
93
102
  2015-03-05 15:04:00 -0800 alert.spectrum: {"event_type":"alert.spectrum","intermediary_source":"spectrumapi001.corp.yourdomain.net","ALARM_ID":"54f8e0e0-e706-12c2-0165-005056a07ac5","CREATION_DATE":"1425596640","SEVERITY":"3","ALARM_TITLE":"LOGMATCH TRAPSEND CRIT","HOSTNAME":"yourhost001.corp.yourdomain.net","IP_ADDRESS":"10.10.0.14","ORIGINATING_EVENT_ATTR":"A SEC logmatch trapsend CRIT Your Alert Message here","MODEL_STRING":"Host_Device","ACKNOWLEDGED":"false","ALARM_STATUS":"","OCCURRENCES":"1","TROUBLE_SHOOTER":"","USER_CLEARABLE":"true","TROUBLE_TICKET_ID":"","PERSISTENT":"true","GC_NAME":"Your_Global_Collection"}
94
103
 
104
+
105
+ ### Output plugin
106
+
107
+ Add the following into your fluentd config.
108
+
109
+ ```
110
+ #set source, here use kafka as example
111
+ <source>
112
+ type kafka
113
+ host localhost #<broker host>
114
+ port 9092 # <broker port: default=9092>
115
+ topics argos-parser #<listening topics(separate with comma',')>
116
+ format json #<input text type (text|json|ltsv|msgpack)>
117
+ </source>
118
+
119
+ # once match specific tag, use spectrum output plugin
120
+ <match argos-parser>
121
+ type spectrum
122
+ endpoint spectrum.yourdomain.com # required, FQDN of endpoint
123
+ user username # required
124
+ pass password # required
125
+ interval 10 # interval in seconds
126
+ model_mh your_model_handler # required, you need to create model in spectrum first
127
+ event_type_id your_event_type_id # required, you need to set event type in spectrum first
128
+ spectrum_key event_type # key in alert to check if alert is from spectrum
129
+ spectrum_value alert.raw.spectrum # value to match is its from spectrum
130
+ alarm_ID_key source_event_id # key in the alert that associate with alarm_ID for calling spectrum PUT alarms api
131
+
132
+ # For 3rd party alerts
133
+ # Create new events in Spectrum
134
+ # Set these parameters according to varbind keys of your event type in spectrum
135
+ # and the keynames in your original event (which you want to push to spectrum)
136
+ <event_rename_rules>
137
+ #key_varbind origin_event_keyname
138
+ 2 source_hostname
139
+ 100 creation_time
140
+ 101 criticality
141
+ 102 source_ip
142
+ 103 alert_description
143
+ 104 application_name
144
+ 105 business_unit_l2
145
+ 106 business_unit_l3
146
+ 107 business_unit_l4
147
+ 108 cmdb_ci_sys_id
148
+ </event_rename_rules>
149
+
150
+
151
+ # Update existing alarms from Spectrum
152
+ # set these parameters according to keys of alarm in spectrum that you want to update
153
+ # and the the keynames in your original event
154
+ <alarm_rename_rules>
155
+ #key_spectrum_alarm origin_event_keyname
156
+ 0xffff00f6 application_name
157
+ 0xffff00f7 business_unit_l2
158
+ 0xffff00f8 business_unit_l3
159
+ 0xffff00f9 business_unit_l4
160
+ 0xffff00fa cmdb_ci_sysid
161
+ </alarm_rename_rules>
162
+
163
+ </match>
164
+ ```
165
+
166
+ Now startup fluentd
167
+
168
+ $ sudo fluentd -c fluent.conf &
169
+
170
+
95
171
  ## To Do
96
172
  * All flag to allow specifying spectrum attributes to get or get _ALL_
97
173
  * Add flag to allow start date/time if users want to backfill data from a specific date. then start loop.
data/fluent-plugin-spectrum.gemspec CHANGED
@@ -3,10 +3,10 @@ $:.push File.expand_path("../lib", __FILE__)
3
3
 
4
4
  Gem::Specification.new do |gem|
5
5
  gem.name = "fluent-plugin-spectrum"
6
- gem.version = "0.0.7"
7
- gem.date = '2015-04-06'
8
- gem.authors = ["Alex Pena"]
9
- gem.email = ["pena.alex@gmail.com"]
6
+ gem.version = "0.0.8"
7
+ gem.date = '2015-05-07'
8
+ gem.authors = ["Alex Pena","Ling Zhang"]
9
+ gem.email = ["pena.alex@gmail.com", "zhangling.ice@gmail.com"]
10
10
  gem.summary = %q{Fluentd plugin for managing monitoring alerts from CA Spectrum}
11
11
  gem.description = %q{Fluentd input/output plugin for managing monitoring alerts from CA Spectrum. Input supports polling CA Spectrum APIs. Output currently only supports updating events retrieved from Spectrum.}
12
12
  gem.homepage = 'https://github.com/Bigel0w/fluent-plugin-spectrum'
@@ -26,4 +26,5 @@ Gem::Specification.new do |gem|
26
26
  gem.add_runtime_dependency "fluentd", '~> 0.12'
27
27
  gem.add_runtime_dependency "json", '~> 1.8'
28
28
  gem.add_runtime_dependency "rest-client", '~> 1.8'
29
+ gem.add_runtime_dependency "highwatermark", '~> 0.1'
29
30
  end
data/lib/fluent/plugin/in_spectrum.rb CHANGED
@@ -8,11 +8,18 @@ module Fluent
8
8
  config_param :endpoint, :string, :default => nil
9
9
  config_param :username, :string, :default => nil
10
10
  config_param :password, :string, :default => nil
11
+
12
+ config_param :state_tag, :string, :default => "spectrum"
13
+ config_param :state_type, :string, :default => "memory"
11
14
  config_param :state_file, :string, :default => nil
12
- config_param :attributes, :string, :default => "ALL"
15
+ config_param :redis_host, :string, :default => nil
16
+ config_param :redis_port, :string, :default => nil
17
+
18
+ config_param :attributes, :string, :default => "__ALL__"
13
19
  config_param :interval, :integer, :default => INTERVAL_MIN
14
20
  config_param :select_limit, :integer, :default => 10000
15
21
  config_param :include_raw, :bool, :default => "false"
22
+
16
23
 
17
24
  # Classes
18
25
  class TimerWatcher < Coolio::TimerWatcher
@@ -29,45 +36,6 @@ module Fluent
29
36
  end # def on_timer
30
37
  end
31
38
 
32
- class StateStore
33
- def initialize(path)
34
- require 'yaml'
35
- @path = path
36
- if File.exists?(@path)
37
- @data = YAML.load_file(@path)
38
- if @data == false || @data == []
39
- @data = {}
40
- elsif !@data.is_a?(Hash)
41
- raise "Spectrum :: ConfigError state_file on #{@path.inspect} is invalid"
42
- end
43
- else
44
- @data = {}
45
- end
46
- end
47
-
48
- def last_records
49
- @data['last_records'] ||= {}
50
- end
51
-
52
- def update!
53
- File.open(@path, 'w') {|f|
54
- f.write YAML.dump(@data)
55
- }
56
- end
57
- end
58
-
59
- class MemoryStateStore
60
- def initialize
61
- @data = {}
62
- end
63
-
64
- def last_records
65
- @data['last_records'] ||= {}
66
- end
67
-
68
- def update!
69
- end
70
- end
71
39
 
72
40
  # function to UTF8 encode
73
41
  def to_utf8(str)
@@ -85,6 +53,8 @@ module Fluent
85
53
  def initialize
86
54
  require 'rest-client'
87
55
  require 'json'
56
+ require 'highwatermark'
57
+ require 'yaml'
88
58
  super
89
59
  end # def initialize
90
60
 
@@ -101,11 +71,21 @@ module Fluent
101
71
  raise ConfigError, "Spectrum :: ConfigError 'interval' must be #{INTERVAL_MIN} or over."
102
72
  end
103
73
  # Warn about optional state file
104
- unless @state_file
105
- $log.warn "Spectrum :: 'state_file PATH' parameter is not set to a valid source."
106
- $log.warn "Spectrum :: this parameter is highly recommended to save the last known good timestamp to resume event consuming"
74
+ unless @state_type == "file" || @state_type =="redis"
75
+ $log.warn "Spectrum :: 'state_type' is not set to file or redis"
76
+ $log.warn "Spectrum :: state file or redis are recommended to save the last known good timestamp to resume event consuming"
107
77
  end
108
- # map of Spectrum attribute codes to names
78
+
79
+ @highwatermark_parameters={
80
+ "state_tag" => @state_tag,
81
+ "state_type" => @state_type,
82
+ "state_file" => @state_file,
83
+ "redis_host" => @redis_host,
84
+ "redis_port" => @redis_port
85
+ }
86
+ $log.info "highwatermark_parameters: #{@highwatermark_parameters}"
87
+
88
+ # default setting for @spectrum_access_code
109
89
  @spectrum_access_code={
110
90
  "0x11f9c" => "ALARM_ID",
111
91
  "0x11f4e" => "CREATION_DATE",
@@ -123,37 +103,63 @@ module Fluent
123
103
  "0x12022" => "TROUBLE_TICKET_ID",
124
104
  "0x12942" => "PERSISTENT",
125
105
  "0x12adb" => "GC_NAME",
126
- "0x57f0105" => "Custom_Project",
127
- #{}"0x11f51" => "CLEARED_BY_USER_NAME",
128
- #{}"0x11f52" => "EVENT_ID_LIST",
129
- #{}"0x11f53" => "MODEL_HANDLE",
130
- #{}"0x11f54" => "PRIMARY_ALARM",
131
- #{}"0x11fc4" => "ALARM_SOURCE",
132
- #{}"0x11fc6" => "TROUBLE_SHOOTER_MH",
133
- #{}"0x12a6c" => "TROUBLE_SHOOTER_EMAIL",
134
- #{}"0x1290d" => "IMPACT_SEVERITY",
135
- #{}"0x1290e" => "IMPACT_SCOPE",
136
- #{}"0x1298a" => "IMPACT_TYPE_LIST",
137
- #{}"0x12948" => "DIAGNOSIS_LOG",
138
- #{}"0x129aa" => "MODEL_ID",
139
- #{}"0x129ab" => "MODEL_TYPE_ID",
140
- #{}"0x129af" => "CLEAR_DATE",
141
- #{}"0x12a04" => "SYMPTOM_LIST_ATTR",
142
- #{}"0x12a6f" => "EVENT_SYMPTOM_LIST_ATTR",
143
- #{}"0x12a05" => "CAUSE_LIST_ATTR",
144
- #{}"0x12a06" => "SYMPTOM_COUNT_ATTR",
145
- #{}"0x12a70" => "EVENT_SYMPTOM_COUNT_ATTR",
146
- #{}"0x12a07" => "CAUSE_COUNT_ATTR",
147
- #{}"0x12a63" => "WEB_CONTEXT_URL",
148
- #{}"0x12a6b" => "COMBINED_IMPACT_TYPE_LIST",
149
- #{}"0x11f50" => "CAUSE_CODE",
150
- #{}"0x10009" => "SECURITY_STRING"
106
+ "0x57f0105" => "CUSTOM_PROJECT",
107
+ "0x11f51" => "CLEARED_BY_USER_NAME",
108
+ "0x11f52" => "EVENT_ID_LIST",
109
+ "0x11f53" => "MODEL_HANDLE",
110
+ "0x11f54" => "PRIMARY_ALARM",
111
+ "0x11fc4" => "ALARM_SOURCE",
112
+ "0x11fc6" => "TROUBLE_SHOOTER_MH",
113
+ "0x12a6c" => "TROUBLE_SHOOTER_EMAIL",
114
+ "0x1290d" => "IMPACT_SEVERITY",
115
+ "0x1290e" => "IMPACT_SCOPE",
116
+ "0x1298a" => "IMPACT_TYPE_LIST",
117
+ "0x12948" => "DIAGNOSIS_LOG",
118
+ "0x129aa" => "MODEL_ID",
119
+ "0x129ab" => "MODEL_TYPE_ID",
120
+ "0x129af" => "CLEAR_DATE",
121
+ "0x12a04" => "SYMPTOM_LIST_ATTR",
122
+ "0x12a6f" => "EVENT_SYMPTOM_LIST_ATTR",
123
+ "0x12a05" => "CAUSE_LIST_ATTR",
124
+ "0x12a06" => "SYMPTOM_COUNT_ATTR",
125
+ "0x12a70" => "EVENT_SYMPTOM_COUNT_ATTR",
126
+ "0x12a07" => "CAUSE_COUNT_ATTR",
127
+ "0x12a63" => "WEB_CONTEXT_URL",
128
+ "0x12a6b" => "COMBINED_IMPACT_TYPE_LIST",
129
+ "0x11f50" => "CAUSE_CODE",
130
+ "0x10009" => "SECURITY_STRING"
151
131
  }
132
+
152
133
  # Create XML chunk for attributes we care about
153
134
  @attr_of_interest=""
154
- @spectrum_access_code.each do |key, array|
155
- @attr_of_interest += " <rs:requested-attribute id=\"#{key}\"/>"
156
- end
135
+ if(@attributes.upcase === "__ALL__")
136
+ $log.info "all attributes"
137
+ @spectrum_access_code.each do |key, value|
138
+ $log.info "key: #{key}, value: #{value}"
139
+ @attr_of_interest += " <rs:requested-attribute id=\"#{key}\"/>"
140
+ end
141
+ else
142
+ $log.info "selected attributes"
143
+ @attributes.split(",").each {|attr|
144
+ key=""
145
+ value=""
146
+ # if it's hex code
147
+ if @spectrum_access_code.has_key?(attr.strip)
148
+ key = attr.strip
149
+ value = @spectrum_access_code.fetch(key)
150
+ # if it's the name
151
+ elsif @spectrum_access_code.has_value?(attr.strip.upcase)
152
+ value = attr.strip.upcase
153
+ key = @spectrum_access_code.key(value)
154
+ # if it's invalid input, not the hex code or name in the map
155
+ else
156
+ raise ConfigError, "Spectrum :: ConfigError attribute '#{attr}' is not in the hash map"
157
+ end
158
+ $log.info "key: #{key}, value: #{value}"
159
+ @attr_of_interest += " <rs:requested-attribute id=\"#{key}\"/>"
160
+ }
161
+ end
162
+
157
163
  # URL Resource
158
164
  def resource
159
165
  @url = 'http://' + @endpoint.to_s + '/spectrum/restful/alarms'
@@ -163,7 +169,7 @@ module Fluent
163
169
 
164
170
  def start
165
171
  @stop_flag = false
166
- @state_store = @state_file.nil? ? MemoryStateStore.new : StateStore.new(@state_file)
172
+ @highwatermark = Highwatermark::HighWaterMark.new(@highwatermark_parameters)
167
173
  @loop = Coolio::Loop.new
168
174
  @loop.attach(TimerWatcher.new(@interval, true, &method(:on_timer)))
169
175
  @thread = Thread.new(&method(:run))
@@ -186,10 +192,12 @@ module Fluent
186
192
  def on_timer
187
193
  if not @stop_flag
188
194
  pollingStart = Engine.now.to_i
189
- if @state_store.last_records.has_key?("spectrum")
190
- alertStartTime = @state_store.last_records['spectrum']
195
+ if @highwatermark.last_records(@state_tag)
196
+ alertStartTime = @highwatermark.last_records(@state_tag)
197
+ $log.info "got hwm form state file: #{alertStartTime.to_i}"
191
198
  else
192
199
  alertStartTime = (pollingStart.to_i - @interval.to_i)
200
+ $log.info "no hwm, got new alert start time: #{alertStartTime.to_i}"
193
201
  end
194
202
  pollingEnd = ''
195
203
  pollingDuration = ''
@@ -218,7 +226,6 @@ module Fluent
218
226
  res=resource.post @xml,:content_type => 'application/xml',:accept => 'application/json'
219
227
  body = JSON.parse(res.body)
220
228
  pollingEnd = Time.parse(res.headers[:date]).to_i
221
- @state_store.last_records['spectrum'] = pollingEnd
222
229
  pollingDuration = Engine.now.to_i - pollingStart
223
230
  end
224
231
 
@@ -273,7 +280,7 @@ module Fluent
273
280
  else
274
281
  $log.info "Spectrum :: returned #{body['ns1.alarm-response-list']['@total-alarms'].to_i} alarms for period < #{alertStartTime.to_i} took #{pollingDuration.to_i} seconds, ended at #{pollingEnd}"
275
282
  end
276
- @state_store.update!
283
+ @highwatermark.update_records(pollingEnd,@state_tag)
277
284
  end
278
285
  end # def input
279
286
  end # class SpectrumInput
data/lib/fluent/plugin/out_spectrum.rb CHANGED
@@ -1,136 +1,86 @@
1
1
  module Fluent
2
- class SpectrumOut < Output
2
+ class SpectrumOut< Output
3
3
  # First, register the plugin. NAME is the name of this plugin
4
4
  # and identifies the plugin in the configuration file.
5
5
  Fluent::Plugin.register_output('spectrum', self)
6
6
 
7
- config_param :tag, :string, default:'alert.spectrum.out'
8
- #config_param :tag, :string, :default => "alert.spectrum"
9
7
  config_param :endpoint, :string, :default => "pleasechangeme.com" #fqdn of endpoint
10
- config_param :interval, :integer, :default => '300' #Default 5 minutes
11
8
  config_param :user, :string, :default => "username"
12
9
  config_param :pass, :string, :default => "password"
13
- config_param :include_raw, :string, :default => "false" #Include original object as raw
14
- config_param :attributes, :string, :default => "ALL" # fields to include, ALL for... well, ALL.
15
-
16
- def parseAttributes(alarmAttribute)
17
- key = @spectrum_access_code[alarmAttribute['@id'].to_s].to_s
18
- value = ((to_utf8(alarmAttribute['$'].to_s)).strip).gsub(/\r?\n/, " ")
19
- return key,value
20
- end
10
+ config_param :interval, :integer, :default => '300' #Default 5 minutes
11
+ config_param :model_mh, :string, :default => "model_mh"
12
+ config_param :event_type_id, :string, :default => "event_type_id"
13
+ config_param :spectrum_key, :string, :default => "event_type" # key in the alert to check if alert is from spectrum
14
+ config_param :spectrum_value, :string, :default => "alert.raw.spectrum"# value to match is its from spectrum
15
+ config_param :alarm_ID_key, :string, :default => "source_event_id" # key in the alert that associate with alarm_ID for calling spectrum PUT alarms api
16
+ config_param :debug, :bool, :default => false
21
17
 
22
18
  def initialize
23
19
  require 'rest-client'
24
20
  require 'json'
25
- require 'pp'
26
- require 'cgi'
21
+ require 'cgi' # verify we need --yes, we need it, to_utf8 could not used to create valid url and xml
27
22
  super
28
23
  end # def initialize
29
24
 
25
+ # function to UTF8 encode
26
+ def to_utf8(str)
27
+ str = str.force_encoding('UTF-8')
28
+ return str if str.valid_encoding?
29
+ str.encode("UTF-8", 'binary', invalid: :replace, undef: :replace, replace: '')
30
+ end
31
+
32
+ def parse_rename_rule rule
33
+ if rule.match /^([^\s]+)\s+(.+)$/
34
+ return $~.captures
35
+ end
36
+ end
30
37
 
31
38
  # This method is called before starting.
32
39
  def configure(conf)
33
- super
34
- # Read property file and create a hash
35
- @rename_rules = []
36
- conf_rename_rules = conf.keys.select { |k| k =~ /^rename_rule(\d+)$/ }
37
- conf_rename_rules.sort_by { |r| r.sub('rename_rule', '').to_i }.each do |r|
38
- key_regexp, new_key = parse_rename_rule conf[r]
40
+ super
41
+ # Read configuration for event_rename_rules and create a hash
42
+ @event_rename_rules = []
43
+ conf.elements.select { |element| element.name == 'event_rename_rules' }.each { |element|
44
+ element.each_pair { |key_varbind, origin_event_keyname|
45
+ element.has_key?(key_varbind) # to suppress unread configuration warning
46
+ @event_rename_rules << { key_varbind: key_varbind, origin_event_keyname: origin_event_keyname }
47
+ $log.info "Added event_rename_rules: #{@event_rename_rules.last}"
48
+ }
49
+ }
39
50
 
40
- if key_regexp.nil? || new_key.nil?
41
- raise Fluent::ConfigError, "Failed to parse: #{r} #{conf[r]}"
42
- end
43
51
 
44
- if @rename_rules.map { |r| r[:key_regexp] }.include? /#{key_regexp}/
45
- raise Fluent::ConfigError, "Duplicated rules for key #{key_regexp}: #{@rename_rules}"
46
- end
52
+ # Read configuration for alarm_rename_rules and create a hash
53
+ @alarm_rename_rules = []
54
+ conf.elements.select { |element| element.name == 'alarm_rename_rules' }.each { |element|
55
+ element.each_pair { |key_spectrum_alarm, origin_event_keyname|
56
+ element.has_key?(key_spectrum_alarm) # to suppress unread configuration warning
57
+ @alarm_rename_rules << { key_spectrum_alarm: key_spectrum_alarm, origin_event_keyname: origin_event_keyname }
58
+ $log.info "Added alarm_rename_rules: #{@alarm_rename_rules.last}"
59
+ }
60
+ }
47
61
 
48
- #@rename_rules << { key_regexp: /#{key_regexp}/, new_key: new_key }
49
- @rename_rules << { key_regexp: key_regexp, new_key: new_key }
50
- $log.info "Added rename key rule: #{r} #{@rename_rules.last}"
62
+
63
+ # Setup URL Resource
64
+ @alarms_url = 'http://' + @endpoint.to_s + '/spectrum/restful/alarms/'
65
+ @events_url = 'http://' + @endpoint.to_s + '/spectrum/restful/events'
66
+ def events_resource
67
+ RestClient::Resource.new(@events_url, :user => @user, :password => @pass, :open_timeout => 5, :timeout => (@interval * 3))
51
68
  end
52
69
 
53
- raise Fluent::ConfigError, "No rename rules are given" if @rename_rules.empty?
54
- @conf = conf
55
- # map of Spectrum attribute codes to names
56
- @spectrum_access_code={
57
- "0x11f9c" => "ALARM_ID",
58
- "0x11f4e" => "CREATION_DATE",
59
- "0x11f56" => "SEVERITY",
60
- "0x12b4c" => "ALARM_TITLE",
61
- "0x1006e" => "HOSTNAME",
62
- "0x12d7f" => "IP_ADDRESS",
63
- "0x1296e" => "ORIGINATING_EVENT_ATTR",
64
- "0x10000" => "MODEL_STRING",
65
- "0x11f4d" => "ACKNOWLEDGED",
66
- "0x11f4f" => "ALARM_STATUS",
67
- "0x11fc5" => "OCCURRENCES",
68
- "0x11f57" => "TROUBLE_SHOOTER",
69
- "0x11f9b" => "USER_CLEARABLE",
70
- "0x12022" => "TROUBLE_TICKET_ID",
71
- "0x12942" => "PERSISTENT",
72
- "0x12adb" => "GC_NAME",
73
- "0x57f0105" => "Custom_Project",
74
- "0x11f4d" => "ACKNOWLEDGED",
75
- "0xffff00ed" => "application_name",
76
- "0xffff00f1" => "business_unit_l1",
77
- "0xffff00f2" => "business_unit_l2",
78
- "0xffff00f3" => "business_unit_l3",
79
- "0xffff00f4" => "business_unit_l4",
80
- "0xffff00f0" => "cmdb_ci_sysid",
81
-
82
- #{}"0x11f51" => "CLEARED_BY_USER_NAME",
83
- #{}"0x11f52" => "EVENT_ID_LIST",
84
- #{}"0x11f53" => "MODEL_HANDLE",
85
- #{}"0x11f54" => "PRIMARY_ALARM",
86
- #{}"0x11fc4" => "ALARM_SOURCE",
87
- #{}"0x11fc6" => "TROUBLE_SHOOTER_MH",
88
- #{}"0x12a6c" => "TROUBLE_SHOOTER_EMAIL",
89
- #{}"0x1290d" => "IMPACT_SEVERITY",
90
- #{}"0x1290e" => "IMPACT_SCOPE",
91
- #{}"0x1298a" => "IMPACT_TYPE_LIST",
92
- #{}"0x12948" => "DIAGNOSIS_LOG",
93
- #{}"0x129aa" => "MODEL_ID",
94
- #{}"0x129ab" => "MODEL_TYPE_ID",
95
- #{}"0x129af" => "CLEAR_DATE",
96
- #{}"0x12a04" => "SYMPTOM_LIST_ATTR",
97
- #{}"0x12a6f" => "EVENT_SYMPTOM_LIST_ATTR",
98
- #{}"0x12a05" => "CAUSE_LIST_ATTR",
99
- #{}"0x12a06" => "SYMPTOM_COUNT_ATTR",
100
- #{}"0x12a70" => "EVENT_SYMPTOM_COUNT_ATTR",
101
- #{}"0x12a07" => "CAUSE_COUNT_ATTR",
102
- #{}"0x12a63" => "WEB_CONTEXT_URL",
103
- #{}"0x12a6b" => "COMBINED_IMPACT_TYPE_LIST",
104
- #{}"0x11f50" => "CAUSE_CODE",
105
- #{}"0x10009" => "SECURITY_STRING"
106
- }
107
- # Create XML chunk for attributes we care about
108
- @attr_of_interest=""
109
- @spectrum_access_code.each do |key, array|
110
- @attr_of_interest += " <rs:requested-attribute id=\"#{key}\"/>"
70
+ def alarms_resource
71
+ RestClient::Resource.new(@alarms_url, :user => @user, :password => @pass, :open_timeout => 5, :timeout => (@interval * 3))
111
72
  end
112
73
 
113
74
 
114
- # Setup URL Resource
115
- @url = 'http://' + @endpoint.to_s + '/spectrum/restful/alarms/'
116
- end
117
-
118
- def parse_rename_rule rule
119
- if rule.match /^([^\s]+)\s+(.+)$/
120
- return $~.captures
121
- end
122
- end
123
-
75
+ end # end of def configure
124
76
  # This method is called when starting.
125
77
  def start
126
78
  super
127
79
  end
128
-
129
80
  # This method is called when shutting down.
130
81
  def shutdown
131
82
  super
132
83
  end
133
-
134
84
  # This method is called when an event reaches Fluentd.
135
85
  # 'es' is a Fluent::EventStream object that includes multiple events.
136
86
  # You can use 'es.each {|time,record["event"]| ... }' to retrieve events.
@@ -141,53 +91,105 @@ module Fluent
141
91
  def emit(tag, es, chain)
142
92
  chain.next
143
93
  es.each {|time,record|
144
- $stderr.puts "OK!"
145
- ## Check if the incoming event already has an event id (alarm id) and a corresponding tag of spectrum
146
- if (record["event"].has_key?("source_event_id") && record["event"].has_key?("event_type"))
147
- # If the value on event_type is spectrum, then it means that ti is already from spectrum and needs an update
148
- if (record["event"].has_value?("alert.spectrum"))
149
-
150
- # Create an empty hash
151
- alertUpdateHash=Hash.new
152
-
153
- # Parse thro the array hash that contains name value pairs for hash mapping and add new records to a new hash
154
- @rename_rules.each { |rule|
155
- pp rule[:new_key]
156
- alertUpdateHash[rule[:key_regexp]]=record["event"][rule[:new_key]]
157
- }
158
- # construct the url and iterate to construct the uri
159
- @urlrest = @url + record["event"]["source_event_id"]
160
- alertUpdateHash.each do |attr, val|
161
- if (val.nil? || val.empty?)
162
- next
163
- else
164
- if (@urlrest =~ /#{record["event"]["source_event_id"]}\s*$/)
165
- @urlrest = @urlrest + "?attr=" + attr + "&val=" + CGI.escape(val.to_s)
166
- else
167
- @urlrest = @urlrest + "&attr=" + attr + "&val=" + CGI.escape(val.to_s)
168
- end
169
- end
170
- end
171
-
172
- $log.info "Rest url " + @urlrest
173
- #RestClient::Resource.new(@user,@pass)
174
- begin
175
- responsePostAffEnt=RestClient::Resource.new(@urlrest,@user,@pass).put(@urlrest,:content_type => 'application/json')
176
- rescue Exception => e
177
- $log.error "Error in restful put call."
178
- log.error e.backtrace.inspect
179
- $log.error responsePostAffEnt
180
- end
181
-
182
- else
183
-
184
- # For now just throw to stdout
185
- $log.info record["event"]
186
-
187
- end
188
-
189
- end
190
- }
191
- end
192
- end
193
- end
94
+
95
+ if (record["event"].has_key?(@alarm_ID_key) && record["event"].has_key?(@spectrum_key) )
96
+ ######native spectrum alert ########################
97
+ ######PUT alarm to update enriched fields
98
+ if (record["event"][@spectrum_key] == @spectrum_value)
99
+ $log.info "The alert is from spectrum"
100
+ # Create an empty hash
101
+ alertUpdateHash=Hash.new
102
+ # Parse thro the array hash that contains name value pairs for hash mapping and add new records to a new hash
103
+ @alarm_rename_rules.each { |rule|
104
+ puts rule[:origin_event_keyname]
105
+ alertUpdateHash[rule[:key_spectrum_alarm]]=record["event"][rule[:origin_event_keyname]]
106
+ }
107
+ # construct the alarms PUT uri for update triggerd alarm withe enriched fields
108
+ @alarms_urlrest = @alarms_url + record["event"][@alarm_ID_key]
109
+ @attr_count = 0
110
+ alertUpdateHash.each do |attr, val|
111
+ if (val.nil? || val.empty?)
112
+ next
113
+ else
114
+ if (@attr_count == 0)
115
+ @alarms_urlrest = @alarms_urlrest + "?attr=" + attr + "&val=" + CGI.escape(val.to_s)
116
+ # @alarms_urlrest = @alarms_urlrest + "?attr=" + attr + "&val=" + to_utf8(val.to_s)
117
+ @attr_count +=1
118
+ else
119
+ @alarms_urlrest = @alarms_urlrest + "&attr=" + attr + "&val=" + CGI.escape(val.to_s)
120
+ # @alarms_urlrest = @alarms_urlrest + "&attr=" + attr + "&val=" + to_utf8(val.to_s)
121
+ @attr_count +=1
122
+ end
123
+ end
124
+ end
125
+ $log.info "Rest url for PUT alarms: " + @alarms_urlrest
126
+
127
+ begin
128
+ # alarmPutRes = alarms_resource.put @alarms_urlrest,:content_type => 'application/json'
129
+ alarmPutRes = RestClient::Resource.new(@alarms_urlrest,@user,@pass).put(@alarms_urlrest,:content_type => 'application/json')
130
+ $log.info alarmPutRes
131
+ end
132
+
133
+ ######3rd party alert #######################
134
+ ######Post an event and then trigger an alarm ######
135
+ else
136
+ $log.info "The alert is from 3rd party"
137
+ # Create an empty hash
138
+ alertNewHash=Hash.new
139
+ # Parse thro the array hash that contains name value pairs for hash mapping and add new records to a new hash
140
+ @event_rename_rules.each { |rule|
141
+ if(debug)
142
+ $log.info rule[:key_varbind]+": "+ rule[:origin_event_keyname]
143
+ end
144
+ alertNewHash[rule[:key_varbind]]=record["event"][rule[:origin_event_keyname]]
145
+ }
146
+ # construct the xml
147
+ @post_event_xml ="<?xml version=\"1.0\" encoding=\"UTF-8\"?>
148
+ <rs:event-request throttlesize=\"10\"
149
+ xmlns:rs=\"http://www.ca.com/spectrum/restful/schema/request\"
150
+ xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"
151
+ xsi:schemaLocation=\"http://www.ca.com/spectrum/restful/schema/request ../../../xsd/Request.xsd\">
152
+ <rs:event>
153
+ <rs:target-models>
154
+ <rs:model mh= \"#{model_mh}\" />
155
+ </rs:target-models>
156
+
157
+ <!-- event ID -->
158
+ <rs:event-type id=\"#{event_type_id}\"/>
159
+
160
+ <!-- attributes/varbinds -->"
161
+ alertNewHash.each do |attr, val|
162
+ if (val.nil? || val.empty?)
163
+ @post_event_xml += "\n <rs:varbind id=\""+ attr + "\"></rs:varbind>"
164
+ else
165
+ @post_event_xml += "\n <rs:varbind id=\""+ attr + "\">"+ CGI.escapeHTML(val) +"</rs:varbind>"
166
+ end
167
+ end
168
+ @post_event_xml += "
169
+ </rs:event>
170
+ </rs:event-request>"
171
+ @triggered_event_id = ''
172
+ if(debug)
173
+ $log.info "Rest url for post events: " + @events_url
174
+ $log.info "xml: " +@post_event_xml
175
+ end
176
+ begin
177
+ # eventPostRes = RestClient::Resource.new(@events_url,@user,@pass).post(@post_event_xml,:content_type => 'application/xml')
178
+ eventPostRes = events_resource.post @post_event_xml,:content_type => 'application/xml',:accept => 'application/json'
179
+ $log.info eventPostRes
180
+ eventPostResBody = JSON.parse(eventPostRes.body)
181
+ @triggered_event_id = eventPostResBody['ns1.event-response-list']['ns1.event-response']['@id']
182
+ # $log.info "event id is: " + @triggered_event_id
183
+ end
184
+
185
+ end #end of checking alerts is from 3rd party or spectrum
186
+
187
+ else # if don't have @alarm_ID_key and @spectrum_key
188
+ $log.info "The alert don't have '#{@alarm_ID_key}' and '#{@spectrum_key}' "
189
+ $log.info record["event"]
190
+ end
191
+
192
+ } # end of loop for each record
193
+ end #end of emit
194
+ end #end of class SpectrumOtherOut
195
+ end #end of module
data/sample/spectrum.access.code.conf.sample ADDED
@@ -0,0 +1,44 @@
1
+ # map of Spectrum attribute codes to names
2
+ spectrum_access_code:{
3
+ "0x11f9c" => "ALARM_ID",
4
+ "0x11f4e" => "CREATION_DATE",
5
+ "0x11f56" => "SEVERITY",
6
+ "0x12b4c" => "ALARM_TITLE",
7
+ "0x1006e" => "HOSTNAME",
8
+ "0x12d7f" => "IP_ADDRESS",
9
+ "0x1296e" => "ORIGINATING_EVENT_ATTR",
10
+ "0x10000" => "MODEL_STRING",
11
+ "0x11f4d" => "ACKNOWLEDGED",
12
+ "0x11f4f" => "ALARM_STATUS",
13
+ "0x11fc5" => "OCCURRENCES",
14
+ "0x11f57" => "TROUBLE_SHOOTER",
15
+ "0x11f9b" => "USER_CLEARABLE",
16
+ "0x12022" => "TROUBLE_TICKET_ID",
17
+ "0x12942" => "PERSISTENT",
18
+ "0x12adb" => "GC_NAME",
19
+ "0x57f0105" => "CUSTOM_PROJECT",
20
+ "0x11f51" => "CLEARED_BY_USER_NAME",
21
+ "0x11f52" => "EVENT_ID_LIST",
22
+ "0x11f53" => "MODEL_HANDLE",
23
+ "0x11f54" => "PRIMARY_ALARM",
24
+ "0x11fc4" => "ALARM_SOURCE",
25
+ "0x11fc6" => "TROUBLE_SHOOTER_MH",
26
+ "0x12a6c" => "TROUBLE_SHOOTER_EMAIL",
27
+ "0x1290d" => "IMPACT_SEVERITY",
28
+ "0x1290e" => "IMPACT_SCOPE",
29
+ "0x1298a" => "IMPACT_TYPE_LIST",
30
+ "0x12948" => "DIAGNOSIS_LOG",
31
+ "0x129aa" => "MODEL_ID",
32
+ "0x129ab" => "MODEL_TYPE_ID",
33
+ "0x129af" => "CLEAR_DATE",
34
+ "0x12a04" => "SYMPTOM_LIST_ATTR",
35
+ "0x12a6f" => "EVENT_SYMPTOM_LIST_ATTR",
36
+ "0x12a05" => "CAUSE_LIST_ATTR",
37
+ "0x12a06" => "SYMPTOM_COUNT_ATTR",
38
+ "0x12a70" => "EVENT_SYMPTOM_COUNT_ATTR",
39
+ "0x12a07" => "CAUSE_COUNT_ATTR",
40
+ "0x12a63" => "WEB_CONTEXT_URL",
41
+ "0x12a6b" => "COMBINED_IMPACT_TYPE_LIST",
42
+ "0x11f50" => "CAUSE_CODE",
43
+ "0x10009" => "SECURITY_STRING"
44
+ }
metadata CHANGED
@@ -1,14 +1,15 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-spectrum
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.7
4
+ version: 0.0.8
5
5
  platform: ruby
6
6
  authors:
7
7
  - Alex Pena
8
+ - Ling Zhang
8
9
  autorequire:
9
10
  bindir: bin
10
11
  cert_chain: []
11
- date: 2015-04-06 00:00:00.000000000 Z
12
+ date: 2015-05-07 00:00:00.000000000 Z
12
13
  dependencies:
13
14
  - !ruby/object:Gem::Dependency
14
15
  name: rake
@@ -108,11 +109,26 @@ dependencies:
108
109
  - - "~>"
109
110
  - !ruby/object:Gem::Version
110
111
  version: '1.8'
112
+ - !ruby/object:Gem::Dependency
113
+ name: highwatermark
114
+ requirement: !ruby/object:Gem::Requirement
115
+ requirements:
116
+ - - "~>"
117
+ - !ruby/object:Gem::Version
118
+ version: '0.1'
119
+ type: :runtime
120
+ prerelease: false
121
+ version_requirements: !ruby/object:Gem::Requirement
122
+ requirements:
123
+ - - "~>"
124
+ - !ruby/object:Gem::Version
125
+ version: '0.1'
111
126
  description: Fluentd input/output plugin for managing monitoring alerts from CA Spectrum.
112
127
  Input supports polling CA Spectrum APIs. Output currently only supports updating
113
128
  events retrieved from Spectrum.
114
129
  email:
115
130
  - pena.alex@gmail.com
131
+ - zhangling.ice@gmail.com
116
132
  executables: []
117
133
  extensions: []
118
134
  extra_rdoc_files: []
@@ -125,6 +141,7 @@ files:
125
141
  - fluent-plugin-spectrum.gemspec
126
142
  - lib/fluent/plugin/in_spectrum.rb
127
143
  - lib/fluent/plugin/out_spectrum.rb
144
+ - sample/spectrum.access.code.conf.sample
128
145
  - sample/spectrum.conf.advanced_sample
129
146
  - sample/spectrum.conf.sample
130
147
  - test/helper.rb