notifu 1.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/bin/notifu +15 -0
- data/lib/notifu.rb +12 -0
- data/lib/notifu/actors/gammu_sms_bridge.rb +41 -0
- data/lib/notifu/actors/pagerduty.rb +0 -0
- data/lib/notifu/actors/slack_chan.rb +29 -0
- data/lib/notifu/actors/slack_msg.rb +29 -0
- data/lib/notifu/actors/smtp.rb +73 -0
- data/lib/notifu/actors/stdout.rb +16 -0
- data/lib/notifu/actors/twilio_call.rb +27 -0
- data/lib/notifu/cli.rb +13 -0
- data/lib/notifu/cli/object.rb +37 -0
- data/lib/notifu/cli/service.rb +53 -0
- data/lib/notifu/config.rb +120 -0
- data/lib/notifu/logger.rb +83 -0
- data/lib/notifu/mixins.rb +49 -0
- data/lib/notifu/model.rb +5 -0
- data/lib/notifu/model/contact.rb +15 -0
- data/lib/notifu/model/event.rb +52 -0
- data/lib/notifu/model/group.rb +14 -0
- data/lib/notifu/model/issue.rb +51 -0
- data/lib/notifu/model/sla.rb +20 -0
- data/lib/notifu/sensu/handler.rb +105 -0
- data/lib/notifu/util.rb +9 -0
- data/lib/notifu/workers/actor.rb +60 -0
- data/lib/notifu/workers/processor.rb +444 -0
- data/lib/notifu/workers/sidekiq_init.rb +24 -0
- metadata +90 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: e84a8d91a613dee08b4ebfa63f7c05082c4230d8
|
4
|
+
data.tar.gz: c0bf2890e4e519adc62b6f76b553fb706451146c
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 9ea47dec858e01b1dcea6586a29a491a0fc5fd0cb98731f0308bd010fe51eae593b4f3f092d454b2ef0d02336b495eea67197c58034bc73ada6751fe2f77bffd
|
7
|
+
data.tar.gz: b212ae242588fea5fc7b9ff199fa0776337b34ab0c5ca3afe24498eb967ae972b7076c4476122debbdf7f653b27111687a01329d8e4953a993d463752450a552
|
data/bin/notifu
ADDED
@@ -0,0 +1,15 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
path = File.symlink?(__FILE__) ? File.readlink(__FILE__) : __FILE__
|
4
|
+
$basepath = '/' + File.dirname(path).split('/')[1..-2].join('/') + '/lib/notifu/'
|
5
|
+
$actorpath = $basepath + "actors/"
|
6
|
+
# $sidekiq_bin = $basepath.sub(/app\//, '') + ".rvm/wrappers/notifu/sidekiq"
|
7
|
+
$sidekiq_bin = "sidekiq"
|
8
|
+
|
9
|
+
require 'notifu'
|
10
|
+
|
11
|
+
Notifu::CONFIG = Notifu::Config.new.get
|
12
|
+
|
13
|
+
Ohm.redis = Redic.new Notifu::CONFIG[:redis_data]
|
14
|
+
|
15
|
+
Notifu::CLI::Root.start(ARGV)
|
data/lib/notifu.rb
ADDED
@@ -0,0 +1,12 @@
|
|
1
|
+
require 'yaml'
|
2
|
+
require 'json'
|
3
|
+
require 'thor'
|
4
|
+
require 'ohm'
|
5
|
+
require "notifu/mixins"
|
6
|
+
require "notifu/util"
|
7
|
+
require "notifu/config"
|
8
|
+
require "notifu/logger"
|
9
|
+
require "notifu/model"
|
10
|
+
require "notifu/cli/service"
|
11
|
+
require "notifu/cli/object"
|
12
|
+
require "notifu/cli"
|
@@ -0,0 +1,41 @@
|
|
1
|
+
module Notifu
|
2
|
+
module Actors
|
3
|
+
class GammuSmsBridge < Notifu::Actor
|
4
|
+
|
5
|
+
self.name = "gammu_sms_bridge"
|
6
|
+
self.desc = "Old-school NetCat-like SMS bridge to Gammu"
|
7
|
+
self.retry = 3
|
8
|
+
|
9
|
+
def act
|
10
|
+
data = OpenStruct.new({
|
11
|
+
notifu_id: self.issue.notifu_id,
|
12
|
+
host: self.issue.host,
|
13
|
+
message: self.issue.message,
|
14
|
+
service: self.issue.service,
|
15
|
+
status: self.issue.code.to_state,
|
16
|
+
first_event: Time.at(self.issue.time_created.to_i),
|
17
|
+
duration: (Time.now.to_i - self.issue.time_created.to_i).duration,
|
18
|
+
occurrences_count: self.issue.occurrences_count,
|
19
|
+
occurrences_trigger: self.issue.occurrences_trigger
|
20
|
+
})
|
21
|
+
message = ERB.new(self.template).result(data.instance_eval {binding})
|
22
|
+
|
23
|
+
self.contacts.each do |contact|
|
24
|
+
cell = contact.cell
|
25
|
+
template = ERB.new File.new("sms.erb").read, nil, "%"
|
26
|
+
message = "template.result(self.issue)"
|
27
|
+
|
28
|
+
# send message to sms-bridge
|
29
|
+
socket = TCPSocket.new Notifu::CONFIG[:actors][:gammu_sms_bridge][:host], Notifu::CONFIG[:actors][:gammu_sms_bridge][port]
|
30
|
+
socket.send contact.cell.to_s + "--" + message
|
31
|
+
socket.close
|
32
|
+
socket = nil
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
def template
|
37
|
+
"<%= data[:status] %> [<%= data[:host] %>/<%= data[:service] %>]: (<%= data[:message] %>) <%= data[:duration] %> [<%= data[:notifu_id] %>]"
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
File without changes
|
@@ -0,0 +1,29 @@
|
|
1
|
+
module Notifu
|
2
|
+
module Actors
|
3
|
+
class SlackChan < Notifu::Actor
|
4
|
+
|
5
|
+
require 'excon'
|
6
|
+
require 'erb'
|
7
|
+
|
8
|
+
self.name = "slack_msg"
|
9
|
+
self.desc = "Sends message to a Slack contact"
|
10
|
+
self.retry = 3
|
11
|
+
|
12
|
+
|
13
|
+
def act
|
14
|
+
cfg = Notifu::CONFIG[:actors][:slack]
|
15
|
+
contacts = self.contacts.map { |contact| contact.cell }
|
16
|
+
req_string = Notifu::CONFIG[:actors][:twilio_call][:api] +
|
17
|
+
"?token=" + Notifu::CONFIG[:actors][:twilio_call][:token] +
|
18
|
+
"&status=" + self.issue.code.to_state +
|
19
|
+
"&hostname=" + self.issue.host +
|
20
|
+
"&service=" + self.issue.service +
|
21
|
+
"&description=" + ERB::Util.url_encode(self.issue.message.to_s) +
|
22
|
+
"&call_group=" + ERB::Util.url_encode(contacts.to_json) +
|
23
|
+
"&init=1"
|
24
|
+
Excon.get req_string if self.issue.code.to_i == 2
|
25
|
+
end
|
26
|
+
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
module Notifu
|
2
|
+
module Actors
|
3
|
+
class SlackMsg < Notifu::Actor
|
4
|
+
|
5
|
+
require 'excon'
|
6
|
+
require 'erb'
|
7
|
+
|
8
|
+
self.name = "slack_msg"
|
9
|
+
self.desc = "Sends message to a Slack contact"
|
10
|
+
self.retry = 3
|
11
|
+
|
12
|
+
|
13
|
+
def act
|
14
|
+
cfg = Notifu::CONFIG[:actors][:slack]
|
15
|
+
contacts = self.contacts.map { |contact| contact.cell }
|
16
|
+
req_string = Notifu::CONFIG[:actors][:twilio_call][:api] +
|
17
|
+
"?token=" + Notifu::CONFIG[:actors][:twilio_call][:token] +
|
18
|
+
"&status=" + self.issue.code.to_state +
|
19
|
+
"&hostname=" + self.issue.host +
|
20
|
+
"&service=" + self.issue.service +
|
21
|
+
"&description=" + ERB::Util.url_encode(self.issue.message.to_s) +
|
22
|
+
"&call_group=" + ERB::Util.url_encode(contacts.to_json) +
|
23
|
+
"&init=1"
|
24
|
+
Excon.get req_string if self.issue.code.to_i == 2
|
25
|
+
end
|
26
|
+
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
@@ -0,0 +1,73 @@
|
|
1
|
+
module Notifu
|
2
|
+
module Actors
|
3
|
+
class Smtp < Notifu::Actor
|
4
|
+
|
5
|
+
require 'ostruct'
|
6
|
+
require 'mail'
|
7
|
+
|
8
|
+
self.name = "smtp"
|
9
|
+
self.desc = "SMTP notifier"
|
10
|
+
self.retry = 2
|
11
|
+
|
12
|
+
def act
|
13
|
+
data = OpenStruct.new({
|
14
|
+
notifu_id: self.issue.notifu_id,
|
15
|
+
host: self.issue.host,
|
16
|
+
message: self.issue.message,
|
17
|
+
service: self.issue.service,
|
18
|
+
status: self.issue.code.to_state,
|
19
|
+
first_event: Time.at(self.issue.time_created.to_i),
|
20
|
+
duration: (Time.now.to_i - self.issue.time_created.to_i).duration,
|
21
|
+
occurrences_count: self.issue.occurrences_count,
|
22
|
+
occurrences_trigger: self.issue.occurrences_trigger
|
23
|
+
})
|
24
|
+
contacts = self.contacts.map { |contact| "#{contact.full_name} <#{contact.mail}>"}
|
25
|
+
text_message = ERB.new(self.text_template).result(data.instance_eval {binding})
|
26
|
+
html_message = ERB.new(self.html_template).result(data.instance_eval {binding})
|
27
|
+
mail = Mail.new do
|
28
|
+
from Notifu::CONFIG[:actors][:smtp][:from]
|
29
|
+
subject "#{data[:status]}/#{data[:host]}/#{data[:service]}"
|
30
|
+
to contacts
|
31
|
+
text_part do
|
32
|
+
body text_message
|
33
|
+
end
|
34
|
+
html_part do
|
35
|
+
content_type 'text/html; charset=UTF-8'
|
36
|
+
body html_message
|
37
|
+
end
|
38
|
+
end
|
39
|
+
mail.delivery_method :sendmail
|
40
|
+
mail.deliver
|
41
|
+
end
|
42
|
+
|
43
|
+
def text_template
|
44
|
+
%{
|
45
|
+
<%= data[:message] %>
|
46
|
+
|
47
|
+
Notifu ID: <%= data[:notifu_id] %>
|
48
|
+
Host: <%= data[:host] %>
|
49
|
+
Service: <%= data[:service] %>
|
50
|
+
Status: <%= data[:status] %>
|
51
|
+
First event: <%= Time.at(data[:first_event]).to_s %>
|
52
|
+
Duration: <%= data[:duration] %>
|
53
|
+
Occurences: <%= data[:occurrences_count] %>/<%= data[:occurrences_trigger] %> (occured/trigger)
|
54
|
+
}
|
55
|
+
end
|
56
|
+
|
57
|
+
def html_template
|
58
|
+
%{
|
59
|
+
<h3><%= data[:message] %></h3><br/>
|
60
|
+
|
61
|
+
<strong>Notifu ID: </strong><%= data[:notifu_id] %><br/>
|
62
|
+
<strong>Host: </strong><%= data[:host] %><br/>
|
63
|
+
<strong>Service: </strong><%= data[:service] %><br/>
|
64
|
+
<strong>Status: </strong><%= data[:status] %><br/>
|
65
|
+
<strong>First event: </strong><%= Time.at(data[:first_event]).to_s %><br/>
|
66
|
+
<strong>Duration: </strong><%= data[:duration] %><br/>
|
67
|
+
<strong>Occurences: </strong><%= data[:occurrences_count] %>/<%= data[:occurrences_trigger] %> (occured/trigger)<br/>
|
68
|
+
}
|
69
|
+
end
|
70
|
+
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
@@ -0,0 +1,16 @@
|
|
1
|
+
module Notifu
|
2
|
+
module Actors
|
3
|
+
class Stdout < Notifu::Actor
|
4
|
+
|
5
|
+
self.name = "stdout"
|
6
|
+
self.desc = "STDOUT notifier, useful for debug only"
|
7
|
+
self.retry = 0
|
8
|
+
|
9
|
+
def act
|
10
|
+
puts self.issue.to_yaml
|
11
|
+
puts self.contacts.to_yaml
|
12
|
+
end
|
13
|
+
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
module Notifu
|
2
|
+
module Actors
|
3
|
+
class TwilioCall < Notifu::Actor
|
4
|
+
|
5
|
+
require 'excon'
|
6
|
+
require 'erb'
|
7
|
+
|
8
|
+
self.name = "twilio_call"
|
9
|
+
self.desc = "POST requst to trigger phone-call"
|
10
|
+
self.retry = 2
|
11
|
+
|
12
|
+
def act
|
13
|
+
contacts = self.contacts.map { |contact| contact.cell }
|
14
|
+
req_string = Notifu::CONFIG[:actors][:twilio_call][:api] +
|
15
|
+
"?token=" + Notifu::CONFIG[:actors][:twilio_call][:token] +
|
16
|
+
"&status=" + self.issue.code.to_state +
|
17
|
+
"&hostname=" + self.issue.host +
|
18
|
+
"&service=" + self.issue.service +
|
19
|
+
"&description=" + ERB::Util.url_encode(self.issue.message.to_s) +
|
20
|
+
"&call_group=" + ERB::Util.url_encode(contacts.to_json) +
|
21
|
+
"&init=1"
|
22
|
+
Excon.get req_string if self.issue.code.to_i == 2
|
23
|
+
end
|
24
|
+
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
data/lib/notifu/cli.rb
ADDED
@@ -0,0 +1,13 @@
|
|
1
|
+
module Notifu
|
2
|
+
module CLI
|
3
|
+
class Root < Thor
|
4
|
+
package_name "Notifu"
|
5
|
+
|
6
|
+
desc "object SUBCOMMAND", "Runtime configuration object management (SLAs, contacts & groups)"
|
7
|
+
subcommand "object", Object
|
8
|
+
|
9
|
+
desc "service SUBCOMMAND", "Notifu service processes"
|
10
|
+
subcommand "service", Service
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
@@ -0,0 +1,37 @@
|
|
1
|
+
module Notifu
|
2
|
+
module CLI
|
3
|
+
class Object < Thor
|
4
|
+
package_name "Notifu object configuration"
|
5
|
+
##
|
6
|
+
# DB Sync task
|
7
|
+
#
|
8
|
+
desc "sync", "Syncs locally defined config objects with DB"
|
9
|
+
def sync
|
10
|
+
puts "Syncing data with Redis..."
|
11
|
+
Notifu::Config.new.ohm_init
|
12
|
+
puts "...done"
|
13
|
+
end
|
14
|
+
|
15
|
+
##
|
16
|
+
# Configure SLA objects
|
17
|
+
#
|
18
|
+
desc "sla ACTION", "Manage SLAs"
|
19
|
+
def sla(name)
|
20
|
+
end
|
21
|
+
|
22
|
+
##
|
23
|
+
# Configure contact objects
|
24
|
+
#
|
25
|
+
desc "contact ACTION", "Manage contacts"
|
26
|
+
def contact(name)
|
27
|
+
end
|
28
|
+
|
29
|
+
##
|
30
|
+
# Configure group objects
|
31
|
+
#
|
32
|
+
desc "group ACTION", "Manage groups"
|
33
|
+
def group(name)
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
@@ -0,0 +1,53 @@
|
|
1
|
+
module Notifu
|
2
|
+
module CLI
|
3
|
+
class Service < Thor
|
4
|
+
package_name "Notifu service"
|
5
|
+
##
|
6
|
+
# API Service
|
7
|
+
#
|
8
|
+
desc "api", "Starts Notifu API"
|
9
|
+
def api
|
10
|
+
# Notifu::API.start
|
11
|
+
end
|
12
|
+
|
13
|
+
##
|
14
|
+
# Processor Service
|
15
|
+
#
|
16
|
+
desc "processor", "Starts Notifu processor"
|
17
|
+
option :concurrency, :type => :numeric, :default => 2, :aliases => :c
|
18
|
+
def processor
|
19
|
+
Process.setproctitle "notifu-processor"
|
20
|
+
puts "Starting #{options[:concurrency].to_s} processor(s)"
|
21
|
+
puts( $sidekiq_bin + " -c " + options[:concurrency].to_s + " -r " + $basepath + "workers/processor.rb -q processor" )
|
22
|
+
system( $sidekiq_bin + " -c " + options[:concurrency].to_s + " -r " + $basepath + "workers/processor.rb -q processor" )
|
23
|
+
end
|
24
|
+
|
25
|
+
##
|
26
|
+
# Actor Service
|
27
|
+
#
|
28
|
+
desc "actor", "Starts Notifu actor"
|
29
|
+
option :actor, :type => :string, :default => nil, :aliases => :a
|
30
|
+
option :concurrency, :type => :numeric, :default => 1, :aliases => :c
|
31
|
+
def actor
|
32
|
+
if ! options[:actor]
|
33
|
+
puts "No actor name specified (-a <actor_name>)! Available actors:"
|
34
|
+
Dir[$actorpath + "*.rb"].each do |name|
|
35
|
+
puts name.gsub(/.*\/([a-zA-Z0-9_]+)\.rb/, " \\1")
|
36
|
+
end
|
37
|
+
else
|
38
|
+
if File.exists?($actorpath + options[:actor] + ".rb") then
|
39
|
+
Process.setproctitle "notifu-actor [#{options[:actor]}]"
|
40
|
+
puts "Starting #{options[:concurrency].to_s} '#{options[:actor]}' actor(s)"
|
41
|
+
puts( $sidekiq_bin + " exec sidekiq -c " + options[:concurrency].to_s + " -r " + $basepath + "workers/actor.rb -q actor-" + options[:actor])
|
42
|
+
system( $sidekiq_bin + " exec sidekiq -c " + options[:concurrency].to_s + " -r " + $basepath + "workers/actor.rb -q actor-" + options[:actor])
|
43
|
+
exit 0
|
44
|
+
else
|
45
|
+
STDERR.puts "Actor '#{options[:actor]}' does not exist"
|
46
|
+
exit 1
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
@@ -0,0 +1,120 @@
|
|
1
|
+
module Notifu
|
2
|
+
class Config
|
3
|
+
|
4
|
+
attr_reader :config
|
5
|
+
|
6
|
+
@@config_path = "/etc/notifu/"
|
7
|
+
|
8
|
+
def initialize
|
9
|
+
begin
|
10
|
+
@config = YAML.load_file(@@config_path + 'notifu.yaml')
|
11
|
+
rescue
|
12
|
+
raise "Failed to load main config file!"
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
def get
|
17
|
+
@get ||= self.config.deep_symbolize_keys
|
18
|
+
end
|
19
|
+
|
20
|
+
def ohm_init
|
21
|
+
contacts_init
|
22
|
+
slas_init
|
23
|
+
groups_init
|
24
|
+
end
|
25
|
+
|
26
|
+
def contacts_init
|
27
|
+
Dir[@@config_path + 'contacts/*.yaml'].each do |path|
|
28
|
+
cfg = YAML.load_file(path).deep_symbolize_keys
|
29
|
+
begin
|
30
|
+
Notifu::Model::Contact.with(:name, cfg[:name]).update(cfg)
|
31
|
+
puts "Updated contact '#{cfg[:name]}'."
|
32
|
+
rescue
|
33
|
+
Notifu::Model::Contact.create(cfg).save
|
34
|
+
puts "Created contact '#{cfg[:name]}'."
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
def slas_init
|
40
|
+
Dir[@@config_path + 'slas/*.yaml'].each do |path|
|
41
|
+
cfg = YAML.load_file(path).deep_symbolize_keys
|
42
|
+
begin
|
43
|
+
Notifu::Model::Sla.with(:name, cfg[:name]).update(cfg)
|
44
|
+
puts "Updated SLA '#{cfg[:name]}'."
|
45
|
+
rescue
|
46
|
+
Notifu::Model::Sla.create(cfg).save
|
47
|
+
puts "Created SLA '#{cfg[:name]}'."
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
def groups_init
|
53
|
+
Dir[@@config_path + 'groups/*.yaml'].each do |path|
|
54
|
+
cfg = YAML.load_file(path).deep_symbolize_keys
|
55
|
+
begin
|
56
|
+
group = Notifu::Model::Group.create(name: cfg[:name])
|
57
|
+
puts "Created group '#{cfg[:name]}'."
|
58
|
+
rescue
|
59
|
+
group = Notifu::Model::Group.with(:name, cfg[:name])
|
60
|
+
puts "Found group '#{cfg[:name]}'."
|
61
|
+
end
|
62
|
+
|
63
|
+
contacts = Array.new
|
64
|
+
|
65
|
+
cfg[:primary].each do |contact_id|
|
66
|
+
begin
|
67
|
+
contacts << Notifu::Model::Contact.with(:name, contact_id)
|
68
|
+
puts "Contact '#{contact_id}' accepted as primary."
|
69
|
+
rescue
|
70
|
+
puts "Failed to load primary contact '#{contact_id}'."
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
group.primary.replace(contacts)
|
75
|
+
group.save
|
76
|
+
puts "Primary contacts for group '#{cfg[:name]}' updated."
|
77
|
+
|
78
|
+
if cfg[:secondary].is_a? Array
|
79
|
+
contacts = Array.new
|
80
|
+
|
81
|
+
cfg[:secondary].each do |contact_id|
|
82
|
+
begin
|
83
|
+
contacts << Notifu::Model::Contact.with(:name, contact_id)
|
84
|
+
puts "Contact '#{contact_id}' accepted as secondary."
|
85
|
+
rescue
|
86
|
+
puts "Failed to load secondary contact '#{contact_id}'."
|
87
|
+
exit 1
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
group.secondary.replace(contacts)
|
92
|
+
group.save
|
93
|
+
puts "Secondary contacts for group '#{cfg[:name]}' updated."
|
94
|
+
|
95
|
+
if cfg[:tertiary].is_a? Array
|
96
|
+
contacts = Array.new
|
97
|
+
|
98
|
+
cfg[:tertiary].each do |contact_id|
|
99
|
+
begin
|
100
|
+
contacts << Notifu::Model::Contact.with(:name, contact_id)
|
101
|
+
puts "Contact '#{contact_id}' accepted as tertiary."
|
102
|
+
rescue
|
103
|
+
puts "Failed to load tertiary contact '#{contact_id}'."
|
104
|
+
exit 1
|
105
|
+
end
|
106
|
+
end
|
107
|
+
|
108
|
+
group.tertiary.replace(contacts)
|
109
|
+
group.save
|
110
|
+
puts "Tertiary contacts for group '#{cfg[:name]}' updated."
|
111
|
+
else
|
112
|
+
group.tertiary.replace([])
|
113
|
+
end
|
114
|
+
else
|
115
|
+
group.secondary.replace([])
|
116
|
+
end
|
117
|
+
end
|
118
|
+
end
|
119
|
+
end
|
120
|
+
end
|