notifu 1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/bin/notifu +15 -0
- data/lib/notifu.rb +12 -0
- data/lib/notifu/actors/gammu_sms_bridge.rb +41 -0
- data/lib/notifu/actors/pagerduty.rb +0 -0
- data/lib/notifu/actors/slack_chan.rb +29 -0
- data/lib/notifu/actors/slack_msg.rb +29 -0
- data/lib/notifu/actors/smtp.rb +73 -0
- data/lib/notifu/actors/stdout.rb +16 -0
- data/lib/notifu/actors/twilio_call.rb +27 -0
- data/lib/notifu/cli.rb +13 -0
- data/lib/notifu/cli/object.rb +37 -0
- data/lib/notifu/cli/service.rb +53 -0
- data/lib/notifu/config.rb +120 -0
- data/lib/notifu/logger.rb +83 -0
- data/lib/notifu/mixins.rb +49 -0
- data/lib/notifu/model.rb +5 -0
- data/lib/notifu/model/contact.rb +15 -0
- data/lib/notifu/model/event.rb +52 -0
- data/lib/notifu/model/group.rb +14 -0
- data/lib/notifu/model/issue.rb +51 -0
- data/lib/notifu/model/sla.rb +20 -0
- data/lib/notifu/sensu/handler.rb +105 -0
- data/lib/notifu/util.rb +9 -0
- data/lib/notifu/workers/actor.rb +60 -0
- data/lib/notifu/workers/processor.rb +444 -0
- data/lib/notifu/workers/sidekiq_init.rb +24 -0
- metadata +90 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: e84a8d91a613dee08b4ebfa63f7c05082c4230d8
|
4
|
+
data.tar.gz: c0bf2890e4e519adc62b6f76b553fb706451146c
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 9ea47dec858e01b1dcea6586a29a491a0fc5fd0cb98731f0308bd010fe51eae593b4f3f092d454b2ef0d02336b495eea67197c58034bc73ada6751fe2f77bffd
|
7
|
+
data.tar.gz: b212ae242588fea5fc7b9ff199fa0776337b34ab0c5ca3afe24498eb967ae972b7076c4476122debbdf7f653b27111687a01329d8e4953a993d463752450a552
|
data/bin/notifu
ADDED
@@ -0,0 +1,15 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
path = File.symlink?(__FILE__) ? File.readlink(__FILE__) : __FILE__
|
4
|
+
$basepath = '/' + File.dirname(path).split('/')[1..-2].join('/') + '/lib/notifu/'
|
5
|
+
$actorpath = $basepath + "actors/"
|
6
|
+
# $sidekiq_bin = $basepath.sub(/app\//, '') + ".rvm/wrappers/notifu/sidekiq"
|
7
|
+
$sidekiq_bin = "sidekiq"
|
8
|
+
|
9
|
+
require 'notifu'
|
10
|
+
|
11
|
+
Notifu::CONFIG = Notifu::Config.new.get
|
12
|
+
|
13
|
+
Ohm.redis = Redic.new Notifu::CONFIG[:redis_data]
|
14
|
+
|
15
|
+
Notifu::CLI::Root.start(ARGV)
|
data/lib/notifu.rb
ADDED
@@ -0,0 +1,12 @@
|
|
1
|
+
require 'yaml'
|
2
|
+
require 'json'
|
3
|
+
require 'thor'
|
4
|
+
require 'ohm'
|
5
|
+
require "notifu/mixins"
|
6
|
+
require "notifu/util"
|
7
|
+
require "notifu/config"
|
8
|
+
require "notifu/logger"
|
9
|
+
require "notifu/model"
|
10
|
+
require "notifu/cli/service"
|
11
|
+
require "notifu/cli/object"
|
12
|
+
require "notifu/cli"
|
@@ -0,0 +1,41 @@
|
|
1
|
+
module Notifu
|
2
|
+
module Actors
|
3
|
+
class GammuSmsBridge < Notifu::Actor
|
4
|
+
|
5
|
+
self.name = "gammu_sms_bridge"
|
6
|
+
self.desc = "Old-school NetCat-like SMS bridge to Gammu"
|
7
|
+
self.retry = 3
|
8
|
+
|
9
|
+
def act
|
10
|
+
data = OpenStruct.new({
|
11
|
+
notifu_id: self.issue.notifu_id,
|
12
|
+
host: self.issue.host,
|
13
|
+
message: self.issue.message,
|
14
|
+
service: self.issue.service,
|
15
|
+
status: self.issue.code.to_state,
|
16
|
+
first_event: Time.at(self.issue.time_created.to_i),
|
17
|
+
duration: (Time.now.to_i - self.issue.time_created.to_i).duration,
|
18
|
+
occurrences_count: self.issue.occurrences_count,
|
19
|
+
occurrences_trigger: self.issue.occurrences_trigger
|
20
|
+
})
|
21
|
+
message = ERB.new(self.template).result(data.instance_eval {binding})
|
22
|
+
|
23
|
+
self.contacts.each do |contact|
|
24
|
+
cell = contact.cell
|
25
|
+
template = ERB.new File.new("sms.erb").read, nil, "%"
|
26
|
+
message = "template.result(self.issue)"
|
27
|
+
|
28
|
+
# send message to sms-bridge
|
29
|
+
socket = TCPSocket.new Notifu::CONFIG[:actors][:gammu_sms_bridge][:host], Notifu::CONFIG[:actors][:gammu_sms_bridge][port]
|
30
|
+
socket.send contact.cell.to_s + "--" + message
|
31
|
+
socket.close
|
32
|
+
socket = nil
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
def template
|
37
|
+
"<%= data[:status] %> [<%= data[:host] %>/<%= data[:service] %>]: (<%= data[:message] %>) <%= data[:duration] %> [<%= data[:notifu_id] %>]"
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
File without changes
|
@@ -0,0 +1,29 @@
|
|
1
|
+
module Notifu
|
2
|
+
module Actors
|
3
|
+
class SlackChan < Notifu::Actor
|
4
|
+
|
5
|
+
require 'excon'
|
6
|
+
require 'erb'
|
7
|
+
|
8
|
+
self.name = "slack_msg"
|
9
|
+
self.desc = "Sends message to a Slack contact"
|
10
|
+
self.retry = 3
|
11
|
+
|
12
|
+
|
13
|
+
def act
|
14
|
+
cfg = Notifu::CONFIG[:actors][:slack]
|
15
|
+
contacts = self.contacts.map { |contact| contact.cell }
|
16
|
+
req_string = Notifu::CONFIG[:actors][:twilio_call][:api] +
|
17
|
+
"?token=" + Notifu::CONFIG[:actors][:twilio_call][:token] +
|
18
|
+
"&status=" + self.issue.code.to_state +
|
19
|
+
"&hostname=" + self.issue.host +
|
20
|
+
"&service=" + self.issue.service +
|
21
|
+
"&description=" + ERB::Util.url_encode(self.issue.message.to_s) +
|
22
|
+
"&call_group=" + ERB::Util.url_encode(contacts.to_json) +
|
23
|
+
"&init=1"
|
24
|
+
Excon.get req_string if self.issue.code.to_i == 2
|
25
|
+
end
|
26
|
+
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
module Notifu
|
2
|
+
module Actors
|
3
|
+
class SlackMsg < Notifu::Actor
|
4
|
+
|
5
|
+
require 'excon'
|
6
|
+
require 'erb'
|
7
|
+
|
8
|
+
self.name = "slack_msg"
|
9
|
+
self.desc = "Sends message to a Slack contact"
|
10
|
+
self.retry = 3
|
11
|
+
|
12
|
+
|
13
|
+
def act
|
14
|
+
cfg = Notifu::CONFIG[:actors][:slack]
|
15
|
+
contacts = self.contacts.map { |contact| contact.cell }
|
16
|
+
req_string = Notifu::CONFIG[:actors][:twilio_call][:api] +
|
17
|
+
"?token=" + Notifu::CONFIG[:actors][:twilio_call][:token] +
|
18
|
+
"&status=" + self.issue.code.to_state +
|
19
|
+
"&hostname=" + self.issue.host +
|
20
|
+
"&service=" + self.issue.service +
|
21
|
+
"&description=" + ERB::Util.url_encode(self.issue.message.to_s) +
|
22
|
+
"&call_group=" + ERB::Util.url_encode(contacts.to_json) +
|
23
|
+
"&init=1"
|
24
|
+
Excon.get req_string if self.issue.code.to_i == 2
|
25
|
+
end
|
26
|
+
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
@@ -0,0 +1,73 @@
|
|
1
|
+
module Notifu
|
2
|
+
module Actors
|
3
|
+
class Smtp < Notifu::Actor
|
4
|
+
|
5
|
+
require 'ostruct'
|
6
|
+
require 'mail'
|
7
|
+
|
8
|
+
self.name = "smtp"
|
9
|
+
self.desc = "SMTP notifier"
|
10
|
+
self.retry = 2
|
11
|
+
|
12
|
+
def act
|
13
|
+
data = OpenStruct.new({
|
14
|
+
notifu_id: self.issue.notifu_id,
|
15
|
+
host: self.issue.host,
|
16
|
+
message: self.issue.message,
|
17
|
+
service: self.issue.service,
|
18
|
+
status: self.issue.code.to_state,
|
19
|
+
first_event: Time.at(self.issue.time_created.to_i),
|
20
|
+
duration: (Time.now.to_i - self.issue.time_created.to_i).duration,
|
21
|
+
occurrences_count: self.issue.occurrences_count,
|
22
|
+
occurrences_trigger: self.issue.occurrences_trigger
|
23
|
+
})
|
24
|
+
contacts = self.contacts.map { |contact| "#{contact.full_name} <#{contact.mail}>"}
|
25
|
+
text_message = ERB.new(self.text_template).result(data.instance_eval {binding})
|
26
|
+
html_message = ERB.new(self.html_template).result(data.instance_eval {binding})
|
27
|
+
mail = Mail.new do
|
28
|
+
from Notifu::CONFIG[:actors][:smtp][:from]
|
29
|
+
subject "#{data[:status]}/#{data[:host]}/#{data[:service]}"
|
30
|
+
to contacts
|
31
|
+
text_part do
|
32
|
+
body text_message
|
33
|
+
end
|
34
|
+
html_part do
|
35
|
+
content_type 'text/html; charset=UTF-8'
|
36
|
+
body html_message
|
37
|
+
end
|
38
|
+
end
|
39
|
+
mail.delivery_method :sendmail
|
40
|
+
mail.deliver
|
41
|
+
end
|
42
|
+
|
43
|
+
def text_template
|
44
|
+
%{
|
45
|
+
<%= data[:message] %>
|
46
|
+
|
47
|
+
Notifu ID: <%= data[:notifu_id] %>
|
48
|
+
Host: <%= data[:host] %>
|
49
|
+
Service: <%= data[:service] %>
|
50
|
+
Status: <%= data[:status] %>
|
51
|
+
First event: <%= Time.at(data[:first_event]).to_s %>
|
52
|
+
Duration: <%= data[:duration] %>
|
53
|
+
Occurences: <%= data[:occurrences_count] %>/<%= data[:occurrences_trigger] %> (occured/trigger)
|
54
|
+
}
|
55
|
+
end
|
56
|
+
|
57
|
+
def html_template
|
58
|
+
%{
|
59
|
+
<h3><%= data[:message] %></h3><br/>
|
60
|
+
|
61
|
+
<strong>Notifu ID: </strong><%= data[:notifu_id] %><br/>
|
62
|
+
<strong>Host: </strong><%= data[:host] %><br/>
|
63
|
+
<strong>Service: </strong><%= data[:service] %><br/>
|
64
|
+
<strong>Status: </strong><%= data[:status] %><br/>
|
65
|
+
<strong>First event: </strong><%= Time.at(data[:first_event]).to_s %><br/>
|
66
|
+
<strong>Duration: </strong><%= data[:duration] %><br/>
|
67
|
+
<strong>Occurences: </strong><%= data[:occurrences_count] %>/<%= data[:occurrences_trigger] %> (occured/trigger)<br/>
|
68
|
+
}
|
69
|
+
end
|
70
|
+
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
@@ -0,0 +1,16 @@
|
|
1
|
+
module Notifu
|
2
|
+
module Actors
|
3
|
+
class Stdout < Notifu::Actor
|
4
|
+
|
5
|
+
self.name = "stdout"
|
6
|
+
self.desc = "STDOUT notifier, useful for debug only"
|
7
|
+
self.retry = 0
|
8
|
+
|
9
|
+
def act
|
10
|
+
puts self.issue.to_yaml
|
11
|
+
puts self.contacts.to_yaml
|
12
|
+
end
|
13
|
+
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
module Notifu
|
2
|
+
module Actors
|
3
|
+
class TwilioCall < Notifu::Actor
|
4
|
+
|
5
|
+
require 'excon'
|
6
|
+
require 'erb'
|
7
|
+
|
8
|
+
self.name = "twilio_call"
|
9
|
+
self.desc = "POST requst to trigger phone-call"
|
10
|
+
self.retry = 2
|
11
|
+
|
12
|
+
def act
|
13
|
+
contacts = self.contacts.map { |contact| contact.cell }
|
14
|
+
req_string = Notifu::CONFIG[:actors][:twilio_call][:api] +
|
15
|
+
"?token=" + Notifu::CONFIG[:actors][:twilio_call][:token] +
|
16
|
+
"&status=" + self.issue.code.to_state +
|
17
|
+
"&hostname=" + self.issue.host +
|
18
|
+
"&service=" + self.issue.service +
|
19
|
+
"&description=" + ERB::Util.url_encode(self.issue.message.to_s) +
|
20
|
+
"&call_group=" + ERB::Util.url_encode(contacts.to_json) +
|
21
|
+
"&init=1"
|
22
|
+
Excon.get req_string if self.issue.code.to_i == 2
|
23
|
+
end
|
24
|
+
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
data/lib/notifu/cli.rb
ADDED
@@ -0,0 +1,13 @@
|
|
1
|
+
module Notifu
|
2
|
+
module CLI
|
3
|
+
class Root < Thor
|
4
|
+
package_name "Notifu"
|
5
|
+
|
6
|
+
desc "object SUBCOMMAND", "Runtime configuration object management (SLAs, contacts & groups)"
|
7
|
+
subcommand "object", Object
|
8
|
+
|
9
|
+
desc "service SUBCOMMAND", "Notifu service processes"
|
10
|
+
subcommand "service", Service
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
@@ -0,0 +1,37 @@
|
|
1
|
+
module Notifu
|
2
|
+
module CLI
|
3
|
+
class Object < Thor
|
4
|
+
package_name "Notifu object configuration"
|
5
|
+
##
|
6
|
+
# DB Sync task
|
7
|
+
#
|
8
|
+
desc "sync", "Syncs locally defined config objects with DB"
|
9
|
+
def sync
|
10
|
+
puts "Syncing data with Redis..."
|
11
|
+
Notifu::Config.new.ohm_init
|
12
|
+
puts "...done"
|
13
|
+
end
|
14
|
+
|
15
|
+
##
|
16
|
+
# Configure SLA objects
|
17
|
+
#
|
18
|
+
desc "sla ACTION", "Manage SLAs"
|
19
|
+
def sla(name)
|
20
|
+
end
|
21
|
+
|
22
|
+
##
|
23
|
+
# Configure contact objects
|
24
|
+
#
|
25
|
+
desc "contact ACTION", "Manage contacts"
|
26
|
+
def contact(name)
|
27
|
+
end
|
28
|
+
|
29
|
+
##
|
30
|
+
# Configure group objects
|
31
|
+
#
|
32
|
+
desc "group ACTION", "Manage groups"
|
33
|
+
def group(name)
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
@@ -0,0 +1,53 @@
|
|
1
|
+
module Notifu
|
2
|
+
module CLI
|
3
|
+
class Service < Thor
|
4
|
+
package_name "Notifu service"
|
5
|
+
##
|
6
|
+
# API Service
|
7
|
+
#
|
8
|
+
desc "api", "Starts Notifu API"
|
9
|
+
def api
|
10
|
+
# Notifu::API.start
|
11
|
+
end
|
12
|
+
|
13
|
+
##
|
14
|
+
# Processor Service
|
15
|
+
#
|
16
|
+
desc "processor", "Starts Notifu processor"
|
17
|
+
option :concurrency, :type => :numeric, :default => 2, :aliases => :c
|
18
|
+
def processor
|
19
|
+
Process.setproctitle "notifu-processor"
|
20
|
+
puts "Starting #{options[:concurrency].to_s} processor(s)"
|
21
|
+
puts( $sidekiq_bin + " -c " + options[:concurrency].to_s + " -r " + $basepath + "workers/processor.rb -q processor" )
|
22
|
+
system( $sidekiq_bin + " -c " + options[:concurrency].to_s + " -r " + $basepath + "workers/processor.rb -q processor" )
|
23
|
+
end
|
24
|
+
|
25
|
+
##
|
26
|
+
# Actor Service
|
27
|
+
#
|
28
|
+
desc "actor", "Starts Notifu actor"
|
29
|
+
option :actor, :type => :string, :default => nil, :aliases => :a
|
30
|
+
option :concurrency, :type => :numeric, :default => 1, :aliases => :c
|
31
|
+
def actor
|
32
|
+
if ! options[:actor]
|
33
|
+
puts "No actor name specified (-a <actor_name>)! Available actors:"
|
34
|
+
Dir[$actorpath + "*.rb"].each do |name|
|
35
|
+
puts name.gsub(/.*\/([a-zA-Z0-9_]+)\.rb/, " \\1")
|
36
|
+
end
|
37
|
+
else
|
38
|
+
if File.exists?($actorpath + options[:actor] + ".rb") then
|
39
|
+
Process.setproctitle "notifu-actor [#{options[:actor]}]"
|
40
|
+
puts "Starting #{options[:concurrency].to_s} '#{options[:actor]}' actor(s)"
|
41
|
+
puts( $sidekiq_bin + " exec sidekiq -c " + options[:concurrency].to_s + " -r " + $basepath + "workers/actor.rb -q actor-" + options[:actor])
|
42
|
+
system( $sidekiq_bin + " exec sidekiq -c " + options[:concurrency].to_s + " -r " + $basepath + "workers/actor.rb -q actor-" + options[:actor])
|
43
|
+
exit 0
|
44
|
+
else
|
45
|
+
STDERR.puts "Actor '#{options[:actor]}' does not exist"
|
46
|
+
exit 1
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
@@ -0,0 +1,120 @@
|
|
1
|
+
module Notifu
|
2
|
+
class Config
|
3
|
+
|
4
|
+
attr_reader :config
|
5
|
+
|
6
|
+
@@config_path = "/etc/notifu/"
|
7
|
+
|
8
|
+
def initialize
|
9
|
+
begin
|
10
|
+
@config = YAML.load_file(@@config_path + 'notifu.yaml')
|
11
|
+
rescue
|
12
|
+
raise "Failed to load main config file!"
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
def get
|
17
|
+
@get ||= self.config.deep_symbolize_keys
|
18
|
+
end
|
19
|
+
|
20
|
+
def ohm_init
|
21
|
+
contacts_init
|
22
|
+
slas_init
|
23
|
+
groups_init
|
24
|
+
end
|
25
|
+
|
26
|
+
def contacts_init
|
27
|
+
Dir[@@config_path + 'contacts/*.yaml'].each do |path|
|
28
|
+
cfg = YAML.load_file(path).deep_symbolize_keys
|
29
|
+
begin
|
30
|
+
Notifu::Model::Contact.with(:name, cfg[:name]).update(cfg)
|
31
|
+
puts "Updated contact '#{cfg[:name]}'."
|
32
|
+
rescue
|
33
|
+
Notifu::Model::Contact.create(cfg).save
|
34
|
+
puts "Created contact '#{cfg[:name]}'."
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
def slas_init
|
40
|
+
Dir[@@config_path + 'slas/*.yaml'].each do |path|
|
41
|
+
cfg = YAML.load_file(path).deep_symbolize_keys
|
42
|
+
begin
|
43
|
+
Notifu::Model::Sla.with(:name, cfg[:name]).update(cfg)
|
44
|
+
puts "Updated SLA '#{cfg[:name]}'."
|
45
|
+
rescue
|
46
|
+
Notifu::Model::Sla.create(cfg).save
|
47
|
+
puts "Created SLA '#{cfg[:name]}'."
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
def groups_init
|
53
|
+
Dir[@@config_path + 'groups/*.yaml'].each do |path|
|
54
|
+
cfg = YAML.load_file(path).deep_symbolize_keys
|
55
|
+
begin
|
56
|
+
group = Notifu::Model::Group.create(name: cfg[:name])
|
57
|
+
puts "Created group '#{cfg[:name]}'."
|
58
|
+
rescue
|
59
|
+
group = Notifu::Model::Group.with(:name, cfg[:name])
|
60
|
+
puts "Found group '#{cfg[:name]}'."
|
61
|
+
end
|
62
|
+
|
63
|
+
contacts = Array.new
|
64
|
+
|
65
|
+
cfg[:primary].each do |contact_id|
|
66
|
+
begin
|
67
|
+
contacts << Notifu::Model::Contact.with(:name, contact_id)
|
68
|
+
puts "Contact '#{contact_id}' accepted as primary."
|
69
|
+
rescue
|
70
|
+
puts "Failed to load primary contact '#{contact_id}'."
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
group.primary.replace(contacts)
|
75
|
+
group.save
|
76
|
+
puts "Primary contacts for group '#{cfg[:name]}' updated."
|
77
|
+
|
78
|
+
if cfg[:secondary].is_a? Array
|
79
|
+
contacts = Array.new
|
80
|
+
|
81
|
+
cfg[:secondary].each do |contact_id|
|
82
|
+
begin
|
83
|
+
contacts << Notifu::Model::Contact.with(:name, contact_id)
|
84
|
+
puts "Contact '#{contact_id}' accepted as secondary."
|
85
|
+
rescue
|
86
|
+
puts "Failed to load secondary contact '#{contact_id}'."
|
87
|
+
exit 1
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
group.secondary.replace(contacts)
|
92
|
+
group.save
|
93
|
+
puts "Secondary contacts for group '#{cfg[:name]}' updated."
|
94
|
+
|
95
|
+
if cfg[:tertiary].is_a? Array
|
96
|
+
contacts = Array.new
|
97
|
+
|
98
|
+
cfg[:tertiary].each do |contact_id|
|
99
|
+
begin
|
100
|
+
contacts << Notifu::Model::Contact.with(:name, contact_id)
|
101
|
+
puts "Contact '#{contact_id}' accepted as tertiary."
|
102
|
+
rescue
|
103
|
+
puts "Failed to load tertiary contact '#{contact_id}'."
|
104
|
+
exit 1
|
105
|
+
end
|
106
|
+
end
|
107
|
+
|
108
|
+
group.tertiary.replace(contacts)
|
109
|
+
group.save
|
110
|
+
puts "Tertiary contacts for group '#{cfg[:name]}' updated."
|
111
|
+
else
|
112
|
+
group.tertiary.replace([])
|
113
|
+
end
|
114
|
+
else
|
115
|
+
group.secondary.replace([])
|
116
|
+
end
|
117
|
+
end
|
118
|
+
end
|
119
|
+
end
|
120
|
+
end
|