torpedo 1.0.19 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/CHANGELOG +9 -0
- data/LICENSE.txt +2 -1
- data/README.md +3 -3
- data/Rakefile +6 -5
- data/VERSION +1 -1
- data/bin/torpedo +0 -1
- data/lib/torpedo.rb +131 -22
- data/lib/torpedo/cleanup.rb +74 -0
- data/lib/torpedo/compute/flavors.rb +22 -22
- data/lib/torpedo/compute/helper.rb +79 -85
- data/lib/torpedo/compute/images.rb +23 -23
- data/lib/torpedo/compute/keypairs.rb +36 -0
- data/lib/torpedo/compute/limits.rb +15 -10
- data/lib/torpedo/compute/servers.rb +458 -382
- data/lib/torpedo/metering/helper.rb +43 -0
- data/lib/torpedo/metering/meters.rb +67 -0
- data/lib/torpedo/net_util.rb +66 -0
- data/lib/torpedo/orchestration/helper.rb +43 -0
- data/lib/torpedo/orchestration/stacks.rb +117 -0
- data/lib/torpedo/orchestration/test_server.hot +31 -0
- data/lib/torpedo/volume/helper.rb +43 -0
- data/lib/torpedo/volume/volumes.rb +114 -0
- data/torpedo.gemspec +30 -19
- metadata +44 -18
@@ -0,0 +1,43 @@
|
|
1
|
+
if RUBY_VERSION =~ /^1.9.*/ then
|
2
|
+
gem 'test-unit'
|
3
|
+
end
|
4
|
+
require 'test/unit'
|
5
|
+
if FOG_VERSION
|
6
|
+
gem 'fog', FOG_VERSION
|
7
|
+
end
|
8
|
+
require 'fog'
|
9
|
+
|
10
|
+
module Torpedo
|
11
|
+
module Metering
|
12
|
+
module Helper
|
13
|
+
|
14
|
+
def self.get_connection
|
15
|
+
|
16
|
+
if ENV['DEBUG'] and ENV['DEBUG'] == 'true' then
|
17
|
+
ENV['EXCON_DEBUG'] = 'true'
|
18
|
+
end
|
19
|
+
|
20
|
+
auth_url = ENV['OS_AUTH_URL']
|
21
|
+
api_key = ENV['OS_PASSWORD']
|
22
|
+
username = ENV['OS_USERNAME']
|
23
|
+
authtenant = ENV['OS_TENANT_NAME']
|
24
|
+
#region = ENV['OS_AUTH_REGION']
|
25
|
+
service_type = ENV['CEILOMETER_SERVICE_TYPE'] || "metering"
|
26
|
+
service_name = ENV['CEILOMETER_SERVICE_NAME'] #nil by default
|
27
|
+
|
28
|
+
Fog::Metering.new(
|
29
|
+
:provider => :openstack,
|
30
|
+
:openstack_auth_url => auth_url+'/tokens',
|
31
|
+
:openstack_username => username,
|
32
|
+
:openstack_tenant => authtenant,
|
33
|
+
:openstack_api_key => api_key,
|
34
|
+
#:openstack_region => region,
|
35
|
+
:openstack_service_type => service_type,
|
36
|
+
:openstack_service_name => service_name
|
37
|
+
)
|
38
|
+
|
39
|
+
end
|
40
|
+
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
@@ -0,0 +1,67 @@
|
|
1
|
+
require 'torpedo/metering/helper'
|
2
|
+
require 'torpedo/compute/helper'
|
3
|
+
require 'tempfile'
|
4
|
+
require 'net/ssh'
|
5
|
+
|
6
|
+
module Torpedo
|
7
|
+
module Metering
|
8
|
+
class Meters < Test::Unit::TestCase
|
9
|
+
|
10
|
+
def setup
|
11
|
+
@conn=Helper::get_connection
|
12
|
+
@compute_conn=Torpedo::Compute::Helper::get_connection
|
13
|
+
end
|
14
|
+
|
15
|
+
def wait_sample_ready(sample_name, resource_id)
|
16
|
+
begin
|
17
|
+
|
18
|
+
timeout(METERING_SAMPLE_TIMEOUT) do
|
19
|
+
|
20
|
+
sample_count = 0
|
21
|
+
until sample_count > 0 do
|
22
|
+
@conn.get_samples(sample_name).body.each do |sample|
|
23
|
+
if sample['resource_id'] == resource_id then
|
24
|
+
sample_count += 1
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
end
|
30
|
+
|
31
|
+
rescue Timeout::Error => te
|
32
|
+
fail('Timeout waiting for metering sample data.')
|
33
|
+
end
|
34
|
+
|
35
|
+
end
|
36
|
+
|
37
|
+
def test_001_check_meters
|
38
|
+
@conn.list_meters.body.each do |meter|
|
39
|
+
assert_not_nil meter['name']
|
40
|
+
assert_not_nil meter['user_id']
|
41
|
+
assert_not_nil meter['resource_id']
|
42
|
+
assert_not_nil meter['project_id']
|
43
|
+
assert_not_nil meter['type']
|
44
|
+
assert_not_nil meter['unit']
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
def test_002_check_compute_memory_samples
|
49
|
+
|
50
|
+
server = Torpedo::Compute::Servers.server
|
51
|
+
server_flavor = server.flavor_ref || server.flavor['id']
|
52
|
+
flavor = @compute_conn.flavors.get(server_flavor)
|
53
|
+
|
54
|
+
wait_sample_ready('memory', server.id)
|
55
|
+
|
56
|
+
@conn.get_samples('memory').body.each do |sample|
|
57
|
+
if sample['resource_id'] == server.id then
|
58
|
+
# convert to a float so they match
|
59
|
+
assert_equal flavor.ram.to_f.to_s, sample['counter_volume'].to_s
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
end
|
64
|
+
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
@@ -0,0 +1,66 @@
|
|
1
|
+
require 'net/ssh'
|
2
|
+
|
3
|
+
module Torpedo
|
4
|
+
class NetUtil < Test::Unit::TestCase
|
5
|
+
|
6
|
+
def self.ssh_test(ip_addr, network_namespace, test_cmd, test_output, admin_pass)
|
7
|
+
|
8
|
+
if network_namespace then
|
9
|
+
out=%x{ip netns exec #{network_namespace} torpedo ssh --ip-address=#{ip_addr} --test-command='#{test_cmd}' --test-output='#{test_output}' --admin-password='#{admin_pass}'}
|
10
|
+
retval=$?
|
11
|
+
if retval.success? then
|
12
|
+
return true
|
13
|
+
else
|
14
|
+
puts out
|
15
|
+
return false
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
ssh_opts = {:paranoid => false}
|
20
|
+
if TEST_ADMIN_PASSWORD then
|
21
|
+
ssh_opts.store(:password, admin_pass)
|
22
|
+
else
|
23
|
+
ssh_identity=SSH_PRIVATE_KEY
|
24
|
+
ssh_opts.store(:keys, ssh_identity)
|
25
|
+
end
|
26
|
+
|
27
|
+
begin
|
28
|
+
Timeout::timeout(SSH_TIMEOUT) do
|
29
|
+
while(1) do
|
30
|
+
begin
|
31
|
+
Net::SSH.start(ip_addr, 'root', ssh_opts) do |ssh|
|
32
|
+
return ssh.exec!(test_cmd) == test_output
|
33
|
+
end
|
34
|
+
rescue Errno::ECONNREFUSED, Errno::EHOSTUNREACH, Errno::ECONNRESET, Net::SSH::Exception
|
35
|
+
next
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
rescue Timeout::Error => te
|
40
|
+
fail("Timeout trying to ssh to server: #{ip_addr}")
|
41
|
+
end
|
42
|
+
|
43
|
+
return false
|
44
|
+
|
45
|
+
end
|
46
|
+
|
47
|
+
def self.ping_test(ip_addr, network_namespace=nil)
|
48
|
+
begin
|
49
|
+
namespace_cmd = network_namespace.nil? ? "" : "ip netns exec #{network_namespace} "
|
50
|
+
ping = TEST_IP_TYPE == 6 ? 'ping6' : 'ping'
|
51
|
+
ping_command = "#{namespace_cmd}#{ping} -c 1 #{ip_addr} > /dev/null 2>&1"
|
52
|
+
Timeout::timeout(PING_TIMEOUT) do
|
53
|
+
while(1) do
|
54
|
+
return true if system(ping_command)
|
55
|
+
end
|
56
|
+
end
|
57
|
+
rescue Timeout::Error => te
|
58
|
+
fail("Timeout pinging server: #{ping_command}")
|
59
|
+
end
|
60
|
+
|
61
|
+
return false
|
62
|
+
|
63
|
+
end
|
64
|
+
|
65
|
+
end
|
66
|
+
end
|
@@ -0,0 +1,43 @@
|
|
1
|
+
if RUBY_VERSION =~ /^1.9.*/ then
|
2
|
+
gem 'test-unit'
|
3
|
+
end
|
4
|
+
require 'test/unit'
|
5
|
+
if FOG_VERSION
|
6
|
+
gem 'fog', FOG_VERSION
|
7
|
+
end
|
8
|
+
require 'fog'
|
9
|
+
|
10
|
+
module Torpedo
|
11
|
+
module Orchestration
|
12
|
+
module Helper
|
13
|
+
|
14
|
+
def self.get_connection
|
15
|
+
|
16
|
+
if ENV['DEBUG'] and ENV['DEBUG'] == 'true' then
|
17
|
+
ENV['EXCON_DEBUG'] = 'true'
|
18
|
+
end
|
19
|
+
|
20
|
+
auth_url = ENV['OS_AUTH_URL']
|
21
|
+
api_key = ENV['OS_PASSWORD']
|
22
|
+
username = ENV['OS_USERNAME']
|
23
|
+
authtenant = ENV['OS_TENANT_NAME']
|
24
|
+
#region = ENV['OS_AUTH_REGION']
|
25
|
+
service_type = ENV['HEAT_SERVICE_TYPE'] || "orchestration"
|
26
|
+
service_name = ENV['HEAT_SERVICE_NAME'] #nil by default
|
27
|
+
|
28
|
+
Fog::Orchestration.new(
|
29
|
+
:provider => :openstack,
|
30
|
+
:openstack_auth_url => auth_url+'/tokens',
|
31
|
+
:openstack_username => username,
|
32
|
+
:openstack_tenant => authtenant,
|
33
|
+
:openstack_api_key => api_key,
|
34
|
+
#:openstack_region => region,
|
35
|
+
:openstack_service_type => service_type,
|
36
|
+
:openstack_service_name => service_name
|
37
|
+
)
|
38
|
+
|
39
|
+
end
|
40
|
+
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
@@ -0,0 +1,117 @@
|
|
1
|
+
require 'torpedo/orchestration/helper'
|
2
|
+
require 'torpedo/compute/helper'
|
3
|
+
require 'torpedo/compute/keypairs'
|
4
|
+
require 'torpedo/compute/servers'
|
5
|
+
require 'tempfile'
|
6
|
+
require 'net/ssh'
|
7
|
+
|
8
|
+
module Torpedo
|
9
|
+
module Orchestration
|
10
|
+
class Stacks < Test::Unit::TestCase
|
11
|
+
|
12
|
+
@@stack = nil
|
13
|
+
@@image_ref = nil
|
14
|
+
@@flavor_ref = nil
|
15
|
+
|
16
|
+
def setup
|
17
|
+
@conn=Helper::get_connection
|
18
|
+
@compute_conn=Torpedo::Compute::Helper::get_connection
|
19
|
+
end
|
20
|
+
|
21
|
+
def test_001_setup
|
22
|
+
|
23
|
+
assert KEYPAIR_ENABLED == true, "Keyairs should be enabled when running Orchestration tests."
|
24
|
+
|
25
|
+
begin
|
26
|
+
@@image_ref = Torpedo::Compute::Servers.image_ref
|
27
|
+
if @@image_ref.nil? then
|
28
|
+
@@image_ref = Torpedo::Compute::Helper::get_image_ref(@compute_conn)
|
29
|
+
end
|
30
|
+
rescue Exception => e
|
31
|
+
fail("Failed get image ref: #{e.message}")
|
32
|
+
end
|
33
|
+
begin
|
34
|
+
@@flavor_ref = Torpedo::Compute::Servers.flavor_ref
|
35
|
+
if @@flavor_ref.nil? then
|
36
|
+
@@flavor_ref = Torpedo::Compute::Helper::get_flavor_ref(@compute_conn)
|
37
|
+
end
|
38
|
+
rescue Exception => e
|
39
|
+
fail("Failed get flavor ref: #{e.message}")
|
40
|
+
end
|
41
|
+
|
42
|
+
end
|
43
|
+
|
44
|
+
def test_002_create_stack
|
45
|
+
|
46
|
+
template = File.join(File.dirname(__FILE__), "test_server.hot")
|
47
|
+
keypair_name = Torpedo::Compute::Keypairs.key_pair.name
|
48
|
+
stack_opts = {
|
49
|
+
:template => IO.read(template),
|
50
|
+
:timeout_mins => (STACK_CREATE_TIMEOUT/60),
|
51
|
+
:parameters => {
|
52
|
+
:server_name => 'torpedo',
|
53
|
+
:key_name => keypair_name,
|
54
|
+
:image => @@image_ref,
|
55
|
+
:flavor => @@flavor_ref
|
56
|
+
}
|
57
|
+
}
|
58
|
+
stack_data = @conn.create_stack('torpedo', stack_opts).body['stack']
|
59
|
+
|
60
|
+
stack = @conn.stacks.get(stack_data['id'])
|
61
|
+
@@stack = stack
|
62
|
+
assert_equal "CREATE_IN_PROGRESS", stack.stack_status
|
63
|
+
|
64
|
+
begin
|
65
|
+
timeout(STACK_CREATE_TIMEOUT) do
|
66
|
+
until stack.stack_status == 'CREATE_COMPLETE' do
|
67
|
+
if stack.stack_status =~ /FAILED/ then
|
68
|
+
fail('Failure status detected when creating stack!')
|
69
|
+
end
|
70
|
+
stack = @conn.stacks.get(stack.id)
|
71
|
+
sleep 1
|
72
|
+
end
|
73
|
+
end
|
74
|
+
rescue Timeout::Error => te
|
75
|
+
fail('Timeout creating stack.')
|
76
|
+
end
|
77
|
+
|
78
|
+
end
|
79
|
+
|
80
|
+
def test_003_update_stack
|
81
|
+
|
82
|
+
template = File.join(File.dirname(__FILE__), "test_server.hot")
|
83
|
+
keypair_name = Torpedo::Compute::Keypairs.key_pair.name
|
84
|
+
stack_opts = {
|
85
|
+
:template => IO.read(template),
|
86
|
+
# update just the stack timeout
|
87
|
+
:timeout_mins => (STACK_CREATE_TIMEOUT/60)+1,
|
88
|
+
:parameters => {
|
89
|
+
:server_name => 'torpedo',
|
90
|
+
:key_name => keypair_name,
|
91
|
+
:image => @@image_ref,
|
92
|
+
:flavor => @@flavor_ref
|
93
|
+
}
|
94
|
+
}
|
95
|
+
@conn.update_stack(@@stack.id, @@stack.stack_name, stack_opts).body['stack']
|
96
|
+
stack = @conn.stacks.get(@@stack.id)
|
97
|
+
assert_equal "UPDATE_IN_PROGRESS", stack.stack_status
|
98
|
+
|
99
|
+
begin
|
100
|
+
timeout(STACK_CREATE_TIMEOUT) do
|
101
|
+
until stack.stack_status == 'UPDATE_COMPLETE' do
|
102
|
+
if stack.stack_status =~ /FAILED/ then
|
103
|
+
fail('Failure status detected when updating stack!')
|
104
|
+
end
|
105
|
+
stack = @conn.stacks.get(stack.id)
|
106
|
+
sleep 1
|
107
|
+
end
|
108
|
+
end
|
109
|
+
rescue Timeout::Error => te
|
110
|
+
fail('Timeout updating stack.')
|
111
|
+
end
|
112
|
+
|
113
|
+
end
|
114
|
+
|
115
|
+
end
|
116
|
+
end
|
117
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
heat_template_version: 2013-05-23
|
2
|
+
|
3
|
+
description: Deploy a single compute instance.
|
4
|
+
|
5
|
+
parameters:
|
6
|
+
server_name:
|
7
|
+
type: string
|
8
|
+
description: Name of server to create.
|
9
|
+
key_name:
|
10
|
+
type: string
|
11
|
+
description: Name of key-pair to be used for compute instance
|
12
|
+
image:
|
13
|
+
type: string
|
14
|
+
description: Image to be used for compute instance
|
15
|
+
flavor:
|
16
|
+
type: string
|
17
|
+
description: Type of instance (flavor) to be used
|
18
|
+
|
19
|
+
resources:
|
20
|
+
my_instance:
|
21
|
+
type: OS::Nova::Server
|
22
|
+
properties:
|
23
|
+
name:
|
24
|
+
Ref: server_name
|
25
|
+
flavor:
|
26
|
+
Ref: flavor
|
27
|
+
image:
|
28
|
+
Ref: image
|
29
|
+
key_name:
|
30
|
+
Ref: key_name
|
31
|
+
metadata: {'key1': 'value1', 'key2': 'value2'}
|
@@ -0,0 +1,43 @@
|
|
1
|
+
if RUBY_VERSION =~ /^1.9.*/ then
|
2
|
+
gem 'test-unit'
|
3
|
+
end
|
4
|
+
require 'test/unit'
|
5
|
+
if FOG_VERSION
|
6
|
+
gem 'fog', FOG_VERSION
|
7
|
+
end
|
8
|
+
require 'fog'
|
9
|
+
|
10
|
+
module Torpedo
|
11
|
+
module Volume
|
12
|
+
module Helper
|
13
|
+
|
14
|
+
def self.get_connection
|
15
|
+
|
16
|
+
if ENV['DEBUG'] and ENV['DEBUG'] == 'true' then
|
17
|
+
ENV['EXCON_DEBUG'] = 'true'
|
18
|
+
end
|
19
|
+
|
20
|
+
auth_url = ENV['OS_AUTH_URL']
|
21
|
+
api_key = ENV['OS_PASSWORD']
|
22
|
+
username = ENV['OS_USERNAME']
|
23
|
+
authtenant = ENV['OS_TENANT_NAME']
|
24
|
+
#region = ENV['OS_AUTH_REGION']
|
25
|
+
service_type = ENV['CINDER_SERVICE_TYPE'] || "volume"
|
26
|
+
service_name = ENV['CINDER_SERVICE_NAME'] #nil by default
|
27
|
+
|
28
|
+
Fog::Volume.new(
|
29
|
+
:provider => :openstack,
|
30
|
+
:openstack_auth_url => auth_url+'/tokens',
|
31
|
+
:openstack_username => username,
|
32
|
+
:openstack_tenant => authtenant,
|
33
|
+
:openstack_api_key => api_key,
|
34
|
+
#:openstack_region => region,
|
35
|
+
:openstack_service_type => service_type,
|
36
|
+
:openstack_service_name => service_name
|
37
|
+
)
|
38
|
+
|
39
|
+
end
|
40
|
+
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
@@ -0,0 +1,114 @@
|
|
1
|
+
require 'torpedo/volume/helper'
|
2
|
+
require 'torpedo/compute/helper'
|
3
|
+
require 'tempfile'
|
4
|
+
require 'net/ssh'
|
5
|
+
|
6
|
+
module Torpedo
|
7
|
+
module Volume
|
8
|
+
class Volumes < Test::Unit::TestCase
|
9
|
+
|
10
|
+
@@volumes = []
|
11
|
+
@@volume = nil #ref to last created volume
|
12
|
+
@@volsize = 1
|
13
|
+
@@volname = "torpedo"
|
14
|
+
@@voldesc = "T0rp3d@! F1r3$"
|
15
|
+
@@snapshot_id = nil
|
16
|
+
|
17
|
+
# public access to the volume ref
|
18
|
+
def self.volume
|
19
|
+
@@volume
|
20
|
+
end
|
21
|
+
|
22
|
+
def setup
|
23
|
+
@conn=Helper::get_connection
|
24
|
+
end
|
25
|
+
|
26
|
+
def create_volume(options)
|
27
|
+
@@volume = @conn.volumes.create(options)
|
28
|
+
@@volumes << @@volume
|
29
|
+
@@volume
|
30
|
+
end
|
31
|
+
|
32
|
+
def check_volume(volume, check_status="available")
|
33
|
+
|
34
|
+
volume = @conn.volumes.get(volume.id)
|
35
|
+
assert_equal(@@volsize, volume.size)
|
36
|
+
assert_equal(@@volname, volume.display_name)
|
37
|
+
assert_equal(@@voldesc, volume.display_description)
|
38
|
+
assert_equal(1, volume.size)
|
39
|
+
|
40
|
+
begin
|
41
|
+
timeout(VOLUME_BUILD_TIMEOUT) do
|
42
|
+
until volume.status == check_status do
|
43
|
+
if volume.status == "error" then
|
44
|
+
fail('Volume ERROR status detected!')
|
45
|
+
end
|
46
|
+
volume = @conn.volumes.get(volume.id)
|
47
|
+
sleep 1
|
48
|
+
end
|
49
|
+
end
|
50
|
+
rescue Timeout::Error => te
|
51
|
+
fail('Timeout creating volume.')
|
52
|
+
end
|
53
|
+
|
54
|
+
volume
|
55
|
+
|
56
|
+
end
|
57
|
+
|
58
|
+
def test_001_create_volume
|
59
|
+
options = {:display_name => @@volname, :display_description => @@voldesc, :size => @@volsize}
|
60
|
+
volume = create_volume(options)
|
61
|
+
|
62
|
+
check_volume(volume)
|
63
|
+
|
64
|
+
end
|
65
|
+
|
66
|
+
def test_002_create_volume_snapshot
|
67
|
+
|
68
|
+
snapshot = @conn.create_volume_snapshot(@@volume.id, "#{@@volname} snap", "#{@@voldesc} snap", true).body['snapshot']
|
69
|
+
assert_not_nil(snapshot['id'])
|
70
|
+
@@snapshot_id = snapshot['id']
|
71
|
+
assert_equal(@@volume.id, snapshot['volume_id'])
|
72
|
+
|
73
|
+
begin
|
74
|
+
timeout(VOLUME_BUILD_TIMEOUT) do
|
75
|
+
until snapshot['status'] == 'available' do
|
76
|
+
if snapshot['status'] == "error" then
|
77
|
+
fail('Volume snapshot ERROR status detected!')
|
78
|
+
end
|
79
|
+
snapshot = @conn.get_snapshot_details(snapshot['id']).body['snapshot']
|
80
|
+
sleep 1
|
81
|
+
end
|
82
|
+
end
|
83
|
+
rescue Timeout::Error => te
|
84
|
+
fail('Timeout creating snapshot.')
|
85
|
+
end
|
86
|
+
|
87
|
+
end if TEST_VOLUME_SNAPSHOTS
|
88
|
+
|
89
|
+
def test_003_del_volume_snapshot
|
90
|
+
|
91
|
+
assert(@conn.delete_snapshot(@@snapshot_id))
|
92
|
+
|
93
|
+
begin
|
94
|
+
snapcount = 1
|
95
|
+
timeout(60) do
|
96
|
+
until snapcount == 0 do
|
97
|
+
snapcount = 0
|
98
|
+
@conn.list_snapshots.body['snapshots'].each do |snap|
|
99
|
+
if snap['name'] == "#{@@volname} snap" then
|
100
|
+
snapcount += 1
|
101
|
+
sleep 1
|
102
|
+
end
|
103
|
+
end
|
104
|
+
end
|
105
|
+
end
|
106
|
+
rescue Timeout::Error => te
|
107
|
+
fail('Timeout waiting for snapshot to be deleted.')
|
108
|
+
end
|
109
|
+
|
110
|
+
end if TEST_VOLUME_SNAPSHOTS
|
111
|
+
|
112
|
+
end
|
113
|
+
end
|
114
|
+
end
|