beaker 2.18.3 → 2.19.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +8 -8
- data/HISTORY.md +439 -2
- data/acceptance/lib/beaker/acceptance/install_utils.rb +58 -0
- data/acceptance/pre_suite/puppet_git/install.rb +6 -65
- data/acceptance/tests/foss_utils/clone_git_repo_on.rb +49 -0
- data/beaker.gemspec +2 -0
- data/lib/beaker/dsl/helpers/web_helpers.rb +2 -1
- data/lib/beaker/dsl/install_utils/aio_defaults.rb +0 -2
- data/lib/beaker/dsl/install_utils/foss_utils.rb +97 -60
- data/lib/beaker/dsl/install_utils/pe_utils.rb +30 -53
- data/lib/beaker/dsl/install_utils/puppet_utils.rb +43 -0
- data/lib/beaker/dsl/install_utils/windows_utils.rb +144 -0
- data/lib/beaker/dsl/roles.rb +20 -3
- data/lib/beaker/dsl/structure.rb +14 -3
- data/lib/beaker/host.rb +24 -3
- data/lib/beaker/host/unix/pkg.rb +9 -0
- data/lib/beaker/host/windows/exec.rb +3 -0
- data/lib/beaker/host_prebuilt_steps.rb +5 -9
- data/lib/beaker/hypervisor/aws_sdk.rb +22 -18
- data/lib/beaker/hypervisor/docker.rb +7 -0
- data/lib/beaker/hypervisor/vmpooler.rb +4 -0
- data/lib/beaker/logger.rb +12 -1
- data/lib/beaker/options/command_line_parser.rb +9 -0
- data/lib/beaker/options/options_hash.rb +3 -296
- data/lib/beaker/options/parser.rb +12 -0
- data/lib/beaker/options/presets.rb +0 -1
- data/lib/beaker/ssh_connection.rb +48 -23
- data/lib/beaker/test_case.rb +1 -1
- data/lib/beaker/version.rb +1 -1
- data/spec/beaker/dsl/helpers/web_helpers_spec.rb +10 -1
- data/spec/beaker/dsl/install_utils/foss_utils_spec.rb +194 -49
- data/spec/beaker/dsl/install_utils/pe_utils_spec.rb +112 -22
- data/spec/beaker/dsl/install_utils/puppet_utils_spec.rb +57 -0
- data/spec/beaker/dsl/install_utils/windows_utils_spec.rb +132 -0
- data/spec/beaker/dsl/roles_spec.rb +36 -5
- data/spec/beaker/dsl/structure_spec.rb +9 -2
- data/spec/beaker/host/unix/pkg_spec.rb +26 -6
- data/spec/beaker/host_prebuilt_steps_spec.rb +3 -2
- data/spec/beaker/host_spec.rb +18 -0
- data/spec/beaker/hypervisor/aixer_spec.rb +1 -1
- data/spec/beaker/hypervisor/aws_sdk_spec.rb +595 -58
- data/spec/beaker/hypervisor/docker_spec.rb +2 -1
- data/spec/beaker/hypervisor/solaris_spec.rb +1 -0
- data/spec/beaker/hypervisor/vagrant_spec.rb +2 -1
- data/spec/beaker/logger_spec.rb +39 -0
- data/spec/beaker/options/command_line_parser_spec.rb +2 -2
- data/spec/beaker/options/options_hash_spec.rb +1 -102
- data/spec/beaker/options/parser_spec.rb +19 -0
- data/spec/beaker/options/pe_version_scaper_spec.rb +11 -1
- data/spec/beaker/options/presets_spec.rb +8 -0
- data/spec/beaker/ssh_connection_spec.rb +39 -21
- data/spec/helpers.rb +9 -3
- data/spec/mocks.rb +2 -0
- metadata +34 -11
- data/lib/beaker/answers.rb +0 -143
- data/lib/beaker/answers/version20.rb +0 -120
- data/lib/beaker/answers/version28.rb +0 -121
- data/lib/beaker/answers/version30.rb +0 -227
- data/lib/beaker/answers/version32.rb +0 -44
- data/lib/beaker/answers/version34.rb +0 -51
- data/lib/beaker/answers/version38.rb +0 -29
- data/lib/beaker/answers/version40.rb +0 -44
- data/spec/beaker/answers_spec.rb +0 -547
@@ -128,14 +128,21 @@ describe ClassMixedWithDSLStructure do
|
|
128
128
|
subject.confine( :to, {} )
|
129
129
|
end
|
130
130
|
|
131
|
-
it 'uses a provided host subset when no criteria is provided' do
|
132
|
-
|
131
|
+
it ':to - uses a provided host subset when no criteria is provided' do
|
133
132
|
subset = ['host1', 'host2']
|
134
133
|
hosts = subset.dup << 'host3'
|
135
134
|
expect( subject ).to receive( :hosts= ).with( subset )
|
136
135
|
subject.confine :to, {}, subset
|
137
136
|
end
|
138
137
|
|
138
|
+
it ':except - excludes provided host subset when no criteria is provided' do
|
139
|
+
subset = ['host1', 'host2']
|
140
|
+
hosts = subset.dup << 'host3'
|
141
|
+
allow( subject ).to receive( :hosts ).and_return(hosts)
|
142
|
+
expect( subject ).to receive( :hosts= ).with( hosts - subset )
|
143
|
+
subject.confine :except, {}, subset
|
144
|
+
end
|
145
|
+
|
139
146
|
it 'raises when given mode is not :to or :except' do
|
140
147
|
allow( subject ).to receive( :hosts )
|
141
148
|
allow( subject ).to receive( :hosts= )
|
@@ -122,9 +122,29 @@ module Beaker
|
|
122
122
|
expect{ instance.check_for_package(pkg) }.to raise_error
|
123
123
|
|
124
124
|
end
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
125
|
+
|
126
|
+
end
|
127
|
+
|
128
|
+
context "install_package" do
|
129
|
+
|
130
|
+
it "uses yum on fedora-20" do
|
131
|
+
@opts = {'platform' => 'fedora-20-is-me'}
|
132
|
+
pkg = 'fedora_package'
|
133
|
+
expect( Beaker::Command ).to receive(:new).with("yum -y install #{pkg}", [], {:prepend_cmds=>nil, :cmdexe=>false}).and_return('')
|
134
|
+
expect( instance ).to receive(:exec).with('', {}).and_return(generate_result("hello", {:exit_code => 0}))
|
135
|
+
expect( instance.install_package(pkg) ).to be == "hello"
|
136
|
+
end
|
137
|
+
|
138
|
+
it "uses dnf on fedora-22" do
|
139
|
+
@opts = {'platform' => 'fedora-22-is-me'}
|
140
|
+
pkg = 'fedora_package'
|
141
|
+
expect( Beaker::Command ).to receive(:new).with("dnf -y install #{pkg}", [], {:prepend_cmds=>nil, :cmdexe=>false}).and_return('')
|
142
|
+
expect( instance ).to receive(:exec).with('', {}).and_return(generate_result("hello", {:exit_code => 0}))
|
143
|
+
expect( instance.install_package(pkg) ).to be == "hello"
|
144
|
+
end
|
145
|
+
|
146
|
+
end
|
147
|
+
|
148
|
+
end
|
149
|
+
end
|
150
|
+
|
@@ -97,7 +97,7 @@ describe Beaker do
|
|
97
97
|
|
98
98
|
expect( Beaker::Command ).to receive( :new ).with("ntpdate -t 20 #{ntpserver}").exactly( 5 ).times
|
99
99
|
|
100
|
-
expect{ subject.timesync( hosts, options ) }.to raise_error
|
100
|
+
expect{ subject.timesync( hosts, options ) }.to raise_error(/NTP date was not successful after/)
|
101
101
|
end
|
102
102
|
|
103
103
|
it "can sync time on windows hosts" do
|
@@ -170,7 +170,7 @@ describe Beaker do
|
|
170
170
|
it "raises an error on non el-5/6 host" do
|
171
171
|
host = make_host( 'testhost', { :platform => Beaker::Platform.new('el-4-platform') } )
|
172
172
|
|
173
|
-
expect{ subject.epel_info_for( host, options )}.to raise_error
|
173
|
+
expect{ subject.epel_info_for( host, options )}.to raise_error(RuntimeError, /epel_info_for does not support el version/)
|
174
174
|
|
175
175
|
end
|
176
176
|
|
@@ -572,6 +572,7 @@ describe Beaker do
|
|
572
572
|
expect( Beaker::Command ).to receive( :new ).with( "mkdir -p #{Pathname.new(host[:ssh_env_file]).dirname}" ).once
|
573
573
|
expect( Beaker::Command ).to receive( :new ).with( "chmod 0600 #{Pathname.new(host[:ssh_env_file]).dirname}" ).once
|
574
574
|
expect( Beaker::Command ).to receive( :new ).with( "touch #{host[:ssh_env_file]}" ).once
|
575
|
+
expect_any_instance_of( Class ).to receive( :extend ).and_return( double( 'class' ).as_null_object )
|
575
576
|
expect( Beaker::Command ).to receive( :new ).with( "cat #{host[:ssh_env_file]}" ).once
|
576
577
|
expect( host ).to receive( :add_env_var ).with( 'PATH', '$PATH' ).once
|
577
578
|
opts.each_pair do |key, value|
|
data/spec/beaker/host_spec.rb
CHANGED
@@ -243,6 +243,9 @@ module Beaker
|
|
243
243
|
host.instance_variable_set :@logger, logger
|
244
244
|
conn = double(:connection)
|
245
245
|
allow( conn ).to receive(:execute).and_return(result)
|
246
|
+
allow( conn ).to receive(:ip).and_return(host['ip'])
|
247
|
+
allow( conn ).to receive(:vmhostname).and_return(host['vmhostname'])
|
248
|
+
allow( conn ).to receive(:hostname).and_return(host.name)
|
246
249
|
host.instance_variable_set :@connection, conn
|
247
250
|
end
|
248
251
|
|
@@ -399,6 +402,9 @@ module Beaker
|
|
399
402
|
|
400
403
|
expect( logger ).to receive(:trace)
|
401
404
|
expect( conn ).to receive(:scp_to).with( *conn_args ).and_return(Beaker::Result.new(host, 'output!'))
|
405
|
+
allow( conn ).to receive(:ip).and_return(host['ip'])
|
406
|
+
allow( conn ).to receive(:vmhostname).and_return(host['vmhostname'])
|
407
|
+
allow( conn ).to receive(:hostname).and_return(host.name)
|
402
408
|
|
403
409
|
host.do_scp_to *args
|
404
410
|
end
|
@@ -466,6 +472,9 @@ module Beaker
|
|
466
472
|
expect( conn ).to_not receive(:scp_to).with( *conn_args )
|
467
473
|
end
|
468
474
|
end
|
475
|
+
allow( conn ).to receive(:ip).and_return(host['ip'])
|
476
|
+
allow( conn ).to receive(:vmhostname).and_return(host['vmhostname'])
|
477
|
+
allow( conn ).to receive(:hostname).and_return(host.name)
|
469
478
|
|
470
479
|
host.do_scp_to *args
|
471
480
|
end
|
@@ -527,6 +536,9 @@ module Beaker
|
|
527
536
|
expect( conn ).to_not receive(:scp_to).with( *conn_args )
|
528
537
|
end
|
529
538
|
end
|
539
|
+
allow( conn ).to receive(:ip).and_return(host['ip'])
|
540
|
+
allow( conn ).to receive(:vmhostname).and_return(host['vmhostname'])
|
541
|
+
allow( conn ).to receive(:hostname).and_return(host.name)
|
530
542
|
|
531
543
|
host.do_scp_to *args
|
532
544
|
end
|
@@ -555,6 +567,9 @@ module Beaker
|
|
555
567
|
expect( conn ).to receive(:scp_to).with( *conn_args ).and_return(Beaker::Result.new(host, 'output!'))
|
556
568
|
end
|
557
569
|
|
570
|
+
allow( conn ).to receive(:ip).and_return(host['ip'])
|
571
|
+
allow( conn ).to receive(:vmhostname).and_return(host['vmhostname'])
|
572
|
+
allow( conn ).to receive(:hostname).and_return(host.name)
|
558
573
|
host.do_scp_to *args
|
559
574
|
end
|
560
575
|
end
|
@@ -572,6 +587,9 @@ module Beaker
|
|
572
587
|
expect( logger ).to receive(:debug)
|
573
588
|
expect( conn ).to receive(:scp_from).with( *conn_args ).and_return(Beaker::Result.new(host, 'output!'))
|
574
589
|
|
590
|
+
allow( conn ).to receive(:ip).and_return(host['ip'])
|
591
|
+
allow( conn ).to receive(:vmhostname).and_return(host['vmhostname'])
|
592
|
+
allow( conn ).to receive(:hostname).and_return(host.name)
|
575
593
|
host.do_scp_from *args
|
576
594
|
end
|
577
595
|
end
|
@@ -44,90 +44,314 @@ module Beaker
|
|
44
44
|
@hosts[4][:user] = 'notroot'
|
45
45
|
end
|
46
46
|
|
47
|
-
|
48
|
-
|
49
|
-
expect(
|
50
|
-
expect(
|
51
|
-
aws.
|
47
|
+
describe '#provision' do
|
48
|
+
before :each do
|
49
|
+
expect(aws).to receive(:launch_all_nodes)
|
50
|
+
expect(aws).to receive(:add_tags)
|
51
|
+
expect(aws).to receive(:populate_dns)
|
52
|
+
expect(aws).to receive(:enable_root_on_hosts)
|
53
|
+
expect(aws).to receive(:set_hostnames)
|
54
|
+
expect(aws).to receive(:configure_hosts)
|
52
55
|
end
|
53
56
|
|
54
|
-
it '
|
55
|
-
|
56
|
-
|
57
|
+
it 'should step through provisioning' do
|
58
|
+
aws.provision
|
59
|
+
end
|
60
|
+
|
61
|
+
it 'should return nil' do
|
62
|
+
expect(aws.provision).to be_nil
|
57
63
|
end
|
64
|
+
end
|
58
65
|
|
59
|
-
|
66
|
+
describe '#kill_instances' do
|
67
|
+
let( :ec2_instance ) { double('ec2_instance', :nil? => false, :exists? => true, :id => "ec2", :terminate => nil) }
|
68
|
+
let( :vpc_instance ) { double('vpc_instance', :nil? => false, :exists? => true, :id => "vpc", :terminate => nil) }
|
69
|
+
let( :nil_instance ) { double('vpc_instance', :nil? => true, :exists? => true, :id => "nil", :terminate => nil) }
|
70
|
+
let( :unreal_instance ) { double('vpc_instance', :nil? => false, :exists? => false, :id => "unreal", :terminate => nil) }
|
71
|
+
subject(:kill_instances) { aws.kill_instances(instance_set) }
|
72
|
+
|
73
|
+
it 'should return nil' do
|
74
|
+
instance_set = [ec2_instance, vpc_instance, nil_instance, unreal_instance]
|
75
|
+
expect(aws.kill_instances(instance_set)).to be_nil
|
76
|
+
end
|
77
|
+
|
78
|
+
it 'cleanly handles an empty instance list' do
|
79
|
+
instance_set = []
|
80
|
+
expect(aws.kill_instances(instance_set)).to be_nil
|
81
|
+
end
|
82
|
+
|
83
|
+
context 'in general use' do
|
84
|
+
let( :instance_set ) { [ec2_instance, vpc_instance] }
|
60
85
|
|
61
|
-
it '
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
86
|
+
it 'terminates each running instance' do
|
87
|
+
instance_set.each do |instance|
|
88
|
+
expect(instance).to receive(:terminate).once
|
89
|
+
end
|
90
|
+
expect(kill_instances).to be_nil
|
91
|
+
end
|
92
|
+
|
93
|
+
it 'verifies instances are not nil' do
|
94
|
+
instance_set.each do |instance|
|
95
|
+
expect(instance).to receive(:nil?)
|
96
|
+
allow(instance).to receive(:terminate).once
|
97
|
+
end
|
98
|
+
expect(kill_instances).to be_nil
|
71
99
|
end
|
100
|
+
|
101
|
+
it 'verifies instances exist in AWS' do
|
102
|
+
instance_set.each do |instance|
|
103
|
+
expect(instance).to receive(:exists?)
|
104
|
+
allow(instance).to receive(:terminate).once
|
105
|
+
end
|
106
|
+
expect(kill_instances).to be_nil
|
107
|
+
end
|
108
|
+
end
|
109
|
+
|
110
|
+
context 'for a single running instance' do
|
111
|
+
let( :instance_set ) { [ec2_instance] }
|
72
112
|
|
73
|
-
it '
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
113
|
+
it 'terminates the running instance' do
|
114
|
+
instance_set.each do |instance|
|
115
|
+
expect(instance).to receive(:terminate).once
|
116
|
+
end
|
117
|
+
expect(kill_instances).to be_nil
|
118
|
+
end
|
119
|
+
|
120
|
+
it 'verifies instance is not nil' do
|
121
|
+
instance_set.each do |instance|
|
122
|
+
expect(instance).to receive(:nil?)
|
123
|
+
allow(instance).to receive(:terminate).once
|
124
|
+
end
|
125
|
+
expect(kill_instances).to be_nil
|
126
|
+
end
|
127
|
+
|
128
|
+
it 'verifies instance exists in AWS' do
|
129
|
+
instance_set.each do |instance|
|
130
|
+
expect(instance).to receive(:exists?)
|
131
|
+
allow(instance).to receive(:terminate).once
|
132
|
+
end
|
133
|
+
expect(kill_instances).to be_nil
|
80
134
|
end
|
135
|
+
end
|
81
136
|
|
137
|
+
context 'when an instance does not exist' do
|
138
|
+
let( :instance_set ) { [unreal_instance] }
|
139
|
+
|
140
|
+
it 'does not call terminate' do
|
141
|
+
instance_set.each do |instance|
|
142
|
+
expect(instance).to receive(:terminate).exactly(0).times
|
143
|
+
end
|
144
|
+
expect(kill_instances).to be_nil
|
145
|
+
end
|
146
|
+
|
147
|
+
it 'verifies instance does not exist' do
|
148
|
+
instance_set.each do |instance|
|
149
|
+
expect(instance).to receive(:exists?).once
|
150
|
+
allow(instance).to receive(:terminate).exactly(0).times
|
151
|
+
end
|
152
|
+
expect(kill_instances).to be_nil
|
153
|
+
end
|
82
154
|
end
|
83
|
-
end
|
84
155
|
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
156
|
+
context 'when an instance is nil' do
|
157
|
+
let( :instance_set ) { [nil_instance] }
|
158
|
+
|
159
|
+
it 'does not call terminate' do
|
160
|
+
instance_set.each do |instance|
|
161
|
+
expect(instance).to receive(:terminate).exactly(0).times
|
162
|
+
end
|
163
|
+
expect(kill_instances).to be_nil
|
164
|
+
end
|
165
|
+
|
166
|
+
it 'verifies instance is nil' do
|
167
|
+
instance_set.each do |instance|
|
168
|
+
expect(instance).to receive(:nil?).once
|
169
|
+
allow(instance).to receive(:terminate).exactly(0).times
|
170
|
+
end
|
171
|
+
expect(kill_instances).to be_nil
|
172
|
+
end
|
89
173
|
end
|
174
|
+
|
90
175
|
end
|
91
176
|
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
allow(File).to receive(:exists?).with(/id_rsa.pub/) { true }
|
96
|
-
allow(File).to receive(:read).with(/id_rsa.pub/) { "foobar" }
|
177
|
+
describe '#cleanup' do
|
178
|
+
subject(:cleanup) { aws.cleanup }
|
179
|
+
let( :ec2_instance ) { double('ec2_instance', :nil? => false, :exists? => true, :terminate => nil, :id => 'id') }
|
97
180
|
|
98
|
-
|
99
|
-
|
181
|
+
context 'with a list of hosts' do
|
182
|
+
before :each do
|
183
|
+
@hosts.each {|host| host['instance'] = ec2_instance}
|
184
|
+
end
|
185
|
+
|
186
|
+
it { is_expected.to be_nil }
|
187
|
+
|
188
|
+
it 'kills instances' do
|
189
|
+
expect(aws).to receive(:kill_instances).once
|
190
|
+
expect(cleanup).to be_nil
|
191
|
+
end
|
100
192
|
end
|
101
193
|
|
102
|
-
|
103
|
-
|
194
|
+
context 'with an empty host list' do
|
195
|
+
before :each do
|
196
|
+
@hosts = []
|
197
|
+
end
|
198
|
+
|
199
|
+
it { is_expected.to be_nil }
|
200
|
+
|
201
|
+
it 'kills instances' do
|
202
|
+
expect(aws).to receive(:kill_instances).once
|
203
|
+
expect(cleanup).to be_nil
|
204
|
+
end
|
104
205
|
end
|
105
206
|
end
|
106
207
|
|
107
|
-
|
108
|
-
|
109
|
-
# Mock out the hostname and local user calls
|
110
|
-
expect( Socket ).to receive(:gethostname) { "foobar" }
|
111
|
-
expect( aws ).to receive(:local_user) { "bob" }
|
208
|
+
describe '#log_instances', :wip do
|
209
|
+
end
|
112
210
|
|
113
|
-
|
114
|
-
|
211
|
+
describe '#instance_by_id' do
|
212
|
+
subject { aws.instance_by_id('my_id') }
|
213
|
+
it { is_expected.to be_instance_of(AWS::EC2::Instance) }
|
214
|
+
end
|
215
|
+
|
216
|
+
describe '#instances' do
|
217
|
+
subject { aws.instances }
|
218
|
+
it { is_expected.to be_instance_of(AWS::EC2::InstanceCollection) }
|
219
|
+
end
|
220
|
+
|
221
|
+
describe '#vpc_by_id' do
|
222
|
+
subject { aws.vpc_by_id('my_id') }
|
223
|
+
it { is_expected.to be_instance_of(AWS::EC2::VPC) }
|
224
|
+
end
|
225
|
+
|
226
|
+
describe '#vpcs' do
|
227
|
+
subject { aws.vpcs }
|
228
|
+
it { is_expected.to be_instance_of(AWS::EC2::VPCCollection) }
|
229
|
+
end
|
230
|
+
|
231
|
+
describe '#security_group_by_id' do
|
232
|
+
subject { aws.security_group_by_id('my_id') }
|
233
|
+
it { is_expected.to be_instance_of(AWS::EC2::SecurityGroup) }
|
234
|
+
end
|
235
|
+
|
236
|
+
describe '#security_groups' do
|
237
|
+
subject { aws.security_groups }
|
238
|
+
it { is_expected.to be_instance_of(AWS::EC2::SecurityGroupCollection) }
|
239
|
+
end
|
240
|
+
|
241
|
+
describe '#kill_zombies', :wip do
|
242
|
+
end
|
243
|
+
|
244
|
+
describe '#kill_zombie_volumes', :wip do
|
245
|
+
end
|
246
|
+
|
247
|
+
describe '#create_instance', :wip do
|
248
|
+
end
|
249
|
+
|
250
|
+
describe '#launch_nodes_on_some_subnet', :wip do
|
251
|
+
end
|
252
|
+
|
253
|
+
describe '#launch_all_nodes', :wip do
|
254
|
+
end
|
255
|
+
|
256
|
+
describe '#wait_for_status' do
|
257
|
+
let( :aws_instance ) { double('aws_instance', :id => "ec2", :terminate => nil) }
|
258
|
+
let( :instance_set ) { [{:instance => aws_instance}] }
|
259
|
+
subject(:wait_for_status) { aws.wait_for_status(:running, instance_set) }
|
260
|
+
|
261
|
+
it 'handles a single instance' do
|
262
|
+
allow(aws_instance).to receive(:status).and_return(:waiting, :waiting, :running)
|
263
|
+
expect(aws).to receive(:backoff_sleep).exactly(3).times
|
264
|
+
expect(wait_for_status).to eq(instance_set)
|
265
|
+
end
|
266
|
+
|
267
|
+
context 'with multiple instances' do
|
268
|
+
let( :instance_set ) { [{:instance => aws_instance}, {:instance => aws_instance}] }
|
269
|
+
|
270
|
+
it 'returns the instance set passed to it' do
|
271
|
+
allow(aws_instance).to receive(:status).and_return(:waiting, :waiting, :running, :waiting, :waiting, :running)
|
272
|
+
allow(aws).to receive(:backoff_sleep).exactly(6).times
|
273
|
+
expect(wait_for_status).to eq(instance_set)
|
274
|
+
end
|
275
|
+
|
276
|
+
it 'calls backoff_sleep once per instance.status call' do
|
277
|
+
allow(aws_instance).to receive(:status).and_return(:waiting, :waiting, :running, :waiting, :waiting, :running)
|
278
|
+
expect(aws).to receive(:backoff_sleep).exactly(6).times
|
279
|
+
expect(wait_for_status).to eq(instance_set)
|
280
|
+
end
|
281
|
+
end
|
282
|
+
|
283
|
+
context 'after 10 tries' do
|
284
|
+
it 'raises RuntimeError' do
|
285
|
+
expect(aws_instance).to receive(:status).and_return(:waiting).exactly(10).times
|
286
|
+
expect(aws).to receive(:backoff_sleep).exactly(9).times
|
287
|
+
expect { wait_for_status }.to raise_error('Instance never reached state running')
|
288
|
+
end
|
289
|
+
end
|
290
|
+
|
291
|
+
context 'with an invalid instance' do
|
292
|
+
it 'raises AWS::EC2::Errors::InvalidInstanceID::NotFound' do
|
293
|
+
expect(aws_instance).to receive(:status).and_raise(AWS::EC2::Errors::InvalidInstanceID::NotFound).exactly(10).times
|
294
|
+
allow(aws).to receive(:backoff_sleep).at_most(10).times
|
295
|
+
expect(wait_for_status).to eq(instance_set)
|
296
|
+
end
|
115
297
|
end
|
116
298
|
end
|
117
299
|
|
118
|
-
|
119
|
-
|
120
|
-
|
300
|
+
describe '#add_tags' do
|
301
|
+
let( :aws_instance ) { double('aws_instance', :add_tag => nil) }
|
302
|
+
subject(:add_tags) { aws.add_tags }
|
303
|
+
|
304
|
+
it 'returns nil' do
|
305
|
+
@hosts.each {|host| host['instance'] = aws_instance}
|
306
|
+
expect(add_tags).to be_nil
|
121
307
|
end
|
122
308
|
|
123
|
-
it '
|
124
|
-
|
309
|
+
it 'handles a single host' do
|
310
|
+
@hosts[0]['instance'] = aws_instance
|
311
|
+
@hosts = [@hosts[0]]
|
312
|
+
expect(add_tags).to be_nil
|
313
|
+
end
|
314
|
+
|
315
|
+
context 'with multiple hosts' do
|
316
|
+
before :each do
|
317
|
+
@hosts.each {|host| host['instance'] = aws_instance}
|
318
|
+
end
|
319
|
+
|
320
|
+
it 'adds tag for jenkins_build_url' do
|
321
|
+
aws.instance_eval('@options[:jenkins_build_url] = "my_build_url"')
|
322
|
+
expect(aws_instance).to receive(:add_tag).with('jenkins_build_url', hash_including(:value => 'my_build_url')).at_least(:once)
|
323
|
+
expect(add_tags).to be_nil
|
324
|
+
end
|
325
|
+
|
326
|
+
it 'adds tag for Name' do
|
327
|
+
expect(aws_instance).to receive(:add_tag).with('Name', hash_including(:value => /vm/)).at_least(@hosts.size).times
|
328
|
+
expect(add_tags).to be_nil
|
329
|
+
end
|
330
|
+
|
331
|
+
it 'adds tag for department' do
|
332
|
+
aws.instance_eval('@options[:department] = "my_department"')
|
333
|
+
expect(aws_instance).to receive(:add_tag).with('department', hash_including(:value => 'my_department')).at_least(:once)
|
334
|
+
expect(add_tags).to be_nil
|
335
|
+
end
|
336
|
+
|
337
|
+
it 'adds tag for project' do
|
338
|
+
aws.instance_eval('@options[:project] = "my_project"')
|
339
|
+
expect(aws_instance).to receive(:add_tag).with('project', hash_including(:value => 'my_project')).at_least(:once)
|
340
|
+
expect(add_tags).to be_nil
|
341
|
+
end
|
342
|
+
|
343
|
+
it 'adds tag for created_by' do
|
344
|
+
aws.instance_eval('@options[:created_by] = "my_created_by"')
|
345
|
+
expect(aws_instance).to receive(:add_tag).with('created_by', hash_including(:value => 'my_created_by')).at_least(:once)
|
346
|
+
expect(add_tags).to be_nil
|
347
|
+
end
|
125
348
|
end
|
126
349
|
end
|
127
350
|
|
128
351
|
describe '#populate_dns' do
|
129
352
|
let( :vpc_instance ) { {ip_address: nil, private_ip_address: "vpc_private_ip", dns_name: "vpc_dns_name"} }
|
130
353
|
let( :ec2_instance ) { {ip_address: "ec2_public_ip", private_ip_address: "ec2_private_ip", dns_name: "ec2_dns_name"} }
|
354
|
+
subject(:populate_dns) { aws.populate_dns }
|
131
355
|
|
132
356
|
context 'on a public EC2 instance' do
|
133
357
|
before :each do
|
@@ -135,7 +359,7 @@ module Beaker
|
|
135
359
|
end
|
136
360
|
|
137
361
|
it 'sets host ip to instance.ip_address' do
|
138
|
-
|
362
|
+
populate_dns
|
139
363
|
hosts = aws.instance_variable_get( :@hosts )
|
140
364
|
hosts.each do |host|
|
141
365
|
expect(host['ip']).to eql(ec2_instance[:ip_address])
|
@@ -143,7 +367,7 @@ module Beaker
|
|
143
367
|
end
|
144
368
|
|
145
369
|
it 'sets host private_ip to instance.private_ip_address' do
|
146
|
-
|
370
|
+
populate_dns
|
147
371
|
hosts = aws.instance_variable_get( :@hosts )
|
148
372
|
hosts.each do |host|
|
149
373
|
expect(host['private_ip']).to eql(ec2_instance[:private_ip_address])
|
@@ -151,7 +375,7 @@ module Beaker
|
|
151
375
|
end
|
152
376
|
|
153
377
|
it 'sets host dns_name to instance.dns_name' do
|
154
|
-
|
378
|
+
populate_dns
|
155
379
|
hosts = aws.instance_variable_get( :@hosts )
|
156
380
|
hosts.each do |host|
|
157
381
|
expect(host['dns_name']).to eql(ec2_instance[:dns_name])
|
@@ -165,7 +389,7 @@ module Beaker
|
|
165
389
|
end
|
166
390
|
|
167
391
|
it 'sets host ip to instance.private_ip_address' do
|
168
|
-
|
392
|
+
populate_dns
|
169
393
|
hosts = aws.instance_variable_get( :@hosts )
|
170
394
|
hosts.each do |host|
|
171
395
|
expect(host['ip']).to eql(vpc_instance[:private_ip_address])
|
@@ -173,7 +397,7 @@ module Beaker
|
|
173
397
|
end
|
174
398
|
|
175
399
|
it 'sets host private_ip to instance.private_ip_address' do
|
176
|
-
|
400
|
+
populate_dns
|
177
401
|
hosts = aws.instance_variable_get( :@hosts )
|
178
402
|
hosts.each do |host|
|
179
403
|
expect(host['private_ip']).to eql(vpc_instance[:private_ip_address])
|
@@ -181,7 +405,7 @@ module Beaker
|
|
181
405
|
end
|
182
406
|
|
183
407
|
it 'sets host dns_name to instance.dns_name' do
|
184
|
-
|
408
|
+
populate_dns
|
185
409
|
hosts = aws.instance_variable_get( :@hosts )
|
186
410
|
hosts.each do |host|
|
187
411
|
expect(host['dns_name']).to eql(vpc_instance[:dns_name])
|
@@ -189,5 +413,318 @@ module Beaker
|
|
189
413
|
end
|
190
414
|
end
|
191
415
|
end
|
416
|
+
|
417
|
+
describe '#etc_hosts_entry' do
|
418
|
+
let( :host ) { @hosts[0] }
|
419
|
+
let( :interface ) { :ip }
|
420
|
+
subject(:etc_hosts_entry) { aws.etc_hosts_entry(host, interface) }
|
421
|
+
|
422
|
+
it 'returns a predictable host entry' do
|
423
|
+
expect(aws).to receive(:get_domain_name).and_return('lan')
|
424
|
+
expect(etc_hosts_entry).to eq("ip.address.for.vm1\tvm1 vm1.lan vm1.box.tld\n")
|
425
|
+
end
|
426
|
+
|
427
|
+
context 'when :private_ip is requested' do
|
428
|
+
let( :interface ) { :private_ip }
|
429
|
+
it 'returns host entry for the private_ip' do
|
430
|
+
host = @hosts[0]
|
431
|
+
expect(aws).to receive(:get_domain_name).and_return('lan')
|
432
|
+
expect(etc_hosts_entry).to eq("private.ip.for.vm1\tvm1 vm1.lan vm1.box.tld\n")
|
433
|
+
end
|
434
|
+
end
|
435
|
+
end
|
436
|
+
|
437
|
+
describe '#configure_hosts' do
|
438
|
+
subject(:configure_hosts) { aws.configure_hosts }
|
439
|
+
|
440
|
+
it { is_expected.to be_nil }
|
441
|
+
|
442
|
+
context 'calls #set_etc_hosts' do
|
443
|
+
it 'for each host' do
|
444
|
+
expect(aws).to receive(:set_etc_hosts).exactly(@hosts.size).times
|
445
|
+
expect(configure_hosts).to be_nil
|
446
|
+
end
|
447
|
+
|
448
|
+
it 'with predictable host entries' do
|
449
|
+
@hosts = [@hosts[0], @hosts[1]]
|
450
|
+
entries = "127.0.0.1\tlocalhost localhost.localdomain\n"\
|
451
|
+
"private.ip.for.vm1\tvm1 vm1.lan vm1.box.tld\n"\
|
452
|
+
"ip.address.for.vm2\tvm2 vm2.lan vm2.box.tld\n"
|
453
|
+
allow(aws).to receive(:get_domain_name).and_return('lan')
|
454
|
+
expect(aws).to receive(:set_etc_hosts).with(@hosts[0], entries)
|
455
|
+
expect(aws).to receive(:set_etc_hosts).with(@hosts[1], anything)
|
456
|
+
expect(configure_hosts).to be_nil
|
457
|
+
end
|
458
|
+
end
|
459
|
+
end
|
460
|
+
|
461
|
+
describe '#enable_root_on_hosts' do
|
462
|
+
context 'enabling root shall be called once for the ubuntu machine' do
|
463
|
+
it "should enable root once" do
|
464
|
+
expect( aws ).to receive(:copy_ssh_to_root).with( @hosts[3], options ).once()
|
465
|
+
expect( aws ).to receive(:enable_root_login).with( @hosts[3], options).once()
|
466
|
+
aws.enable_root_on_hosts();
|
467
|
+
end
|
468
|
+
end
|
469
|
+
|
470
|
+
it 'enables root once on the f5 host through its code path' do
|
471
|
+
expect( aws ).to receive(:enable_root_f5).with( @hosts[4] ).once()
|
472
|
+
aws.enable_root_on_hosts()
|
473
|
+
end
|
474
|
+
end
|
475
|
+
|
476
|
+
describe '#enable_root_f5' do
|
477
|
+
let( :f5_host ) { @hosts[4] }
|
478
|
+
subject(:enable_root_f5) { aws.enable_root_f5(f5_host) }
|
479
|
+
|
480
|
+
it 'creates a password on the host' do
|
481
|
+
result_mock = Beaker::Result.new(f5_host, '')
|
482
|
+
result_mock.exit_code = 0
|
483
|
+
allow( f5_host ).to receive( :exec ).and_return(result_mock)
|
484
|
+
allow( aws ).to receive( :backoff_sleep )
|
485
|
+
sha_mock = Object.new
|
486
|
+
allow( Digest::SHA256 ).to receive( :new ).and_return(sha_mock)
|
487
|
+
expect( sha_mock ).to receive( :hexdigest ).once()
|
488
|
+
enable_root_f5
|
489
|
+
end
|
490
|
+
|
491
|
+
it 'tries 10x before failing correctly' do
|
492
|
+
result_mock = Beaker::Result.new(f5_host, '')
|
493
|
+
result_mock.exit_code = 2
|
494
|
+
allow( f5_host ).to receive( :exec ).and_return(result_mock)
|
495
|
+
expect( aws ).to receive( :backoff_sleep ).exactly(9).times
|
496
|
+
expect{ enable_root_f5 }.to raise_error( RuntimeError, /unable/ )
|
497
|
+
end
|
498
|
+
end
|
499
|
+
|
500
|
+
describe '#set_hostnames' do
|
501
|
+
subject(:set_hostnames) { aws.set_hostnames }
|
502
|
+
it 'returns @hosts' do
|
503
|
+
expect(set_hostnames).to eq(@hosts)
|
504
|
+
end
|
505
|
+
|
506
|
+
context 'for each host' do
|
507
|
+
it 'calls exec' do
|
508
|
+
@hosts.each {|host| expect(host).to receive(:exec).once}
|
509
|
+
expect(set_hostnames).to eq(@hosts)
|
510
|
+
end
|
511
|
+
|
512
|
+
it 'passes a Command instance to exec' do
|
513
|
+
@hosts.each do |host|
|
514
|
+
expect(host).to receive(:exec).with( instance_of(Beaker::Command) ).once
|
515
|
+
end
|
516
|
+
expect(set_hostnames).to eq(@hosts)
|
517
|
+
end
|
518
|
+
end
|
519
|
+
end
|
520
|
+
|
521
|
+
describe '#backoff_sleep' do
|
522
|
+
it "should call sleep 1024 times at attempt 10" do
|
523
|
+
expect_any_instance_of( Object ).to receive(:sleep).with(1024)
|
524
|
+
aws.backoff_sleep(10)
|
525
|
+
end
|
526
|
+
end
|
527
|
+
|
528
|
+
describe '#public_key' do
|
529
|
+
subject(:public_key) { aws.public_key }
|
530
|
+
|
531
|
+
it "retrieves contents from local ~/.ssh/ssh_rsa.pub file" do
|
532
|
+
# Stub calls to file read/exists
|
533
|
+
allow(File).to receive(:exists?).with(/id_rsa.pub/) { true }
|
534
|
+
allow(File).to receive(:read).with(/id_rsa.pub/) { "foobar" }
|
535
|
+
|
536
|
+
# Should return contents of allow( previously ).to receivebed id_rsa.pub
|
537
|
+
expect(public_key).to eq("foobar")
|
538
|
+
end
|
539
|
+
|
540
|
+
it "should return an error if the files do not exist" do
|
541
|
+
expect { public_key }.to raise_error(RuntimeError, /Expected either/)
|
542
|
+
end
|
543
|
+
end
|
544
|
+
|
545
|
+
describe '#key_name' do
|
546
|
+
it 'returns a key name from the local hostname' do
|
547
|
+
# Mock out the hostname and local user calls
|
548
|
+
expect( Socket ).to receive(:gethostname) { "foobar" }
|
549
|
+
expect( aws ).to receive(:local_user) { "bob" }
|
550
|
+
|
551
|
+
# Should match the expected composite key name
|
552
|
+
expect(aws.key_name).to eq("Beaker-bob-foobar")
|
553
|
+
end
|
554
|
+
end
|
555
|
+
|
556
|
+
describe '#local_user' do
|
557
|
+
it 'returns ENV["USER"]' do
|
558
|
+
stub_const('ENV', ENV.to_hash.merge('USER' => 'SuperUser'))
|
559
|
+
expect(aws.local_user).to eq("SuperUser")
|
560
|
+
end
|
561
|
+
end
|
562
|
+
|
563
|
+
describe '#ensure_key_pair' do
|
564
|
+
let( :region ) { double('region') }
|
565
|
+
subject(:ensure_key_pair) { aws.ensure_key_pair(region) }
|
566
|
+
|
567
|
+
context 'when a beaker keypair already exists' do
|
568
|
+
it 'returns the keypair if available' do
|
569
|
+
stub_const('ENV', ENV.to_hash.merge('USER' => 'rspec'))
|
570
|
+
key_pair = double(:exists? => true, :secret => 'supersekritkey')
|
571
|
+
key_pairs = { "Beaker-rspec-SUT" => key_pair }
|
572
|
+
|
573
|
+
expect( region ).to receive(:key_pairs).and_return(key_pairs).once
|
574
|
+
expect( Socket ).to receive(:gethostname).and_return("SUT")
|
575
|
+
expect(ensure_key_pair).to eq(key_pair)
|
576
|
+
end
|
577
|
+
end
|
578
|
+
|
579
|
+
context 'when a pre-existing keypair cannot be found' do
|
580
|
+
let( :key_name ) { "Beaker-rspec-SUT" }
|
581
|
+
let( :key_pair ) { double(:exists? => false) }
|
582
|
+
let( :key_pairs ) { { key_name => key_pair } }
|
583
|
+
let( :pubkey ) { "Beaker-rspec-SUT_secret-key" }
|
584
|
+
|
585
|
+
before :each do
|
586
|
+
stub_const('ENV', ENV.to_hash.merge('USER' => 'rspec'))
|
587
|
+
expect( region ).to receive(:key_pairs).and_return(key_pairs).once
|
588
|
+
expect( Socket ).to receive(:gethostname).and_return("SUT")
|
589
|
+
end
|
590
|
+
|
591
|
+
it 'imports a new key based on user pubkey' do
|
592
|
+
allow(aws).to receive(:public_key).and_return(pubkey)
|
593
|
+
expect( key_pairs ).to receive(:import).with(key_name, pubkey)
|
594
|
+
expect(ensure_key_pair)
|
595
|
+
end
|
596
|
+
|
597
|
+
it 'returns imported keypair' do
|
598
|
+
allow(aws).to receive(:public_key)
|
599
|
+
expect( key_pairs ).to receive(:import).and_return(key_pair).once
|
600
|
+
expect(ensure_key_pair).to eq(key_pair)
|
601
|
+
end
|
602
|
+
end
|
603
|
+
end
|
604
|
+
|
605
|
+
describe '#group_id' do
|
606
|
+
it 'should return a predicatable group_id from a port list' do
|
607
|
+
expect(aws.group_id([22, 1024])).to eq("Beaker-2799478787")
|
608
|
+
end
|
609
|
+
|
610
|
+
it 'should return a predicatable group_id from an empty list' do
|
611
|
+
expect { aws.group_id([]) }.to raise_error(ArgumentError, "Ports list cannot be nil or empty")
|
612
|
+
end
|
613
|
+
end
|
614
|
+
|
615
|
+
describe '#ensure_group' do
|
616
|
+
let( :vpc ) { double('vpc') }
|
617
|
+
let( :ports ) { [22, 80, 8080] }
|
618
|
+
subject(:ensure_group) { aws.ensure_group(vpc, ports) }
|
619
|
+
|
620
|
+
context 'for an existing group' do
|
621
|
+
before :each do
|
622
|
+
@group = double(:nil? => false)
|
623
|
+
end
|
624
|
+
|
625
|
+
it 'returns group from vpc lookup' do
|
626
|
+
expect(vpc).to receive_message_chain('security_groups.filter.first').and_return(@group)
|
627
|
+
expect(ensure_group).to eq(@group)
|
628
|
+
end
|
629
|
+
|
630
|
+
context 'during group lookup' do
|
631
|
+
it 'performs group_id lookup for ports' do
|
632
|
+
expect(aws).to receive(:group_id).with(ports)
|
633
|
+
expect(vpc).to receive_message_chain('security_groups.filter.first').and_return(@group)
|
634
|
+
expect(ensure_group).to eq(@group)
|
635
|
+
end
|
636
|
+
|
637
|
+
it 'filters on group_id' do
|
638
|
+
expect(vpc).to receive(:security_groups).and_return(vpc)
|
639
|
+
expect(vpc).to receive(:filter).with('group-name', 'Beaker-1521896090').and_return(vpc)
|
640
|
+
expect(vpc).to receive(:first).and_return(@group)
|
641
|
+
expect(ensure_group).to eq(@group)
|
642
|
+
end
|
643
|
+
end
|
644
|
+
end
|
645
|
+
|
646
|
+
context 'when group does not exist' do
|
647
|
+
it 'creates group if group.nil?' do
|
648
|
+
group = double(:nil? => true)
|
649
|
+
expect(aws).to receive(:create_group).with(vpc, ports).and_return(group)
|
650
|
+
expect(vpc).to receive_message_chain('security_groups.filter.first').and_return(group)
|
651
|
+
expect(ensure_group).to eq(group)
|
652
|
+
end
|
653
|
+
end
|
654
|
+
end
|
655
|
+
|
656
|
+
describe '#create_group' do
|
657
|
+
let( :rv ) { double('rv') }
|
658
|
+
let( :ports ) { [22, 80, 8080] }
|
659
|
+
subject(:create_group) { aws.create_group(rv, ports) }
|
660
|
+
|
661
|
+
before :each do
|
662
|
+
@group = double(:nil? => false)
|
663
|
+
end
|
664
|
+
|
665
|
+
it 'returns a newly created group' do
|
666
|
+
allow(rv).to receive_message_chain('security_groups.create').and_return(@group)
|
667
|
+
allow(@group).to receive(:authorize_ingress).at_least(:once)
|
668
|
+
expect(create_group).to eq(@group)
|
669
|
+
end
|
670
|
+
|
671
|
+
it 'requests group_id for ports given' do
|
672
|
+
expect(aws).to receive(:group_id).with(ports)
|
673
|
+
allow(rv).to receive_message_chain('security_groups.create').and_return(@group)
|
674
|
+
allow(@group).to receive(:authorize_ingress).at_least(:once)
|
675
|
+
expect(create_group).to eq(@group)
|
676
|
+
end
|
677
|
+
|
678
|
+
it 'creates group with expected arguments' do
|
679
|
+
group_name = "Beaker-1521896090"
|
680
|
+
group_desc = "Custom Beaker security group for #{ports.to_a}"
|
681
|
+
expect(rv).to receive_message_chain('security_groups.create')
|
682
|
+
.with(group_name, :description => group_desc)
|
683
|
+
.and_return(@group)
|
684
|
+
allow(@group).to receive(:authorize_ingress).at_least(:once)
|
685
|
+
expect(create_group).to eq(@group)
|
686
|
+
end
|
687
|
+
|
688
|
+
it 'authorizes requested ports for group' do
|
689
|
+
expect(rv).to receive_message_chain('security_groups.create').and_return(@group)
|
690
|
+
ports.each do |port|
|
691
|
+
expect(@group).to receive(:authorize_ingress).with(:tcp, port).once
|
692
|
+
end
|
693
|
+
expect(create_group).to eq(@group)
|
694
|
+
end
|
695
|
+
end
|
696
|
+
|
697
|
+
describe '#load_fog_credentials' do
|
698
|
+
# Receive#and_call_original below allows us to test the core load_fog_credentials method
|
699
|
+
let(:creds) { {:access_key => 'awskey', :secret_key => 'awspass'} }
|
700
|
+
let(:dot_fog) { '.fog' }
|
701
|
+
subject(:load_fog_credentials) { aws.load_fog_credentials(dot_fog) }
|
702
|
+
|
703
|
+
it 'returns loaded fog credentials' do
|
704
|
+
fog_hash = {:default => {:aws_access_key_id => 'awskey', :aws_secret_access_key => 'awspass'}}
|
705
|
+
expect(aws).to receive(:load_fog_credentials).and_call_original
|
706
|
+
expect(YAML).to receive(:load_file).and_return(fog_hash)
|
707
|
+
expect(load_fog_credentials).to eq(creds)
|
708
|
+
end
|
709
|
+
|
710
|
+
context 'raises errors' do
|
711
|
+
it 'if missing access_key credential' do
|
712
|
+
fog_hash = {:default => {:aws_secret_access_key => 'awspass'}}
|
713
|
+
err_text = "You must specify an aws_access_key_id in your .fog file (#{dot_fog}) for ec2 instances!"
|
714
|
+
expect(aws).to receive(:load_fog_credentials).and_call_original
|
715
|
+
expect(YAML).to receive(:load_file).and_return(fog_hash)
|
716
|
+
expect { load_fog_credentials }.to raise_error(err_text)
|
717
|
+
end
|
718
|
+
|
719
|
+
it 'if missing secret_key credential' do
|
720
|
+
dot_fog = '.fog'
|
721
|
+
fog_hash = {:default => {:aws_access_key_id => 'awskey'}}
|
722
|
+
err_text = "You must specify an aws_secret_access_key in your .fog file (#{dot_fog}) for ec2 instances!"
|
723
|
+
expect(aws).to receive(:load_fog_credentials).and_call_original
|
724
|
+
expect(YAML).to receive(:load_file).and_return(fog_hash)
|
725
|
+
expect { load_fog_credentials }.to raise_error(err_text)
|
726
|
+
end
|
727
|
+
end
|
728
|
+
end
|
192
729
|
end
|
193
730
|
end
|