openstax_aws 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +19 -0
- data/.travis.yml +12 -0
- data/CHANGELOG.md +11 -0
- data/Gemfile +6 -0
- data/Gemfile.lock +120 -0
- data/LICENSE.txt +1 -0
- data/README.md +927 -0
- data/Rakefile +6 -0
- data/TODO.md +1 -0
- data/assets/secrets_sequence_diagram.png +0 -0
- data/bin/console +14 -0
- data/bin/create_development_environment +26 -0
- data/bin/get_latest_ubuntu_ami +31 -0
- data/bin/setup +8 -0
- data/bin/templates/aws_ruby_development.yml +221 -0
- data/examples/deployment.rb +90 -0
- data/ideas.md +15 -0
- data/lib/openstax/aws/auto_scaling_group.rb +28 -0
- data/lib/openstax/aws/auto_scaling_instance.rb +96 -0
- data/lib/openstax/aws/build_image_command_1.rb +53 -0
- data/lib/openstax/aws/change_set.rb +100 -0
- data/lib/openstax/aws/deployment_base.rb +372 -0
- data/lib/openstax/aws/distribution.rb +56 -0
- data/lib/openstax/aws/ec2_instance_data.rb +18 -0
- data/lib/openstax/aws/extensions.rb +19 -0
- data/lib/openstax/aws/git_helper.rb +18 -0
- data/lib/openstax/aws/image.rb +34 -0
- data/lib/openstax/aws/msk_cluster.rb +19 -0
- data/lib/openstax/aws/packer_1_2_5.rb +63 -0
- data/lib/openstax/aws/packer_1_4_1.rb +72 -0
- data/lib/openstax/aws/packer_factory.rb +25 -0
- data/lib/openstax/aws/rds_instance.rb +25 -0
- data/lib/openstax/aws/s3_text_file.rb +50 -0
- data/lib/openstax/aws/sam_stack.rb +85 -0
- data/lib/openstax/aws/secrets.rb +302 -0
- data/lib/openstax/aws/secrets_factory.rb +126 -0
- data/lib/openstax/aws/secrets_set.rb +21 -0
- data/lib/openstax/aws/secrets_specification.rb +68 -0
- data/lib/openstax/aws/stack.rb +465 -0
- data/lib/openstax/aws/stack_event.rb +28 -0
- data/lib/openstax/aws/stack_factory.rb +153 -0
- data/lib/openstax/aws/stack_parameters.rb +19 -0
- data/lib/openstax/aws/stack_status.rb +125 -0
- data/lib/openstax/aws/system.rb +21 -0
- data/lib/openstax/aws/tag.rb +31 -0
- data/lib/openstax/aws/template.rb +129 -0
- data/lib/openstax/aws/version.rb +5 -0
- data/lib/openstax/aws/wait_message.rb +20 -0
- data/lib/openstax_aws.rb +154 -0
- data/openstax_aws.gemspec +58 -0
- metadata +350 -0
@@ -0,0 +1,28 @@
|
|
1
|
+
|
2
|
+
require_relative 'stack_status'
|
3
|
+
|
4
|
+
module OpenStax::Aws
|
5
|
+
class Stack
|
6
|
+
class Event
|
7
|
+
def initialize(aws_stack_event)
|
8
|
+
@aws_stack_event = aws_stack_event
|
9
|
+
end
|
10
|
+
|
11
|
+
def status_text
|
12
|
+
@aws_stack_event.data.resource_status
|
13
|
+
end
|
14
|
+
|
15
|
+
def status_reason
|
16
|
+
@aws_stack_event.data.resource_status_reason
|
17
|
+
end
|
18
|
+
|
19
|
+
def failed?
|
20
|
+
Status.failure_status_texts.include?(status_text)
|
21
|
+
end
|
22
|
+
|
23
|
+
def user_initiated?
|
24
|
+
status_reason == "User Initiated"
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,153 @@
|
|
1
|
+
module OpenStax::Aws
|
2
|
+
class StackFactory
|
3
|
+
attr_reader :attributes, :id
|
4
|
+
|
5
|
+
def initialize(id:, deployment:)
|
6
|
+
raise "`deployment` cannot be nil" if deployment.nil?
|
7
|
+
|
8
|
+
@id = id
|
9
|
+
@deployment = deployment
|
10
|
+
@attributes = {
|
11
|
+
parameter_defaults: {},
|
12
|
+
tags: {}
|
13
|
+
}
|
14
|
+
end
|
15
|
+
|
16
|
+
def self.build(id:, deployment:, &block)
|
17
|
+
factory = new(id: id, deployment: deployment)
|
18
|
+
factory.instance_eval(&block) if block_given?
|
19
|
+
factory.build
|
20
|
+
end
|
21
|
+
|
22
|
+
def method_missing(name, *args, &block)
|
23
|
+
if args.empty? && !block_given?
|
24
|
+
attributes[name.to_sym]
|
25
|
+
else
|
26
|
+
attributes[name.to_sym] = args.empty? ?
|
27
|
+
@deployment.instance_eval(&block) :
|
28
|
+
args[0]
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
def template_directory(*directory_parts)
|
33
|
+
if directory_parts.empty? && !block_given?
|
34
|
+
attributes[:template_directory]
|
35
|
+
else
|
36
|
+
directory_parts = yield if directory_parts.empty?
|
37
|
+
attributes[:template_directory] = File.join(*directory_parts)
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
def autoset_absolute_template_path(fallback_base_directory)
|
42
|
+
base_directory = template_directory || fallback_base_directory
|
43
|
+
|
44
|
+
if base_directory.blank?
|
45
|
+
raise "Tried to autoset the absolute_template_path but didn't have " \
|
46
|
+
"access to a base directory (e.g. set with template_directory)"
|
47
|
+
end
|
48
|
+
|
49
|
+
if relative_template_path
|
50
|
+
path = File.join(base_directory, relative_template_path)
|
51
|
+
else
|
52
|
+
path = File.join(base_directory, "#{@id}.yml")
|
53
|
+
|
54
|
+
if !File.file?(path)
|
55
|
+
path = File.join(base_directory, "#{@id}.json")
|
56
|
+
if !File.file?(path)
|
57
|
+
raise "Couldn't infer an existing template file for stack #{@id}"
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
self.absolute_template_path(path)
|
63
|
+
end
|
64
|
+
|
65
|
+
def parameter_defaults(&block)
|
66
|
+
factory = ParameterDefaultsFactory.new(@deployment)
|
67
|
+
factory.instance_eval(&block) if block_given?
|
68
|
+
attributes[:parameter_defaults].merge!(factory.attributes)
|
69
|
+
end
|
70
|
+
|
71
|
+
def volatile_parameters(&block)
|
72
|
+
attributes[:volatile_parameters_block] = block
|
73
|
+
end
|
74
|
+
|
75
|
+
def secrets(&block)
|
76
|
+
attributes[:secrets_blocks] ||= []
|
77
|
+
attributes[:secrets_blocks].push(block)
|
78
|
+
end
|
79
|
+
|
80
|
+
def cycle_if_different_parameter(name=nil, &block)
|
81
|
+
attributes[:cycle_if_different_parameter] = block.present? ? block.call : name
|
82
|
+
end
|
83
|
+
|
84
|
+
def tag(key, value)
|
85
|
+
raise 'The first argument to `tag` must not be blank' if key.blank?
|
86
|
+
(attributes[:tags] ||= {})[key] = value
|
87
|
+
end
|
88
|
+
|
89
|
+
def build
|
90
|
+
autoset_absolute_template_path(nil) if absolute_template_path.blank?
|
91
|
+
|
92
|
+
initializer_args = {
|
93
|
+
id: id,
|
94
|
+
name: attributes[:name],
|
95
|
+
tags: @deployment.tags.merge(attributes[:tags]),
|
96
|
+
region: attributes[:region],
|
97
|
+
enable_termination_protection: attributes[:enable_termination_protection],
|
98
|
+
absolute_template_path: attributes[:absolute_template_path],
|
99
|
+
capabilities: attributes[:capabilities],
|
100
|
+
parameter_defaults: attributes[:parameter_defaults],
|
101
|
+
volatile_parameters_block: attributes[:volatile_parameters_block],
|
102
|
+
secrets_blocks: attributes[:secrets_blocks],
|
103
|
+
secrets_context: @deployment,
|
104
|
+
secrets_namespace: [@deployment.env_name, @deployment.name],
|
105
|
+
shared_secrets_substitutions_block: @deployment.shared_secrets_substitutions_block,
|
106
|
+
cycle_if_different_parameter: attributes[:cycle_if_different_parameter],
|
107
|
+
dry_run: attributes[:dry_run]
|
108
|
+
}
|
109
|
+
|
110
|
+
template = Template.from_absolute_file_path(absolute_template_path)
|
111
|
+
if template.is_sam?
|
112
|
+
if !@deployment.respond_to?(:sam_build_directory)
|
113
|
+
raise "You must set the SAM build directory with a call to `sam_build_directory`"
|
114
|
+
end
|
115
|
+
|
116
|
+
initializer_args[:build_directory] = @deployment.sam_build_directory
|
117
|
+
SamStack.new(**initializer_args)
|
118
|
+
else
|
119
|
+
Stack.new(**initializer_args)
|
120
|
+
end
|
121
|
+
end
|
122
|
+
|
123
|
+
class ParameterDefaultsFactory
|
124
|
+
attr_reader :attributes
|
125
|
+
|
126
|
+
def initialize(context)
|
127
|
+
@context = context
|
128
|
+
@attributes = {}
|
129
|
+
end
|
130
|
+
|
131
|
+
def method_missing(name, *args, &block)
|
132
|
+
attributes[name.to_sym] = args[0] || @context.instance_eval(&block)
|
133
|
+
end
|
134
|
+
end
|
135
|
+
|
136
|
+
class VolatileParametersFactory
|
137
|
+
attr_reader :attributes
|
138
|
+
|
139
|
+
def initialize(context)
|
140
|
+
raise "context cannot be nil" if context.nil?
|
141
|
+
@context = context
|
142
|
+
@attributes = {}
|
143
|
+
end
|
144
|
+
|
145
|
+
def method_missing(name, *args, &block)
|
146
|
+
raise "Volatile parameter `#{name}` cannot be called with arguments, only a block" if !args.empty?
|
147
|
+
raise "Volatile parameter `#{name}` must be called with a block to set the parameter value" if !block_given?
|
148
|
+
attributes[name.to_sym] = @context.instance_eval(&block)
|
149
|
+
end
|
150
|
+
end
|
151
|
+
|
152
|
+
end
|
153
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
module OpenStax::Aws
|
2
|
+
class StackParameters
|
3
|
+
|
4
|
+
def initialize(stack:, params:, recover_previous_values: true)
|
5
|
+
@stack = stack
|
6
|
+
@raw_params = params
|
7
|
+
@recover_previous_values = recover_previous_values
|
8
|
+
end
|
9
|
+
|
10
|
+
def [](key)
|
11
|
+
if @recover_previous_values && @raw_params[key] == :use_previous_value
|
12
|
+
@stack.deployed_parameters[key]
|
13
|
+
else
|
14
|
+
@raw_params[key]
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
end
|
19
|
+
end
|
@@ -0,0 +1,125 @@
|
|
1
|
+
module OpenStax::Aws
|
2
|
+
class Stack
|
3
|
+
class Status
|
4
|
+
def initialize(stack)
|
5
|
+
@stack = stack
|
6
|
+
end
|
7
|
+
|
8
|
+
def status_text
|
9
|
+
begin
|
10
|
+
@stack.aws_stack.stack_status
|
11
|
+
rescue Aws::CloudFormation::Errors::ValidationError => ee
|
12
|
+
case ee.message
|
13
|
+
when /Stack.*does not exist/
|
14
|
+
self.class.does_not_exist_status_text
|
15
|
+
else
|
16
|
+
raise
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
def failed?
|
22
|
+
self.class.failure_status_texts.include?(status_text)
|
23
|
+
end
|
24
|
+
|
25
|
+
def succeeded?
|
26
|
+
self.class.success_status_texts.include?(status_text)
|
27
|
+
end
|
28
|
+
|
29
|
+
def updating?
|
30
|
+
%w(
|
31
|
+
UPDATE_COMPLETE_CLEANUP_IN_PROGRESS
|
32
|
+
UPDATE_IN_PROGRESS
|
33
|
+
UPDATE_ROLLBACK_COMPLETE_CLEANUP_IN_PROGRESS
|
34
|
+
UPDATE_ROLLBACK_IN_PROGRESS
|
35
|
+
).include?(status_text)
|
36
|
+
end
|
37
|
+
|
38
|
+
def creating?
|
39
|
+
"CREATE_IN_PROGRESS" == status_text
|
40
|
+
end
|
41
|
+
|
42
|
+
def deleting?
|
43
|
+
"DELETE_IN_PROGRESS" == status_text
|
44
|
+
end
|
45
|
+
|
46
|
+
def exists?
|
47
|
+
self.class.does_not_exist_status_text != status_text
|
48
|
+
end
|
49
|
+
|
50
|
+
def self.does_not_exist_status_text
|
51
|
+
"DOES_NOT_EXIST"
|
52
|
+
end
|
53
|
+
|
54
|
+
def self.all_status_texts
|
55
|
+
%w(
|
56
|
+
CREATE_IN_PROGRESS
|
57
|
+
CREATE_FAILED
|
58
|
+
CREATE_COMPLETE
|
59
|
+
ROLLBACK_IN_PROGRESS
|
60
|
+
ROLLBACK_FAILED
|
61
|
+
ROLLBACK_COMPLETE
|
62
|
+
DELETE_IN_PROGRESS
|
63
|
+
DELETE_FAILED
|
64
|
+
DELETE_COMPLETE
|
65
|
+
UPDATE_IN_PROGRESS
|
66
|
+
UPDATE_COMPLETE_CLEANUP_IN_PROGRESS
|
67
|
+
UPDATE_COMPLETE
|
68
|
+
UPDATE_ROLLBACK_IN_PROGRESS
|
69
|
+
UPDATE_ROLLBACK_FAILED
|
70
|
+
UPDATE_ROLLBACK_COMPLETE_CLEANUP_IN_PROGRESS
|
71
|
+
UPDATE_ROLLBACK_COMPLETE
|
72
|
+
REVIEW_IN_PROGRESS
|
73
|
+
IMPORT_IN_PROGRESS
|
74
|
+
IMPORT_COMPLETE
|
75
|
+
IMPORT_ROLLBACK_IN_PROGRESS
|
76
|
+
IMPORT_ROLLBACK_FAILED
|
77
|
+
IMPORT_ROLLBACK_COMPLETE
|
78
|
+
)
|
79
|
+
end
|
80
|
+
|
81
|
+
def self.active_status_texts
|
82
|
+
all_status_texts - %w(CREATE_FAILED DELETE_COMPLETE)
|
83
|
+
end
|
84
|
+
|
85
|
+
def self.failure_status_texts
|
86
|
+
%w(
|
87
|
+
UPDATE_ROLLBACK_COMPLETE
|
88
|
+
ROLLBACK_COMPLETE
|
89
|
+
ROLLBACK_IN_PROGRESS
|
90
|
+
CREATE_FAILED
|
91
|
+
ROLLBACK_FAILED
|
92
|
+
DELETE_FAILED
|
93
|
+
UPDATE_ROLLBACK_FAILED
|
94
|
+
IMPORT_ROLLBACK_FAILED
|
95
|
+
)
|
96
|
+
end
|
97
|
+
|
98
|
+
def self.success_status_texts
|
99
|
+
%w(
|
100
|
+
UPDATE_COMPLETE
|
101
|
+
DELETE_COMPLETE
|
102
|
+
CREATE_COMPLETE
|
103
|
+
)
|
104
|
+
end
|
105
|
+
|
106
|
+
def failed_events_since_last_user_event
|
107
|
+
@stack.events.each_with_object([]) do |event, array|
|
108
|
+
array.push(event) if event.failed? && event.status_reason #if nil, don't push
|
109
|
+
return array if event.user_initiated?
|
110
|
+
end
|
111
|
+
end
|
112
|
+
|
113
|
+
def to_h
|
114
|
+
{
|
115
|
+
status: status_text,
|
116
|
+
failed_events_since_last_user_event: failed? ? failed_events_since_last_user_event : []
|
117
|
+
}
|
118
|
+
end
|
119
|
+
|
120
|
+
def to_json
|
121
|
+
to_h.to_json
|
122
|
+
end
|
123
|
+
end
|
124
|
+
end
|
125
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
module OpenStax::Aws
|
2
|
+
module System
|
3
|
+
|
4
|
+
def self.call(command, logger: nil, dry_run:)
|
5
|
+
logger&.info("**** DRY RUN ****") if dry_run
|
6
|
+
logger&.info("Running: #{command}")
|
7
|
+
|
8
|
+
if !dry_run
|
9
|
+
Open3.popen2e(command) do |stdin, stdout_err, wait_thr|
|
10
|
+
while line=stdout_err.gets do
|
11
|
+
STDERR.puts(line)
|
12
|
+
end
|
13
|
+
|
14
|
+
exit_status = wait_thr.value.exitstatus
|
15
|
+
exit(exit_status) if exit_status != 0
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
end
|
21
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
module OpenStax::Aws
|
2
|
+
class Tag
|
3
|
+
|
4
|
+
attr_reader :key, :value
|
5
|
+
|
6
|
+
AWS_TAG_KEY_REGEX = /\A[\w\-\/.+=:@]{1,128}\z/
|
7
|
+
AWS_TAG_VALUE_REGEX = /\A[\w\-\/.+=:@ ]{0,256}\z/
|
8
|
+
|
9
|
+
def initialize(key, value)
|
10
|
+
if key.nil? || !key.match(AWS_TAG_KEY_REGEX)
|
11
|
+
raise "The tag key '#{key}' is invalid: must be a non-blank ID matching #{AWS_TAG_KEY_REGEX}"
|
12
|
+
end
|
13
|
+
|
14
|
+
@key = key.to_s
|
15
|
+
|
16
|
+
if @key.starts_with?("aws:")
|
17
|
+
raise "The tag key '#{@key}' is invalid: it cannot start with 'aws:'"
|
18
|
+
end
|
19
|
+
|
20
|
+
if value.nil?
|
21
|
+
raise "The tag value for key '#{key}' cannot be nil"
|
22
|
+
end
|
23
|
+
|
24
|
+
if !value.match(AWS_TAG_VALUE_REGEX)
|
25
|
+
raise "The tag value '#{value}' must be a tag value matching #{AWS_TAG_VALUE_REGEX}"
|
26
|
+
end
|
27
|
+
|
28
|
+
@value = value.to_s
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
@@ -0,0 +1,129 @@
|
|
1
|
+
module OpenStax::Aws
|
2
|
+
class Template
|
3
|
+
|
4
|
+
class TemplateInvalid < StandardError
|
5
|
+
attr_reader :template_path, :template_body
|
6
|
+
|
7
|
+
def initialize(msg, template_path, template_body)
|
8
|
+
@template_path = template_path
|
9
|
+
@template_body = template_body
|
10
|
+
|
11
|
+
super(msg)
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
attr_reader :absolute_file_path
|
16
|
+
|
17
|
+
def self.from_absolute_file_path(absolute_file_path)
|
18
|
+
new(absolute_file_path: absolute_file_path)
|
19
|
+
end
|
20
|
+
|
21
|
+
def self.from_body(body)
|
22
|
+
new(body: body)
|
23
|
+
end
|
24
|
+
|
25
|
+
def basename
|
26
|
+
File.basename(absolute_file_path)
|
27
|
+
end
|
28
|
+
|
29
|
+
def body
|
30
|
+
@body ||= File.read(absolute_file_path)
|
31
|
+
end
|
32
|
+
|
33
|
+
def hash
|
34
|
+
json_hash || yml_hash || raise("Cannot read template #{absolute_file_path}")
|
35
|
+
end
|
36
|
+
|
37
|
+
def valid?
|
38
|
+
begin
|
39
|
+
validate
|
40
|
+
rescue TemplateInvalid
|
41
|
+
return false
|
42
|
+
end
|
43
|
+
|
44
|
+
true
|
45
|
+
end
|
46
|
+
|
47
|
+
def parameter_names
|
48
|
+
hash["Parameters"].try(:keys) || []
|
49
|
+
end
|
50
|
+
|
51
|
+
def required_capabilities
|
52
|
+
has_an_iam_resource = hash["Resources"].any? do |resource_id, resource_body|
|
53
|
+
resource_body["Type"].starts_with?("AWS::IAM::")
|
54
|
+
end
|
55
|
+
|
56
|
+
# A bit of a cop out to claim named_iam since we haven't checked
|
57
|
+
# to see if any of the IAM resources have custom names, but it will
|
58
|
+
# work
|
59
|
+
has_an_iam_resource ? [:named_iam] : []
|
60
|
+
end
|
61
|
+
|
62
|
+
def s3_key
|
63
|
+
[
|
64
|
+
OpenStax::Aws.configuration.cfn_template_bucket_folder,
|
65
|
+
s3_folder,
|
66
|
+
basename
|
67
|
+
].compact.join("/")
|
68
|
+
end
|
69
|
+
|
70
|
+
def s3_folder
|
71
|
+
@unique_s3_folder ||=
|
72
|
+
OpenStax::Aws.configuration.fixed_s3_template_folder ||
|
73
|
+
Time.now.utc.strftime("%Y%m%d_%H%M%S_#{SecureRandom.hex(4)}")
|
74
|
+
end
|
75
|
+
|
76
|
+
def s3_url
|
77
|
+
upload_once
|
78
|
+
"https://s3.amazonaws.com/#{OpenStax::Aws.configuration.cfn_template_bucket_name}/#{s3_key}"
|
79
|
+
end
|
80
|
+
|
81
|
+
def upload_once
|
82
|
+
return if @uploaded
|
83
|
+
|
84
|
+
validate
|
85
|
+
upload
|
86
|
+
|
87
|
+
@uploaded = true
|
88
|
+
end
|
89
|
+
|
90
|
+
def is_sam?
|
91
|
+
body.match(/Transform: AWS::Serverless/).present?
|
92
|
+
end
|
93
|
+
|
94
|
+
protected
|
95
|
+
|
96
|
+
def initialize(absolute_file_path: nil, body: nil)
|
97
|
+
raise "One of `absolute_file_path` or `body` must be set" if absolute_file_path.blank? && body.nil?
|
98
|
+
@absolute_file_path = absolute_file_path
|
99
|
+
@body = body.try(:dup)
|
100
|
+
end
|
101
|
+
|
102
|
+
def validate
|
103
|
+
begin
|
104
|
+
# any region works here
|
105
|
+
Aws::CloudFormation::Client.new(region: "us-west-2").validate_template(template_body: body)
|
106
|
+
rescue Aws::CloudFormation::Errors::ValidationError => ee
|
107
|
+
raise TemplateInvalid.new(basename + ": " + ee.message, absolute_file_path, body)
|
108
|
+
end
|
109
|
+
end
|
110
|
+
|
111
|
+
def upload
|
112
|
+
client = Aws::S3::Client.new(region: OpenStax::Aws.configuration.cfn_template_bucket_region)
|
113
|
+
client.put_object({
|
114
|
+
body: body,
|
115
|
+
bucket: OpenStax::Aws.configuration.cfn_template_bucket_name,
|
116
|
+
key: s3_key
|
117
|
+
})
|
118
|
+
end
|
119
|
+
|
120
|
+
def json_hash
|
121
|
+
JSON.parse(body) rescue nil
|
122
|
+
end
|
123
|
+
|
124
|
+
def yml_hash
|
125
|
+
YAML.load(body) rescue nil
|
126
|
+
end
|
127
|
+
|
128
|
+
end
|
129
|
+
end
|