cloud-mu 3.2.0 → 3.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Dockerfile +1 -1
- data/bin/mu-adopt +12 -1
- data/bin/mu-load-config.rb +2 -1
- data/bin/mu-run-tests +14 -2
- data/cloud-mu.gemspec +3 -3
- data/modules/mu.rb +2 -2
- data/modules/mu/adoption.rb +5 -5
- data/modules/mu/cleanup.rb +47 -25
- data/modules/mu/cloud.rb +29 -1
- data/modules/mu/cloud/dnszone.rb +0 -2
- data/modules/mu/cloud/resource_base.rb +9 -3
- data/modules/mu/cloud/wrappers.rb +4 -0
- data/modules/mu/config.rb +1 -1
- data/modules/mu/config/bucket.rb +31 -2
- data/modules/mu/config/cache_cluster.rb +1 -1
- data/modules/mu/config/cdn.rb +100 -0
- data/modules/mu/config/container_cluster.rb +1 -1
- data/modules/mu/config/database.rb +1 -1
- data/modules/mu/config/dnszone.rb +4 -3
- data/modules/mu/config/endpoint.rb +1 -0
- data/modules/mu/config/function.rb +16 -7
- data/modules/mu/config/job.rb +89 -0
- data/modules/mu/config/notifier.rb +7 -18
- data/modules/mu/config/ref.rb +53 -7
- data/modules/mu/config/server.rb +1 -1
- data/modules/mu/config/vpc.rb +1 -0
- data/modules/mu/defaults/AWS.yaml +26 -26
- data/modules/mu/deploy.rb +13 -0
- data/modules/mu/master.rb +21 -0
- data/modules/mu/mommacat.rb +1 -0
- data/modules/mu/mommacat/daemon.rb +13 -7
- data/modules/mu/providers/aws.rb +115 -16
- data/modules/mu/providers/aws/alarm.rb +2 -2
- data/modules/mu/providers/aws/bucket.rb +274 -40
- data/modules/mu/providers/aws/cache_cluster.rb +4 -4
- data/modules/mu/providers/aws/cdn.rb +782 -0
- data/modules/mu/providers/aws/collection.rb +2 -2
- data/modules/mu/providers/aws/container_cluster.rb +57 -37
- data/modules/mu/providers/aws/database.rb +11 -11
- data/modules/mu/providers/aws/dnszone.rb +24 -7
- data/modules/mu/providers/aws/endpoint.rb +535 -50
- data/modules/mu/providers/aws/firewall_rule.rb +6 -3
- data/modules/mu/providers/aws/folder.rb +1 -1
- data/modules/mu/providers/aws/function.rb +288 -125
- data/modules/mu/providers/aws/group.rb +9 -7
- data/modules/mu/providers/aws/habitat.rb +2 -2
- data/modules/mu/providers/aws/job.rb +466 -0
- data/modules/mu/providers/aws/loadbalancer.rb +9 -8
- data/modules/mu/providers/aws/log.rb +3 -3
- data/modules/mu/providers/aws/msg_queue.rb +12 -3
- data/modules/mu/providers/aws/nosqldb.rb +96 -5
- data/modules/mu/providers/aws/notifier.rb +135 -63
- data/modules/mu/providers/aws/role.rb +51 -37
- data/modules/mu/providers/aws/search_domain.rb +165 -29
- data/modules/mu/providers/aws/server.rb +12 -9
- data/modules/mu/providers/aws/server_pool.rb +26 -13
- data/modules/mu/providers/aws/storage_pool.rb +2 -2
- data/modules/mu/providers/aws/user.rb +4 -4
- data/modules/mu/providers/aws/userdata/linux.erb +5 -4
- data/modules/mu/providers/aws/vpc.rb +3 -3
- data/modules/mu/providers/azure/server.rb +2 -1
- data/modules/mu/providers/google.rb +1 -0
- data/modules/mu/providers/google/bucket.rb +1 -1
- data/modules/mu/providers/google/container_cluster.rb +1 -1
- data/modules/mu/providers/google/database.rb +1 -1
- data/modules/mu/providers/google/firewall_rule.rb +1 -1
- data/modules/mu/providers/google/folder.rb +1 -1
- data/modules/mu/providers/google/function.rb +1 -1
- data/modules/mu/providers/google/group.rb +1 -1
- data/modules/mu/providers/google/habitat.rb +1 -1
- data/modules/mu/providers/google/loadbalancer.rb +1 -1
- data/modules/mu/providers/google/role.rb +4 -2
- data/modules/mu/providers/google/server.rb +1 -1
- data/modules/mu/providers/google/server_pool.rb +1 -1
- data/modules/mu/providers/google/user.rb +1 -1
- data/modules/mu/providers/google/vpc.rb +1 -1
- data/modules/tests/aws-jobs-functions.yaml +46 -0
- data/modules/tests/centos6.yaml +4 -0
- data/modules/tests/centos7.yaml +4 -0
- data/modules/tests/ecs.yaml +2 -2
- data/modules/tests/eks.yaml +1 -1
- data/modules/tests/functions/node-function/lambda_function.js +10 -0
- data/modules/tests/functions/python-function/lambda_function.py +12 -0
- data/modules/tests/microservice_app.yaml +288 -0
- data/modules/tests/rds.yaml +5 -5
- data/modules/tests/regrooms/rds.yaml +5 -5
- data/modules/tests/server-with-scrub-muisms.yaml +1 -1
- data/modules/tests/super_complex_bok.yml +2 -2
- data/modules/tests/super_simple_bok.yml +2 -2
- metadata +12 -4
@@ -381,14 +381,14 @@ module MU
|
|
381
381
|
# @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
|
382
382
|
# @param region [String]: The cloud provider region
|
383
383
|
# @return [void]
|
384
|
-
def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
|
384
|
+
def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
|
385
385
|
filters = if flags and flags["vpc_id"]
|
386
386
|
[
|
387
387
|
{name: "vpc-id", values: [flags["vpc_id"]]}
|
388
388
|
]
|
389
389
|
else
|
390
390
|
filters = [
|
391
|
-
{name: "tag:MU-ID", values: [
|
391
|
+
{name: "tag:MU-ID", values: [deploy_id]}
|
392
392
|
]
|
393
393
|
if !ignoremaster
|
394
394
|
filters << {name: "tag:MU-MASTER-IP", values: [MU.mu_public_ip]}
|
@@ -860,8 +860,11 @@ module MU
|
|
860
860
|
p_start = rule['port'].to_i
|
861
861
|
p_end = rule['port'].to_i
|
862
862
|
elsif rule['proto'] != "icmp"
|
863
|
-
|
863
|
+
MU.log "Can't create a TCP or UDP security group rule without specifying ports, assuming 'all'", MU::WARN, details: rule
|
864
|
+
p_start = "0"
|
865
|
+
p_end = "65535"
|
864
866
|
end
|
867
|
+
|
865
868
|
if rule['proto'] != "icmp"
|
866
869
|
if p_start.nil? or p_end.nil?
|
867
870
|
raise MuError, "Got nil ports out of rule #{rule}"
|
@@ -59,7 +59,7 @@ module MU
|
|
59
59
|
# @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
|
60
60
|
# @param region [String]: The cloud provider region
|
61
61
|
# @return [void]
|
62
|
-
def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
|
62
|
+
def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
|
63
63
|
end
|
64
64
|
|
65
65
|
# Locate an existing AWS organization. If no identifying parameters are specified, this will return a description of the Organization which owns the account for our credentials.
|
@@ -18,6 +18,18 @@ module MU
|
|
18
18
|
# A function as configured in {MU::Config::BasketofKittens::functions}
|
19
19
|
class Function < MU::Cloud::Function
|
20
20
|
|
21
|
+
# If we have sibling resources in our deployment, automatically inject
|
22
|
+
# interesting things about them into our function's environment
|
23
|
+
# variables.
|
24
|
+
SIBLING_VARS = {
|
25
|
+
"servers" => ["private_ip_address", "public_ip_address"],
|
26
|
+
"search_domains" => ["endpoint"],
|
27
|
+
"databases" => ["endpoint"],
|
28
|
+
"endpoints" => ["url"],
|
29
|
+
"notifiers" => ["TopicArn"],
|
30
|
+
"nosqldbs" => ["table_arn"]
|
31
|
+
}
|
32
|
+
|
21
33
|
# Initialize this cloud resource object. Calling +super+ will invoke the initializer defined under {MU::Cloud}, which should set the attribtues listed in {MU::Cloud::PUBLIC_ATTRS} as well as applicable dependency shortcuts, like +@vpc+, for us.
|
22
34
|
# @param args [Hash]: Hash of named arguments passed via Ruby's double-splat
|
23
35
|
def initialize(**args)
|
@@ -42,92 +54,44 @@ module MU
|
|
42
54
|
|
43
55
|
# Called automatically by {MU::Deploy#createResources}
|
44
56
|
def create
|
45
|
-
role_arn = get_role_arn(@config['iam_role'])
|
46
57
|
|
47
|
-
lambda_properties =
|
48
|
-
code: {},
|
49
|
-
function_name: @mu_name,
|
50
|
-
handler: @config['handler'],
|
51
|
-
publish: true,
|
52
|
-
role: role_arn,
|
53
|
-
runtime: @config['runtime'],
|
54
|
-
}
|
58
|
+
lambda_properties = get_properties
|
55
59
|
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
lambda_properties[:code][:zip_file] = zip
|
60
|
-
else
|
61
|
-
lambda_properties[:code][:s3_bucket] = @config['code']['s3_bucket']
|
62
|
-
lambda_properties[:code][:s3_key] = @config['code']['s3_key']
|
63
|
-
if @config['code']['s3_object_version']
|
64
|
-
lambda_properties[:code][:s3_object_version] = @config['code']['s3_object_version']
|
65
|
-
end
|
66
|
-
end
|
67
|
-
|
68
|
-
if @config.has_key?('timeout')
|
69
|
-
lambda_properties[:timeout] = @config['timeout'].to_i ## secs
|
70
|
-
end
|
71
|
-
|
72
|
-
if @config.has_key?('memory')
|
73
|
-
lambda_properties[:memory_size] = @config['memory'].to_i
|
74
|
-
end
|
75
|
-
|
76
|
-
if @config.has_key?('environment_variables')
|
77
|
-
lambda_properties[:environment] = {
|
78
|
-
variables: {@config['environment_variables'][0]['key'] => @config['environment_variables'][0]['value']}
|
79
|
-
}
|
80
|
-
end
|
81
|
-
|
82
|
-
lambda_properties[:tags] = {}
|
83
|
-
MU::MommaCat.listStandardTags.each_pair { |k, v|
|
84
|
-
lambda_properties[:tags][k] = v
|
60
|
+
MU.retrier([Aws::Lambda::Errors::InvalidParameterValueException], max: 5, wait: 10) {
|
61
|
+
resp = MU::Cloud::AWS.lambda(region: @config['region'], credentials: @config['credentials']).create_function(lambda_properties)
|
62
|
+
@cloud_id = resp.function_name
|
85
63
|
}
|
86
|
-
if @config['tags']
|
87
|
-
@config['tags'].each { |tag|
|
88
|
-
lambda_properties[:tags][tag.key.first] = tag.values.first
|
89
|
-
}
|
90
|
-
end
|
91
64
|
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
}
|
99
|
-
end
|
100
|
-
if !@vpc
|
101
|
-
raise MuError, "Function #{@config['name']} had a VPC configured, but none was loaded"
|
102
|
-
end
|
103
|
-
lambda_properties[:vpc_config] = {
|
104
|
-
:subnet_ids => @vpc.subnets.map { |s| s.cloud_id },
|
105
|
-
:security_group_ids => sgs
|
106
|
-
}
|
107
|
-
end
|
108
|
-
|
109
|
-
retries = 0
|
110
|
-
resp = begin
|
111
|
-
MU::Cloud::AWS.lambda(region: @config['region'], credentials: @config['credentials']).create_function(lambda_properties)
|
112
|
-
rescue Aws::Lambda::Errors::InvalidParameterValueException => e
|
113
|
-
# Freshly-made IAM roles sometimes aren't really ready
|
114
|
-
if retries < 5
|
115
|
-
sleep 10
|
116
|
-
retries += 1
|
117
|
-
retry
|
118
|
-
end
|
119
|
-
raise e
|
120
|
-
end
|
121
|
-
|
122
|
-
@cloud_id = resp.function_name
|
65
|
+
# the console does this and docs expect it to be there, so mimic the
|
66
|
+
# behavior
|
67
|
+
MU::Cloud::AWS.cloudwatchlogs(region: @config["region"], credentials: @credentials).create_log_group(
|
68
|
+
log_group_name: "/aws/lambda/#{@cloud_id}",
|
69
|
+
tags: @tags
|
70
|
+
)
|
123
71
|
end
|
124
72
|
|
125
73
|
# Called automatically by {MU::Deploy#createResources}
|
126
74
|
def groom
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
75
|
+
old_props = MU.structToHash(cloud_desc)
|
76
|
+
|
77
|
+
new_props = get_properties
|
78
|
+
code_block = new_props[:code]
|
79
|
+
new_props.reject! { |k, _v| [:code, :publish, :tags].include?(k) }
|
80
|
+
changes = {}
|
81
|
+
new_props.each_pair { |k, v|
|
82
|
+
changes[k] = v if v != old_props[k]
|
83
|
+
}
|
84
|
+
if !changes.empty?
|
85
|
+
MU.log "Updating Lambda #{@mu_name}", MU::NOTICE, details: changes
|
86
|
+
MU::Cloud::AWS.lambda(region: @config['region'], credentials: @config['credentials']).update_function_configuration(new_props)
|
87
|
+
end
|
88
|
+
|
89
|
+
if @code_sha256 and @code_sha256 != cloud_desc.code_sha_256.chomp
|
90
|
+
MU.log "Updating code in Lambda #{@mu_name}", MU::NOTICE, details: { "old" => @code_sha256, "new" => cloud_desc.code_sha_256 }
|
91
|
+
code_block[:publish] = true
|
92
|
+
code_block[:function_name] = @cloud_id
|
93
|
+
MU::Cloud::AWS.lambda(region: @config['region'], credentials: @config['credentials']).update_function_code(code_block)
|
94
|
+
end
|
131
95
|
|
132
96
|
# tag_function = assign_tag(lambda_func.function_arn, @config['tags'])
|
133
97
|
|
@@ -141,7 +105,7 @@ module MU
|
|
141
105
|
### triggers must exist prior
|
142
106
|
if @config['triggers']
|
143
107
|
@config['triggers'].each { |tr|
|
144
|
-
trigger_arn =
|
108
|
+
trigger_arn = resolveARN(tr['service'], tr['name'])
|
145
109
|
|
146
110
|
trigger_properties = {
|
147
111
|
action: "lambda:InvokeFunction",
|
@@ -151,15 +115,33 @@ module MU
|
|
151
115
|
statement_id: "#{@mu_name}-ID-1",
|
152
116
|
}
|
153
117
|
|
154
|
-
MU.log
|
118
|
+
MU.log "Adding #{tr['service']} #{tr['name']} trigger to Lambda function #{@cloud_id}", details: trigger_properties
|
155
119
|
begin
|
156
120
|
MU::Cloud::AWS.lambda(region: @config['region'], credentials: @config['credentials']).add_permission(trigger_properties)
|
157
121
|
rescue Aws::Lambda::Errors::ResourceConflictException
|
122
|
+
# just means the permission is already there
|
158
123
|
end
|
159
|
-
adjust_trigger(tr['service'], trigger_arn,
|
124
|
+
adjust_trigger(tr['service'], trigger_arn, arn, @mu_name)
|
160
125
|
}
|
161
126
|
|
162
127
|
end
|
128
|
+
|
129
|
+
if @config['invoke_on_completion']
|
130
|
+
invoke_params = {
|
131
|
+
function_name: @cloud_id,
|
132
|
+
invocation_type: @config['invoke_on_completion']['invocation_type'],
|
133
|
+
log_type: "Tail"
|
134
|
+
}
|
135
|
+
if @config['invoke_on_completion']['payload']
|
136
|
+
invoke_params[:payload] = JSON.generate(@config['invoke_on_completion']['payload'])
|
137
|
+
end
|
138
|
+
resp = MU::Cloud::AWS.lambda(region: @config['region'], credentials: @config['credentials']).invoke(invoke_params)
|
139
|
+
if resp.status_code == 200
|
140
|
+
MU.log "Invoked #{@cloud_id}", MU::NOTICE, details: Base64.decode64(resp.log_result)
|
141
|
+
else
|
142
|
+
MU.log "Invoked #{@cloud_id} and got #{resp.status_code} (#{resp.function_error})", MU::WARN, details: Base64.decode64(resp.log_result)
|
143
|
+
end
|
144
|
+
end
|
163
145
|
end
|
164
146
|
|
165
147
|
# Intended to be called by other Mu resources, such as Endpoints (API
|
@@ -170,13 +152,16 @@ module MU
|
|
170
152
|
function_name: @mu_name,
|
171
153
|
principal: "#{calling_service}.amazonaws.com",
|
172
154
|
source_arn: calling_arn,
|
173
|
-
statement_id: "#{calling_service}-#{calling_name}",
|
155
|
+
statement_id: "#{calling_service}-#{calling_name.gsub(/[^a-z0-9\-_]/i, '_')}",
|
174
156
|
}
|
175
157
|
|
176
158
|
begin
|
177
159
|
# XXX There doesn't seem to be an API call to list or view existing
|
178
160
|
# permissions, wtaf. This means we can't intelligently guard this.
|
179
161
|
MU::Cloud::AWS.lambda(region: @config['region'], credentials: @config['credentials']).add_permission(trigger)
|
162
|
+
rescue Aws::Lambda::Errors::ValidationException => e
|
163
|
+
MU.log e.message+" (calling_arn: #{calling_arn}, calling_service: #{calling_service}, calling_name: #{calling_name})", MU::ERR, details: trigger
|
164
|
+
raise e
|
180
165
|
rescue Aws::Lambda::Errors::ResourceConflictException => e
|
181
166
|
if e.message.match(/already exists/)
|
182
167
|
MU::Cloud::AWS.lambda(region: @config['region'], credentials: @config['credentials']).remove_permission(
|
@@ -192,17 +177,23 @@ module MU
|
|
192
177
|
end
|
193
178
|
|
194
179
|
# Look up an ARN for a given trigger type and resource name
|
195
|
-
def
|
196
|
-
supported_triggers = %w(apigateway sns events event cloudwatch_event)
|
180
|
+
def resolveARN(svc, name)
|
181
|
+
supported_triggers = %w(apigateway sns events event cloudwatch_event dynamodb)
|
197
182
|
if supported_triggers.include?(svc.downcase)
|
198
183
|
arn = nil
|
199
184
|
case svc.downcase
|
200
185
|
when 'sns'
|
201
|
-
|
186
|
+
sib_sns = @deploy.findLitterMate(name: name, type: "notifiers")
|
187
|
+
arn = sib_sns ? sib_sns.arn : "arn:aws:sns:#{@config['region']}:#{MU::Cloud::AWS.credToAcct(@config['credentials'])}:#{name}"
|
202
188
|
when 'alarm','events', 'event', 'cloudwatch_event'
|
203
|
-
|
189
|
+
sib_event = @deploy.findLitterMate(name: name, type: "job")
|
190
|
+
arn = sib_event ? sib_event.arn : "arn:aws:events:#{@config['region']}:#{MU::Cloud::AWS.credToAcct(@config['credentials'])}:rule/#{name}"
|
191
|
+
when 'dynamodb'
|
192
|
+
sib_dynamo = @deploy.findLitterMate(name: name, type: "nosqldb")
|
193
|
+
arn = sib_dynamo ? sib_dynamo.arn : "arn:aws:dynamodb:#{@config['region']}:#{MU::Cloud::AWS.credToAcct(@config['credentials'])}:table/#{name}"
|
204
194
|
when 'apigateway'
|
205
|
-
|
195
|
+
sib_apig = @deploy.findLitterMate(name: name, type: "endpoints")
|
196
|
+
arn = sib_apig ? sib_apig.arn : "arn:aws:apigateway:#{@config['region']}:#{MU::Cloud::AWS.credToAcct(@config['credentials'])}:#{name}"
|
206
197
|
when 's3'
|
207
198
|
arn = ''
|
208
199
|
end
|
@@ -219,13 +210,21 @@ module MU
|
|
219
210
|
case trig_type
|
220
211
|
|
221
212
|
when 'sns'
|
222
|
-
|
223
|
-
|
224
|
-
|
225
|
-
|
226
|
-
|
227
|
-
|
228
|
-
|
213
|
+
MU::Cloud.resourceClass("AWS", "Notifier").subscribe(trig_arn, arn, "lambda", region: @config['region'], credentials: @credentials)
|
214
|
+
when 'dynamodb'
|
215
|
+
stream = MU::Cloud::AWS.dynamostream(region: @config['region'], credentials: @config['credentials']).list_streams(table_name: trig_arn.sub(/.*?:table\//, '')).streams.first
|
216
|
+
# XXX guard this
|
217
|
+
MU.log "Adding DynamoDB Stream from #{stream.stream_arn} as trigger for #{@cloud_id}"
|
218
|
+
begin
|
219
|
+
MU::Cloud::AWS.lambda(region: @config['region'], credentials: @config['credentials']).create_event_source_mapping(
|
220
|
+
event_source_arn: stream.stream_arn,
|
221
|
+
function_name: @cloud_id,
|
222
|
+
starting_position: "TRIM_HORIZON" # ...whatever that is
|
223
|
+
)
|
224
|
+
rescue ::Aws::Lambda::Errors::ResourceConflictException
|
225
|
+
end
|
226
|
+
|
227
|
+
# MU::Cloud.resourceClass("AWS", "NoSQLDB").subscribe(trig_arn, arn, "lambda", region: @config['region'], credentials: @credentials)
|
229
228
|
when 'event','cloudwatch_event', 'events'
|
230
229
|
# XXX don't do this, use MU::Cloud::AWS::Log
|
231
230
|
MU::Cloud::AWS.cloudwatch_events(region: region, credentials: @config['credentials']).put_targets({
|
@@ -237,9 +236,8 @@ module MU
|
|
237
236
|
}
|
238
237
|
]
|
239
238
|
})
|
240
|
-
|
241
|
-
|
242
|
-
# MU.log "Creation of API Gateway integrations not yet implemented, you'll have to do this manually", MU::WARN, details: "(because we'll basically have to implement all of APIG for this)"
|
239
|
+
when 'apigateway'
|
240
|
+
addTrigger(trig_arn, "lambda", trig_arn.sub(/.*?([a-z0-9\-_]+)$/i, '\1'))
|
243
241
|
end
|
244
242
|
end
|
245
243
|
|
@@ -247,9 +245,8 @@ module MU
|
|
247
245
|
# Return the metadata for this Function rule
|
248
246
|
# @return [Hash]
|
249
247
|
def notify
|
250
|
-
|
251
|
-
|
252
|
-
return deploy_struct
|
248
|
+
return nil if !cloud_desc
|
249
|
+
MU.structToHash(cloud_desc, stringify_keys: true)
|
253
250
|
end
|
254
251
|
|
255
252
|
# Does this resource type exist as a global (cloud-wide) artifact, or
|
@@ -270,14 +267,14 @@ module MU
|
|
270
267
|
# @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
|
271
268
|
# @param region [String]: The cloud provider region
|
272
269
|
# @return [void]
|
273
|
-
def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
|
270
|
+
def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
|
274
271
|
MU.log "AWS::Function.cleanup: need to support flags['known']", MU::DEBUG, details: flags
|
275
272
|
|
276
273
|
MU::Cloud::AWS.lambda(credentials: credentials, region: region).list_functions.functions.each { |f|
|
277
274
|
desc = MU::Cloud::AWS.lambda(credentials: credentials, region: region).get_function(
|
278
275
|
function_name: f.function_name
|
279
276
|
)
|
280
|
-
if desc.tags and desc.tags["MU-ID"] ==
|
277
|
+
if desc.tags and desc.tags["MU-ID"] == deploy_id and (desc.tags["MU-MASTER-IP"] == MU.mu_public_ip or ignoremaster)
|
281
278
|
MU.log "Deleting Lambda function #{f.function_name}"
|
282
279
|
if !noop
|
283
280
|
MU::Cloud::AWS.lambda(credentials: credentials, region: region).delete_function(
|
@@ -292,7 +289,7 @@ module MU
|
|
292
289
|
# Canonical Amazon Resource Number for this resource
|
293
290
|
# @return [String]
|
294
291
|
def arn
|
295
|
-
cloud_desc.function_arn
|
292
|
+
cloud_desc ? cloud_desc.function_arn : nil
|
296
293
|
end
|
297
294
|
|
298
295
|
# Locate an existing function.
|
@@ -334,6 +331,20 @@ module MU
|
|
334
331
|
bok['timeout'] = cloud_desc.timeout
|
335
332
|
|
336
333
|
function = MU::Cloud::AWS.lambda(region: @config['region'], credentials: @credentials).get_function(function_name: bok['name'])
|
334
|
+
# event_srcs = MU::Cloud::AWS.lambda(region: @config['region'], credentials: @credentials).list_event_source_mappings(function_name: @cloud_id)
|
335
|
+
# if event_srcs and !event_srcs.event_source_mappings.empty?
|
336
|
+
# MU.log "dem mappings tho #{@cloud_id}", MU::WARN, details: event_srcs
|
337
|
+
# end
|
338
|
+
|
339
|
+
# begin
|
340
|
+
# invoke_cfg = MU::Cloud::AWS.lambda(region: @config['region'], credentials: @credentials).get_function_event_invoke_config(function_name: @cloud_id)
|
341
|
+
# MU.log "invoke config #{@cloud_id}", MU::WARN, details: invoke_cfg
|
342
|
+
# rescue ::Aws::Lambda::Errors::ResourceNotFoundException
|
343
|
+
# end
|
344
|
+
|
345
|
+
# MU.log @cloud_id, MU::WARN, details: cloud_desc if @cloud_id == "Espier-Scheduled-Scanner"
|
346
|
+
# MU.log "configuration #{@cloud_id}", MU::WARN, details: MU::Cloud::AWS.lambda(region: @config['region'], credentials: @credentials).get_function_configuration(function_name: @cloud_id) if @cloud_id == "Espier-Scheduled-Scanner"
|
347
|
+
|
337
348
|
|
338
349
|
if function.code.repository_type == "S3"
|
339
350
|
bok['code'] = {}
|
@@ -393,16 +404,29 @@ module MU
|
|
393
404
|
|
394
405
|
if function.configuration.role
|
395
406
|
shortname = function.configuration.role.sub(/.*?role\/([^\/]+)$/, '\1')
|
396
|
-
MU.log shortname, MU::NOTICE, details: function.configuration.role
|
397
407
|
bok['role'] = MU::Config::Ref.get(
|
398
408
|
id: shortname,
|
399
|
-
name: shortname,
|
400
409
|
cloud: "AWS",
|
401
410
|
type: "roles"
|
402
411
|
)
|
403
412
|
end
|
413
|
+
|
414
|
+
begin
|
415
|
+
pol = MU::Cloud::AWS.lambda(region: @config['region'], credentials: @credentials).get_policy(function_name: @cloud_id).policy
|
416
|
+
MU.log @cloud_id, MU::WARN, details: JSON.parse(pol) if @cloud_id == "ESPIER-DEV-2020080900-LN-ON-DEMAND-SCANNER"
|
417
|
+
if pol
|
418
|
+
bok['triggers'] ||= []
|
419
|
+
JSON.parse(pol)["Statement"].each { |s|
|
420
|
+
bok['triggers'] << {
|
421
|
+
"service" => s["Principal"]["Service"].sub(/\..*/, ''),
|
422
|
+
"name" => s["Resource"].sub(/.*?[:\/]([^:\/]+)$/, '\1')
|
423
|
+
}
|
424
|
+
}
|
425
|
+
end
|
426
|
+
rescue ::Aws::Lambda::Errors::ResourceNotFoundException
|
427
|
+
end
|
404
428
|
#MU.log @cloud_id, MU::NOTICE, details: function
|
405
|
-
# XXX
|
429
|
+
# XXX permissions
|
406
430
|
|
407
431
|
bok
|
408
432
|
end
|
@@ -414,6 +438,22 @@ MU.log shortname, MU::NOTICE, details: function.configuration.role
|
|
414
438
|
def self.schema(_config)
|
415
439
|
toplevel_required = ["runtime"]
|
416
440
|
schema = {
|
441
|
+
"invoke_on_completion" => {
|
442
|
+
"type" => "object",
|
443
|
+
"description" => "Setting this will cause this Lambda function to be invoked when its groom phase is complete.",
|
444
|
+
"required" => ["invocation_type"],
|
445
|
+
"properties" => {
|
446
|
+
"invocation_type" => {
|
447
|
+
"type" => "string",
|
448
|
+
"enum" => ["RequestResponse", "Event", "Dryrun"],
|
449
|
+
"default" => "RequestReponse"
|
450
|
+
},
|
451
|
+
"payload" => {
|
452
|
+
"type" => "object",
|
453
|
+
"description" => "Optional input to the function, which will be formatted as JSON and sent for execution"
|
454
|
+
}
|
455
|
+
}
|
456
|
+
},
|
417
457
|
"triggers" => {
|
418
458
|
"type" => "array",
|
419
459
|
"items" => {
|
@@ -423,7 +463,7 @@ MU.log shortname, MU::NOTICE, details: function.configuration.role
|
|
423
463
|
"properties" => {
|
424
464
|
"service" => {
|
425
465
|
"type" => "string",
|
426
|
-
"enum" => %w{apigateway events s3 sns sqs dynamodb kinesis ses cognito alexa iot},
|
466
|
+
"enum" => %w{apigateway events s3 sns sqs dynamodb kinesis ses cognito alexa iot lex},
|
427
467
|
"description" => "The name of the AWS service that will trigger this function"
|
428
468
|
},
|
429
469
|
"name" => {
|
@@ -482,6 +522,28 @@ MU.log shortname, MU::NOTICE, details: function.configuration.role
|
|
482
522
|
def self.validateConfig(function, configurator)
|
483
523
|
ok = true
|
484
524
|
|
525
|
+
if function['triggers']
|
526
|
+
function['triggers'].each { |t|
|
527
|
+
mu_type = if t["service"] == "sns"
|
528
|
+
"notifiers"
|
529
|
+
elsif t["service"] == "apigateway"
|
530
|
+
"endpoints"
|
531
|
+
elsif t["service"] == "s3"
|
532
|
+
"buckets"
|
533
|
+
elsif t["service"] == "dynamodb"
|
534
|
+
"nosqldbs"
|
535
|
+
elsif t["service"] == "events"
|
536
|
+
"jobs"
|
537
|
+
elsif t["service"] == "sqs"
|
538
|
+
"msg_queues"
|
539
|
+
end
|
540
|
+
|
541
|
+
if mu_type
|
542
|
+
MU::Config.addDependency(function, t['name'], mu_type, no_create_wait: true)
|
543
|
+
end
|
544
|
+
}
|
545
|
+
end
|
546
|
+
|
485
547
|
if function['vpc']
|
486
548
|
fwname = "lambda-#{function['name']}"
|
487
549
|
# default to allowing pings, if no ingress_rules were specified
|
@@ -508,7 +570,10 @@ MU.log shortname, MU::NOTICE, details: function.configuration.role
|
|
508
570
|
MU::Config.addDependency(function, fwname, "firewall_rule")
|
509
571
|
end
|
510
572
|
|
511
|
-
|
573
|
+
function['role'] ||= function['iam_role']
|
574
|
+
function.delete("iam_role")
|
575
|
+
|
576
|
+
if !function['role']
|
512
577
|
policy_map = {
|
513
578
|
"basic" => "AWSLambdaBasicExecutionRole",
|
514
579
|
"kinesis" => "AWSLambdaKinesisExecutionRole",
|
@@ -537,9 +602,21 @@ MU.log shortname, MU::NOTICE, details: function.configuration.role
|
|
537
602
|
}
|
538
603
|
configurator.insertKitten(roledesc, "roles")
|
539
604
|
|
540
|
-
function['
|
605
|
+
function['role'] = function['name']+"execrole"
|
541
606
|
|
542
|
-
|
607
|
+
end
|
608
|
+
|
609
|
+
if function['role'].is_a?(String)
|
610
|
+
function['role'] = MU::Config::Ref.get(
|
611
|
+
name: function['role'],
|
612
|
+
type: "roles",
|
613
|
+
cloud: "AWS",
|
614
|
+
credentials: function['credentials']
|
615
|
+
)
|
616
|
+
end
|
617
|
+
|
618
|
+
if function['role']['name']
|
619
|
+
MU::Config.addDependency(function, function['role']['name'], "role")
|
543
620
|
end
|
544
621
|
|
545
622
|
ok
|
@@ -547,23 +624,109 @@ MU.log shortname, MU::NOTICE, details: function.configuration.role
|
|
547
624
|
|
548
625
|
private
|
549
626
|
|
550
|
-
|
551
|
-
|
552
|
-
|
553
|
-
# @param name [String]
|
554
|
-
def get_role_arn(name)
|
555
|
-
sib_role = @deploy.findLitterMate(name: name, type: "roles")
|
556
|
-
return sib_role.cloudobj.arn if sib_role
|
627
|
+
def get_properties
|
628
|
+
role_obj = MU::Config::Ref.get(@config['role']).kitten(@deploy, cloud: "AWS")
|
629
|
+
raise MuError.new "Failed to fetch object from role reference", details: @config['role'].to_h if !role_obj
|
557
630
|
|
558
|
-
|
559
|
-
|
560
|
-
|
561
|
-
|
562
|
-
|
563
|
-
|
564
|
-
|
631
|
+
lambda_properties = {
|
632
|
+
code: {},
|
633
|
+
function_name: @mu_name,
|
634
|
+
handler: @config['handler'],
|
635
|
+
publish: true,
|
636
|
+
role: role_obj.arn,
|
637
|
+
runtime: @config['runtime'],
|
638
|
+
}
|
639
|
+
|
640
|
+
if @config['code']['zip_file'] or @config['code']['path']
|
641
|
+
tempfile = nil
|
642
|
+
if @config['code']['path']
|
643
|
+
tempfile = Tempfile.new
|
644
|
+
MU.log "#{@mu_name} using code at #{@config['code']['path']}"
|
645
|
+
MU::Master.zipDir(@config['code']['path'], tempfile.path)
|
646
|
+
@config['code']['zip_file'] = tempfile.path
|
647
|
+
else
|
648
|
+
MU.log "#{@mu_name} using code packaged at #{@config['code']['zip_file']}"
|
649
|
+
end
|
650
|
+
zip = File.read(@config['code']['zip_file'])
|
651
|
+
@code_sha256 = Base64.encode64(Digest::SHA256.digest(zip)).chomp
|
652
|
+
lambda_properties[:code][:zip_file] = zip
|
653
|
+
if tempfile
|
654
|
+
tempfile.close
|
655
|
+
tempfile.unlink
|
656
|
+
end
|
657
|
+
else
|
658
|
+
lambda_properties[:code][:s3_bucket] = @config['code']['s3_bucket']
|
659
|
+
lambda_properties[:code][:s3_key] = @config['code']['s3_key']
|
660
|
+
if @config['code']['s3_object_version']
|
661
|
+
lambda_properties[:code][:s3_object_version] = @config['code']['s3_object_version']
|
662
|
+
end
|
663
|
+
# XXX need to download to a temporarily file, read it in, and calculate the digest in order to trigger updates in groom
|
664
|
+
end
|
665
|
+
|
666
|
+
if @config.has_key?('timeout')
|
667
|
+
lambda_properties[:timeout] = @config['timeout'].to_i ## secs
|
668
|
+
end
|
669
|
+
|
670
|
+
if @config.has_key?('memory')
|
671
|
+
lambda_properties[:memory_size] = @config['memory'].to_i
|
672
|
+
end
|
673
|
+
|
674
|
+
SIBLING_VARS.each_key { |sib_type|
|
675
|
+
siblings = @deploy.findLitterMate(return_all: true, type: sib_type, cloud: "AWS")
|
676
|
+
if siblings
|
677
|
+
siblings.each_value { |sibling|
|
678
|
+
metadata = sibling.notify
|
679
|
+
if !metadata
|
680
|
+
MU.log "Failed to extract metadata from sibling #{sibling}", MU::WARN
|
681
|
+
next
|
682
|
+
end
|
683
|
+
SIBLING_VARS[sib_type].each { |var|
|
684
|
+
if metadata[var]
|
685
|
+
@config['environment_variables'] ||= []
|
686
|
+
@config['environment_variables'] << {
|
687
|
+
"key" => (sibling.config['name']+"_"+var).gsub(/[^a-z0-9_]/i, '_'),
|
688
|
+
"value" => metadata[var]
|
689
|
+
}
|
690
|
+
end
|
691
|
+
}
|
692
|
+
}
|
693
|
+
end
|
694
|
+
}
|
695
|
+
|
696
|
+
if @config.has_key?('environment_variables')
|
697
|
+
lambda_properties[:environment] = {
|
698
|
+
variables: Hash[@config['environment_variables'].map { |v| [v['key'], v['value']] }]
|
699
|
+
}
|
700
|
+
end
|
701
|
+
|
702
|
+
lambda_properties[:tags] = {}
|
703
|
+
MU::MommaCat.listStandardTags.each_pair { |k, v|
|
704
|
+
lambda_properties[:tags][k] = v
|
705
|
+
}
|
706
|
+
if @config['tags']
|
707
|
+
@config['tags'].each { |tag|
|
708
|
+
lambda_properties[:tags][tag.key.first] = tag.values.first
|
709
|
+
}
|
565
710
|
end
|
566
|
-
|
711
|
+
|
712
|
+
if @config.has_key?('vpc')
|
713
|
+
sgs = []
|
714
|
+
if @config['add_firewall_rules']
|
715
|
+
@config['add_firewall_rules'].each { |sg|
|
716
|
+
sg = @deploy.findLitterMate(type: "firewall_rule", name: sg['name'])
|
717
|
+
sgs << sg.cloud_id if sg and sg.cloud_id
|
718
|
+
}
|
719
|
+
end
|
720
|
+
if !@vpc
|
721
|
+
raise MuError, "Function #{@config['name']} had a VPC configured, but none was loaded"
|
722
|
+
end
|
723
|
+
lambda_properties[:vpc_config] = {
|
724
|
+
:subnet_ids => @vpc.subnets.map { |s| s.cloud_id },
|
725
|
+
:security_group_ids => sgs
|
726
|
+
}
|
727
|
+
end
|
728
|
+
|
729
|
+
lambda_properties
|
567
730
|
end
|
568
731
|
|
569
732
|
end
|