cloud-mu 3.1.4 → 3.3.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Dockerfile +5 -1
- data/ansible/roles/mu-windows/README.md +33 -0
- data/ansible/roles/mu-windows/defaults/main.yml +2 -0
- data/ansible/roles/mu-windows/files/LaunchConfig.json +9 -0
- data/ansible/roles/mu-windows/files/config.xml +76 -0
- data/ansible/roles/mu-windows/handlers/main.yml +2 -0
- data/ansible/roles/mu-windows/meta/main.yml +53 -0
- data/ansible/roles/mu-windows/tasks/main.yml +36 -0
- data/ansible/roles/mu-windows/tests/inventory +2 -0
- data/ansible/roles/mu-windows/tests/test.yml +5 -0
- data/ansible/roles/mu-windows/vars/main.yml +2 -0
- data/bin/mu-adopt +16 -12
- data/bin/mu-azure-tests +57 -0
- data/bin/mu-cleanup +2 -4
- data/bin/mu-configure +52 -0
- data/bin/mu-deploy +3 -3
- data/bin/mu-findstray-tests +25 -0
- data/bin/mu-gen-docs +2 -4
- data/bin/mu-load-config.rb +2 -1
- data/bin/mu-node-manage +15 -16
- data/bin/mu-run-tests +37 -12
- data/cloud-mu.gemspec +5 -3
- data/cookbooks/mu-activedirectory/resources/domain.rb +4 -4
- data/cookbooks/mu-activedirectory/resources/domain_controller.rb +4 -4
- data/cookbooks/mu-tools/libraries/helper.rb +1 -1
- data/cookbooks/mu-tools/recipes/apply_security.rb +14 -14
- data/cookbooks/mu-tools/recipes/aws_api.rb +9 -0
- data/cookbooks/mu-tools/recipes/eks.rb +2 -2
- data/cookbooks/mu-tools/recipes/selinux.rb +2 -1
- data/cookbooks/mu-tools/recipes/windows-client.rb +163 -164
- data/cookbooks/mu-tools/resources/windows_users.rb +44 -43
- data/extras/clean-stock-amis +25 -19
- data/extras/generate-stock-images +1 -0
- data/extras/image-generators/AWS/win2k12.yaml +18 -13
- data/extras/image-generators/AWS/win2k16.yaml +18 -13
- data/extras/image-generators/AWS/win2k19.yaml +21 -0
- data/modules/mommacat.ru +1 -1
- data/modules/mu.rb +158 -107
- data/modules/mu/adoption.rb +386 -59
- data/modules/mu/cleanup.rb +214 -303
- data/modules/mu/cloud.rb +128 -1632
- data/modules/mu/cloud/database.rb +49 -0
- data/modules/mu/cloud/dnszone.rb +44 -0
- data/modules/mu/cloud/machine_images.rb +212 -0
- data/modules/mu/cloud/providers.rb +81 -0
- data/modules/mu/cloud/resource_base.rb +926 -0
- data/modules/mu/cloud/server.rb +40 -0
- data/modules/mu/cloud/server_pool.rb +1 -0
- data/modules/mu/cloud/ssh_sessions.rb +228 -0
- data/modules/mu/cloud/winrm_sessions.rb +237 -0
- data/modules/mu/cloud/wrappers.rb +169 -0
- data/modules/mu/config.rb +135 -82
- data/modules/mu/config/alarm.rb +2 -6
- data/modules/mu/config/bucket.rb +32 -3
- data/modules/mu/config/cache_cluster.rb +2 -2
- data/modules/mu/config/cdn.rb +100 -0
- data/modules/mu/config/collection.rb +1 -1
- data/modules/mu/config/container_cluster.rb +7 -2
- data/modules/mu/config/database.rb +84 -105
- data/modules/mu/config/database.yml +1 -2
- data/modules/mu/config/dnszone.rb +5 -4
- data/modules/mu/config/doc_helpers.rb +5 -6
- data/modules/mu/config/endpoint.rb +2 -1
- data/modules/mu/config/firewall_rule.rb +3 -19
- data/modules/mu/config/folder.rb +1 -1
- data/modules/mu/config/function.rb +17 -8
- data/modules/mu/config/group.rb +1 -1
- data/modules/mu/config/habitat.rb +1 -1
- data/modules/mu/config/job.rb +89 -0
- data/modules/mu/config/loadbalancer.rb +57 -11
- data/modules/mu/config/log.rb +1 -1
- data/modules/mu/config/msg_queue.rb +1 -1
- data/modules/mu/config/nosqldb.rb +1 -1
- data/modules/mu/config/notifier.rb +8 -19
- data/modules/mu/config/ref.rb +92 -14
- data/modules/mu/config/role.rb +1 -1
- data/modules/mu/config/schema_helpers.rb +38 -37
- data/modules/mu/config/search_domain.rb +1 -1
- data/modules/mu/config/server.rb +12 -13
- data/modules/mu/config/server.yml +1 -0
- data/modules/mu/config/server_pool.rb +3 -7
- data/modules/mu/config/storage_pool.rb +1 -1
- data/modules/mu/config/tail.rb +11 -0
- data/modules/mu/config/user.rb +1 -1
- data/modules/mu/config/vpc.rb +27 -23
- data/modules/mu/config/vpc.yml +0 -1
- data/modules/mu/defaults/AWS.yaml +91 -68
- data/modules/mu/defaults/Azure.yaml +1 -0
- data/modules/mu/defaults/Google.yaml +1 -0
- data/modules/mu/deploy.rb +33 -19
- data/modules/mu/groomer.rb +16 -1
- data/modules/mu/groomers/ansible.rb +123 -21
- data/modules/mu/groomers/chef.rb +64 -11
- data/modules/mu/logger.rb +120 -144
- data/modules/mu/master.rb +97 -4
- data/modules/mu/master/ssl.rb +0 -1
- data/modules/mu/mommacat.rb +154 -867
- data/modules/mu/mommacat/daemon.rb +23 -14
- data/modules/mu/mommacat/naming.rb +110 -3
- data/modules/mu/mommacat/search.rb +495 -0
- data/modules/mu/mommacat/storage.rb +225 -192
- data/modules/mu/{clouds → providers}/README.md +1 -1
- data/modules/mu/{clouds → providers}/aws.rb +281 -64
- data/modules/mu/{clouds → providers}/aws/alarm.rb +3 -3
- data/modules/mu/{clouds → providers}/aws/bucket.rb +275 -41
- data/modules/mu/{clouds → providers}/aws/cache_cluster.rb +14 -50
- data/modules/mu/providers/aws/cdn.rb +782 -0
- data/modules/mu/{clouds → providers}/aws/collection.rb +5 -5
- data/modules/mu/{clouds → providers}/aws/container_cluster.rb +708 -749
- data/modules/mu/providers/aws/database.rb +1744 -0
- data/modules/mu/{clouds → providers}/aws/dnszone.rb +75 -57
- data/modules/mu/providers/aws/endpoint.rb +1072 -0
- data/modules/mu/{clouds → providers}/aws/firewall_rule.rb +212 -242
- data/modules/mu/{clouds → providers}/aws/folder.rb +1 -1
- data/modules/mu/{clouds → providers}/aws/function.rb +289 -134
- data/modules/mu/{clouds → providers}/aws/group.rb +18 -20
- data/modules/mu/{clouds → providers}/aws/habitat.rb +3 -3
- data/modules/mu/providers/aws/job.rb +466 -0
- data/modules/mu/{clouds → providers}/aws/loadbalancer.rb +50 -41
- data/modules/mu/{clouds → providers}/aws/log.rb +5 -5
- data/modules/mu/{clouds → providers}/aws/msg_queue.rb +14 -11
- data/modules/mu/{clouds → providers}/aws/nosqldb.rb +96 -5
- data/modules/mu/{clouds → providers}/aws/notifier.rb +135 -63
- data/modules/mu/{clouds → providers}/aws/role.rb +94 -57
- data/modules/mu/{clouds → providers}/aws/search_domain.rb +173 -42
- data/modules/mu/{clouds → providers}/aws/server.rb +782 -1107
- data/modules/mu/{clouds → providers}/aws/server_pool.rb +36 -46
- data/modules/mu/{clouds → providers}/aws/storage_pool.rb +21 -38
- data/modules/mu/{clouds → providers}/aws/user.rb +12 -16
- data/modules/mu/{clouds → providers}/aws/userdata/README.md +0 -0
- data/modules/mu/{clouds → providers}/aws/userdata/linux.erb +5 -4
- data/modules/mu/{clouds → providers}/aws/userdata/windows.erb +2 -1
- data/modules/mu/{clouds → providers}/aws/vpc.rb +429 -849
- data/modules/mu/providers/aws/vpc_subnet.rb +286 -0
- data/modules/mu/{clouds → providers}/azure.rb +13 -0
- data/modules/mu/{clouds → providers}/azure/container_cluster.rb +1 -5
- data/modules/mu/{clouds → providers}/azure/firewall_rule.rb +8 -1
- data/modules/mu/{clouds → providers}/azure/habitat.rb +0 -0
- data/modules/mu/{clouds → providers}/azure/loadbalancer.rb +0 -0
- data/modules/mu/{clouds → providers}/azure/role.rb +0 -0
- data/modules/mu/{clouds → providers}/azure/server.rb +32 -24
- data/modules/mu/{clouds → providers}/azure/user.rb +1 -1
- data/modules/mu/{clouds → providers}/azure/userdata/README.md +0 -0
- data/modules/mu/{clouds → providers}/azure/userdata/linux.erb +0 -0
- data/modules/mu/{clouds → providers}/azure/userdata/windows.erb +0 -0
- data/modules/mu/{clouds → providers}/azure/vpc.rb +4 -6
- data/modules/mu/{clouds → providers}/cloudformation.rb +10 -0
- data/modules/mu/{clouds → providers}/cloudformation/alarm.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/cache_cluster.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/collection.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/database.rb +6 -17
- data/modules/mu/{clouds → providers}/cloudformation/dnszone.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/firewall_rule.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/loadbalancer.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/log.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/server.rb +7 -7
- data/modules/mu/{clouds → providers}/cloudformation/server_pool.rb +5 -5
- data/modules/mu/{clouds → providers}/cloudformation/vpc.rb +3 -3
- data/modules/mu/{clouds → providers}/docker.rb +0 -0
- data/modules/mu/{clouds → providers}/google.rb +29 -6
- data/modules/mu/{clouds → providers}/google/bucket.rb +5 -5
- data/modules/mu/{clouds → providers}/google/container_cluster.rb +59 -37
- data/modules/mu/{clouds → providers}/google/database.rb +5 -12
- data/modules/mu/{clouds → providers}/google/firewall_rule.rb +5 -5
- data/modules/mu/{clouds → providers}/google/folder.rb +5 -9
- data/modules/mu/{clouds → providers}/google/function.rb +14 -8
- data/modules/mu/{clouds → providers}/google/group.rb +9 -17
- data/modules/mu/{clouds → providers}/google/habitat.rb +4 -8
- data/modules/mu/{clouds → providers}/google/loadbalancer.rb +5 -5
- data/modules/mu/{clouds → providers}/google/role.rb +50 -31
- data/modules/mu/{clouds → providers}/google/server.rb +142 -55
- data/modules/mu/{clouds → providers}/google/server_pool.rb +14 -14
- data/modules/mu/{clouds → providers}/google/user.rb +34 -24
- data/modules/mu/{clouds → providers}/google/userdata/README.md +0 -0
- data/modules/mu/{clouds → providers}/google/userdata/linux.erb +0 -0
- data/modules/mu/{clouds → providers}/google/userdata/windows.erb +0 -0
- data/modules/mu/{clouds → providers}/google/vpc.rb +46 -15
- data/modules/tests/aws-jobs-functions.yaml +46 -0
- data/modules/tests/centos6.yaml +15 -0
- data/modules/tests/centos7.yaml +15 -0
- data/modules/tests/centos8.yaml +12 -0
- data/modules/tests/ecs.yaml +23 -0
- data/modules/tests/eks.yaml +1 -1
- data/modules/tests/functions/node-function/lambda_function.js +10 -0
- data/modules/tests/functions/python-function/lambda_function.py +12 -0
- data/modules/tests/includes-and-params.yaml +2 -1
- data/modules/tests/microservice_app.yaml +288 -0
- data/modules/tests/rds.yaml +108 -0
- data/modules/tests/regrooms/rds.yaml +123 -0
- data/modules/tests/server-with-scrub-muisms.yaml +2 -1
- data/modules/tests/super_complex_bok.yml +2 -2
- data/modules/tests/super_simple_bok.yml +3 -5
- data/modules/tests/win2k12.yaml +25 -0
- data/modules/tests/win2k16.yaml +25 -0
- data/modules/tests/win2k19.yaml +25 -0
- data/requirements.txt +1 -0
- data/spec/mu/clouds/azure_spec.rb +2 -2
- metadata +169 -93
- data/extras/image-generators/AWS/windows.yaml +0 -18
- data/modules/mu/clouds/aws/database.rb +0 -1974
- data/modules/mu/clouds/aws/endpoint.rb +0 -596
- data/modules/tests/needwork/win2k12.yaml +0 -13
@@ -59,7 +59,7 @@ module MU
|
|
59
59
|
# @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
|
60
60
|
# @param region [String]: The cloud provider region
|
61
61
|
# @return [void]
|
62
|
-
def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
|
62
|
+
def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
|
63
63
|
end
|
64
64
|
|
65
65
|
# Locate an existing AWS organization. If no identifying parameters are specified, this will return a description of the Organization which owns the account for our credentials.
|
@@ -18,6 +18,18 @@ module MU
|
|
18
18
|
# A function as configured in {MU::Config::BasketofKittens::functions}
|
19
19
|
class Function < MU::Cloud::Function
|
20
20
|
|
21
|
+
# If we have sibling resources in our deployment, automatically inject
|
22
|
+
# interesting things about them into our function's environment
|
23
|
+
# variables.
|
24
|
+
SIBLING_VARS = {
|
25
|
+
"servers" => ["private_ip_address", "public_ip_address"],
|
26
|
+
"search_domains" => ["endpoint"],
|
27
|
+
"databases" => ["endpoint"],
|
28
|
+
"endpoints" => ["url"],
|
29
|
+
"notifiers" => ["TopicArn"],
|
30
|
+
"nosqldbs" => ["table_arn"]
|
31
|
+
}
|
32
|
+
|
21
33
|
# Initialize this cloud resource object. Calling +super+ will invoke the initializer defined under {MU::Cloud}, which should set the attribtues listed in {MU::Cloud::PUBLIC_ATTRS} as well as applicable dependency shortcuts, like +@vpc+, for us.
|
22
34
|
# @param args [Hash]: Hash of named arguments passed via Ruby's double-splat
|
23
35
|
def initialize(**args)
|
@@ -42,92 +54,44 @@ module MU
|
|
42
54
|
|
43
55
|
# Called automatically by {MU::Deploy#createResources}
|
44
56
|
def create
|
45
|
-
role_arn = get_role_arn(@config['iam_role'])
|
46
57
|
|
47
|
-
lambda_properties =
|
48
|
-
code: {},
|
49
|
-
function_name: @mu_name,
|
50
|
-
handler: @config['handler'],
|
51
|
-
publish: true,
|
52
|
-
role: role_arn,
|
53
|
-
runtime: @config['runtime'],
|
54
|
-
}
|
58
|
+
lambda_properties = get_properties
|
55
59
|
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
lambda_properties[:code][:zip_file] = zip
|
60
|
-
else
|
61
|
-
lambda_properties[:code][:s3_bucket] = @config['code']['s3_bucket']
|
62
|
-
lambda_properties[:code][:s3_key] = @config['code']['s3_key']
|
63
|
-
if @config['code']['s3_object_version']
|
64
|
-
lambda_properties[:code][:s3_object_version] = @config['code']['s3_object_version']
|
65
|
-
end
|
66
|
-
end
|
67
|
-
|
68
|
-
if @config.has_key?('timeout')
|
69
|
-
lambda_properties[:timeout] = @config['timeout'].to_i ## secs
|
70
|
-
end
|
71
|
-
|
72
|
-
if @config.has_key?('memory')
|
73
|
-
lambda_properties[:memory_size] = @config['memory'].to_i
|
74
|
-
end
|
75
|
-
|
76
|
-
if @config.has_key?('environment_variables')
|
77
|
-
lambda_properties[:environment] = {
|
78
|
-
variables: {@config['environment_variables'][0]['key'] => @config['environment_variables'][0]['value']}
|
79
|
-
}
|
80
|
-
end
|
81
|
-
|
82
|
-
lambda_properties[:tags] = {}
|
83
|
-
MU::MommaCat.listStandardTags.each_pair { |k, v|
|
84
|
-
lambda_properties[:tags][k] = v
|
60
|
+
MU.retrier([Aws::Lambda::Errors::InvalidParameterValueException], max: 5, wait: 10) {
|
61
|
+
resp = MU::Cloud::AWS.lambda(region: @config['region'], credentials: @config['credentials']).create_function(lambda_properties)
|
62
|
+
@cloud_id = resp.function_name
|
85
63
|
}
|
86
|
-
if @config['tags']
|
87
|
-
@config['tags'].each { |tag|
|
88
|
-
lambda_properties[:tags][tag.key.first] = tag.values.first
|
89
|
-
}
|
90
|
-
end
|
91
64
|
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
}
|
99
|
-
end
|
100
|
-
if !@vpc
|
101
|
-
raise MuError, "Function #{@config['name']} had a VPC configured, but none was loaded"
|
102
|
-
end
|
103
|
-
lambda_properties[:vpc_config] = {
|
104
|
-
:subnet_ids => @vpc.subnets.map { |s| s.cloud_id },
|
105
|
-
:security_group_ids => sgs
|
106
|
-
}
|
107
|
-
end
|
108
|
-
|
109
|
-
retries = 0
|
110
|
-
resp = begin
|
111
|
-
MU::Cloud::AWS.lambda(region: @config['region'], credentials: @config['credentials']).create_function(lambda_properties)
|
112
|
-
rescue Aws::Lambda::Errors::InvalidParameterValueException => e
|
113
|
-
# Freshly-made IAM roles sometimes aren't really ready
|
114
|
-
if retries < 5
|
115
|
-
sleep 10
|
116
|
-
retries += 1
|
117
|
-
retry
|
118
|
-
end
|
119
|
-
raise e
|
120
|
-
end
|
121
|
-
|
122
|
-
@cloud_id = resp.function_name
|
65
|
+
# the console does this and docs expect it to be there, so mimic the
|
66
|
+
# behavior
|
67
|
+
MU::Cloud::AWS.cloudwatchlogs(region: @config["region"], credentials: @credentials).create_log_group(
|
68
|
+
log_group_name: "/aws/lambda/#{@cloud_id}",
|
69
|
+
tags: @tags
|
70
|
+
)
|
123
71
|
end
|
124
72
|
|
125
73
|
# Called automatically by {MU::Deploy#createResources}
|
126
74
|
def groom
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
75
|
+
old_props = MU.structToHash(cloud_desc)
|
76
|
+
|
77
|
+
new_props = get_properties
|
78
|
+
code_block = new_props[:code]
|
79
|
+
new_props.reject! { |k, _v| [:code, :publish, :tags].include?(k) }
|
80
|
+
changes = {}
|
81
|
+
new_props.each_pair { |k, v|
|
82
|
+
changes[k] = v if v != old_props[k]
|
83
|
+
}
|
84
|
+
if !changes.empty?
|
85
|
+
MU.log "Updating Lambda #{@mu_name}", MU::NOTICE, details: changes
|
86
|
+
MU::Cloud::AWS.lambda(region: @config['region'], credentials: @config['credentials']).update_function_configuration(new_props)
|
87
|
+
end
|
88
|
+
|
89
|
+
if @code_sha256 and @code_sha256 != cloud_desc.code_sha_256.chomp
|
90
|
+
MU.log "Updating code in Lambda #{@mu_name}", MU::NOTICE, details: { "old" => @code_sha256, "new" => cloud_desc.code_sha_256 }
|
91
|
+
code_block[:publish] = true
|
92
|
+
code_block[:function_name] = @cloud_id
|
93
|
+
MU::Cloud::AWS.lambda(region: @config['region'], credentials: @config['credentials']).update_function_code(code_block)
|
94
|
+
end
|
131
95
|
|
132
96
|
# tag_function = assign_tag(lambda_func.function_arn, @config['tags'])
|
133
97
|
|
@@ -141,7 +105,7 @@ module MU
|
|
141
105
|
### triggers must exist prior
|
142
106
|
if @config['triggers']
|
143
107
|
@config['triggers'].each { |tr|
|
144
|
-
trigger_arn =
|
108
|
+
trigger_arn = resolveARN(tr['service'], tr['name'])
|
145
109
|
|
146
110
|
trigger_properties = {
|
147
111
|
action: "lambda:InvokeFunction",
|
@@ -151,15 +115,33 @@ module MU
|
|
151
115
|
statement_id: "#{@mu_name}-ID-1",
|
152
116
|
}
|
153
117
|
|
154
|
-
MU.log
|
118
|
+
MU.log "Adding #{tr['service']} #{tr['name']} trigger to Lambda function #{@cloud_id}", details: trigger_properties
|
155
119
|
begin
|
156
120
|
MU::Cloud::AWS.lambda(region: @config['region'], credentials: @config['credentials']).add_permission(trigger_properties)
|
157
121
|
rescue Aws::Lambda::Errors::ResourceConflictException
|
122
|
+
# just means the permission is already there
|
158
123
|
end
|
159
|
-
adjust_trigger(tr['service'], trigger_arn,
|
124
|
+
adjust_trigger(tr['service'], trigger_arn, arn, @mu_name)
|
160
125
|
}
|
161
126
|
|
162
127
|
end
|
128
|
+
|
129
|
+
if @config['invoke_on_completion']
|
130
|
+
invoke_params = {
|
131
|
+
function_name: @cloud_id,
|
132
|
+
invocation_type: @config['invoke_on_completion']['invocation_type'],
|
133
|
+
log_type: "Tail"
|
134
|
+
}
|
135
|
+
if @config['invoke_on_completion']['payload']
|
136
|
+
invoke_params[:payload] = JSON.generate(@config['invoke_on_completion']['payload'])
|
137
|
+
end
|
138
|
+
resp = MU::Cloud::AWS.lambda(region: @config['region'], credentials: @config['credentials']).invoke(invoke_params)
|
139
|
+
if resp.status_code == 200
|
140
|
+
MU.log "Invoked #{@cloud_id}", MU::NOTICE, details: Base64.decode64(resp.log_result)
|
141
|
+
else
|
142
|
+
MU.log "Invoked #{@cloud_id} and got #{resp.status_code} (#{resp.function_error})", MU::WARN, details: Base64.decode64(resp.log_result)
|
143
|
+
end
|
144
|
+
end
|
163
145
|
end
|
164
146
|
|
165
147
|
# Intended to be called by other Mu resources, such as Endpoints (API
|
@@ -170,13 +152,16 @@ module MU
|
|
170
152
|
function_name: @mu_name,
|
171
153
|
principal: "#{calling_service}.amazonaws.com",
|
172
154
|
source_arn: calling_arn,
|
173
|
-
statement_id: "#{calling_service}-#{calling_name}",
|
155
|
+
statement_id: "#{calling_service}-#{calling_name.gsub(/[^a-z0-9\-_]/i, '_')}",
|
174
156
|
}
|
175
157
|
|
176
158
|
begin
|
177
159
|
# XXX There doesn't seem to be an API call to list or view existing
|
178
160
|
# permissions, wtaf. This means we can't intelligently guard this.
|
179
161
|
MU::Cloud::AWS.lambda(region: @config['region'], credentials: @config['credentials']).add_permission(trigger)
|
162
|
+
rescue Aws::Lambda::Errors::ValidationException => e
|
163
|
+
MU.log e.message+" (calling_arn: #{calling_arn}, calling_service: #{calling_service}, calling_name: #{calling_name})", MU::ERR, details: trigger
|
164
|
+
raise e
|
180
165
|
rescue Aws::Lambda::Errors::ResourceConflictException => e
|
181
166
|
if e.message.match(/already exists/)
|
182
167
|
MU::Cloud::AWS.lambda(region: @config['region'], credentials: @config['credentials']).remove_permission(
|
@@ -192,17 +177,23 @@ module MU
|
|
192
177
|
end
|
193
178
|
|
194
179
|
# Look up an ARN for a given trigger type and resource name
|
195
|
-
def
|
196
|
-
supported_triggers = %w(apigateway sns events event cloudwatch_event)
|
180
|
+
def resolveARN(svc, name)
|
181
|
+
supported_triggers = %w(apigateway sns events event cloudwatch_event dynamodb)
|
197
182
|
if supported_triggers.include?(svc.downcase)
|
198
183
|
arn = nil
|
199
184
|
case svc.downcase
|
200
185
|
when 'sns'
|
201
|
-
|
186
|
+
sib_sns = @deploy.findLitterMate(name: name, type: "notifiers")
|
187
|
+
arn = sib_sns ? sib_sns.arn : "arn:aws:sns:#{@config['region']}:#{MU::Cloud::AWS.credToAcct(@config['credentials'])}:#{name}"
|
202
188
|
when 'alarm','events', 'event', 'cloudwatch_event'
|
203
|
-
|
189
|
+
sib_event = @deploy.findLitterMate(name: name, type: "job")
|
190
|
+
arn = sib_event ? sib_event.arn : "arn:aws:events:#{@config['region']}:#{MU::Cloud::AWS.credToAcct(@config['credentials'])}:rule/#{name}"
|
191
|
+
when 'dynamodb'
|
192
|
+
sib_dynamo = @deploy.findLitterMate(name: name, type: "nosqldb")
|
193
|
+
arn = sib_dynamo ? sib_dynamo.arn : "arn:aws:dynamodb:#{@config['region']}:#{MU::Cloud::AWS.credToAcct(@config['credentials'])}:table/#{name}"
|
204
194
|
when 'apigateway'
|
205
|
-
|
195
|
+
sib_apig = @deploy.findLitterMate(name: name, type: "endpoints")
|
196
|
+
arn = sib_apig ? sib_apig.arn : "arn:aws:apigateway:#{@config['region']}:#{MU::Cloud::AWS.credToAcct(@config['credentials'])}:#{name}"
|
206
197
|
when 's3'
|
207
198
|
arn = ''
|
208
199
|
end
|
@@ -219,13 +210,21 @@ module MU
|
|
219
210
|
case trig_type
|
220
211
|
|
221
212
|
when 'sns'
|
222
|
-
|
223
|
-
|
224
|
-
|
225
|
-
|
226
|
-
|
227
|
-
|
228
|
-
|
213
|
+
MU::Cloud.resourceClass("AWS", "Notifier").subscribe(trig_arn, arn, "lambda", region: @config['region'], credentials: @credentials)
|
214
|
+
when 'dynamodb'
|
215
|
+
stream = MU::Cloud::AWS.dynamostream(region: @config['region'], credentials: @config['credentials']).list_streams(table_name: trig_arn.sub(/.*?:table\//, '')).streams.first
|
216
|
+
# XXX guard this
|
217
|
+
MU.log "Adding DynamoDB Stream from #{stream.stream_arn} as trigger for #{@cloud_id}"
|
218
|
+
begin
|
219
|
+
MU::Cloud::AWS.lambda(region: @config['region'], credentials: @config['credentials']).create_event_source_mapping(
|
220
|
+
event_source_arn: stream.stream_arn,
|
221
|
+
function_name: @cloud_id,
|
222
|
+
starting_position: "TRIM_HORIZON" # ...whatever that is
|
223
|
+
)
|
224
|
+
rescue ::Aws::Lambda::Errors::ResourceConflictException
|
225
|
+
end
|
226
|
+
|
227
|
+
# MU::Cloud.resourceClass("AWS", "NoSQLDB").subscribe(trig_arn, arn, "lambda", region: @config['region'], credentials: @credentials)
|
229
228
|
when 'event','cloudwatch_event', 'events'
|
230
229
|
# XXX don't do this, use MU::Cloud::AWS::Log
|
231
230
|
MU::Cloud::AWS.cloudwatch_events(region: region, credentials: @config['credentials']).put_targets({
|
@@ -237,9 +236,8 @@ module MU
|
|
237
236
|
}
|
238
237
|
]
|
239
238
|
})
|
240
|
-
|
241
|
-
|
242
|
-
# MU.log "Creation of API Gateway integrations not yet implemented, you'll have to do this manually", MU::WARN, details: "(because we'll basically have to implement all of APIG for this)"
|
239
|
+
when 'apigateway'
|
240
|
+
addTrigger(trig_arn, "lambda", trig_arn.sub(/.*?([a-z0-9\-_]+)$/i, '\1'))
|
243
241
|
end
|
244
242
|
end
|
245
243
|
|
@@ -247,9 +245,8 @@ module MU
|
|
247
245
|
# Return the metadata for this Function rule
|
248
246
|
# @return [Hash]
|
249
247
|
def notify
|
250
|
-
|
251
|
-
|
252
|
-
return deploy_struct
|
248
|
+
return nil if !cloud_desc
|
249
|
+
MU.structToHash(cloud_desc, stringify_keys: true)
|
253
250
|
end
|
254
251
|
|
255
252
|
# Does this resource type exist as a global (cloud-wide) artifact, or
|
@@ -270,14 +267,14 @@ module MU
|
|
270
267
|
# @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
|
271
268
|
# @param region [String]: The cloud provider region
|
272
269
|
# @return [void]
|
273
|
-
def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
|
270
|
+
def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
|
274
271
|
MU.log "AWS::Function.cleanup: need to support flags['known']", MU::DEBUG, details: flags
|
275
272
|
|
276
273
|
MU::Cloud::AWS.lambda(credentials: credentials, region: region).list_functions.functions.each { |f|
|
277
274
|
desc = MU::Cloud::AWS.lambda(credentials: credentials, region: region).get_function(
|
278
275
|
function_name: f.function_name
|
279
276
|
)
|
280
|
-
if desc.tags and desc.tags["MU-ID"] ==
|
277
|
+
if desc.tags and desc.tags["MU-ID"] == deploy_id and (desc.tags["MU-MASTER-IP"] == MU.mu_public_ip or ignoremaster)
|
281
278
|
MU.log "Deleting Lambda function #{f.function_name}"
|
282
279
|
if !noop
|
283
280
|
MU::Cloud::AWS.lambda(credentials: credentials, region: region).delete_function(
|
@@ -292,7 +289,7 @@ module MU
|
|
292
289
|
# Canonical Amazon Resource Number for this resource
|
293
290
|
# @return [String]
|
294
291
|
def arn
|
295
|
-
cloud_desc.function_arn
|
292
|
+
cloud_desc ? cloud_desc.function_arn : nil
|
296
293
|
end
|
297
294
|
|
298
295
|
# Locate an existing function.
|
@@ -334,6 +331,20 @@ module MU
|
|
334
331
|
bok['timeout'] = cloud_desc.timeout
|
335
332
|
|
336
333
|
function = MU::Cloud::AWS.lambda(region: @config['region'], credentials: @credentials).get_function(function_name: bok['name'])
|
334
|
+
# event_srcs = MU::Cloud::AWS.lambda(region: @config['region'], credentials: @credentials).list_event_source_mappings(function_name: @cloud_id)
|
335
|
+
# if event_srcs and !event_srcs.event_source_mappings.empty?
|
336
|
+
# MU.log "dem mappings tho #{@cloud_id}", MU::WARN, details: event_srcs
|
337
|
+
# end
|
338
|
+
|
339
|
+
# begin
|
340
|
+
# invoke_cfg = MU::Cloud::AWS.lambda(region: @config['region'], credentials: @credentials).get_function_event_invoke_config(function_name: @cloud_id)
|
341
|
+
# MU.log "invoke config #{@cloud_id}", MU::WARN, details: invoke_cfg
|
342
|
+
# rescue ::Aws::Lambda::Errors::ResourceNotFoundException
|
343
|
+
# end
|
344
|
+
|
345
|
+
# MU.log @cloud_id, MU::WARN, details: cloud_desc if @cloud_id == "Espier-Scheduled-Scanner"
|
346
|
+
# MU.log "configuration #{@cloud_id}", MU::WARN, details: MU::Cloud::AWS.lambda(region: @config['region'], credentials: @credentials).get_function_configuration(function_name: @cloud_id) if @cloud_id == "Espier-Scheduled-Scanner"
|
347
|
+
|
337
348
|
|
338
349
|
if function.code.repository_type == "S3"
|
339
350
|
bok['code'] = {}
|
@@ -393,16 +404,29 @@ module MU
|
|
393
404
|
|
394
405
|
if function.configuration.role
|
395
406
|
shortname = function.configuration.role.sub(/.*?role\/([^\/]+)$/, '\1')
|
396
|
-
MU.log shortname, MU::NOTICE, details: function.configuration.role
|
397
407
|
bok['role'] = MU::Config::Ref.get(
|
398
408
|
id: shortname,
|
399
|
-
name: shortname,
|
400
409
|
cloud: "AWS",
|
401
410
|
type: "roles"
|
402
411
|
)
|
403
412
|
end
|
413
|
+
|
414
|
+
begin
|
415
|
+
pol = MU::Cloud::AWS.lambda(region: @config['region'], credentials: @credentials).get_policy(function_name: @cloud_id).policy
|
416
|
+
MU.log @cloud_id, MU::WARN, details: JSON.parse(pol) if @cloud_id == "ESPIER-DEV-2020080900-LN-ON-DEMAND-SCANNER"
|
417
|
+
if pol
|
418
|
+
bok['triggers'] ||= []
|
419
|
+
JSON.parse(pol)["Statement"].each { |s|
|
420
|
+
bok['triggers'] << {
|
421
|
+
"service" => s["Principal"]["Service"].sub(/\..*/, ''),
|
422
|
+
"name" => s["Resource"].sub(/.*?[:\/]([^:\/]+)$/, '\1')
|
423
|
+
}
|
424
|
+
}
|
425
|
+
end
|
426
|
+
rescue ::Aws::Lambda::Errors::ResourceNotFoundException
|
427
|
+
end
|
404
428
|
#MU.log @cloud_id, MU::NOTICE, details: function
|
405
|
-
# XXX
|
429
|
+
# XXX permissions
|
406
430
|
|
407
431
|
bok
|
408
432
|
end
|
@@ -414,6 +438,22 @@ MU.log shortname, MU::NOTICE, details: function.configuration.role
|
|
414
438
|
def self.schema(_config)
|
415
439
|
toplevel_required = ["runtime"]
|
416
440
|
schema = {
|
441
|
+
"invoke_on_completion" => {
|
442
|
+
"type" => "object",
|
443
|
+
"description" => "Setting this will cause this Lambda function to be invoked when its groom phase is complete.",
|
444
|
+
"required" => ["invocation_type"],
|
445
|
+
"properties" => {
|
446
|
+
"invocation_type" => {
|
447
|
+
"type" => "string",
|
448
|
+
"enum" => ["RequestResponse", "Event", "Dryrun"],
|
449
|
+
"default" => "RequestReponse"
|
450
|
+
},
|
451
|
+
"payload" => {
|
452
|
+
"type" => "object",
|
453
|
+
"description" => "Optional input to the function, which will be formatted as JSON and sent for execution"
|
454
|
+
}
|
455
|
+
}
|
456
|
+
},
|
417
457
|
"triggers" => {
|
418
458
|
"type" => "array",
|
419
459
|
"items" => {
|
@@ -423,7 +463,7 @@ MU.log shortname, MU::NOTICE, details: function.configuration.role
|
|
423
463
|
"properties" => {
|
424
464
|
"service" => {
|
425
465
|
"type" => "string",
|
426
|
-
"enum" => %w{apigateway events s3 sns sqs dynamodb kinesis ses cognito alexa iot},
|
466
|
+
"enum" => %w{apigateway events s3 sns sqs dynamodb kinesis ses cognito alexa iot lex},
|
427
467
|
"description" => "The name of the AWS service that will trigger this function"
|
428
468
|
},
|
429
469
|
"name" => {
|
@@ -482,6 +522,28 @@ MU.log shortname, MU::NOTICE, details: function.configuration.role
|
|
482
522
|
def self.validateConfig(function, configurator)
|
483
523
|
ok = true
|
484
524
|
|
525
|
+
if function['triggers']
|
526
|
+
function['triggers'].each { |t|
|
527
|
+
mu_type = if t["service"] == "sns"
|
528
|
+
"notifiers"
|
529
|
+
elsif t["service"] == "apigateway"
|
530
|
+
"endpoints"
|
531
|
+
elsif t["service"] == "s3"
|
532
|
+
"buckets"
|
533
|
+
elsif t["service"] == "dynamodb"
|
534
|
+
"nosqldbs"
|
535
|
+
elsif t["service"] == "events"
|
536
|
+
"jobs"
|
537
|
+
elsif t["service"] == "sqs"
|
538
|
+
"msg_queues"
|
539
|
+
end
|
540
|
+
|
541
|
+
if mu_type
|
542
|
+
MU::Config.addDependency(function, t['name'], mu_type, no_create_wait: true)
|
543
|
+
end
|
544
|
+
}
|
545
|
+
end
|
546
|
+
|
485
547
|
if function['vpc']
|
486
548
|
fwname = "lambda-#{function['name']}"
|
487
549
|
# default to allowing pings, if no ingress_rules were specified
|
@@ -505,14 +567,13 @@ MU.log shortname, MU::NOTICE, details: function.configuration.role
|
|
505
567
|
function["add_firewall_rules"] << {"name" => fwname}
|
506
568
|
function["permissions"] ||= []
|
507
569
|
function["permissions"] << "network"
|
508
|
-
function
|
509
|
-
function['dependencies'] << {
|
510
|
-
"name" => fwname,
|
511
|
-
"type" => "firewall_rule"
|
512
|
-
}
|
570
|
+
MU::Config.addDependency(function, fwname, "firewall_rule")
|
513
571
|
end
|
514
572
|
|
515
|
-
|
573
|
+
function['role'] ||= function['iam_role']
|
574
|
+
function.delete("iam_role")
|
575
|
+
|
576
|
+
if !function['role']
|
516
577
|
policy_map = {
|
517
578
|
"basic" => "AWSLambdaBasicExecutionRole",
|
518
579
|
"kinesis" => "AWSLambdaKinesisExecutionRole",
|
@@ -541,13 +602,21 @@ MU.log shortname, MU::NOTICE, details: function.configuration.role
|
|
541
602
|
}
|
542
603
|
configurator.insertKitten(roledesc, "roles")
|
543
604
|
|
544
|
-
function['
|
545
|
-
function['iam_role'] = function['name']+"execrole"
|
605
|
+
function['role'] = function['name']+"execrole"
|
546
606
|
|
547
|
-
|
548
|
-
|
549
|
-
|
550
|
-
|
607
|
+
end
|
608
|
+
|
609
|
+
if function['role'].is_a?(String)
|
610
|
+
function['role'] = MU::Config::Ref.get(
|
611
|
+
name: function['role'],
|
612
|
+
type: "roles",
|
613
|
+
cloud: "AWS",
|
614
|
+
credentials: function['credentials']
|
615
|
+
)
|
616
|
+
end
|
617
|
+
|
618
|
+
if function['role']['name']
|
619
|
+
MU::Config.addDependency(function, function['role']['name'], "role")
|
551
620
|
end
|
552
621
|
|
553
622
|
ok
|
@@ -555,23 +624,109 @@ MU.log shortname, MU::NOTICE, details: function.configuration.role
|
|
555
624
|
|
556
625
|
private
|
557
626
|
|
558
|
-
|
559
|
-
|
560
|
-
|
561
|
-
# @param name [String]
|
562
|
-
def get_role_arn(name)
|
563
|
-
sib_role = @deploy.findLitterMate(name: name, type: "roles")
|
564
|
-
return sib_role.cloudobj.arn if sib_role
|
627
|
+
def get_properties
|
628
|
+
role_obj = MU::Config::Ref.get(@config['role']).kitten(@deploy, cloud: "AWS")
|
629
|
+
raise MuError.new "Failed to fetch object from role reference", details: @config['role'].to_h if !role_obj
|
565
630
|
|
566
|
-
|
567
|
-
|
568
|
-
|
569
|
-
|
570
|
-
|
571
|
-
|
572
|
-
|
631
|
+
lambda_properties = {
|
632
|
+
code: {},
|
633
|
+
function_name: @mu_name,
|
634
|
+
handler: @config['handler'],
|
635
|
+
publish: true,
|
636
|
+
role: role_obj.arn,
|
637
|
+
runtime: @config['runtime'],
|
638
|
+
}
|
639
|
+
|
640
|
+
if @config['code']['zip_file'] or @config['code']['path']
|
641
|
+
tempfile = nil
|
642
|
+
if @config['code']['path']
|
643
|
+
tempfile = Tempfile.new
|
644
|
+
MU.log "#{@mu_name} using code at #{@config['code']['path']}"
|
645
|
+
MU::Master.zipDir(@config['code']['path'], tempfile.path)
|
646
|
+
@config['code']['zip_file'] = tempfile.path
|
647
|
+
else
|
648
|
+
MU.log "#{@mu_name} using code packaged at #{@config['code']['zip_file']}"
|
649
|
+
end
|
650
|
+
zip = File.read(@config['code']['zip_file'])
|
651
|
+
@code_sha256 = Base64.encode64(Digest::SHA256.digest(zip)).chomp
|
652
|
+
lambda_properties[:code][:zip_file] = zip
|
653
|
+
if tempfile
|
654
|
+
tempfile.close
|
655
|
+
tempfile.unlink
|
656
|
+
end
|
657
|
+
else
|
658
|
+
lambda_properties[:code][:s3_bucket] = @config['code']['s3_bucket']
|
659
|
+
lambda_properties[:code][:s3_key] = @config['code']['s3_key']
|
660
|
+
if @config['code']['s3_object_version']
|
661
|
+
lambda_properties[:code][:s3_object_version] = @config['code']['s3_object_version']
|
662
|
+
end
|
663
|
+
# XXX need to download to a temporarily file, read it in, and calculate the digest in order to trigger updates in groom
|
573
664
|
end
|
574
|
-
|
665
|
+
|
666
|
+
if @config.has_key?('timeout')
|
667
|
+
lambda_properties[:timeout] = @config['timeout'].to_i ## secs
|
668
|
+
end
|
669
|
+
|
670
|
+
if @config.has_key?('memory')
|
671
|
+
lambda_properties[:memory_size] = @config['memory'].to_i
|
672
|
+
end
|
673
|
+
|
674
|
+
SIBLING_VARS.each_key { |sib_type|
|
675
|
+
siblings = @deploy.findLitterMate(return_all: true, type: sib_type, cloud: "AWS")
|
676
|
+
if siblings
|
677
|
+
siblings.each_value { |sibling|
|
678
|
+
metadata = sibling.notify
|
679
|
+
if !metadata
|
680
|
+
MU.log "Failed to extract metadata from sibling #{sibling}", MU::WARN
|
681
|
+
next
|
682
|
+
end
|
683
|
+
SIBLING_VARS[sib_type].each { |var|
|
684
|
+
if metadata[var]
|
685
|
+
@config['environment_variables'] ||= []
|
686
|
+
@config['environment_variables'] << {
|
687
|
+
"key" => (sibling.config['name']+"_"+var).gsub(/[^a-z0-9_]/i, '_'),
|
688
|
+
"value" => metadata[var]
|
689
|
+
}
|
690
|
+
end
|
691
|
+
}
|
692
|
+
}
|
693
|
+
end
|
694
|
+
}
|
695
|
+
|
696
|
+
if @config.has_key?('environment_variables')
|
697
|
+
lambda_properties[:environment] = {
|
698
|
+
variables: Hash[@config['environment_variables'].map { |v| [v['key'], v['value']] }]
|
699
|
+
}
|
700
|
+
end
|
701
|
+
|
702
|
+
lambda_properties[:tags] = {}
|
703
|
+
MU::MommaCat.listStandardTags.each_pair { |k, v|
|
704
|
+
lambda_properties[:tags][k] = v
|
705
|
+
}
|
706
|
+
if @config['tags']
|
707
|
+
@config['tags'].each { |tag|
|
708
|
+
lambda_properties[:tags][tag['key']] = tag['value']
|
709
|
+
}
|
710
|
+
end
|
711
|
+
|
712
|
+
if @config.has_key?('vpc')
|
713
|
+
sgs = []
|
714
|
+
if @config['add_firewall_rules']
|
715
|
+
@config['add_firewall_rules'].each { |sg|
|
716
|
+
sg = @deploy.findLitterMate(type: "firewall_rule", name: sg['name'])
|
717
|
+
sgs << sg.cloud_id if sg and sg.cloud_id
|
718
|
+
}
|
719
|
+
end
|
720
|
+
if !@vpc
|
721
|
+
raise MuError, "Function #{@config['name']} had a VPC configured, but none was loaded"
|
722
|
+
end
|
723
|
+
lambda_properties[:vpc_config] = {
|
724
|
+
:subnet_ids => @vpc.subnets.map { |s| s.cloud_id },
|
725
|
+
:security_group_ids => sgs
|
726
|
+
}
|
727
|
+
end
|
728
|
+
|
729
|
+
lambda_properties
|
575
730
|
end
|
576
731
|
|
577
732
|
end
|