cloud-mu 3.4.0 → 3.5.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (104) hide show
  1. checksums.yaml +4 -4
  2. data/ansible/roles/mu-nat/tasks/main.yml +3 -0
  3. data/bin/mu-aws-setup +41 -7
  4. data/bin/mu-azure-setup +34 -0
  5. data/bin/mu-configure +214 -119
  6. data/bin/mu-gcp-setup +37 -2
  7. data/bin/mu-node-manage +3 -0
  8. data/bin/mu-refresh-ssl +67 -0
  9. data/bin/mu-run-tests +14 -4
  10. data/bin/mu-self-update +30 -10
  11. data/bin/mu-upload-chef-artifacts +30 -26
  12. data/cloud-mu.gemspec +8 -6
  13. data/cookbooks/mu-master/attributes/default.rb +5 -1
  14. data/cookbooks/mu-master/metadata.rb +2 -2
  15. data/cookbooks/mu-master/recipes/default.rb +81 -26
  16. data/cookbooks/mu-master/recipes/init.rb +197 -62
  17. data/cookbooks/mu-master/recipes/update_nagios_only.rb +1 -1
  18. data/cookbooks/mu-master/recipes/vault.rb +78 -77
  19. data/cookbooks/mu-master/templates/default/mods/rewrite.conf.erb +1 -0
  20. data/cookbooks/mu-master/templates/default/nagios.conf.erb +103 -0
  21. data/cookbooks/mu-master/templates/default/web_app.conf.erb +14 -30
  22. data/cookbooks/mu-tools/attributes/default.rb +5 -0
  23. data/cookbooks/mu-tools/files/centos-6/CentOS-Base.repo +47 -0
  24. data/cookbooks/mu-tools/libraries/helper.rb +12 -2
  25. data/cookbooks/mu-tools/libraries/monkey.rb +1 -1
  26. data/cookbooks/mu-tools/recipes/apply_security.rb +6 -0
  27. data/cookbooks/mu-tools/recipes/aws_api.rb +6 -4
  28. data/cookbooks/mu-tools/recipes/base_repositories.rb +1 -1
  29. data/cookbooks/mu-tools/recipes/gcloud.rb +2 -9
  30. data/cookbooks/mu-tools/recipes/google_api.rb +5 -2
  31. data/cookbooks/mu-tools/resources/disk.rb +108 -58
  32. data/extras/Gemfile.lock.bootstrap +394 -0
  33. data/extras/bucketstubs/error.html +0 -0
  34. data/extras/bucketstubs/index.html +0 -0
  35. data/extras/clean-stock-amis +9 -9
  36. data/extras/git_rpm/build.sh +20 -0
  37. data/extras/git_rpm/mugit.spec +53 -0
  38. data/extras/image-generators/VMWare/centos8.yaml +15 -0
  39. data/extras/openssl_rpm/build.sh +19 -0
  40. data/extras/openssl_rpm/mussl.spec +46 -0
  41. data/extras/python_rpm/muthon.spec +14 -4
  42. data/extras/ruby_rpm/muby.spec +9 -5
  43. data/extras/sqlite_rpm/build.sh +19 -0
  44. data/extras/sqlite_rpm/muqlite.spec +47 -0
  45. data/install/installer +7 -5
  46. data/modules/mu.rb +12 -5
  47. data/modules/mu/cloud/machine_images.rb +1 -1
  48. data/modules/mu/cloud/providers.rb +6 -1
  49. data/modules/mu/cloud/resource_base.rb +1 -1
  50. data/modules/mu/cloud/ssh_sessions.rb +4 -0
  51. data/modules/mu/config.rb +28 -12
  52. data/modules/mu/config/database.rb +2 -2
  53. data/modules/mu/config/firewall_rule.rb +1 -1
  54. data/modules/mu/config/ref.rb +2 -2
  55. data/modules/mu/config/schema_helpers.rb +12 -3
  56. data/modules/mu/config/server.rb +10 -4
  57. data/modules/mu/config/server_pool.rb +2 -2
  58. data/modules/mu/config/vpc.rb +10 -10
  59. data/modules/mu/defaults/AWS.yaml +32 -32
  60. data/modules/mu/deploy.rb +23 -10
  61. data/modules/mu/groomers/chef.rb +2 -2
  62. data/modules/mu/master.rb +49 -3
  63. data/modules/mu/mommacat.rb +8 -5
  64. data/modules/mu/mommacat/naming.rb +2 -2
  65. data/modules/mu/mommacat/storage.rb +22 -27
  66. data/modules/mu/providers/aws.rb +142 -48
  67. data/modules/mu/providers/aws/alarm.rb +3 -3
  68. data/modules/mu/providers/aws/bucket.rb +19 -19
  69. data/modules/mu/providers/aws/cache_cluster.rb +22 -22
  70. data/modules/mu/providers/aws/cdn.rb +2 -2
  71. data/modules/mu/providers/aws/collection.rb +14 -14
  72. data/modules/mu/providers/aws/container_cluster.rb +27 -27
  73. data/modules/mu/providers/aws/database.rb +40 -39
  74. data/modules/mu/providers/aws/dnszone.rb +5 -5
  75. data/modules/mu/providers/aws/endpoint.rb +35 -35
  76. data/modules/mu/providers/aws/firewall_rule.rb +26 -23
  77. data/modules/mu/providers/aws/function.rb +28 -28
  78. data/modules/mu/providers/aws/group.rb +7 -7
  79. data/modules/mu/providers/aws/habitat.rb +2 -2
  80. data/modules/mu/providers/aws/job.rb +6 -6
  81. data/modules/mu/providers/aws/loadbalancer.rb +34 -34
  82. data/modules/mu/providers/aws/log.rb +14 -14
  83. data/modules/mu/providers/aws/msg_queue.rb +10 -10
  84. data/modules/mu/providers/aws/nosqldb.rb +8 -8
  85. data/modules/mu/providers/aws/notifier.rb +7 -7
  86. data/modules/mu/providers/aws/role.rb +17 -15
  87. data/modules/mu/providers/aws/search_domain.rb +10 -10
  88. data/modules/mu/providers/aws/server.rb +176 -95
  89. data/modules/mu/providers/aws/server_pool.rb +65 -105
  90. data/modules/mu/providers/aws/storage_pool.rb +17 -9
  91. data/modules/mu/providers/aws/user.rb +1 -1
  92. data/modules/mu/providers/aws/vpc.rb +103 -51
  93. data/modules/mu/providers/aws/vpc_subnet.rb +43 -39
  94. data/modules/mu/providers/azure.rb +78 -12
  95. data/modules/mu/providers/azure/server.rb +18 -3
  96. data/modules/mu/providers/cloudformation/server.rb +1 -1
  97. data/modules/mu/providers/google.rb +19 -4
  98. data/modules/mu/providers/google/folder.rb +6 -2
  99. data/modules/mu/providers/google/function.rb +65 -30
  100. data/modules/mu/providers/google/role.rb +1 -1
  101. data/modules/mu/providers/google/vpc.rb +27 -2
  102. data/modules/tests/aws-servers-with-handrolled-iam.yaml +37 -0
  103. data/modules/tests/k8s.yaml +1 -1
  104. metadata +24 -8
@@ -30,13 +30,13 @@ module MU
30
30
  @config["log_group_name"] = @mu_name
31
31
  @config["log_stream_name"] =
32
32
  if @config["enable_cloudtrail_logging"]
33
- "#{MU::Cloud::AWS.credToAcct(@config['credentials'])}_CloudTrail_#{@config["region"]}"
33
+ "#{MU::Cloud::AWS.credToAcct(@credentials)}_CloudTrail_#{@region}"
34
34
  else
35
35
  @mu_name
36
36
  end
37
37
 
38
38
  MU.log "Creating log group #{@mu_name}"
39
- MU::Cloud::AWS.cloudwatchlogs(region: @config["region"], credentials: @config["credentials"]).create_log_group(
39
+ MU::Cloud::AWS.cloudwatchlogs(region: @region, credentials: @credentials).create_log_group(
40
40
  log_group_name: @config["log_group_name"],
41
41
  tags: @tags
42
42
  )
@@ -45,7 +45,7 @@ module MU
45
45
  retries = 0
46
46
  max_retries = 5
47
47
  begin
48
- resp = MU::Cloud::AWS::Log.getLogGroupByName(@config["log_group_name"], region: @config["region"])
48
+ resp = MU::Cloud::AWS::Log.getLogGroupByName(@config["log_group_name"], region: @region)
49
49
  if resp.nil?
50
50
  if retries >= max_retries
51
51
  raise MuError, "Cloudwatch Logs group #{@config["log_group_name"]} creation hasn't succeeded after #{(retries*max_retries).to_s}s"
@@ -56,19 +56,19 @@ module MU
56
56
  end
57
57
  end while resp.nil?
58
58
 
59
- MU::Cloud::AWS.cloudwatchlogs(region: @config["region"], credentials: @config["credentials"]).create_log_stream(
59
+ MU::Cloud::AWS.cloudwatchlogs(region: @region, credentials: @credentials).create_log_stream(
60
60
  log_group_name: @config["log_group_name"],
61
61
  log_stream_name: @config["log_stream_name"]
62
62
  )
63
63
 
64
- MU::Cloud::AWS.cloudwatchlogs(region: @config["region"], credentials: @config["credentials"]).put_retention_policy(
64
+ MU::Cloud::AWS.cloudwatchlogs(region: @region, credentials: @credentials).put_retention_policy(
65
65
  log_group_name: @config["log_group_name"],
66
66
  retention_in_days: @config["retention_period"]
67
67
  )
68
68
 
69
69
  if @config["filters"] && !@config["filters"].empty?
70
70
  @config["filters"].each{ |filter|
71
- MU::Cloud::AWS.cloudwatchlogs(region: @config["region"], credentials: @config["credentials"]).put_metric_filter(
71
+ MU::Cloud::AWS.cloudwatchlogs(region: @region, credentials: @credentials).put_metric_filter(
72
72
  log_group_name: @config["log_group_name"],
73
73
  filter_name: filter["name"],
74
74
  filter_pattern: filter["search_pattern"],
@@ -82,8 +82,8 @@ module MU
82
82
  end
83
83
 
84
84
  if @config["enable_cloudtrail_logging"]
85
- trail_resp = MU::Cloud::AWS.cloudtrail(region: @config["region"], credentials: @config["credentials"]).describe_trails.trail_list.first
86
- raise MuError, "Can't find a cloudtrail in #{MU::Cloud::AWS.credToAcct(@config['credentials'])}/#{@config["region"]}. Please create cloudtrail before enabling logging on it" unless trail_resp
85
+ trail_resp = MU::Cloud::AWS.cloudtrail(region: @region, credentials: @credentials).describe_trails.trail_list.first
86
+ raise MuError, "Can't find a cloudtrail in #{MU::Cloud::AWS.credToAcct(@credentials)}/#{@region}. Please create cloudtrail before enabling logging on it" unless trail_resp
87
87
 
88
88
  iam_policy = '{
89
89
  "Version": "2012-10-17",
@@ -96,7 +96,7 @@ module MU
96
96
  "logs:PutLogEventsBatch",
97
97
  "logs:PutLogEvents"
98
98
  ],
99
- "Resource": "arn:'+(MU::Cloud::AWS.isGovCloud?(@config["region"]) ? "aws-us-gov" : "aws")+':logs:'+@config["region"]+':'+MU::Cloud::AWS.credToAcct(@config['credentials'])+':log-group:'+@config["log_group_name"]+':log-stream:'+@config["log_stream_name"]+'*"
99
+ "Resource": "arn:'+(MU::Cloud::AWS.isGovCloud?(@region) ? "aws-us-gov" : "aws")+':logs:'+@region+':'+MU::Cloud::AWS.credToAcct(@credentials)+':log-group:'+@config["log_group_name"]+':log-stream:'+@config["log_stream_name"]+'*"
100
100
  }
101
101
  ]
102
102
  }'
@@ -132,11 +132,11 @@ module MU
132
132
  policy_document: iam_policy
133
133
  )
134
134
 
135
- log_group_resp = MU::Cloud::AWS::Log.getLogGroupByName(@config["log_group_name"], region: @config["region"])
135
+ log_group_resp = MU::Cloud::AWS::Log.getLogGroupByName(@config["log_group_name"], region: @region)
136
136
 
137
137
  retries = 0
138
138
  begin
139
- MU::Cloud::AWS.cloudtrail(region: @config["region"], credentials: @config["credentials"]).update_trail(
139
+ MU::Cloud::AWS.cloudtrail(region: @region, credentials: @credentials).update_trail(
140
140
  name: trail_resp.name,
141
141
  cloud_watch_logs_log_group_arn: log_group_resp.arn,
142
142
  cloud_watch_logs_role_arn: iam_resp.role.arn
@@ -270,9 +270,9 @@ module MU
270
270
  def toKitten(**_args)
271
271
  bok = {
272
272
  "cloud" => "AWS",
273
- "credentials" => @config['credentials'],
273
+ "credentials" => @credentials,
274
274
  "cloud_id" => @cloud_id,
275
- "region" => @config['region']
275
+ "region" => @region
276
276
  }
277
277
 
278
278
  if !cloud_desc
@@ -283,7 +283,7 @@ module MU
283
283
  bok['name'] = cloud_desc.log_group_name.sub(/.*?\/([^\/]+)$/, '\1')
284
284
 
285
285
  if cloud_desc.metric_filter_count > 0
286
- resp = MU::Cloud::AWS.cloudwatchlogs(region: @config['region'], credentials: @credentials).describe_metric_filters(
286
+ resp = MU::Cloud::AWS.cloudwatchlogs(region: @region, credentials: @credentials).describe_metric_filters(
287
287
  log_group_name: @cloud_id
288
288
  )
289
289
  resp.metric_filters.each { |filter|
@@ -33,7 +33,7 @@ module MU
33
33
  namestr += ".fifo" if attrs['FifoQueue']
34
34
 
35
35
  MU.log "Creating SQS queue #{namestr}", details: attrs
36
- resp = MU::Cloud::AWS.sqs(region: @config['region'], credentials: @config['credentials']).create_queue(
36
+ resp = MU::Cloud::AWS.sqs(region: @region, credentials: @credentials).create_queue(
37
37
  queue_name: namestr,
38
38
  attributes: attrs
39
39
  )
@@ -60,7 +60,7 @@ module MU
60
60
  }
61
61
  if changed
62
62
  MU.log "Updating SQS queue #{@mu_name}", MU::NOTICE, details: new_attrs
63
- MU::Cloud::AWS.sqs(region: @config['region'], credentials: @config['credentials']).set_queue_attributes(
63
+ MU::Cloud::AWS.sqs(region: @region, credentials: @credentials).set_queue_attributes(
64
64
  queue_url: @cloud_id,
65
65
  attributes: new_attrs
66
66
  )
@@ -71,7 +71,7 @@ module MU
71
71
  # Canonical Amazon Resource Number for this resource
72
72
  # @return [String]
73
73
  def arn
74
- "arn:"+(MU::Cloud::AWS.isGovCloud?(@config["region"]) ? "aws-us-gov" : "aws")+":sqs:"+@config['region']+":"+MU::Cloud::AWS.credToAcct(@config['credentials'])+":"+@cloud_id
74
+ "arn:"+(MU::Cloud::AWS.isGovCloud?(@region) ? "aws-us-gov" : "aws")+":sqs:"+@region+":"+MU::Cloud::AWS.credToAcct(@credentials)+":"+@cloud_id
75
75
  end
76
76
 
77
77
  @cloud_desc_cache = nil
@@ -83,7 +83,7 @@ module MU
83
83
  return nil if !@cloud_id
84
84
 
85
85
  if !@cloud_id
86
- resp = MU::Cloud::AWS.sqs(region: @config['region'], credentials: @config['credentials']).list_queues(
86
+ resp = MU::Cloud::AWS.sqs(region: @region, credentials: @credentials).list_queues(
87
87
  queue_name_prefix: @mu_name
88
88
  )
89
89
  return nil if !resp or !resp.queue_urls
@@ -98,8 +98,8 @@ module MU
98
98
  return nil if !@cloud_id
99
99
  @cloud_desc_cache = MU::Cloud::AWS::MsgQueue.find(
100
100
  cloud_id: @cloud_id.dup,
101
- region: @config['region'],
102
- credentials: @config['credentials']
101
+ region: @region,
102
+ credentials: @credentials
103
103
  )
104
104
  @cloud_desc_cache
105
105
  end
@@ -110,8 +110,8 @@ module MU
110
110
  cloud_desc
111
111
  deploy_struct = MU::Cloud::AWS::MsgQueue.find(
112
112
  cloud_id: @cloud_id,
113
- region: @config['region'],
114
- credentials: @config['credentials']
113
+ region: @region,
114
+ credentials: @credentials
115
115
  )
116
116
  return deploy_struct
117
117
  end
@@ -426,7 +426,7 @@ module MU
426
426
  if sibling # resolve sibling queues to something useful
427
427
  id = sibling.cloud_id
428
428
  end
429
- desc = MU::Cloud::AWS::MsgQueue.find(cloud_id: id, credentials: @config['credentials'])
429
+ desc = MU::Cloud::AWS::MsgQueue.find(cloud_id: id, credentials: @credentials)
430
430
  if !desc
431
431
  raise MuError, "Failed to get cloud descriptor for SQS queue #{@config['failqueue']['name']}"
432
432
  end
@@ -484,7 +484,7 @@ module MU
484
484
  end
485
485
 
486
486
  begin
487
- MU::Cloud::AWS.sqs(region: @config['region'], credentials: @config['credentials']).tag_queue(
487
+ MU::Cloud::AWS.sqs(region: @region, credentials: @credentials).tag_queue(
488
488
  queue_url: url,
489
489
  tags: tags
490
490
  )
@@ -114,11 +114,11 @@ module MU
114
114
 
115
115
  MU.log "Creating DynamoDB table #{@mu_name}", MU::NOTICE, details: params
116
116
 
117
- resp = MU::Cloud::AWS.dynamo(credentials: @config['credentials'], region: @config['region']).create_table(params)
117
+ resp = MU::Cloud::AWS.dynamo(credentials: @credentials, region: @region).create_table(params)
118
118
  @cloud_id = @mu_name
119
119
 
120
120
  begin
121
- resp = MU::Cloud::AWS.dynamo(credentials: @config['credentials'], region: @config['region']).describe_table(table_name: @cloud_id)
121
+ resp = MU::Cloud::AWS.dynamo(credentials: @credentials, region: @region).describe_table(table_name: @cloud_id)
122
122
  sleep 5 if resp.table.table_status == "CREATING"
123
123
  end while resp.table.table_status == "CREATING"
124
124
 
@@ -130,7 +130,7 @@ module MU
130
130
  begin
131
131
  batch = items_to_write.slice!(0, (items_to_write.length >= 25 ? 25 : items_to_write.length))
132
132
  begin
133
- MU::Cloud::AWS.dynamo(credentials: @config['credentials'], region: @config['region']).batch_write_item(
133
+ MU::Cloud::AWS.dynamo(credentials: @credentials, region: @region).batch_write_item(
134
134
  request_items: {
135
135
  @cloud_id => batch.map { |i| { put_request: { item: i } } }
136
136
  }
@@ -162,7 +162,7 @@ module MU
162
162
  }
163
163
  end
164
164
 
165
- MU::Cloud::AWS.dynamo(credentials: @config['credentials'], region: @config['region']).tag_resource(
165
+ MU::Cloud::AWS.dynamo(credentials: @credentials, region: @region).tag_resource(
166
166
  resource_arn: arn,
167
167
  tags: tagset
168
168
  )
@@ -281,9 +281,9 @@ module MU
281
281
  def toKitten(**_args)
282
282
  bok = {
283
283
  "cloud" => "AWS",
284
- "credentials" => @config['credentials'],
284
+ "credentials" => @credentials,
285
285
  "cloud_id" => @cloud_id,
286
- "region" => @config['region']
286
+ "region" => @region
287
287
  }
288
288
 
289
289
  if !cloud_desc
@@ -318,10 +318,10 @@ module MU
318
318
 
319
319
  bok['stream'] = cloud_desc.stream_specification.stream_view_type
320
320
  # cloud_desc.latest_stream_arn
321
- # MU::Cloud::AWS.dynamostream(credentials: @credentials, region: @config['region']).list_streams
321
+ # MU::Cloud::AWS.dynamostream(credentials: @credentials, region: @region).list_streams
322
322
  end
323
323
 
324
- bok["populate"] = MU::Cloud::AWS.dynamo(credentials: @credentials, region: @config['region']).scan(
324
+ bok["populate"] = MU::Cloud::AWS.dynamo(credentials: @credentials, region: @region).scan(
325
325
  table_name: @cloud_id
326
326
  ).items
327
327
 
@@ -28,7 +28,7 @@ module MU
28
28
  # Called automatically by {MU::Deploy#createResources}
29
29
  def create
30
30
  @cloud_id = @mu_name
31
- MU::Cloud::AWS.sns(region: @config['region'], credentials: @config['credentials']).create_topic(name: @cloud_id)
31
+ MU::Cloud::AWS.sns(region: @region, credentials: @credentials).create_topic(name: @cloud_id)
32
32
  MU.log "Created SNS topic #{@mu_name}"
33
33
  end
34
34
 
@@ -52,7 +52,7 @@ module MU
52
52
  # @param endpoint [String]: The address, identifier, or ARN of the resource being subscribed
53
53
  # @param protocol [String]: The protocol being subscribed
54
54
  def subscribe(endpoint, protocol)
55
- self.class.subscribe(arn, endpoint, protocol, region: @config['region'], credentials: @credentials)
55
+ self.class.subscribe(arn, endpoint, protocol, region: @region, credentials: @credentials)
56
56
  end
57
57
 
58
58
  # Subscribe something to an SNS topic
@@ -116,14 +116,14 @@ module MU
116
116
  # @return [String]
117
117
  def arn
118
118
  @cloud_id ||= @mu_name
119
- "arn:"+(MU::Cloud::AWS.isGovCloud?(@config["region"]) ? "aws-us-gov" : "aws")+":sns:"+@config['region']+":"+MU::Cloud::AWS.credToAcct(@config['credentials'])+":"+@cloud_id
119
+ "arn:"+(MU::Cloud::AWS.isGovCloud?(@region) ? "aws-us-gov" : "aws")+":sns:"+@region+":"+MU::Cloud::AWS.credToAcct(@credentials)+":"+@cloud_id
120
120
  end
121
121
 
122
122
  # Return the metadata for this user cofiguration
123
123
  # @return [Hash]
124
124
  def notify
125
125
  return nil if !@cloud_id or !cloud_desc(use_cache: false)
126
- desc = MU::Cloud::AWS.sns(region: @config["region"], credentials: @config["credentials"]).get_topic_attributes(topic_arn: arn).attributes
126
+ desc = MU::Cloud::AWS.sns(region: @region, credentials: @credentials).get_topic_attributes(topic_arn: arn).attributes
127
127
  MU.structToHash(desc)
128
128
  end
129
129
 
@@ -165,9 +165,9 @@ module MU
165
165
  def toKitten(**_args)
166
166
  bok = {
167
167
  "cloud" => "AWS",
168
- "credentials" => @config['credentials'],
168
+ "credentials" => @credentials,
169
169
  "cloud_id" => @cloud_id,
170
- "region" => @config['region']
170
+ "region" => @region
171
171
  }
172
172
 
173
173
  if !cloud_desc
@@ -180,7 +180,7 @@ module MU
180
180
  "lambda" => "functions",
181
181
  "sqs" => "msg_queues"
182
182
  }
183
- MU::Cloud::AWS.sns(region: @config['region'], credentials: @credentials).list_subscriptions_by_topic(topic_arn: cloud_desc["TopicArn"]).subscriptions.each { |sub|
183
+ MU::Cloud::AWS.sns(region: @region, credentials: @credentials).list_subscriptions_by_topic(topic_arn: cloud_desc["TopicArn"]).subscriptions.each { |sub|
184
184
  bok['subscriptions'] ||= []
185
185
 
186
186
  bok['subscriptions'] << if sub.endpoint.match(/^arn:[^:]+:(sqs|lambda):([^:]+):(\d+):.*?([^:\/]+)$/)
@@ -30,7 +30,7 @@ module MU
30
30
  end
31
31
  end
32
32
 
33
- @mu_name ||= @deploy.getResourceName(@config["name"], max_length: 64)
33
+ @mu_name ||= @config['scrub_mu_isms'] ? @config['name'] : @deploy.getResourceName(@config["name"], max_length: 64)
34
34
  end
35
35
 
36
36
  # Called automatically by {MU::Deploy#createResources}
@@ -60,7 +60,7 @@ module MU
60
60
  :role_name => @mu_name,
61
61
  :description => "Generated by Mu",
62
62
  :assume_role_policy_document => gen_assume_role_policy_doc,
63
- :tags => get_tag_params
63
+ :tags => get_tag_params(@config['scrub_mu_isms'])
64
64
  }
65
65
 
66
66
  MU.log "Creating IAM role #{@mu_name} (#{@credentials})", details: params
@@ -166,7 +166,7 @@ module MU
166
166
  version_id: desc.policy.default_version_id
167
167
  )
168
168
 
169
- ext = JSON.parse(URI.decode(version.policy_version.document))
169
+ ext = JSON.parse(CGI.unescape(version.policy_version.document))
170
170
  if ext != policy.values.first
171
171
  # Special exception- we don't want to overwrite extra rules
172
172
  # in MuSecrets policies, because our siblings might have
@@ -194,7 +194,6 @@ module MU
194
194
  )
195
195
  MU.retrier([Aws::IAM::Errors::NoSuchEntity], loop_if: Proc.new { desc.nil? }) {
196
196
  desc = MU::Cloud::AWS.iam(credentials: credentials).get_policy(policy_arn: arn)
197
- pp desc
198
197
  }
199
198
  desc
200
199
  end
@@ -223,6 +222,7 @@ module MU
223
222
  # populated with one or both depending on what this resource has
224
223
  # defined.
225
224
  def cloud_desc(use_cache: true)
225
+ require 'aws-sdk-iam'
226
226
 
227
227
  # we might inherit a naive cached description from the base cloud
228
228
  # layer; rearrange it to our tastes
@@ -312,8 +312,8 @@ end
312
312
  # Insert a new target entity into an existing policy.
313
313
  # @param policy [String]: The name of the policy to which we're appending, which must already exist as part of this role resource
314
314
  # @param targets [Array<String>]: The target resource. If +target_type+ isn't specified, this should be a fully-resolved ARN.
315
- def injectPolicyTargets(policy, targets)
316
- if !policy.match(/^#{@deploy.deploy_id}/)
315
+ def injectPolicyTargets(policy, targets, attach: false)
316
+ if @deploy and !policy.match(/^#{@deploy.deploy_id}/)
317
317
  policy = @mu_name+"-"+policy.upcase
318
318
  end
319
319
  my_policies = cloud_desc(use_cache: false)["policies"]
@@ -328,14 +328,14 @@ end
328
328
  version_id: p.default_version_id
329
329
  ).policy_version
330
330
 
331
- doc = JSON.parse URI.decode_www_form_component old.document
331
+ doc = JSON.parse CGI.unescape_www_form_component old.document
332
332
  need_update = false
333
333
 
334
334
  doc["Statement"].each { |s|
335
335
  targets.each { |target|
336
336
  target_string = target
337
337
 
338
- if target['type']
338
+ if target['type'] and @deploy
339
339
  sibling = @deploy.findLitterMate(
340
340
  name: target["identifier"],
341
341
  type: target["type"]
@@ -616,7 +616,7 @@ end
616
616
  policy_name: pol.policy_name
617
617
  )
618
618
  if resp and resp.policy_document
619
- JSON.parse(URI.decode(resp.policy_document))
619
+ JSON.parse(CGI.unescape(resp.policy_document))
620
620
  end
621
621
  rescue ::Aws::IAM::Errors::NoSuchEntity, ::Aws::IAM::Errors::ValidationError
622
622
  resp = MU::Cloud::AWS.iam(credentials: @credentials).get_policy(
@@ -626,7 +626,7 @@ end
626
626
  policy_arn: pol.arn,
627
627
  version_id: resp.policy.default_version_id
628
628
  )
629
- JSON.parse(URI.decode(version.policy_version.document))
629
+ JSON.parse(CGI.unescape(version.policy_version.document))
630
630
  end
631
631
  bok["policies"] = MU::Cloud::AWS::Role.doc2MuPolicies(pol.policy_name, doc, bok["policies"])
632
632
  end
@@ -642,7 +642,7 @@ end
642
642
  bok["strip_path"] = true if desc.path == "/"
643
643
 
644
644
  if desc.assume_role_policy_document
645
- assume_doc = JSON.parse(URI.decode(desc.assume_role_policy_document))
645
+ assume_doc = JSON.parse(CGI.unescape(desc.assume_role_policy_document))
646
646
  assume_doc["Statement"].each { |s|
647
647
  bok["can_assume"] ||= []
648
648
  method = if s["Action"] == "sts:AssumeRoleWithWebIdentity"
@@ -794,14 +794,14 @@ end
794
794
  path_prefix: "/"+@deploy.deploy_id+"/"
795
795
  ).policies
796
796
  mypolicies.reject! { |p|
797
- !p.policy_name.match(/^#{Regexp.quote(@mu_name)}-/)
797
+ !p.policy_name.match(/^#{Regexp.quote(@mu_name)}(-|$)/)
798
798
  }
799
799
 
800
800
  if @config['attachable_policies']
801
801
  @config['attachable_policies'].each { |policy_hash|
802
802
  policy = policy_hash["id"]
803
803
  p_arn = if !policy.match(/^arn:/i)
804
- "arn:"+(MU::Cloud::AWS.isGovCloud?(@config["region"]) ? "aws-us-gov" : "aws")+":iam::aws:policy/"+policy
804
+ "arn:"+(MU::Cloud::AWS.isGovCloud?(@region) ? "aws-us-gov" : "aws")+":iam::aws:policy/"+policy
805
805
  else
806
806
  policy
807
807
  end
@@ -813,7 +813,7 @@ end
813
813
  ).policy
814
814
  rescue Aws::IAM::Errors::NoSuchEntity => e
815
815
  if subpaths.size > 0
816
- p_arn = "arn:"+(MU::Cloud::AWS.isGovCloud?(@config["region"]) ? "aws-us-gov" : "aws")+":iam::aws:policy/#{subpaths.shift}/"+policy
816
+ p_arn = "arn:"+(MU::Cloud::AWS.isGovCloud?(@region) ? "aws-us-gov" : "aws")+":iam::aws:policy/#{subpaths.shift}/"+policy
817
817
  retry
818
818
  end
819
819
  raise e
@@ -1087,6 +1087,8 @@ end
1087
1087
  role.delete("import")
1088
1088
  end
1089
1089
 
1090
+ role['strip_path'] = true if role['scrub_mu_isms']
1091
+
1090
1092
  # If we're attaching some managed policies, make sure all of the ones
1091
1093
  # that should already exist do indeed exist
1092
1094
  if role['attachable_policies']
@@ -1117,7 +1119,7 @@ end
1117
1119
  role['policies'].each { |policy|
1118
1120
  policy['targets'].each { |target|
1119
1121
  if target['type']
1120
- MU::Config.addDependency(role, target['identifier'], target['type'], no_create_wait: true)
1122
+ MU::Config.addDependency(role, target['identifier'], target['type'], my_phase: "groom")
1121
1123
  end
1122
1124
  }
1123
1125
  }
@@ -36,7 +36,7 @@ module MU
36
36
 
37
37
  MU.log "Creating ElasticSearch domain #{@config['domain_name']}", details: params
38
38
  @cloud_id = @config['domain_name']
39
- MU::Cloud::AWS.elasticsearch(region: @config['region'], credentials: @credentials).create_elasticsearch_domain(params).domain_status
39
+ MU::Cloud::AWS.elasticsearch(region: @region, credentials: @credentials).create_elasticsearch_domain(params).domain_status
40
40
 
41
41
  tagDomain
42
42
 
@@ -52,7 +52,7 @@ module MU
52
52
  waitWhileProcessing # wait until the create finishes, if still going
53
53
 
54
54
  MU.log "Updating ElasticSearch domain #{@config['domain_name']}", MU::NOTICE, details: params
55
- MU::Cloud::AWS.elasticsearch(region: @config['region'], credentials: @credentials).update_elasticsearch_domain_config(params)
55
+ MU::Cloud::AWS.elasticsearch(region: @region, credentials: @credentials).update_elasticsearch_domain_config(params)
56
56
  end
57
57
 
58
58
  waitWhileProcessing # don't return until creation/updating is complete
@@ -68,7 +68,7 @@ module MU
68
68
  @cloud_id ||= @config['domain_name']
69
69
  return nil if !@cloud_id
70
70
  MU.retrier([::Aws::ElasticsearchService::Errors::ResourceNotFoundException], wait: 10, max: 12) {
71
- @cloud_desc_cache = MU::Cloud::AWS.elasticsearch(region: @config['region'], credentials: @credentials).describe_elasticsearch_domain(
71
+ @cloud_desc_cache = MU::Cloud::AWS.elasticsearch(region: @region, credentials: @credentials).describe_elasticsearch_domain(
72
72
  domain_name: @cloud_id
73
73
  ).domain_status
74
74
  }
@@ -88,7 +88,7 @@ module MU
88
88
  def notify
89
89
  return nil if !cloud_desc(use_cache: false)
90
90
  deploy_struct = MU.structToHash(cloud_desc, stringify_keys: true)
91
- tags = MU::Cloud::AWS.elasticsearch(region: @config['region'], credentials: @credentials).list_tags(arn: arn).tag_list
91
+ tags = MU::Cloud::AWS.elasticsearch(region: @region, credentials: @credentials).list_tags(arn: arn).tag_list
92
92
  deploy_struct['tags'] = tags.map { |t| { t.key => t.value } }
93
93
  if deploy_struct['endpoint']
94
94
  deploy_struct['kibana'] = deploy_struct['endpoint']+"/_plugin/kibana/"
@@ -200,7 +200,7 @@ module MU
200
200
  "cloud" => "AWS",
201
201
  "credentials" => @credentials,
202
202
  "cloud_id" => @cloud_id,
203
- "region" => @config['region']
203
+ "region" => @region
204
204
  }
205
205
 
206
206
  if !cloud_desc
@@ -241,7 +241,7 @@ module MU
241
241
  bok['identity_pool_id'] = cloud_desc.cognito_options.identity_pool_id
242
242
  end
243
243
 
244
- tags = MU::Cloud::AWS.elasticsearch(region: @config['region'], credentials: @credentials).list_tags(arn: cloud_desc.arn).tag_list
244
+ tags = MU::Cloud::AWS.elasticsearch(region: @region, credentials: @credentials).list_tags(arn: cloud_desc.arn).tag_list
245
245
  if tags and !tags.empty?
246
246
  bok['tags'] = MU.structToHash(tags)
247
247
  end
@@ -252,7 +252,7 @@ module MU
252
252
  cloud: "AWS",
253
253
  credentials: @credentials,
254
254
  type: "vpcs",
255
- region: @config['region'],
255
+ region: @region,
256
256
  subnets: cloud_desc.vpc_options.subnet_ids.map { |s| { "subnet_id" => s } }
257
257
  )
258
258
  if cloud_desc.vpc_options.security_group_ids and
@@ -262,7 +262,7 @@ module MU
262
262
  id: sg,
263
263
  cloud: "AWS",
264
264
  credentials: @credentials,
265
- region: @config['region'],
265
+ region: @region,
266
266
  type: "firewall_rules",
267
267
  )
268
268
  }
@@ -683,7 +683,7 @@ module MU
683
683
  params[:log_publishing_options]["SEARCH_SLOW_LOGS"] = {}
684
684
  params[:log_publishing_options]["SEARCH_SLOW_LOGS"][:enabled] = true
685
685
  params[:log_publishing_options]["SEARCH_SLOW_LOGS"][:cloud_watch_logs_log_group_arn] = arn
686
- MU::Cloud.resourceClass("AWS", "Log").allowService("es.amazonaws.com", arn, @config['region'])
686
+ MU::Cloud.resourceClass("AWS", "Log").allowService("es.amazonaws.com", arn, @region)
687
687
  end
688
688
  end
689
689
 
@@ -813,7 +813,7 @@ module MU
813
813
  raise MU::MuError, "Can't tag ElasticSearch domain, cloud descriptor came back without an ARN"
814
814
  end
815
815
 
816
- MU::Cloud::AWS.elasticsearch(region: @config['region'], credentials: @credentials).add_tags(
816
+ MU::Cloud::AWS.elasticsearch(region: @region, credentials: @credentials).add_tags(
817
817
  arn: domain.arn,
818
818
  tag_list: tags
819
819
  )