cloud-mu 3.3.1 → 3.3.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 9ccd46dd31e8b95994aa2f8c2ac1ac486549e542ea5f782149f5fac61a3db3f3
4
- data.tar.gz: '0020871875d52fb5c2988a2f7793d37bfc53681d97820d6e12d0488ce48bb857'
3
+ metadata.gz: 831bdab20f2750e0080d2282d6bbb14710339b442bd484ae539b496f9e9cbe07
4
+ data.tar.gz: 1e8369e3e7937fa4bda0002a13cada2d7d99eb5c9bede0bd11b5009d5ac62add
5
5
  SHA512:
6
- metadata.gz: 7ac6aca25c92e455f3d3c3dc5d1a55306f4a0071669b8d4ebb2d7c2cef78ec56aba234a4f12db86e556b1aa1f001f4832fbdb2b55c31263da72c236f8ba1bae3
7
- data.tar.gz: 60fe5e02d425d2379eb88db7454d53729ef1e25f57ea636f4464d032e721eee9e5a0e3d1855e03f24f07d71e266f68cfc3fbf227714ca98ae59a1ec53be72dbb
6
+ metadata.gz: 7677217cea3460bccc8bfcfc6cd39400e6efc659ceaf35c80742885b541988ad42776385dc232d72848acd34d9dcbed5d44abe6ea6e5af688b3d6ec86ac5a6d6
7
+ data.tar.gz: 2e26064eb275a1f68f73766558bd39513c50a4218aeb8968779f6592d3b1210c4b84ee2808455d81bc3eac3bf8dad3027e97f451a99c9dd5bd87d05c77fb8880
@@ -17,8 +17,8 @@ end
17
17
 
18
18
  Gem::Specification.new do |s|
19
19
  s.name = 'cloud-mu'
20
- s.version = '3.3.1'
21
- s.date = '2020-09-29'
20
+ s.version = '3.3.2'
21
+ s.date = '2020-10-04'
22
22
  s.require_paths = ['modules']
23
23
  s.required_ruby_version = '>= 2.4'
24
24
  s.summary = "The eGTLabs Mu toolkit for unified cloud deployments"
@@ -738,7 +738,7 @@ module MU
738
738
  MU.log "Couldn't find existing resource #{ext_deploy["cloud_id"]}, #{ext_deploy["cloud_type"]}", MU::ERR if found.nil?
739
739
  @deploy.notify(ext_deploy["cloud_type"], found.config["name"], found.deploydata, mu_name: found.mu_name, triggering_node: @mu_name)
740
740
  elsif ext_deploy["mu_name"] && ext_deploy["deploy_id"]
741
- MU.log "#{ext_deploy["mu_name"]} / #{ext_deploy["deploy_id"]}"
741
+ MU.log "#{self}: Importing metadata for #{ext_deploy["cloud_type"]} #{ext_deploy["mu_name"]} from #{ext_deploy["deploy_id"]}"
742
742
  found = MU::MommaCat.findStray(
743
743
  @config['cloud'],
744
744
  ext_deploy["cloud_type"],
@@ -748,8 +748,11 @@ module MU
748
748
  dummy_ok: false
749
749
  ).first
750
750
 
751
- MU.log "Couldn't find existing resource #{ext_deploy["mu_name"]}/#{ext_deploy["deploy_id"]}, #{ext_deploy["cloud_type"]}", MU::ERR if found.nil?
752
- @deploy.notify(ext_deploy["cloud_type"], found.config["name"], found.deploydata, mu_name: ext_deploy["mu_name"], triggering_node: @mu_name)
751
+ if found.nil?
752
+ MU.log "Couldn't find existing resource #{ext_deploy["mu_name"]}/#{ext_deploy["deploy_id"]}, #{ext_deploy["cloud_type"]}", MU::ERR
753
+ else
754
+ @deploy.notify(ext_deploy["cloud_type"], found.config["name"], found.deploydata, mu_name: ext_deploy["mu_name"], triggering_node: @mu_name)
755
+ end
753
756
  else
754
757
  MU.log "Trying to find existing deploy, but either the cloud_id is not valid or no mu_name and deploy_id where provided", MU::ERR
755
758
  end
@@ -125,7 +125,7 @@ module MU
125
125
  end
126
126
  end
127
127
  end
128
- rescue RuntimeError => e
128
+ rescue RuntimeError, IOError => e
129
129
  raise MU::Cloud::BootstrapTempFail, "Got #{e.inspect} performing initial SSH connect tasks, will try again"
130
130
  end
131
131
 
@@ -634,7 +634,7 @@ MESSAGE_END
634
634
  begin
635
635
  if myservice['#MUOBJECT'].nil?
636
636
  if @mommacat
637
- ext_obj = @mommacat.findLitterMate(type: myservice["#MU_CLOUDCLASS"].cfg_plural, name: myservice['name'], credentials: myservice['credentials'], created_only: true, return_all: false)
637
+ ext_obj = @mommacat.findLitterMate(type: myservice["#MU_CLOUDCLASS"].cfg_plural, name: myservice['name'], credentials: myservice['credentials'], created_only: true, return_all: false, ignore_missing: !@updating)
638
638
  if @updating and ext_obj
639
639
  ext_obj.config!(myservice)
640
640
  end
@@ -547,9 +547,14 @@ module MU
547
547
  # @param remove [Boolean]: Remove this resource from the deploy structure, instead of adding it.
548
548
  # @return [void]
549
549
  def notify(type, key, data, mu_name: nil, remove: false, triggering_node: nil, delayed_save: false)
550
+ no_write = (@no_artifacts or caller.grep(/\/mommacat\.rb:\d+:in `notify'/))
550
551
 
551
552
  begin
552
- MU::MommaCat.lock("deployment-notification", deploy_id: @deploy_id) if !@no_artifacts
553
+ if !no_write
554
+ if !MU::MommaCat.lock("deployment-notification", deploy_id: @deploy_id, retries: 10)
555
+ raise MuError, "Failed to get deployment-notifcation lock for #{@deploy_id}"
556
+ end
557
+ end
553
558
 
554
559
  if !@need_deploy_flush or @deployment.nil? or @deployment.empty?
555
560
  loadDeploy(true) # make sure we're saving the latest and greatest
@@ -588,7 +593,9 @@ module MU
588
593
  @deployment[type][key] = data
589
594
  MU.log "Adding to @deployment[#{type}][#{key}]", MU::DEBUG, details: data
590
595
  end
591
- save!(key) if !delayed_save and !@no_artifacts
596
+ if !delayed_save and !no_write
597
+ save!(key)
598
+ end
592
599
  else
593
600
  have_deploy = true
594
601
  if @deployment[type].nil? or @deployment[type][key].nil?
@@ -613,10 +620,10 @@ module MU
613
620
  end
614
621
  }
615
622
  end
616
- save! if !delayed_save and !@no_artifacts
623
+ save! if !delayed_save and !no_write
617
624
  end
618
625
  ensure
619
- MU::MommaCat.unlock("deployment-notification", deploy_id: @deploy_id) if !@no_artifacts
626
+ MU::MommaCat.unlock("deployment-notification", deploy_id: @deploy_id) if !no_write
620
627
  end
621
628
  end
622
629
 
@@ -138,7 +138,7 @@ module MU
138
138
  # @param created_only [Boolean]: Only return the littermate if its cloud_id method returns a value
139
139
  # @param return_all [Boolean]: Return a Hash of matching objects indexed by their mu_name, instead of a single match. Only valid for resource types where has_multiples is true.
140
140
  # @return [MU::Cloud]
141
- def findLitterMate(type: nil, name: nil, mu_name: nil, cloud_id: nil, created_only: false, return_all: false, credentials: nil, habitat: nil, debug: false, **flags)
141
+ def findLitterMate(type: nil, name: nil, mu_name: nil, cloud_id: nil, created_only: false, return_all: false, credentials: nil, habitat: nil, ignore_missing: false, debug: false, **flags)
142
142
  _shortclass, _cfg_name, type, _classname, attrs = MU::Cloud.getResourceNames(type)
143
143
 
144
144
  # If we specified a habitat, which we may also have done by its shorthand
@@ -173,8 +173,10 @@ module MU
173
173
  end
174
174
  end
175
175
  if @object_load_fails or !@kittens[type]
176
- MU.log "#{@deploy_id}'s original config has #{@original_config[type].size == 1 ? "a" : @original_config[type].size.to_s} #{type}, but loadObjects could not populate anything from deployment metadata", MU::ERR if !@object_load_fails
177
- @object_load_fails = true
176
+ if !ignore_missing
177
+ MU.log "#{@deploy_id}'s original config has #{@original_config[type].size == 1 ? "a" : @original_config[type].size.to_s} #{type}, but loadObjects could not populate anything from deployment metadata", MU::ERR if !@object_load_fails
178
+ @object_load_fails = true
179
+ end
178
180
  return nil
179
181
  end
180
182
  end
@@ -123,11 +123,12 @@ module MU
123
123
  MU::Cloud.resource_types.each_pair { |res_type, attrs|
124
124
  next if !@deployment.has_key?(attrs[:cfg_plural])
125
125
  deletia = []
126
+ # existing_deploys
126
127
  @deployment[attrs[:cfg_plural]].each_pair { |res_name, data|
127
128
  orig_cfg = findResourceConfig(attrs[:cfg_plural], res_name, (scrub_with || @original_config))
128
129
 
129
- if orig_cfg.nil?
130
- MU.log "#{res_type} #{res_name} no longer configured, will remove deployment metadata", MU::NOTICE
130
+ if orig_cfg.nil? and (!data['mu_name'] or data['mu_name'] =~ /^#{Regexp.quote(@deploy_id)}/)
131
+ MU.log "#{res_type} #{res_name} no longer configured, will remove deployment metadata", MU::NOTICE, details: data
131
132
  deletia << res_name
132
133
  end
133
134
  }
@@ -176,7 +177,7 @@ module MU
176
177
  # @param id [String]: The lock identifier to release.
177
178
  # @param nonblock [Boolean]: Whether to block while waiting for the lock. In non-blocking mode, we simply return false if the lock is not available.
178
179
  # return [false, nil]
179
- def self.lock(id, nonblock = false, global = false, deploy_id: MU.deploy_id)
180
+ def self.lock(id, nonblock = false, global = false, retries: 0, deploy_id: MU.deploy_id)
180
181
  raise MuError, "Can't pass a nil id to MU::MommaCat.lock" if id.nil?
181
182
 
182
183
  if !global
@@ -189,6 +190,7 @@ module MU
189
190
  MU.log "Creating #{lockdir}", MU::DEBUG
190
191
  Dir.mkdir(lockdir, 0700)
191
192
  end
193
+ nonblock = true if retries > 0
192
194
 
193
195
  @lock_semaphore.synchronize {
194
196
  if @locks[Thread.current.object_id].nil?
@@ -197,11 +199,34 @@ module MU
197
199
 
198
200
  @locks[Thread.current.object_id][id] = File.open("#{lockdir}/#{id}.lock", File::CREAT|File::RDWR, 0600)
199
201
  }
200
- MU.log "Getting a lock on #{lockdir}/#{id}.lock (thread #{Thread.current.object_id})...", MU::DEBUG
202
+
203
+ MU.log "Getting a lock on #{lockdir}/#{id}.lock (thread #{Thread.current.object_id})...", MU::DEBUG, details: caller
204
+ show_relevant = Proc.new {
205
+ @locks.each_pair { |thread_id, lock|
206
+ lock.each_pair { |lockid, lockpath|
207
+ if lockid == id
208
+ thread = Thread.list.select { |t| t.object_id == thread_id }.first
209
+ if thread.object_id != Thread.current.object_id
210
+ MU.log "#{thread_id} sitting on #{id}", MU::WARN, thread.backtrace
211
+ end
212
+ end
213
+ }
214
+ }
215
+ }
201
216
  begin
202
217
  if nonblock
203
218
  if !@locks[Thread.current.object_id][id].flock(File::LOCK_EX|File::LOCK_NB)
204
- return false
219
+ if retries > 0
220
+ success = false
221
+ MU.retrier([], loop_if: Proc.new { !success }, loop_msg: "Waiting for lock on #{lockdir}/#{id}.lock...", max: retries) {
222
+ success = @locks[Thread.current.object_id][id].flock(File::LOCK_EX|File::LOCK_NB)
223
+ show_relevant.call() if !success
224
+ }
225
+ show_relevant.call() if !success
226
+ return success
227
+ else
228
+ return false
229
+ end
205
230
  end
206
231
  else
207
232
  @locks[Thread.current.object_id][id].flock(File::LOCK_EX)
@@ -593,13 +593,34 @@ module MU
593
593
  @cloud_desc_cache = MU::Cloud::AWS.elb2(region: @config['region'], credentials: @config['credentials']).describe_load_balancers(
594
594
  names: [@cloud_id]
595
595
  ).load_balancers.first
596
- if @targetgroups.nil? and !@deploy.nil? and
597
- @deploy.deployment['loadbalancers'].has_key?(@config['name']) and
598
- @deploy.deployment['loadbalancers'][@config['name']].has_key?("targetgroups")
596
+ if @targetgroups.nil?
599
597
  @targetgroups = {}
600
- @deploy.deployment['loadbalancers'][@config['name']]["targetgroups"].each_pair { |tg_name, tg_arn|
601
- @targetgroups[tg_name] = MU::Cloud::AWS.elb2(region: @config['region'], credentials: @config['credentials']).describe_target_groups(target_group_arns: [tg_arn]).target_groups.first
602
- }
598
+ if !@deploy.nil? and
599
+ @deploy.deployment['loadbalancers'] and
600
+ @deploy.deployment['loadbalancers'][@config['name']] and
601
+ @deploy.deployment['loadbalancers'][@config['name']]["targetgroups"]
602
+ @deploy.deployment['loadbalancers'][@config['name']]["targetgroups"].each_pair { |tg_name, tg_arn|
603
+ @targetgroups[tg_name] = MU::Cloud::AWS.elb2(region: @config['region'], credentials: @config['credentials']).describe_target_groups(target_group_arns: [tg_arn]).target_groups.first
604
+ }
605
+ else
606
+ pp @config['targetgroups']
607
+ MU::Cloud::AWS.elb2(region: @config['region'], credentials: @config['credentials']).describe_target_groups(load_balancer_arn: @cloud_desc_cache.load_balancer_arn).target_groups.each { |tg|
608
+ tg_name = tg.target_group_name
609
+ if @config['targetgroups']
610
+ @config['targetgroups'].each { |tg_cfg|
611
+ if tg_name = @deploy.getResourceName(tg_cfg["name"], max_length: 32, disallowed_chars: /[^A-Za-z0-9-]/)
612
+ tg_name = tg_cfg['name']
613
+ break
614
+ end
615
+ }
616
+ end
617
+ @targetgroups[tg_name] = tg
618
+ }
619
+ # @config['targetgroups'].each { |tg|
620
+ # tg_name = @deploy.getResourceName(tg["name"], max_length: 32, disallowed_chars: /[^A-Za-z0-9-]/)
621
+ # @targetgroups[tg_name] = MU::Cloud::AWS.elb2(region: @config['region'], credentials: @config['credentials']).describe_target_groups(target_group_arns: [tg_arn]).target_groups.first
622
+ # }
623
+ end
603
624
  end
604
625
 
605
626
  return @cloud_desc_cache
@@ -137,7 +137,7 @@ module MU
137
137
 
138
138
  if !@config['bare_policies'] and
139
139
  (@config['raw_policies'] or @config['attachable_policies'])
140
- # bindTo("role", @mu_name)
140
+ bindTo("role", @mu_name)
141
141
  end
142
142
  end
143
143
 
@@ -1416,20 +1416,12 @@ module MU
1416
1416
  # XXX probably have to query API to get the DNS name of this one
1417
1417
  }
1418
1418
  elsif lb["concurrent_load_balancer"]
1419
- raise MuError, "No loadbalancers exist! I need one named #{lb['concurrent_load_balancer']}" if !@deploy.deployment["loadbalancers"]
1420
- found = false
1421
- @deploy.deployment["loadbalancers"].each_pair { |lb_name, deployed_lb|
1422
- if lb_name == lb['concurrent_load_balancer']
1423
- lbs << deployed_lb["awsname"] # XXX check for classic
1424
- if deployed_lb.has_key?("targetgroups")
1425
- deployed_lb["targetgroups"].values.each { |tg_arn|
1426
- tg_arns << tg_arn
1427
- }
1428
- end
1429
- found = true
1430
- end
1431
- }
1432
- raise MuError, "I need a loadbalancer named #{lb['concurrent_load_balancer']}, but none seems to have been created!" if !found
1419
+ lb = @deploy.findLitterMate(name: lb['concurrent_load_balancer'], type: "loadbalancers")
1420
+ raise MuError, "No loadbalancers exist! I need one named #{lb['concurrent_load_balancer']}" if !lb
1421
+ lbs << lb.mu_name
1422
+ if lb.targetgroups
1423
+ tg_arns = lb.targetgroups.values.map { |tg| tg.target_group_arn }
1424
+ end
1433
1425
  end
1434
1426
  }
1435
1427
  if tg_arns.size > 0
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: cloud-mu
3
3
  version: !ruby/object:Gem::Version
4
- version: 3.3.1
4
+ version: 3.3.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - John Stange
@@ -11,7 +11,7 @@ authors:
11
11
  autorequire:
12
12
  bindir: bin
13
13
  cert_chain: []
14
- date: 2020-09-29 00:00:00.000000000 Z
14
+ date: 2020-10-04 00:00:00.000000000 Z
15
15
  dependencies:
16
16
  - !ruby/object:Gem::Dependency
17
17
  name: addressable