cloud-mu 3.3.2 → 3.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (33) hide show
  1. checksums.yaml +4 -4
  2. data/cloud-mu.gemspec +3 -3
  3. data/cookbooks/mu-tools/attributes/default.rb +7 -0
  4. data/cookbooks/mu-tools/libraries/helper.rb +86 -2
  5. data/cookbooks/mu-tools/recipes/apply_security.rb +25 -9
  6. data/cookbooks/mu-tools/recipes/aws_api.rb +4 -0
  7. data/cookbooks/mu-tools/recipes/google_api.rb +4 -0
  8. data/cookbooks/mu-tools/recipes/rsyslog.rb +8 -1
  9. data/cookbooks/mu-tools/resources/disk.rb +33 -12
  10. data/cookbooks/mu-tools/resources/mommacat_request.rb +1 -2
  11. data/cookbooks/mu-tools/templates/centos-8/sshd_config.erb +215 -0
  12. data/extras/clean-stock-amis +10 -2
  13. data/extras/generate-stock-images +6 -3
  14. data/extras/image-generators/AWS/centos7.yaml +19 -16
  15. data/extras/image-generators/AWS/{rhel7.yaml → rhel71.yaml} +0 -0
  16. data/extras/image-generators/AWS/{win2k12.yaml → win2k12r2.yaml} +0 -0
  17. data/modules/mommacat.ru +2 -2
  18. data/modules/mu/cloud/wrappers.rb +16 -7
  19. data/modules/mu/config/ref.rb +1 -1
  20. data/modules/mu/defaults/AWS.yaml +96 -96
  21. data/modules/mu/mommacat.rb +10 -2
  22. data/modules/mu/mommacat/search.rb +11 -2
  23. data/modules/mu/mommacat/storage.rb +30 -15
  24. data/modules/mu/providers/aws.rb +43 -23
  25. data/modules/mu/providers/aws/database.rb +9 -6
  26. data/modules/mu/providers/aws/function.rb +8 -5
  27. data/modules/mu/providers/aws/job.rb +29 -26
  28. data/modules/mu/providers/aws/role.rb +38 -32
  29. data/modules/mu/providers/aws/server.rb +58 -51
  30. data/modules/mu/providers/aws/vpc.rb +3 -0
  31. data/modules/mu/providers/google.rb +1 -1
  32. data/modules/mu/providers/google/role.rb +1 -0
  33. metadata +10 -9
@@ -23,7 +23,7 @@ $opts = Optimist::options do
23
23
  #{$0} [-c credentials] [-i imagename]
24
24
  EOS
25
25
  opt :credentials, "Use these AWS credentials from mu.yaml instead of the default set", :required => false, :type => :string
26
- opt :image, "Purge a specific image, instead of just scrubing old ones", :required => false, :type => :string
26
+ opt :image, "Purge a specific image, instead of just scrubbing old ones", :required => false, :type => :string
27
27
  end
28
28
 
29
29
  filters = [
@@ -33,12 +33,14 @@ filters = [
33
33
  }
34
34
  ]
35
35
 
36
+ in_use = MU::Cloud.getStockImage("AWS").values.flatten.map { |h| h.values }.flatten
36
37
 
37
38
  MU::Cloud::AWS.listRegions.each { | r|
38
39
  images = MU::Cloud::AWS.ec2(region: r, credentials: $opts[:credentials]).describe_images(
39
40
  filters: filters + [{ "name" => "state", "values" => ["available"]}]
40
41
  ).images
41
42
  images.each { |ami|
43
+ next if in_use.include?(ami)
42
44
  if ($opts[:image] and ami.name == $opts[:image]) or
43
45
  ((DateTime.now.to_time - DateTime.parse(ami.creation_date).to_time) > 15552000 and ami.name.match(/^MU-(PROD|DEV)/))
44
46
  snaps = []
@@ -53,7 +55,13 @@ MU::Cloud::AWS.listRegions.each { | r|
53
55
  rescue Aws::EC2::Errors::InvalidAMIIDUnavailable
54
56
  end
55
57
  snaps.each { |snap_id|
56
- MU::Cloud::AWS.ec2(region: r, credentials: $opts[:credentials]).delete_snapshot(snapshot_id: snap_id)
58
+ begin
59
+ MU::Cloud::AWS.ec2(region: r, credentials: $opts[:credentials]).delete_snapshot(snapshot_id: snap_id)
60
+ rescue Aws::EC2::Errors::InvalidSnapshotInUse
61
+ sleep 5
62
+ retry
63
+ rescue Aws::EC2::Errors::InvalidSnapshotNotFound
64
+ end
57
65
  }
58
66
  end
59
67
  }
@@ -63,6 +63,7 @@ end
63
63
  now = DateTime.now
64
64
 
65
65
  exitcode = 0
66
+ succeeded = 0
66
67
  $opts[:clouds].each { |cloud|
67
68
  current_images = MU::Cloud.getStockImage(cloud, fail_hard: true)
68
69
  $opts[:platforms].each { |platform|
@@ -114,6 +115,7 @@ $opts[:clouds].each { |cloud|
114
115
  # Scrub any loose metadata left over from our image deployment. It's
115
116
  # ok, this won't touch the images we just made.
116
117
  MU::Cleanup.run(deployer.mommacat.deploy_id, skipsnapshots: true, verbosity: MU::Logger::QUIET)
118
+ succeeded += 1
117
119
  rescue Exception => e
118
120
  MU.log e.message, MU::ERR
119
121
  exitcode = 1
@@ -122,10 +124,11 @@ $opts[:clouds].each { |cloud|
122
124
  end
123
125
  }
124
126
 
125
- if !available_clouds.keys.include?("AWS") # XXX or if we don't have permissions
127
+ if !$opts[:dryrun] and succeeded > 0
126
128
  puts current_images.to_yaml
127
- elsif !$opts[:dryrun]
128
- MU::Cloud::AWS::Bucket.upload($opts[:upload_to]+"/"+cloud+".yaml", data: current_images.to_yaml, credentials: $opts[:aws_creds], acl: "public-read")
129
+ if available_clouds.keys.include?("AWS")
130
+ MU::Cloud::AWS::Bucket.upload($opts[:upload_to]+"/"+cloud+".yaml", data: current_images.to_yaml, credentials: $opts[:aws_creds], acl: "public-read")
131
+ end
129
132
  end
130
133
  }
131
134
 
@@ -1,17 +1,20 @@
1
1
  ---
2
- appname: mu
3
- servers:
4
- -
5
- name: centos7
6
- platform: centos7
7
- size: m3.medium
8
- scrub_groomer: true
9
- run_list:
10
- - recipe[mu-tools::apply_security]
11
- - recipe[mu-tools::updates]
12
- - recipe[mu-tools::split_var_partitions]
13
- create_image:
14
- image_then_destroy: true
15
- public: true
16
- copy_to_regions:
17
- - "#ALL"
2
+ appname: mu
3
+ servers:
4
+ - name: centos7
5
+ platform: centos7
6
+ size: m4.large
7
+ vpc:
8
+ name: c7vpc
9
+ scrub_groomer: true
10
+ run_list:
11
+ - recipe[mu-tools::apply_security]
12
+ - recipe[mu-tools::updates]
13
+ - recipe[mu-tools::split_var_partitions]
14
+ create_image:
15
+ image_then_destroy: true
16
+ public: true
17
+ copy_to_regions:
18
+ - "#ALL"
19
+ vpcs:
20
+ - name: c7vpc
@@ -387,7 +387,7 @@ app = proc do |env|
387
387
 
388
388
  # XXX We can't assume AWS anymore. What does this look like otherwise?
389
389
  # If this is an already-groomed instance, try to get a real object for it
390
- instance = MU::MommaCat.findStray("AWS", "server", cloud_id: req["mu_instance_id"], region: server_cfg["region"], deploy_id: req["mu_id"], name: req["mu_resource_name"], dummy_ok: false, calling_deploy: kittenpile).first
390
+ instance = MU::MommaCat.findStray("AWS", "server", cloud_id: req["mu_instance_id"], region: server_cfg["region"], deploy_id: req["mu_id"], name: req["mu_resource_name"], dummy_ok: true, calling_deploy: kittenpile).first
391
391
  mu_name = nil
392
392
  if instance.nil?
393
393
  # Now we're just checking for existence in the cloud provider, really
@@ -416,7 +416,7 @@ app = proc do |env|
416
416
  if instance.respond_to?(:addVolume)
417
417
  # XXX make sure we handle mangled input safely
418
418
  params = JSON.parse(Base64.decode64(req["add_volume"]))
419
- MU.log "ADDVOLUME REQUEST", MU::WARN, details: params
419
+ MU.log "add_volume request", MU::NOTICE, details: params
420
420
  instance.addVolume(params["dev"], params["size"], delete_on_termination: params["delete_on_termination"])
421
421
  else
422
422
  returnval = throw500 "I don't know how to add a volume for #{instance}"
@@ -103,14 +103,23 @@ module MU
103
103
  next
104
104
  end
105
105
 
106
- found = cloudclass.find(args)
107
- if !found.nil?
108
- if found.is_a?(Hash)
109
- allfound.merge!(found)
110
- else
111
- raise MuError, "#{cloudclass}.find returned a non-Hash result"
112
- end
106
+ credsets = if args[:credentials]
107
+ [args[:credentials]]
108
+ else
109
+ cloudbase.listCredentials
113
110
  end
111
+
112
+ credsets.each { |creds|
113
+ args[:credentials] = creds
114
+ found = cloudclass.find(args)
115
+ if !found.nil?
116
+ if found.is_a?(Hash)
117
+ allfound.merge!(found)
118
+ else
119
+ raise MuError, "#{cloudclass}.find returned a non-Hash result"
120
+ end
121
+ end
122
+ }
114
123
  rescue MuCloudResourceNotImplemented
115
124
  end
116
125
  }
@@ -310,7 +310,7 @@ module MU
310
310
  return @obj
311
311
  end
312
312
 
313
- if mommacat and !caller.grep(/`findLitterMate'/) # XXX the dumbest
313
+ if mommacat and caller.grep(/`findLitterMate'/).empty? # XXX the dumbest
314
314
  MU.log "Looking for #{@type} #{@name} #{@id} in deploy #{mommacat.deploy_id}", loglevel
315
315
  begin
316
316
  @obj = mommacat.findLitterMate(type: @type, name: @name, cloud_id: @id, credentials: @credentials, debug: debug)
@@ -1,55 +1,55 @@
1
1
  ---
2
2
  rhel71: &5
3
- us-east-1: ami-0f05fce24aa75ba9f
4
- ap-northeast-1: ami-0c0ec19eb19055763
5
- ap-northeast-2: ami-0717ac5c67c99f745
6
- ap-south-1: ami-03454a4bef3ec6a9a
7
- ap-southeast-1: ami-0f3aa03320c0f6524
8
- ap-southeast-2: ami-0aa5e6888260cdb3c
9
- ca-central-1: ami-03e72964d7646b689
10
- eu-central-1: ami-02df259ca785eff54
11
- eu-north-1: ami-05253c445bdf7777d
12
- eu-west-1: ami-0c21c559f6d0f2401
13
- eu-west-2: ami-057c8d4259087594f
14
- eu-west-3: ami-05a428dc7a7f4ba46
15
- sa-east-1: ami-0a1d1cf6a89a2db56
16
- us-east-2: ami-02f6682c7816b3cfc
17
- us-west-1: ami-04898e596c06e802b
18
- us-west-2: ami-02db5457189a8a8c2
3
+ us-east-1: ami-0c834836b3bd45e2f
4
+ ap-northeast-1: ami-036bb589253fe929e
5
+ ap-northeast-2: ami-0e39b4957dbc7e14d
6
+ ap-south-1: ami-0cd0554d9a05dddc9
7
+ ap-southeast-1: ami-0bae2684e9ed09b8b
8
+ ap-southeast-2: ami-0711ccf93abe3989b
9
+ ca-central-1: ami-0d75d90f73e417c25
10
+ eu-central-1: ami-0cae3cb53b9bbd783
11
+ eu-north-1: ami-0cd30fe8547a809f7
12
+ eu-west-1: ami-01441cca97c35eb0e
13
+ eu-west-2: ami-051aaf1b532b3e6bc
14
+ eu-west-3: ami-09a7af6793a3e8d09
15
+ sa-east-1: ami-0c4064cfe711311d5
16
+ us-east-2: ami-0124fd8917f59f8ce
17
+ us-west-1: ami-00457c55541605cb4
18
+ us-west-2: ami-02211d4e254a9e10f
19
19
  centos6: &4
20
- us-east-1: ami-009723c5c7f8fbc75
21
- us-east-2: ami-0781f11395714cd39
22
- ap-northeast-1: ami-07fa5a8795da2b6bc
23
- ap-northeast-2: ami-0219f0a7c979ff63f
24
- ap-south-1: ami-0f24817242c401740
25
- ap-southeast-1: ami-042ef2e0643e8e207
26
- ap-southeast-2: ami-09fc51de648afa168
27
- ca-central-1: ami-0dc643db74edc5aa5
28
- eu-central-1: ami-0628759cb297569d5
29
- eu-north-1: ami-0aed023791f886315
30
- eu-west-1: ami-0f87f0f252ff03622
31
- eu-west-2: ami-00abb555d5a460afe
32
- eu-west-3: ami-0ccd93d454c2418a2
33
- sa-east-1: ami-01e10ea6ea72534ae
34
- us-west-1: ami-01fee56b9ee690ffe
35
- us-west-2: ami-08bcdb944f185e2a8
20
+ us-east-1: ami-0ccdc671f12147a1d
21
+ us-east-2: ami-00d0e8bc2f05ab949
22
+ ap-northeast-1: ami-0726801ceef87f5f8
23
+ ap-northeast-2: ami-05fa4afc4a0493b0a
24
+ ap-south-1: ami-0d6e4f3b6592b3139
25
+ ap-southeast-1: ami-0c988e3dc80b14653
26
+ ap-southeast-2: ami-02ac856fd094675ef
27
+ ca-central-1: ami-0ce7e343953af2292
28
+ eu-central-1: ami-0ce8317423cea27b8
29
+ eu-north-1: ami-0a923b493d5fc9743
30
+ eu-west-1: ami-06e0f02328921c865
31
+ eu-west-2: ami-07ae118c8814df140
32
+ eu-west-3: ami-03c1017cd1ccc6e9d
33
+ sa-east-1: ami-05212ae133b9c3ba1
34
+ us-west-1: ami-0b05ec54412b9f8b0
35
+ us-west-2: ami-0447e036b102b2ca0
36
36
  centos7:
37
- us-east-1: ami-067256ca1497c924d
38
- ap-northeast-1: ami-07c1e51354fdfd362
39
- ap-northeast-2: ami-042b761c93d6df2f1
40
- ap-south-1: ami-02e879f52322e7c98
41
- ap-southeast-1: ami-0487e9f84d0ffde89
42
- ap-southeast-2: ami-0e854dab39fd6a427
43
- ca-central-1: ami-05a27d311b585a70b
44
- eu-central-1: ami-0e396d00c787b4f47
45
- eu-north-1: ami-087763a2ba60b2bfe
46
- eu-west-1: ami-04e3bd9335a14e635
47
- eu-west-2: ami-0efd34a8d1fc2b104
48
- eu-west-3: ami-08d0bcbc780448cf8
49
- sa-east-1: ami-0284f4a0968263cf0
50
- us-east-2: ami-0292786917d1e3015
51
- us-west-1: ami-0ba622529dcdff2bb
52
- us-west-2: ami-079a309ca6261d7f6
37
+ us-east-1: ami-0be9d646b29a9f51d
38
+ ap-northeast-1: ami-0ea500fc488406ad8
39
+ ap-northeast-2: ami-062d5536e5ebf04e9
40
+ ap-south-1: ami-0c2bf51384a5dcd92
41
+ ap-southeast-1: ami-0a8022e9bb353022f
42
+ ap-southeast-2: ami-03d47c88f2e01203b
43
+ ca-central-1: ami-0f0dc8a3e18a28544
44
+ eu-central-1: ami-0ea618bc5a1f372a2
45
+ eu-north-1: ami-02b062056e7dd4741
46
+ eu-west-1: ami-03851110db1a143e7
47
+ eu-west-2: ami-03800a8fe524171d2
48
+ eu-west-3: ami-074acd461ca002f0a
49
+ sa-east-1: ami-00c88550221a205f9
50
+ us-east-2: ami-098da5da2eef484e5
51
+ us-west-1: ami-0eea632474ef51860
52
+ us-west-2: ami-075aad0e6e9fc5654
53
53
  ubuntu16: &3
54
54
  us-east-1: ami-bcdc16c6
55
55
  us-west-1: ami-1b17257b
@@ -73,56 +73,56 @@ ubuntu14:
73
73
  ap-southeast-1: ami-2855964b
74
74
  ap-southeast-2: ami-d19fc4b2
75
75
  win2k12r2: &1
76
- us-east-1: ami-003aea65bc2e7136a
77
- us-east-2: ami-0163293e39ba504c2
78
- ca-central-1: ami-055689dd92f29d2aa
79
- us-west-2: ami-0ce87dda2c9244e57
80
- us-west-1: ami-00d9cf64bd2fafa44
81
- eu-west-1: ami-026d7427b9fadad40
82
- eu-west-2: ami-036a22c0780551794
83
- eu-west-3: ami-05e3d9b79bdc10861
84
- eu-north-1: ami-063eb48504c7d73f1
85
- sa-east-1: ami-0a8c1829a5e650bc5
86
- eu-central-1: ami-0ea20cef52335b008
87
- ap-northeast-1: ami-08db2dc67228dbb90
88
- ap-south-1: ami-012241411db3f09c3
89
- ap-northeast-2: ami-0368c224de1d20502
90
- ap-southeast-1: ami-028ef74e1edc3943a
91
- ap-southeast-2: ami-09e03eab1b1bc151b
76
+ us-east-1: ami-0d28b9a40ed446e35
77
+ us-east-2: ami-010d247b7ee850d55
78
+ ca-central-1: ami-0c223858875f62d11
79
+ us-west-2: ami-01d188c5c06078fee
80
+ us-west-1: ami-0dbcc051c49ec24ec
81
+ eu-west-1: ami-080c7b4d6e32bf9f3
82
+ eu-west-2: ami-01b1edb5894a54bcc
83
+ eu-west-3: ami-09445cdc7a2acb1c0
84
+ eu-north-1: ami-0093cc63496e435df
85
+ sa-east-1: ami-04d015c8b371ba7b3
86
+ eu-central-1: ami-04d6144bcbb029141
87
+ ap-northeast-1: ami-07fad72f121aa157c
88
+ ap-south-1: ami-0b3241fd09b1ce87f
89
+ ap-northeast-2: ami-0ba7cd822e36dc0df
90
+ ap-southeast-1: ami-0648522a4cb50953c
91
+ ap-southeast-2: ami-0fc0bd73cd61a970f
92
92
  win2k16: &2
93
- us-east-1: ami-02801a2c8dcbfb883
94
- us-east-2: ami-0ca4f779a2a58a7ea
95
- ca-central-1: ami-05d3854d9d6e9bcc5
96
- us-west-2: ami-091f4a88ce32d28b6
97
- eu-west-1: ami-0b938c9b23ed7d18c
98
- us-west-1: ami-0fd744c3fbe8260f2
99
- eu-west-2: ami-071a89b959c5eda27
100
- eu-west-3: ami-0b206e3dbda9ff9eb
101
- eu-central-1: ami-0dd9bdad31dd0d3ce
102
- sa-east-1: ami-0d69b8d6c0f9a7bae
103
- ap-northeast-1: ami-02eb4a6f519bc3190
104
- ap-south-1: ami-0666fd543ac8b5501
105
- ap-northeast-2: ami-01277c81f9b91cf77
106
- ap-southeast-2: ami-0426a246f9b0ccadd
107
- ap-southeast-1: ami-07ecb0d55c2eb7247
108
- eu-north-1: ami-047811530583b6d08
93
+ us-east-1: ami-018151f8c8339a093
94
+ us-east-2: ami-09e045936c7d9ecd5
95
+ ca-central-1: ami-0a16abf0f1c35667a
96
+ us-west-2: ami-00d4216f80a82894d
97
+ eu-west-1: ami-07afc1525928ccad7
98
+ us-west-1: ami-0d2f7d4198b79a625
99
+ eu-west-2: ami-04eb62f17efc84a37
100
+ eu-west-3: ami-0b4affcd5848cf50c
101
+ eu-central-1: ami-0a92ae047ebc7a3da
102
+ sa-east-1: ami-0cc9a87c95fb37832
103
+ ap-northeast-1: ami-014c730050acef11d
104
+ ap-south-1: ami-0c3c8739263aa4844
105
+ ap-northeast-2: ami-0018a05eb15503b5e
106
+ ap-southeast-2: ami-0b50bfbb507285a89
107
+ ap-southeast-1: ami-073c677bcf225774a
108
+ eu-north-1: ami-09bb6d618593f2e7f
109
109
  win2k19:
110
- us-east-1: ami-00820419bf212df7e
111
- us-east-2: ami-0a7916b90aa4629d5
112
- ca-central-1: ami-0d704529661e19185
113
- us-west-2: ami-0ee6a198d7ac35eb1
114
- eu-west-2: ami-0f6ac1634bd7add92
115
- us-west-1: ami-039e3816b4cac1e27
116
- eu-west-1: ami-03a771d99091199b7
117
- eu-central-1: ami-03b648d5b45f51a4f
118
- eu-west-3: ami-068839907c18c3a6e
119
- eu-north-1: ami-0db851ee76f7deefb
120
- sa-east-1: ami-0c2cc60c62159f87c
121
- ap-northeast-2: ami-06bdf8ae9ae9add92
122
- ap-northeast-1: ami-02306d959c7f175b9
123
- ap-southeast-1: ami-0d5b4a3d73e0f471f
124
- ap-southeast-2: ami-00fa88caff4f64937
125
- ap-south-1: ami-0b44feae4bb9f497a
110
+ us-east-1: ami-09c18c34c341f2b6a
111
+ us-east-2: ami-030371d5ee8881350
112
+ ca-central-1: ami-018e5bf45c30fa58f
113
+ us-west-2: ami-07d1e5c4f906877e1
114
+ eu-west-2: ami-0b9d95fef44aa7c11
115
+ us-west-1: ami-0e7e082d6fa1769f3
116
+ eu-west-1: ami-05573fafa080144b6
117
+ eu-central-1: ami-0122b027c265988ea
118
+ eu-west-3: ami-0ba7c0a3dc4148b6a
119
+ eu-north-1: ami-03d8c3307f72f9847
120
+ sa-east-1: ami-0d0f66c3e0dfc09ee
121
+ ap-northeast-2: ami-07eff56de9293ab16
122
+ ap-northeast-1: ami-020fb790a3bed4cda
123
+ ap-southeast-1: ami-0b5d036d6d711a4c8
124
+ ap-southeast-2: ami-0fff96935fef7bf60
125
+ ap-south-1: ami-030ec249497f66a33
126
126
  amazon:
127
127
  us-east-1: ami-b73b63a0
128
128
  us-east-2: ami-58277d3d
@@ -547,7 +547,7 @@ module MU
547
547
  # @param remove [Boolean]: Remove this resource from the deploy structure, instead of adding it.
548
548
  # @return [void]
549
549
  def notify(type, key, data, mu_name: nil, remove: false, triggering_node: nil, delayed_save: false)
550
- no_write = (@no_artifacts or caller.grep(/\/mommacat\.rb:\d+:in `notify'/))
550
+ no_write = (@no_artifacts or !caller.grep(/\/mommacat\.rb:\d+:in `notify'/).empty?)
551
551
 
552
552
  begin
553
553
  if !no_write
@@ -560,7 +560,13 @@ module MU
560
560
  loadDeploy(true) # make sure we're saving the latest and greatest
561
561
  end
562
562
 
563
- _shortclass, _cfg_name, type, _classname, attrs = MU::Cloud.getResourceNames(type, false)
563
+ @timestamp ||= @deployment['timestamp']
564
+ @seed ||= @deployment['seed']
565
+ @appname ||= @deployment['appname']
566
+ @handle ||= @deployment['handle']
567
+
568
+ _shortclass, _cfg_name, mu_type, _classname, attrs = MU::Cloud.getResourceNames(type, false)
569
+ type = mu_type if mu_type
564
570
  has_multiples = attrs[:has_multiples] ? true : false
565
571
 
566
572
  mu_name ||= if !data.nil? and !data["mu_name"].nil?
@@ -574,6 +580,7 @@ module MU
574
580
  end
575
581
 
576
582
  @need_deploy_flush = true
583
+ @last_modified = Time.now
577
584
 
578
585
  if !remove
579
586
  if data.nil?
@@ -903,6 +910,7 @@ MAIL_HEAD_END
903
910
  ###########################################################################
904
911
  ###########################################################################
905
912
  def setThreadContextToMe
913
+
906
914
  ["appname", "environment", "timestamp", "seed", "handle"].each { |var|
907
915
  @deployment[var] ||= instance_variable_get("@#{var}".to_sym)
908
916
  if @deployment[var]
@@ -107,8 +107,17 @@ module MU
107
107
  matches = []
108
108
 
109
109
  credlist.each { |creds|
110
- # next if region and region.is_a?(Array) and !region.empty? and !region.include?(r)
111
- cloud_descs = search_cloud_provider(type, cloud, habitats, region, cloud_id: cloud_id, tag_key: tag_key, tag_value: tag_value, credentials: creds, flags: flags)
110
+ cur_habitats = []
111
+
112
+ if habitats and !habitats.empty? and habitats != [nil]
113
+ valid_habitats = cloudclass.listHabitats(creds)
114
+ cur_habitats = (habitats & valid_habitats)
115
+ next if cur_habitats.empty?
116
+ else
117
+ cur_habitats = cloudclass.listHabitats(creds)
118
+ end
119
+
120
+ cloud_descs = search_cloud_provider(type, cloud, cur_habitats, region, cloud_id: cloud_id, tag_key: tag_key, tag_value: tag_value, credentials: creds, flags: flags)
112
121
 
113
122
  cloud_descs.each_pair.each { |p, regions|
114
123
  regions.each_pair.each { |r, results|
@@ -202,14 +202,16 @@ module MU
202
202
 
203
203
  MU.log "Getting a lock on #{lockdir}/#{id}.lock (thread #{Thread.current.object_id})...", MU::DEBUG, details: caller
204
204
  show_relevant = Proc.new {
205
- @locks.each_pair { |thread_id, lock|
206
- lock.each_pair { |lockid, lockpath|
207
- if lockid == id
208
- thread = Thread.list.select { |t| t.object_id == thread_id }.first
209
- if thread.object_id != Thread.current.object_id
210
- MU.log "#{thread_id} sitting on #{id}", MU::WARN, thread.backtrace
205
+ @lock_semaphore.synchronize {
206
+ @locks.each_pair { |thread_id, lock|
207
+ lock.each_pair { |lockid, lockpath|
208
+ if lockid == id
209
+ thread = Thread.list.select { |t| t.object_id == thread_id }.first
210
+ if thread.object_id != Thread.current.object_id
211
+ MU.log "#{thread_id} sitting on #{id}", MU::WARN, thread.backtrace
212
+ end
211
213
  end
212
- end
214
+ }
213
215
  }
214
216
  }
215
217
  }
@@ -218,11 +220,13 @@ module MU
218
220
  if !@locks[Thread.current.object_id][id].flock(File::LOCK_EX|File::LOCK_NB)
219
221
  if retries > 0
220
222
  success = false
221
- MU.retrier([], loop_if: Proc.new { !success }, loop_msg: "Waiting for lock on #{lockdir}/#{id}.lock...", max: retries) {
223
+ MU.retrier([], loop_if: Proc.new { !success }, loop_msg: "Waiting for lock on #{lockdir}/#{id}.lock...", max: retries) { |cur_retries, _wait|
222
224
  success = @locks[Thread.current.object_id][id].flock(File::LOCK_EX|File::LOCK_NB)
223
- show_relevant.call() if !success
225
+ if !success and cur_retries > 0 and (cur_retries % 3) == 0
226
+ show_relevant.call(cur_retries)
227
+ end
224
228
  }
225
- show_relevant.call() if !success
229
+ show_relevant.call(cur_retries) if !success
226
230
  return success
227
231
  else
228
232
  return false
@@ -415,6 +419,7 @@ module MU
415
419
  deploy.flock(File::LOCK_UN)
416
420
  deploy.close
417
421
  @need_deploy_flush = false
422
+ @last_modified = nil
418
423
  MU::MommaCat.updateLitter(@deploy_id, self)
419
424
  end
420
425
 
@@ -643,6 +648,14 @@ module MU
643
648
  def loadDeployFromCache(set_context_to_me = true)
644
649
  return false if !File.size?(deploy_dir+"/deployment.json")
645
650
 
651
+ lastmod = File.mtime("#{deploy_dir}/deployment.json")
652
+ if @last_modified and lastmod < @last_modified
653
+ MU.log "#{deploy_dir}/deployment.json last written at #{lastmod}, live meta at #{@last_modified}, not loading", MU::WARN if @last_modified
654
+ # this is a weird place for this
655
+ setThreadContextToMe if set_context_to_me
656
+ return true
657
+ end
658
+
646
659
  deploy = File.open("#{deploy_dir}/deployment.json", File::RDONLY)
647
660
  MU.log "Getting lock to read #{deploy_dir}/deployment.json", MU::DEBUG
648
661
  # deploy.flock(File::LOCK_EX)
@@ -654,6 +667,7 @@ module MU
654
667
 
655
668
  begin
656
669
  @deployment = JSON.parse(File.read("#{deploy_dir}/deployment.json"))
670
+ # XXX is it worthwhile to merge fuckery?
657
671
  rescue JSON::ParserError => e
658
672
  MU.log "JSON parse failed on #{deploy_dir}/deployment.json", MU::ERR, details: e.message
659
673
  end
@@ -663,20 +677,21 @@ module MU
663
677
 
664
678
  setThreadContextToMe if set_context_to_me
665
679
 
666
- @timestamp = @deployment['timestamp']
667
- @seed = @deployment['seed']
668
- @appname = @deployment['appname']
669
- @handle = @deployment['handle']
670
-
671
680
  true
672
681
  end
673
682
 
683
+
674
684
  ###########################################################################
675
685
  ###########################################################################
676
686
  def loadDeploy(deployment_json_only = false, set_context_to_me: true)
677
687
  MU::MommaCat.deploy_struct_semaphore.synchronize {
678
688
  success = loadDeployFromCache(set_context_to_me)
679
689
 
690
+ @timestamp ||= @deployment['timestamp']
691
+ @seed ||= @deployment['seed']
692
+ @appname ||= @deployment['appname']
693
+ @handle ||= @deployment['handle']
694
+
680
695
  return if deployment_json_only and success
681
696
 
682
697
  if File.exist?(deploy_dir+"/private_key")