cloud-mu 3.3.2 → 3.4.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (33) hide show
  1. checksums.yaml +4 -4
  2. data/cloud-mu.gemspec +3 -3
  3. data/cookbooks/mu-tools/attributes/default.rb +7 -0
  4. data/cookbooks/mu-tools/libraries/helper.rb +86 -2
  5. data/cookbooks/mu-tools/recipes/apply_security.rb +25 -9
  6. data/cookbooks/mu-tools/recipes/aws_api.rb +4 -0
  7. data/cookbooks/mu-tools/recipes/google_api.rb +4 -0
  8. data/cookbooks/mu-tools/recipes/rsyslog.rb +8 -1
  9. data/cookbooks/mu-tools/resources/disk.rb +33 -12
  10. data/cookbooks/mu-tools/resources/mommacat_request.rb +1 -2
  11. data/cookbooks/mu-tools/templates/centos-8/sshd_config.erb +215 -0
  12. data/extras/clean-stock-amis +10 -2
  13. data/extras/generate-stock-images +6 -3
  14. data/extras/image-generators/AWS/centos7.yaml +19 -16
  15. data/extras/image-generators/AWS/{rhel7.yaml → rhel71.yaml} +0 -0
  16. data/extras/image-generators/AWS/{win2k12.yaml → win2k12r2.yaml} +0 -0
  17. data/modules/mommacat.ru +2 -2
  18. data/modules/mu/cloud/wrappers.rb +16 -7
  19. data/modules/mu/config/ref.rb +1 -1
  20. data/modules/mu/defaults/AWS.yaml +96 -96
  21. data/modules/mu/mommacat.rb +10 -2
  22. data/modules/mu/mommacat/search.rb +11 -2
  23. data/modules/mu/mommacat/storage.rb +30 -15
  24. data/modules/mu/providers/aws.rb +43 -23
  25. data/modules/mu/providers/aws/database.rb +9 -6
  26. data/modules/mu/providers/aws/function.rb +8 -5
  27. data/modules/mu/providers/aws/job.rb +29 -26
  28. data/modules/mu/providers/aws/role.rb +38 -32
  29. data/modules/mu/providers/aws/server.rb +58 -51
  30. data/modules/mu/providers/aws/vpc.rb +3 -0
  31. data/modules/mu/providers/google.rb +1 -1
  32. data/modules/mu/providers/google/role.rb +1 -0
  33. metadata +10 -9
@@ -23,7 +23,7 @@ $opts = Optimist::options do
23
23
  #{$0} [-c credentials] [-i imagename]
24
24
  EOS
25
25
  opt :credentials, "Use these AWS credentials from mu.yaml instead of the default set", :required => false, :type => :string
26
- opt :image, "Purge a specific image, instead of just scrubing old ones", :required => false, :type => :string
26
+ opt :image, "Purge a specific image, instead of just scrubbing old ones", :required => false, :type => :string
27
27
  end
28
28
 
29
29
  filters = [
@@ -33,12 +33,14 @@ filters = [
33
33
  }
34
34
  ]
35
35
 
36
+ in_use = MU::Cloud.getStockImage("AWS").values.flatten.map { |h| h.values }.flatten
36
37
 
37
38
  MU::Cloud::AWS.listRegions.each { | r|
38
39
  images = MU::Cloud::AWS.ec2(region: r, credentials: $opts[:credentials]).describe_images(
39
40
  filters: filters + [{ "name" => "state", "values" => ["available"]}]
40
41
  ).images
41
42
  images.each { |ami|
43
+ next if in_use.include?(ami)
42
44
  if ($opts[:image] and ami.name == $opts[:image]) or
43
45
  ((DateTime.now.to_time - DateTime.parse(ami.creation_date).to_time) > 15552000 and ami.name.match(/^MU-(PROD|DEV)/))
44
46
  snaps = []
@@ -53,7 +55,13 @@ MU::Cloud::AWS.listRegions.each { | r|
53
55
  rescue Aws::EC2::Errors::InvalidAMIIDUnavailable
54
56
  end
55
57
  snaps.each { |snap_id|
56
- MU::Cloud::AWS.ec2(region: r, credentials: $opts[:credentials]).delete_snapshot(snapshot_id: snap_id)
58
+ begin
59
+ MU::Cloud::AWS.ec2(region: r, credentials: $opts[:credentials]).delete_snapshot(snapshot_id: snap_id)
60
+ rescue Aws::EC2::Errors::InvalidSnapshotInUse
61
+ sleep 5
62
+ retry
63
+ rescue Aws::EC2::Errors::InvalidSnapshotNotFound
64
+ end
57
65
  }
58
66
  end
59
67
  }
@@ -63,6 +63,7 @@ end
63
63
  now = DateTime.now
64
64
 
65
65
  exitcode = 0
66
+ succeeded = 0
66
67
  $opts[:clouds].each { |cloud|
67
68
  current_images = MU::Cloud.getStockImage(cloud, fail_hard: true)
68
69
  $opts[:platforms].each { |platform|
@@ -114,6 +115,7 @@ $opts[:clouds].each { |cloud|
114
115
  # Scrub any loose metadata left over from our image deployment. It's
115
116
  # ok, this won't touch the images we just made.
116
117
  MU::Cleanup.run(deployer.mommacat.deploy_id, skipsnapshots: true, verbosity: MU::Logger::QUIET)
118
+ succeeded += 1
117
119
  rescue Exception => e
118
120
  MU.log e.message, MU::ERR
119
121
  exitcode = 1
@@ -122,10 +124,11 @@ $opts[:clouds].each { |cloud|
122
124
  end
123
125
  }
124
126
 
125
- if !available_clouds.keys.include?("AWS") # XXX or if we don't have permissions
127
+ if !$opts[:dryrun] and succeeded > 0
126
128
  puts current_images.to_yaml
127
- elsif !$opts[:dryrun]
128
- MU::Cloud::AWS::Bucket.upload($opts[:upload_to]+"/"+cloud+".yaml", data: current_images.to_yaml, credentials: $opts[:aws_creds], acl: "public-read")
129
+ if available_clouds.keys.include?("AWS")
130
+ MU::Cloud::AWS::Bucket.upload($opts[:upload_to]+"/"+cloud+".yaml", data: current_images.to_yaml, credentials: $opts[:aws_creds], acl: "public-read")
131
+ end
129
132
  end
130
133
  }
131
134
 
@@ -1,17 +1,20 @@
1
1
  ---
2
- appname: mu
3
- servers:
4
- -
5
- name: centos7
6
- platform: centos7
7
- size: m3.medium
8
- scrub_groomer: true
9
- run_list:
10
- - recipe[mu-tools::apply_security]
11
- - recipe[mu-tools::updates]
12
- - recipe[mu-tools::split_var_partitions]
13
- create_image:
14
- image_then_destroy: true
15
- public: true
16
- copy_to_regions:
17
- - "#ALL"
2
+ appname: mu
3
+ servers:
4
+ - name: centos7
5
+ platform: centos7
6
+ size: m4.large
7
+ vpc:
8
+ name: c7vpc
9
+ scrub_groomer: true
10
+ run_list:
11
+ - recipe[mu-tools::apply_security]
12
+ - recipe[mu-tools::updates]
13
+ - recipe[mu-tools::split_var_partitions]
14
+ create_image:
15
+ image_then_destroy: true
16
+ public: true
17
+ copy_to_regions:
18
+ - "#ALL"
19
+ vpcs:
20
+ - name: c7vpc
@@ -387,7 +387,7 @@ app = proc do |env|
387
387
 
388
388
  # XXX We can't assume AWS anymore. What does this look like otherwise?
389
389
  # If this is an already-groomed instance, try to get a real object for it
390
- instance = MU::MommaCat.findStray("AWS", "server", cloud_id: req["mu_instance_id"], region: server_cfg["region"], deploy_id: req["mu_id"], name: req["mu_resource_name"], dummy_ok: false, calling_deploy: kittenpile).first
390
+ instance = MU::MommaCat.findStray("AWS", "server", cloud_id: req["mu_instance_id"], region: server_cfg["region"], deploy_id: req["mu_id"], name: req["mu_resource_name"], dummy_ok: true, calling_deploy: kittenpile).first
391
391
  mu_name = nil
392
392
  if instance.nil?
393
393
  # Now we're just checking for existence in the cloud provider, really
@@ -416,7 +416,7 @@ app = proc do |env|
416
416
  if instance.respond_to?(:addVolume)
417
417
  # XXX make sure we handle mangled input safely
418
418
  params = JSON.parse(Base64.decode64(req["add_volume"]))
419
- MU.log "ADDVOLUME REQUEST", MU::WARN, details: params
419
+ MU.log "add_volume request", MU::NOTICE, details: params
420
420
  instance.addVolume(params["dev"], params["size"], delete_on_termination: params["delete_on_termination"])
421
421
  else
422
422
  returnval = throw500 "I don't know how to add a volume for #{instance}"
@@ -103,14 +103,23 @@ module MU
103
103
  next
104
104
  end
105
105
 
106
- found = cloudclass.find(args)
107
- if !found.nil?
108
- if found.is_a?(Hash)
109
- allfound.merge!(found)
110
- else
111
- raise MuError, "#{cloudclass}.find returned a non-Hash result"
112
- end
106
+ credsets = if args[:credentials]
107
+ [args[:credentials]]
108
+ else
109
+ cloudbase.listCredentials
113
110
  end
111
+
112
+ credsets.each { |creds|
113
+ args[:credentials] = creds
114
+ found = cloudclass.find(args)
115
+ if !found.nil?
116
+ if found.is_a?(Hash)
117
+ allfound.merge!(found)
118
+ else
119
+ raise MuError, "#{cloudclass}.find returned a non-Hash result"
120
+ end
121
+ end
122
+ }
114
123
  rescue MuCloudResourceNotImplemented
115
124
  end
116
125
  }
@@ -310,7 +310,7 @@ module MU
310
310
  return @obj
311
311
  end
312
312
 
313
- if mommacat and !caller.grep(/`findLitterMate'/) # XXX the dumbest
313
+ if mommacat and caller.grep(/`findLitterMate'/).empty? # XXX the dumbest
314
314
  MU.log "Looking for #{@type} #{@name} #{@id} in deploy #{mommacat.deploy_id}", loglevel
315
315
  begin
316
316
  @obj = mommacat.findLitterMate(type: @type, name: @name, cloud_id: @id, credentials: @credentials, debug: debug)
@@ -1,55 +1,55 @@
1
1
  ---
2
2
  rhel71: &5
3
- us-east-1: ami-0f05fce24aa75ba9f
4
- ap-northeast-1: ami-0c0ec19eb19055763
5
- ap-northeast-2: ami-0717ac5c67c99f745
6
- ap-south-1: ami-03454a4bef3ec6a9a
7
- ap-southeast-1: ami-0f3aa03320c0f6524
8
- ap-southeast-2: ami-0aa5e6888260cdb3c
9
- ca-central-1: ami-03e72964d7646b689
10
- eu-central-1: ami-02df259ca785eff54
11
- eu-north-1: ami-05253c445bdf7777d
12
- eu-west-1: ami-0c21c559f6d0f2401
13
- eu-west-2: ami-057c8d4259087594f
14
- eu-west-3: ami-05a428dc7a7f4ba46
15
- sa-east-1: ami-0a1d1cf6a89a2db56
16
- us-east-2: ami-02f6682c7816b3cfc
17
- us-west-1: ami-04898e596c06e802b
18
- us-west-2: ami-02db5457189a8a8c2
3
+ us-east-1: ami-0c834836b3bd45e2f
4
+ ap-northeast-1: ami-036bb589253fe929e
5
+ ap-northeast-2: ami-0e39b4957dbc7e14d
6
+ ap-south-1: ami-0cd0554d9a05dddc9
7
+ ap-southeast-1: ami-0bae2684e9ed09b8b
8
+ ap-southeast-2: ami-0711ccf93abe3989b
9
+ ca-central-1: ami-0d75d90f73e417c25
10
+ eu-central-1: ami-0cae3cb53b9bbd783
11
+ eu-north-1: ami-0cd30fe8547a809f7
12
+ eu-west-1: ami-01441cca97c35eb0e
13
+ eu-west-2: ami-051aaf1b532b3e6bc
14
+ eu-west-3: ami-09a7af6793a3e8d09
15
+ sa-east-1: ami-0c4064cfe711311d5
16
+ us-east-2: ami-0124fd8917f59f8ce
17
+ us-west-1: ami-00457c55541605cb4
18
+ us-west-2: ami-02211d4e254a9e10f
19
19
  centos6: &4
20
- us-east-1: ami-009723c5c7f8fbc75
21
- us-east-2: ami-0781f11395714cd39
22
- ap-northeast-1: ami-07fa5a8795da2b6bc
23
- ap-northeast-2: ami-0219f0a7c979ff63f
24
- ap-south-1: ami-0f24817242c401740
25
- ap-southeast-1: ami-042ef2e0643e8e207
26
- ap-southeast-2: ami-09fc51de648afa168
27
- ca-central-1: ami-0dc643db74edc5aa5
28
- eu-central-1: ami-0628759cb297569d5
29
- eu-north-1: ami-0aed023791f886315
30
- eu-west-1: ami-0f87f0f252ff03622
31
- eu-west-2: ami-00abb555d5a460afe
32
- eu-west-3: ami-0ccd93d454c2418a2
33
- sa-east-1: ami-01e10ea6ea72534ae
34
- us-west-1: ami-01fee56b9ee690ffe
35
- us-west-2: ami-08bcdb944f185e2a8
20
+ us-east-1: ami-0ccdc671f12147a1d
21
+ us-east-2: ami-00d0e8bc2f05ab949
22
+ ap-northeast-1: ami-0726801ceef87f5f8
23
+ ap-northeast-2: ami-05fa4afc4a0493b0a
24
+ ap-south-1: ami-0d6e4f3b6592b3139
25
+ ap-southeast-1: ami-0c988e3dc80b14653
26
+ ap-southeast-2: ami-02ac856fd094675ef
27
+ ca-central-1: ami-0ce7e343953af2292
28
+ eu-central-1: ami-0ce8317423cea27b8
29
+ eu-north-1: ami-0a923b493d5fc9743
30
+ eu-west-1: ami-06e0f02328921c865
31
+ eu-west-2: ami-07ae118c8814df140
32
+ eu-west-3: ami-03c1017cd1ccc6e9d
33
+ sa-east-1: ami-05212ae133b9c3ba1
34
+ us-west-1: ami-0b05ec54412b9f8b0
35
+ us-west-2: ami-0447e036b102b2ca0
36
36
  centos7:
37
- us-east-1: ami-067256ca1497c924d
38
- ap-northeast-1: ami-07c1e51354fdfd362
39
- ap-northeast-2: ami-042b761c93d6df2f1
40
- ap-south-1: ami-02e879f52322e7c98
41
- ap-southeast-1: ami-0487e9f84d0ffde89
42
- ap-southeast-2: ami-0e854dab39fd6a427
43
- ca-central-1: ami-05a27d311b585a70b
44
- eu-central-1: ami-0e396d00c787b4f47
45
- eu-north-1: ami-087763a2ba60b2bfe
46
- eu-west-1: ami-04e3bd9335a14e635
47
- eu-west-2: ami-0efd34a8d1fc2b104
48
- eu-west-3: ami-08d0bcbc780448cf8
49
- sa-east-1: ami-0284f4a0968263cf0
50
- us-east-2: ami-0292786917d1e3015
51
- us-west-1: ami-0ba622529dcdff2bb
52
- us-west-2: ami-079a309ca6261d7f6
37
+ us-east-1: ami-0be9d646b29a9f51d
38
+ ap-northeast-1: ami-0ea500fc488406ad8
39
+ ap-northeast-2: ami-062d5536e5ebf04e9
40
+ ap-south-1: ami-0c2bf51384a5dcd92
41
+ ap-southeast-1: ami-0a8022e9bb353022f
42
+ ap-southeast-2: ami-03d47c88f2e01203b
43
+ ca-central-1: ami-0f0dc8a3e18a28544
44
+ eu-central-1: ami-0ea618bc5a1f372a2
45
+ eu-north-1: ami-02b062056e7dd4741
46
+ eu-west-1: ami-03851110db1a143e7
47
+ eu-west-2: ami-03800a8fe524171d2
48
+ eu-west-3: ami-074acd461ca002f0a
49
+ sa-east-1: ami-00c88550221a205f9
50
+ us-east-2: ami-098da5da2eef484e5
51
+ us-west-1: ami-0eea632474ef51860
52
+ us-west-2: ami-075aad0e6e9fc5654
53
53
  ubuntu16: &3
54
54
  us-east-1: ami-bcdc16c6
55
55
  us-west-1: ami-1b17257b
@@ -73,56 +73,56 @@ ubuntu14:
73
73
  ap-southeast-1: ami-2855964b
74
74
  ap-southeast-2: ami-d19fc4b2
75
75
  win2k12r2: &1
76
- us-east-1: ami-003aea65bc2e7136a
77
- us-east-2: ami-0163293e39ba504c2
78
- ca-central-1: ami-055689dd92f29d2aa
79
- us-west-2: ami-0ce87dda2c9244e57
80
- us-west-1: ami-00d9cf64bd2fafa44
81
- eu-west-1: ami-026d7427b9fadad40
82
- eu-west-2: ami-036a22c0780551794
83
- eu-west-3: ami-05e3d9b79bdc10861
84
- eu-north-1: ami-063eb48504c7d73f1
85
- sa-east-1: ami-0a8c1829a5e650bc5
86
- eu-central-1: ami-0ea20cef52335b008
87
- ap-northeast-1: ami-08db2dc67228dbb90
88
- ap-south-1: ami-012241411db3f09c3
89
- ap-northeast-2: ami-0368c224de1d20502
90
- ap-southeast-1: ami-028ef74e1edc3943a
91
- ap-southeast-2: ami-09e03eab1b1bc151b
76
+ us-east-1: ami-0d28b9a40ed446e35
77
+ us-east-2: ami-010d247b7ee850d55
78
+ ca-central-1: ami-0c223858875f62d11
79
+ us-west-2: ami-01d188c5c06078fee
80
+ us-west-1: ami-0dbcc051c49ec24ec
81
+ eu-west-1: ami-080c7b4d6e32bf9f3
82
+ eu-west-2: ami-01b1edb5894a54bcc
83
+ eu-west-3: ami-09445cdc7a2acb1c0
84
+ eu-north-1: ami-0093cc63496e435df
85
+ sa-east-1: ami-04d015c8b371ba7b3
86
+ eu-central-1: ami-04d6144bcbb029141
87
+ ap-northeast-1: ami-07fad72f121aa157c
88
+ ap-south-1: ami-0b3241fd09b1ce87f
89
+ ap-northeast-2: ami-0ba7cd822e36dc0df
90
+ ap-southeast-1: ami-0648522a4cb50953c
91
+ ap-southeast-2: ami-0fc0bd73cd61a970f
92
92
  win2k16: &2
93
- us-east-1: ami-02801a2c8dcbfb883
94
- us-east-2: ami-0ca4f779a2a58a7ea
95
- ca-central-1: ami-05d3854d9d6e9bcc5
96
- us-west-2: ami-091f4a88ce32d28b6
97
- eu-west-1: ami-0b938c9b23ed7d18c
98
- us-west-1: ami-0fd744c3fbe8260f2
99
- eu-west-2: ami-071a89b959c5eda27
100
- eu-west-3: ami-0b206e3dbda9ff9eb
101
- eu-central-1: ami-0dd9bdad31dd0d3ce
102
- sa-east-1: ami-0d69b8d6c0f9a7bae
103
- ap-northeast-1: ami-02eb4a6f519bc3190
104
- ap-south-1: ami-0666fd543ac8b5501
105
- ap-northeast-2: ami-01277c81f9b91cf77
106
- ap-southeast-2: ami-0426a246f9b0ccadd
107
- ap-southeast-1: ami-07ecb0d55c2eb7247
108
- eu-north-1: ami-047811530583b6d08
93
+ us-east-1: ami-018151f8c8339a093
94
+ us-east-2: ami-09e045936c7d9ecd5
95
+ ca-central-1: ami-0a16abf0f1c35667a
96
+ us-west-2: ami-00d4216f80a82894d
97
+ eu-west-1: ami-07afc1525928ccad7
98
+ us-west-1: ami-0d2f7d4198b79a625
99
+ eu-west-2: ami-04eb62f17efc84a37
100
+ eu-west-3: ami-0b4affcd5848cf50c
101
+ eu-central-1: ami-0a92ae047ebc7a3da
102
+ sa-east-1: ami-0cc9a87c95fb37832
103
+ ap-northeast-1: ami-014c730050acef11d
104
+ ap-south-1: ami-0c3c8739263aa4844
105
+ ap-northeast-2: ami-0018a05eb15503b5e
106
+ ap-southeast-2: ami-0b50bfbb507285a89
107
+ ap-southeast-1: ami-073c677bcf225774a
108
+ eu-north-1: ami-09bb6d618593f2e7f
109
109
  win2k19:
110
- us-east-1: ami-00820419bf212df7e
111
- us-east-2: ami-0a7916b90aa4629d5
112
- ca-central-1: ami-0d704529661e19185
113
- us-west-2: ami-0ee6a198d7ac35eb1
114
- eu-west-2: ami-0f6ac1634bd7add92
115
- us-west-1: ami-039e3816b4cac1e27
116
- eu-west-1: ami-03a771d99091199b7
117
- eu-central-1: ami-03b648d5b45f51a4f
118
- eu-west-3: ami-068839907c18c3a6e
119
- eu-north-1: ami-0db851ee76f7deefb
120
- sa-east-1: ami-0c2cc60c62159f87c
121
- ap-northeast-2: ami-06bdf8ae9ae9add92
122
- ap-northeast-1: ami-02306d959c7f175b9
123
- ap-southeast-1: ami-0d5b4a3d73e0f471f
124
- ap-southeast-2: ami-00fa88caff4f64937
125
- ap-south-1: ami-0b44feae4bb9f497a
110
+ us-east-1: ami-09c18c34c341f2b6a
111
+ us-east-2: ami-030371d5ee8881350
112
+ ca-central-1: ami-018e5bf45c30fa58f
113
+ us-west-2: ami-07d1e5c4f906877e1
114
+ eu-west-2: ami-0b9d95fef44aa7c11
115
+ us-west-1: ami-0e7e082d6fa1769f3
116
+ eu-west-1: ami-05573fafa080144b6
117
+ eu-central-1: ami-0122b027c265988ea
118
+ eu-west-3: ami-0ba7c0a3dc4148b6a
119
+ eu-north-1: ami-03d8c3307f72f9847
120
+ sa-east-1: ami-0d0f66c3e0dfc09ee
121
+ ap-northeast-2: ami-07eff56de9293ab16
122
+ ap-northeast-1: ami-020fb790a3bed4cda
123
+ ap-southeast-1: ami-0b5d036d6d711a4c8
124
+ ap-southeast-2: ami-0fff96935fef7bf60
125
+ ap-south-1: ami-030ec249497f66a33
126
126
  amazon:
127
127
  us-east-1: ami-b73b63a0
128
128
  us-east-2: ami-58277d3d
@@ -547,7 +547,7 @@ module MU
547
547
  # @param remove [Boolean]: Remove this resource from the deploy structure, instead of adding it.
548
548
  # @return [void]
549
549
  def notify(type, key, data, mu_name: nil, remove: false, triggering_node: nil, delayed_save: false)
550
- no_write = (@no_artifacts or caller.grep(/\/mommacat\.rb:\d+:in `notify'/))
550
+ no_write = (@no_artifacts or !caller.grep(/\/mommacat\.rb:\d+:in `notify'/).empty?)
551
551
 
552
552
  begin
553
553
  if !no_write
@@ -560,7 +560,13 @@ module MU
560
560
  loadDeploy(true) # make sure we're saving the latest and greatest
561
561
  end
562
562
 
563
- _shortclass, _cfg_name, type, _classname, attrs = MU::Cloud.getResourceNames(type, false)
563
+ @timestamp ||= @deployment['timestamp']
564
+ @seed ||= @deployment['seed']
565
+ @appname ||= @deployment['appname']
566
+ @handle ||= @deployment['handle']
567
+
568
+ _shortclass, _cfg_name, mu_type, _classname, attrs = MU::Cloud.getResourceNames(type, false)
569
+ type = mu_type if mu_type
564
570
  has_multiples = attrs[:has_multiples] ? true : false
565
571
 
566
572
  mu_name ||= if !data.nil? and !data["mu_name"].nil?
@@ -574,6 +580,7 @@ module MU
574
580
  end
575
581
 
576
582
  @need_deploy_flush = true
583
+ @last_modified = Time.now
577
584
 
578
585
  if !remove
579
586
  if data.nil?
@@ -903,6 +910,7 @@ MAIL_HEAD_END
903
910
  ###########################################################################
904
911
  ###########################################################################
905
912
  def setThreadContextToMe
913
+
906
914
  ["appname", "environment", "timestamp", "seed", "handle"].each { |var|
907
915
  @deployment[var] ||= instance_variable_get("@#{var}".to_sym)
908
916
  if @deployment[var]
@@ -107,8 +107,17 @@ module MU
107
107
  matches = []
108
108
 
109
109
  credlist.each { |creds|
110
- # next if region and region.is_a?(Array) and !region.empty? and !region.include?(r)
111
- cloud_descs = search_cloud_provider(type, cloud, habitats, region, cloud_id: cloud_id, tag_key: tag_key, tag_value: tag_value, credentials: creds, flags: flags)
110
+ cur_habitats = []
111
+
112
+ if habitats and !habitats.empty? and habitats != [nil]
113
+ valid_habitats = cloudclass.listHabitats(creds)
114
+ cur_habitats = (habitats & valid_habitats)
115
+ next if cur_habitats.empty?
116
+ else
117
+ cur_habitats = cloudclass.listHabitats(creds)
118
+ end
119
+
120
+ cloud_descs = search_cloud_provider(type, cloud, cur_habitats, region, cloud_id: cloud_id, tag_key: tag_key, tag_value: tag_value, credentials: creds, flags: flags)
112
121
 
113
122
  cloud_descs.each_pair.each { |p, regions|
114
123
  regions.each_pair.each { |r, results|
@@ -202,14 +202,16 @@ module MU
202
202
 
203
203
  MU.log "Getting a lock on #{lockdir}/#{id}.lock (thread #{Thread.current.object_id})...", MU::DEBUG, details: caller
204
204
  show_relevant = Proc.new {
205
- @locks.each_pair { |thread_id, lock|
206
- lock.each_pair { |lockid, lockpath|
207
- if lockid == id
208
- thread = Thread.list.select { |t| t.object_id == thread_id }.first
209
- if thread.object_id != Thread.current.object_id
210
- MU.log "#{thread_id} sitting on #{id}", MU::WARN, thread.backtrace
205
+ @lock_semaphore.synchronize {
206
+ @locks.each_pair { |thread_id, lock|
207
+ lock.each_pair { |lockid, lockpath|
208
+ if lockid == id
209
+ thread = Thread.list.select { |t| t.object_id == thread_id }.first
210
+ if thread.object_id != Thread.current.object_id
211
+ MU.log "#{thread_id} sitting on #{id}", MU::WARN, thread.backtrace
212
+ end
211
213
  end
212
- end
214
+ }
213
215
  }
214
216
  }
215
217
  }
@@ -218,11 +220,13 @@ module MU
218
220
  if !@locks[Thread.current.object_id][id].flock(File::LOCK_EX|File::LOCK_NB)
219
221
  if retries > 0
220
222
  success = false
221
- MU.retrier([], loop_if: Proc.new { !success }, loop_msg: "Waiting for lock on #{lockdir}/#{id}.lock...", max: retries) {
223
+ MU.retrier([], loop_if: Proc.new { !success }, loop_msg: "Waiting for lock on #{lockdir}/#{id}.lock...", max: retries) { |cur_retries, _wait|
222
224
  success = @locks[Thread.current.object_id][id].flock(File::LOCK_EX|File::LOCK_NB)
223
- show_relevant.call() if !success
225
+ if !success and cur_retries > 0 and (cur_retries % 3) == 0
226
+ show_relevant.call(cur_retries)
227
+ end
224
228
  }
225
- show_relevant.call() if !success
229
+ show_relevant.call(cur_retries) if !success
226
230
  return success
227
231
  else
228
232
  return false
@@ -415,6 +419,7 @@ module MU
415
419
  deploy.flock(File::LOCK_UN)
416
420
  deploy.close
417
421
  @need_deploy_flush = false
422
+ @last_modified = nil
418
423
  MU::MommaCat.updateLitter(@deploy_id, self)
419
424
  end
420
425
 
@@ -643,6 +648,14 @@ module MU
643
648
  def loadDeployFromCache(set_context_to_me = true)
644
649
  return false if !File.size?(deploy_dir+"/deployment.json")
645
650
 
651
+ lastmod = File.mtime("#{deploy_dir}/deployment.json")
652
+ if @last_modified and lastmod < @last_modified
653
+ MU.log "#{deploy_dir}/deployment.json last written at #{lastmod}, live meta at #{@last_modified}, not loading", MU::WARN if @last_modified
654
+ # this is a weird place for this
655
+ setThreadContextToMe if set_context_to_me
656
+ return true
657
+ end
658
+
646
659
  deploy = File.open("#{deploy_dir}/deployment.json", File::RDONLY)
647
660
  MU.log "Getting lock to read #{deploy_dir}/deployment.json", MU::DEBUG
648
661
  # deploy.flock(File::LOCK_EX)
@@ -654,6 +667,7 @@ module MU
654
667
 
655
668
  begin
656
669
  @deployment = JSON.parse(File.read("#{deploy_dir}/deployment.json"))
670
+ # XXX is it worthwhile to merge fuckery?
657
671
  rescue JSON::ParserError => e
658
672
  MU.log "JSON parse failed on #{deploy_dir}/deployment.json", MU::ERR, details: e.message
659
673
  end
@@ -663,20 +677,21 @@ module MU
663
677
 
664
678
  setThreadContextToMe if set_context_to_me
665
679
 
666
- @timestamp = @deployment['timestamp']
667
- @seed = @deployment['seed']
668
- @appname = @deployment['appname']
669
- @handle = @deployment['handle']
670
-
671
680
  true
672
681
  end
673
682
 
683
+
674
684
  ###########################################################################
675
685
  ###########################################################################
676
686
  def loadDeploy(deployment_json_only = false, set_context_to_me: true)
677
687
  MU::MommaCat.deploy_struct_semaphore.synchronize {
678
688
  success = loadDeployFromCache(set_context_to_me)
679
689
 
690
+ @timestamp ||= @deployment['timestamp']
691
+ @seed ||= @deployment['seed']
692
+ @appname ||= @deployment['appname']
693
+ @handle ||= @deployment['handle']
694
+
680
695
  return if deployment_json_only and success
681
696
 
682
697
  if File.exist?(deploy_dir+"/private_key")