cloud-mu 2.0.0.pre.alpha → 2.0.0.pre.alpha2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f1059c013e5f5e7c2ece20c2aa9ffcb60facab001cc864c0c26f60baf384d7c4
-  data.tar.gz: ebca2309c37e8a8038df432be18bb2b4f54e44b52da42029a79461515603f899
+  metadata.gz: ea07140b34d57b4dea2bc06b6a9184896585db834782e6b2b4c40184b6cdee71
+  data.tar.gz: 5382c8d82180c39d6797ccbfabb4903877f29f914f2c968aa4550033c6607db8
 SHA512:
-  metadata.gz: d73ffd1304213dc06b04f0f350ef9d70cf99e48a3dd1c0ea7db0424606437b83e710bd87c0b9df471ca0ed0998991cf4e0e7a7e482d859ce647d0e0e09f84f6e
-  data.tar.gz: ad2503195884645217ee178aa1a9d94fb12520c83132f355cc1042535059d3b8d89cd8ce0066f1eae3d4e665aa6b678599ba2d174acbd7f9701430a1c242b464
+  metadata.gz: b4f27d9340d4146cadcd2eeb0a9a2abe650a0822158b136560ff3c8f409975f1a83e8106372c0e8c1f51e35a5868de3764492026f50e6aed9faa410fc133199a
+  data.tar.gz: 411b9b7d3e8db61459dfc480a1d2d874603580d87e3597956d882a0e9e9e41fc0eff61d3af3d01924a4ddd8c6f10916bcb6df8a18576bd3fd5f3a187a2d460b7

data/bin/mu-configure

@@ -28,6 +28,11 @@ require 'fileutils'
 require 'erb'
 require 'tmpdir'
 
+$IN_GEM = false
+if Gem.paths and Gem.paths.home and File.dirname(__FILE__).match(/^#{Gem.paths.home}/)
+  $IN_GEM = true
+end
+
 GIT_PATTERN = /(((git|ssh|http(s)?)|(git@[\w\.]+))(:(\/\/)?))?([\w\.@\:\/\-~]+)(\.git)?(\/)?/
 
 # Top-level keys in $MU_CFG for which we'll provide interactive, menu-driven
@@ -197,6 +202,11 @@ $CONFIGURABLES = {
         "required" => true,
         "changes" => ["chefrun"]
       },
+      "masequerade_as" => {
+        "title" => "GSuite Masquerade User",
+        "required" => false,
+        "desc" => "For Google Cloud projects which are attached to a GSuite domain. GCP service accounts cannot view or manage GSuite resources (groups, users, etc) directly, but must instead masquerade as a GSuite user which has delegated authority to the service account. See also: https://developers.google.com/identity/protocols/OAuth2ServiceAccount#delegatingauthority"
+      },
       "default" => {
         "title" => "Is Default Account",
         "default" => false,
@@ -281,10 +291,11 @@ end
 
 $INITIALIZE = (!File.size?("#{MU_BASE}/etc/mu.yaml") or $opts[:force])
 $HAVE_GLOBAL_CONFIG = File.size?("#{MU_BASE}/etc/mu.yaml")
-if !AMROOT and ($INITIALIZE or !$HAVE_GLOBAL_CONFIG)
+if !AMROOT and ($INITIALIZE or !$HAVE_GLOBAL_CONFIG) and !$IN_GEM
   puts "Global configuration has not been initialized or is missing. Must run as root to correct."
   exit 1
 end
+
 if !$HAVE_GLOBAL_CONFIG and $opts[:noninteractive] and (!$opts[:public_address] or !$opts[:mu_admin_email])
   puts "Specify --public-address and --mu-admin-email on new non-interactive configs"
   exit 1
@@ -315,7 +326,7 @@ begin
     instance_id = open("http://169.254.169.254/metadata/instance/compute").read
     $IN_AWS = true if !instance_id.nil? and instance_id.size > 0
   end
-rescue OpenURI::HTTPError, Timeout::Error, SocketError, Errno::ENETUNREACH
+rescue OpenURI::HTTPError, Timeout::Error, SocketError, Errno::ENETUNREACH, Errno::EHOSTUNREACH
 end
 
 
@@ -385,6 +396,7 @@ def assignMenuEntries(tree = $CONFIGURABLES, map = $MENU_MAP)
           "required" => true
         }
         map[count.to_s]["#addnew"] = true
+        map[count.to_s]["#title"] = data['title']
         map[count.to_s]["#key"] = key
 
         # Now the menu entries for the existing ones
@@ -480,7 +492,7 @@ def cloneGitRepo(repo)
   puts "Testing ability to check out Git repository #{repo.bold}"
   fullrepo = repo
   if !repo.match(/@|:\/\//) # we try ssh first
-    fullrepo = "git@github.com:"+repo 
+    fullrepo = "git@github.com:"+repo
     puts "Doesn't look like a full URL, trying SSH to #{fullrepo}"
   end
   cwd = Dir.pwd
@@ -511,7 +523,7 @@ def cloneGitRepo(repo)
       end
     end
     if !repo.match(/@|:\/\//)
-      fullrepo = "git://github.com/"+repo 
+      fullrepo = "git://github.com/"+repo
       puts ""
       puts "No luck there, trying #{fullrepo}".bold
       puts "/usr/bin/git clone #{fullrepo}"
@@ -522,7 +534,7 @@ def cloneGitRepo(repo)
         return fullrepo
       else
         puts output.red.on_black
-        fullrepo = "https://github.com/"+repo 
+        fullrepo = "https://github.com/"+repo
         puts "Final attempt, trying #{fullrepo}"
         puts "/usr/bin/git clone #{fullrepo}"
         output = %x{/usr/bin/git clone #{fullrepo} 2>&1}
@@ -745,8 +757,8 @@ def displayCurrentOpts(tree = $CONFIGURABLES)
   count = 1
   optlist = []
   tree.each_pair { |key, data|
-    next if !AMROOT and data['rootonly']
     next if !data.is_a?(Hash)
+    next if !AMROOT and data['rootonly']
     if data["title"].nil? or data["#menu"].nil?
       next
     end
@@ -780,7 +792,7 @@ end
 ###############################################################################
 
 trap("INT"){ puts "" ; exit }
-importCurrentValues if !$INITIALIZE or $HAVE_GLOBAL_CONFIG
+importCurrentValues if !$INITIALIZE or $HAVE_GLOBAL_CONFIG or $IN_GEM
 importCLIValues
 setDefaults
 assignMenuEntries # populates $MENU_MAP
@@ -821,7 +833,7 @@ def validate(newval, reqs, addnewline = true, in_use: [])
       puts "\nInvalid value '#{newval.bold}' for #{reqs['title'].bold} (must be true or false)".light_red.on_black
       puts "\n\n" if addnewline
       ok = false
-    elsif in_use.size > 0 and in_use.include?(newval)
+    elsif in_use and in_use.size > 0 and in_use.include?(newval)
       puts "\n##{reqs['title'].bold} #{newval} not available".light_red.on_black
       puts "\n\n" if addnewline
       ok = false
@@ -830,7 +842,7 @@ def validate(newval, reqs, addnewline = true, in_use: [])
         puts "\nSupplied value for #{reqs['title'].bold} did not pass validation".light_red.on_black
         puts "\n\n" if addnewline
         ok = false
-      elsif reqs['negate_pattern'] 
+      elsif reqs['negate_pattern']
         if newval.to_s.match(reqs['pattern'])
           puts "\nInvalid value '#{newval.bold}' for #{reqs['title'].bold} (must NOT match #{reqs['pattern']})".light_red.on_black
           puts "\n\n" if addnewline
@@ -915,7 +927,9 @@ def menu(tree = $CONFIGURABLES, map = $MENU_MAP, submenu_name = nil, in_use_name
         map[answer],
         minimap,
         map[answer]['#title']+" (NEW)",
-        map[answer]['#entries'].keys.reject { |k| k.match(/^#/) }
+        if map[answer]['#entries']
+          map[answer]['#entries'].keys.reject { |k| k.match(/^#/) }
+        end
       )
       if newtree
         newname = newtree["name"]["value"]
@@ -945,7 +959,7 @@ def menu(tree = $CONFIGURABLES, map = $MENU_MAP, submenu_name = nil, in_use_name
       newtree, newmap = menu(
         map[answer],
         minimap,
-        map[answer]["#title"], 
+        map[answer]["#title"],
         (map[answer]['#entries'].keys - [map[answer]['#title']])
       )
       map[answer] = newtree if newtree
@@ -1070,7 +1084,9 @@ if AMROOT
 end
 
 if $INITIALIZE
-  %x{/sbin/service iptables stop} # Chef run will set up correct rules later
+  if AMROOT
+    %x{/sbin/service iptables stop} # Chef run will set up correct rules later
+  end
   $MU_SET_DEFAULTS = setConfigTree
   require File.realpath(File.expand_path(File.dirname(__FILE__)+"/mu-load-config.rb"))
 else
@@ -1093,6 +1109,12 @@ rescue LoadError
   require 'mu'
 end
 
+if $IN_GEM
+  puts $MU_CFG.to_yaml
+  saveMuConfig($MU_CFG)
+  exit
+end
+
 if AMROOT and ($INITIALIZE or $CHANGES.include?("hostname"))
   system("/bin/hostname #{$MU_CFG['hostname']}")
 end
@@ -1130,6 +1152,7 @@ def updateChefRbs
   user = AMROOT ? "mu" : Etc.getpwuid(Process.uid).name
   chefuser = user.gsub(/\./, "")
   templates = { HOMEDIR+"/.chef/knife.rb" => KNIFE_TEMPLATE }
+  Dir.mkdir(HOMEDIR+"/.chef") if !Dir.exists?(HOMEDIR+"/.chef")
   if AMROOT
     templates["/etc/chef/client.rb"] = CLIENT_TEMPLATE
     templates["/etc/opscode/pivotal.rb"] = PIVOTAL_TEMPLATE
@@ -1252,7 +1275,7 @@ if $INITIALIZE or $CHANGES.include?("vault")
   system("chef-client -o 'recipe[mu-master::vault]'")
 end
 
-if $MU_CFG['ldap']['type'] == "389 Directory Services" 
+if $MU_CFG['ldap']['type'] == "389 Directory Services"
   begin
     MU::Master::LDAP.listUsers
   rescue Exception => e # XXX lazy exception handling is lazy

data/cloud-mu.gemspec

@@ -17,7 +17,7 @@ end
 
 Gem::Specification.new do |s|
   s.name        = 'cloud-mu'
-  s.version     = '2.0.0-alpha'
+  s.version     = '2.0.0-alpha2'
   s.date        = '2018-12-11'
   s.require_paths = ['modules']
   s.required_ruby_version = '>= 2.4'

data/modules/mu/cleanup.rb

@@ -61,8 +61,7 @@ module MU
       end
 
 
-# XXX AWS needs to check MU::Cloud::AWS.isGovCloud? on some things, or gracefully handle the API not existing
-      types_in_order = ["Collection", "Function", "ServerPool", "ContainerCluster", "SearchDomain", "Server", "MsgQueue", "Database", "CacheCluster", "StoragePool", "LoadBalancer", "FirewallRule", "Alarm", "Notifier", "Log", "VPC", "DNSZone", "Collection"]
+      types_in_order = ["Collection", "Function", "ServerPool", "ContainerCluster", "SearchDomain", "Server", "MsgQueue", "Database", "CacheCluster", "StoragePool", "LoadBalancer", "FirewallRule", "Alarm", "Notifier", "Log", "VPC", "DNSZone", "Collection", "Habitat"]
 
       # Load up our deployment metadata
       if !mommacat.nil?
@@ -161,6 +160,7 @@ module MU
 
                       if @mommacat.nil? or @mommacat.numKittens(types: [t]) > 0
                         begin
+                        puts t if t == "Habitat"
                           resclass = Object.const_get("MU").const_get("Cloud").const_get(t)
                           resclass.cleanup(
                             noop: @noop,
@@ -184,7 +184,7 @@ module MU
                 # XXX move to MU::AWS
                 if provider == "AWS"
                   resp = MU::Cloud::AWS.ec2(region: r, credentials: credset).describe_key_pairs(
-                      filters: [{name: "key-name", values: [keyname]}]
+                    filters: [{name: "key-name", values: [keyname]}]
                   )
                   resp.data.key_pairs.each { |keypair|
                     MU.log "Deleting key pair #{keypair.key_name} from #{r}"

data/modules/mu/cloud.rb

@@ -1424,6 +1424,7 @@ MU.log "in dependencies() and findLitterMate gave me "+sib_by_name.to_s+" on beh
         # Wrapper for the cleanup class method of underlying cloud object implementations.
         def self.cleanup(*flags)
           params = flags.first
+
           clouds = MU::Cloud.supportedClouds
           if params[:cloud]
             clouds = [params[:cloud]]
@@ -1433,10 +1434,12 @@ MU.log "in dependencies() and findLitterMate gave me "+sib_by_name.to_s+" on beh
             begin
               cloudclass = MU::Cloud.loadCloudType(cloud, shortname)
               raise MuCloudResourceNotImplemented if !cloudclass.respond_to?(:cleanup) or cloudclass.method(:cleanup).owner.to_s != "#<Class:#{cloudclass}>"
-              MU.log "Invoking #{cloudclass}.cleanup from #{shortname}", MU::DEBUG, details: flags
+if cfg_name == "habitat"
+              MU.log "Invoking #{cloudclass}.cleanup from #{shortname}", MU::WARN, details: flags
+end
               cloudclass.cleanup(params)
             rescue MuCloudResourceNotImplemented
-              MU.log "No #{cloud} implementation of #{shortname}.cleanup, skipping", MU::DEBUG, details: flags
+              MU.log "No #{cloud} implementation of #{shortname}.cleanup, skipping", MU::WARN, details: flags
             end
           }
           MU::MommaCat.unlockAll

data/modules/mu/clouds/aws/habitat.rb

@@ -96,6 +96,16 @@ module MU
         # @param region [String]: The cloud provider region
         # @return [void]
         def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
+          puts "IN HABITAT CLEANUP"
+          return if !orgMasterCreds?(credentials)
+
+          resp = MU::Cloud::AWS.orgs(credentials: credentials).list_accounts
+
+          if resp and resp.accounts
+            resp.accounts.each { |acct|
+            pp acct
+            }
+          end
         end
 
         # Locate an existing account
@@ -121,6 +131,19 @@ module MU
           [toplevel_required, schema]
         end
 
+        # Figure out what account we're calling from, and then figure out if
+        # it's the organization's master account- the only place from which
+        # we can create accounts, amongst other things.
+        # @param credentials [String]
+        # @return [Boolean]
+        def self.orgMasterCreds?(credentials = nil)
+          user_list = MU::Cloud::AWS.iam(credentials:  credentials).list_users.users
+          acct_num = MU::Cloud::AWS.iam(credentials:  credentials).list_users.users.first.arn.split(/:/)[4]
+
+          parentorg = MU::Cloud::AWS::Folder.find(credentials: credentials).values.first
+          acct_num == parentorg.master_account_id
+        end
+
         # Cloud-specific pre-processing of {MU::Config::BasketofKittens::habitats}, bare and unvalidated.
         # @param habitat [Hash]: The resource to process and validate
         # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
@@ -132,14 +155,7 @@ module MU
             MU.log "No email address specified in habitat #{habitat['name']}, and AWS requires a unique contact email. Will generate an alias to #{$MU_CFG['mu_admin_email']} at run time.", MU::NOTICE
           end
 
-          # Figure out what account we're calling from, and then figure out if
-          # it's the organization's master account- the only place from which
-          # we can create accounts.
-          user_list = MU::Cloud::AWS.iam(credentials:  habitat['credentials']).list_users.users
-          acct_num = MU::Cloud::AWS.iam(credentials:  habitat['credentials']).list_users.users.first.arn.split(/:/)[4]
-
-          parentorg = MU::Cloud::AWS::Folder.find(credentials: habitat['credentials']).values.first
-          if acct_num != parentorg.master_account_id
+          if !orgMasterCreds?(habitat['credentials'])
             MU.log "The Organization master account for habitat #{habitat["name"]} is #{parentorg.master_account_id}, but my credentials (#{ habitat['credentials'] ?  habitat['credentials'] : "default"}) are for a non-master account (#{acct_num}). AWS accounts can only be created and managed with credentials from an Organization's master account.", MU::ERR
             ok = false
           end

data/modules/mu/clouds/aws.rb

@@ -487,7 +487,10 @@ module MU
 #          MU.log "Got #{e.inspect} while trying to figure out our account number", MU::WARN, details: caller
 #        end
 #        if user_list.nil? or user_list.size == 0
-          mac = MU::Cloud::AWS.getAWSMetaData("network/interfaces/macs/").split(/\n/)[0]
+          resp = MU::Cloud::AWS.getAWSMetaData("network/interfaces/macs/")
+          return nil if !resp
+
+          mac = resp.split(/\n/)[0]
           acct_num = MU::Cloud::AWS.getAWSMetaData("network/interfaces/macs/#{mac}owner-id")
           acct_num.chomp!
 #        else

data/modules/mu/clouds/google.rb

@@ -25,6 +25,7 @@ module MU
       @@authtoken = nil
       @@default_project = nil
       @@myRegion_var = nil
+      @@my_hosted_cfg = nil
       @@authorizers = {}
       @@acct_to_profile_map = {}
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cloud-mu
 version: !ruby/object:Gem::Version
-  version: 2.0.0.pre.alpha
+  version: 2.0.0.pre.alpha2
 platform: ruby
 authors:
 - John Stange