cloud-mu 3.0.0 → 3.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ec38717ba0e001faf5500c971fad6d2ce978302e11c9b7dfc587b4d7f1e7c7a3
-  data.tar.gz: 57a212554afb4c30752e471211b515252524dc5a4e4cddabafe29f66bd6a377f
+  metadata.gz: c0ab46aed6ed1aaf58e3d8de76de9ea27adc49cbd754b51cc4ef7e6354a78a26
+  data.tar.gz: 5b341646d7533692813097af86fb37c46f1ba23fb0faf6acfe09688a3456e13c
 SHA512:
-  metadata.gz: 5fa39a5ac28414a186e1a1b620e2f097d3d662989dfd0501e02f01ad8f406f289e397a9b506c79c3511c3a659a316a1920d8ca1b8ecc5995bfe8e5f076e98f6c
-  data.tar.gz: 663202c112b0cf2c816a578da15c1d82724ddd577077b7d15075d0e1c6c683b8048dbf216f044a6a623cf82d3ba2ce370ff87a66627c052e9fd996a3b372244e
+  metadata.gz: e9c62154b428aa93f247fbf61589372744a167814f0403a339640e1e96042c8ece774b57617127860e7cfa46dc95c0c666022c3872beac77987f496cea65ad00
+  data.tar.gz: 12a4ced1bb6cbfe64ef666699a34bafe1aaf0228d080d6f8ee3584bcaec967084eca3d3a4167a405e05d1f8bc0d171093f7bab46e5648d1ff3339750a36c60e8

data/Dockerfile

@@ -0,0 +1,25 @@
+FROM ruby:2.5-slim
+
+RUN mkdir -p /opt/mu/etc/ /home/mu /usr/local/ruby-current/lib/ruby/gems/2.5.0/gems/var/
+
+WORKDIR /home/mu
+
+RUN apt-get update
+
+RUN apt-get install -y ruby2.5-dev dnsutils ansible build-essential
+
+RUN apt-get upgrade -y
+
+COPY ./cloud-mu-*.gem /home/mu
+
+RUN gem install ./cloud-mu-*.gem thin -N
+
+RUN rm cloud-mu-*.gem
+
+RUN apt-get remove -y build-essential ruby2.5-dev
+
+RUN apt-get autoremove -y
+
+EXPOSE 2260
+
+CMD /usr/sbin/init

data/README.md

@@ -1,9 +1,9 @@
 mu -- Cloudamatic Automation Tooling
 ===
 [![pipeline status](https://gitlab.com/cloudamatic/mu/badges/master/pipeline.svg)](https://gitlab.com/cloudamatic/mu/commits/master)
-[![Gem Version](https://badge.fury.io/rb/cloud-mu.svg)](https://badge.fury.io/rb/cloud-mu)
 [![Maintainability](https://api.codeclimate.com/v1/badges/dd4e5d867890336accd1/maintainability)](https://codeclimate.com/github/cloudamatic/mu/maintainability)
-[![Inline docs](http://inch-ci.org/github/cloudamatic/mu.svg?branch=master)](http://inch-ci.org/github/cloudamatic/mu)
+[![Gem Version](https://badge.fury.io/rb/cloud-mu.svg)](https://badge.fury.io/rb/cloud-mu)
+[![Docker Version](https://images.microbadger.com/badges/version/egtlabs/mu.svg)](https://microbadger.com/images/egtlabs/mu)
 
 # About mu
 **Mu**  is the deployer and developer toolset for the Cloudamatic suite of services, designed to provision, orchestrate and manage complex platforms and applications. At [eGT Labs](https://www.eglobaltech.com/egt-labs/), we use mu for rapid prototyping of cloud migration efforts for federal customers, for managing cloud applications throughout their lifecycles, and as a tools library for cloud maintenance tasks.

data/bin/mu-node-manage

@@ -599,7 +599,7 @@ def updateAWSMetaData(deploys = MU::MommaCat.listDeploys, nodes = [])
 
       MU.log "Updating #{nodename} userdata (#{server["conf"]["platform"]})"
       begin
-        MU::Cloud::AWS.ec2(server['region']).modify_instance_attribute(
+        MU::Cloud::AWS.ec2(region: server['region']).modify_instance_attribute(
           instance_id: id,
           attribute: "userData",
           value: Base64.encode64(userdata)

data/chefignore

@@ -0,0 +1 @@
+.kitchen

data/cloud-mu.gemspec

@@ -17,8 +17,8 @@ end
 
 Gem::Specification.new do |s|
   s.name        = 'cloud-mu'
-  s.version     = '3.0.0'
-  s.date        = '2019-11-11'
+  s.version     = '3.0.1'
+  s.date        = '2019-11-22'
   s.require_paths = ['modules']
   s.required_ruby_version = '>= 2.4'
   s.summary     = "The eGTLabs Mu toolkit for unified cloud deployments"

data/kitchen.yml

@@ -0,0 +1,97 @@
+driver:
+  name: ec2
+  region: us-east-1
+  subnet_id: subnet-00f6ebfde53bbdccd
+  instance_type: t2.medium
+  iam_profile_name: mu-master-role
+  associate_public_ip: true
+  interface: dns
+  shared_credentials_profile: test_kitchen
+  instance_initiated_shutdown_behavior: terminate
+  tags:
+      Name: kitchen-mu-node
+      created-by: mu-pipeline
+  sudo: true
+
+provisioner:
+  name: chef_zero
+  roles_path: roles
+  chef_license: accept
+
+verifier:
+  name: shell
+
+platforms:
+  - name: centos-7
+    vm_tags:
+      osdistro: centos7
+
+  - name: amazon2
+    vm_tags:
+      osdistro: amazon2
+
+  - name: centos-7-az
+    driver:
+      name: azurerm
+      subscription_id: '3d20ddd8-4652-4074-adda-0d127ef1f0e0'
+      location: 'East US'
+      #image_urn: 'tunnelbiz:centos70-min:centos7-min:0.1.1'
+      machine_size: 'Standard_B2s'
+      vm_name: mu-install
+      vm_tags:
+        osdistro: centos7
+
+suites:
+  - name: mu-install-aws
+    provisioner:
+      name:   shell
+      script:
+      - "./kitchen_vars"
+      - "./install/installer"
+      - "./install/mu-master.yaml"
+      command:
+      - sudo yum install -y bind-utils
+      - sudo chown root:root /tmp/installer
+      - sudo chmod u+x /tmp/installer
+      - sudo chown root:root /tmp/mu-master.yaml
+      - source /tmp/kitchen_vars
+      - env
+      - myip="$(dig +short myip.opendns.com @resolver1.opendns.com)"
+      - sudo /tmp/installer -p $myip -m zach@zach.systems -o 2260 -h "$(echo $HOSTNAME | sed s/\\./-/g)" -n
+      root_path: '/tmp/'
+    includes:
+      - centos-7
+      - amazon2
+
+  - name: mu-install-azure
+    provisioner:
+      name:   shell
+      script:
+      - "./install/installer"
+      - "./install/mu-master.yaml"
+      command:
+      - myip="$(dig +short myip.opendns.com @resolver1.opendns.com)"
+      - sudo ll /tmp/ && sudo MU_BRANCH=development /tmp/installer -p $myip -m zach@zach.systems -o 2260 -h "$(echo $HOSTNAME | sed s/\\./-/g)" -n
+      - sudo mu-deploy /tmp/mu-master.yaml -p name=stange-mu -p cloud=Azure
+      root_path: '/tmp/'
+    includes:
+      - centos-7-az
+
+  #- name: mu-node
+  #  run_list:
+  #    - recipe[mu-tools::newclient]
+  #    - recipe[mu-tools::gcloud]
+  #    - role[mu-node]
+  #  includes:
+      # - ubuntu
+  #    - centos-7
+      # - centos-6
+      # - rhel-7
+      # - rhel-6
+      # - amazon
+  #    - amazon2
+      # - windows
+
+  # - name: mu-master
+  #   run_list:
+  #     - role[mu-master]

data/modules/mu.rb

@@ -343,6 +343,17 @@ module MU
     ((Gem.paths and Gem.paths.home and File.realpath(File.expand_path(File.dirname(__FILE__))).match(/^#{Gem.paths.home}/)) or !Dir.exist?("/opt/mu"))
   end
 
+  # Are we operating in a gem?
+  def self.inGem?
+    return @in_gem if defined? @in_gem
+
+    if Gem.paths and Gem.paths.home and File.dirname(__FILE__).match(/^#{Gem.paths.home}/)
+      @in_gem = true
+    else
+      @in_gem = false
+    end
+  end
+
   # The main (root) Mu user's data directory.
   @@mainDataDir = File.expand_path(@@myRoot+"/../var")
   # The main (root) Mu user's data directory.
@@ -631,8 +642,9 @@ module MU
      !$MU_CFG['public_address'].empty? and @@my_public_ip != $MU_CFG['public_address']
     @@mu_public_addr = $MU_CFG['public_address']
     if !@@mu_public_addr.match(/^\d+\.\d+\.\d+\.\d+$/)
-      resolver = Resolv::DNS.new
-      @@mu_public_ip = resolver.getaddress(@@mu_public_addr).to_s
+      hostname = IO.readlines("/etc/hostname")[0].gsub /\n/, ''
+
+      @@mu_public_ip = File.open('/etc/hosts').grep(/.*#{hostname}.*/).first.match(/^\d+\.\d+\.\d+\.\d+/)[0]
     else
       @@mu_public_ip = @@mu_public_addr
     end

data/modules/mu/cloud.rb

@@ -1142,6 +1142,30 @@ module MU
 
         # Remove all metadata and cloud resources associated with this object
         def destroy
+          if self.class.cfg_name == "server"
+            begin
+              ip = canonicalIP
+              MU::MommaCat.removeIPFromSSHKnownHosts(ip) if ip
+              if @deploy and @deploy.deployment and
+                 @deploy.deployment['servers'] and @config['name'] and
+                me = @deploy.deployment['servers'][@config['name']][@mu_name]
+                if me
+                  ["private_ip_address", "public_ip_address"].each { |field|
+                    if me[field]
+                      MU::MommaCat.removeIPFromSSHKnownHosts(me[field])
+                    end
+                  }
+                  if me["private_ip_list"]
+                    me["private_ip_list"].each { |ip|
+                      MU::MommaCat.removeIPFromSSHKnownHosts(ip)
+                    }
+                  end
+                end
+              end
+            rescue MU::MuError => e
+              MU.log e.message, MU::WARN
+            end
+          end
           if !@cloudobj.nil? and !@cloudobj.groomer.nil?
             @cloudobj.groomer.cleanup
           elsif !@groomer.nil?
@@ -1952,6 +1976,7 @@ puts "CHOOSING #{@vpc.to_s} 'cause it has #{@config['vpc']['subnet_name']}"
                     :keys => [ssh_keydir+"/"+nat_ssh_key, ssh_keydir+"/"+@deploy.ssh_key_name],
                     :verify_host_key => false,
                     #           :verbose => :info,
+                    :host_key => "ssh-rsa",
                     :port => 22,
                     :auth_methods => ['publickey'],
                     :proxy => proxy
@@ -1967,6 +1992,7 @@ puts "CHOOSING #{@vpc.to_s} 'cause it has #{@config['vpc']['subnet_name']}"
                     :keys => [ssh_keydir+"/"+@deploy.ssh_key_name],
                     :verify_host_key => false,
                     #           :verbose => :info,
+                    :host_key => "ssh-rsa",
                     :port => 22,
                     :auth_methods => ['publickey']
                 )

data/modules/mu/clouds/aws.rb

@@ -164,7 +164,6 @@ module MU
       # @param r [String]
       # @return [String]
       def self.validate_region(r, credentials: nil)
-        require "aws-sdk-core"
         begin
           MU::Cloud::AWS.ec2(region: r, credentials: credentials).describe_availability_zones.availability_zones.first.region_name
         rescue ::Aws::EC2::Errors::UnauthorizedOperation => e
@@ -1339,8 +1338,8 @@ module MU
         # @param region [String]: Amazon region so we know what endpoint to use
         # @param api [String]: Which API are we wrapping?
         def initialize(region: MU.curRegion, api: "EC2", credentials: nil)
-          @credentials = MU::Cloud::AWS.credConfig(credentials, name_only: true)
           @cred_obj = MU::Cloud::AWS.loadCredentials(credentials)
+          @credentials = MU::Cloud::AWS.credConfig(credentials, name_only: true)
 
           if !@cred_obj
             raise MuError, "Unable to locate valid AWS credentials for #{api} API. #{credentials ? "Credentials requested were '#{credentials}'": ""}"

data/modules/mu/clouds/aws/container_cluster.rb

@@ -66,6 +66,7 @@ module MU
                   ]
                 }
               end
+              params.delete(:version) if params[:version] == "latest"
 
               MU.log "Creating EKS cluster #{@mu_name}", details: params
               resp = MU::Cloud::AWS.eks(region: @config['region'], credentials: @config['credentials']).create_cluster(params)
@@ -91,7 +92,7 @@ module MU
                 sleep 5
                 retry
               else
-                MU.log e.message, MU::WARN, details: role_arn
+                MU.log e.message, MU::WARN, details: params
                 sleep 5
                 retry
               end

data/modules/mu/clouds/aws/database.rb

@@ -928,7 +928,7 @@ module MU
               basename[0..29].gsub(/[^a-z0-9]/i, "")
             elsif config["engine"].match(/^sqlserver/)
               basename[0..127].gsub(/[^a-z0-9]/i, "")
-            elsif config["engine"].match(/^mysql/)
+            elsif config["engine"].match(/^(mysql|maria)/)
               basename[0..15].gsub(/[^a-z0-9]/i, "")
             elsif config["engine"].match(/^aurora/)
               basename[0..15].gsub(/[^a-z0-9]/i, "")
@@ -1603,7 +1603,7 @@ module MU
           db["license_model"] ||=
             if ["postgres", "postgresql", "aurora-postgresql"].include?(db["engine"])
               "postgresql-license"
-            elsif db["engine"] == "mysql"
+            elsif ["mysql", "mariadb"].include?(db["engine"])
               "general-public-license"
             else
               "license-included"

data/modules/mu/clouds/aws/loadbalancer.rb

@@ -188,7 +188,7 @@ module MU
             if @config['targetgroups']
               MU.log "Configuring target groups and health checks check for ELB #{@mu_name}", details: @config['healthcheck']
               @config['targetgroups'].each { |tg|
-                tg_name = @deploy.getResourceName(tg["name"], max_length: 32)
+                tg_name = @deploy.getResourceName(tg["name"], max_length: 32, allowed_chars: /[A-Za-z0-9-]/)
                 tg_descriptor = {
                   :name => tg_name,
                   :protocol => tg['proto'],

data/modules/mu/clouds/aws/search_domain.rb

@@ -67,7 +67,7 @@ module MU
               domain_name: @deploydata['domain_name']
             ).domain_status
           else
-            raise MU::MuError "#{@mu_name} can't find its official Elasticsearch domain name!"
+            raise MuError "#{@mu_name} can't find its official Elasticsearch domain name!"
           end
         end
 

data/modules/mu/clouds/aws/server.rb

@@ -120,8 +120,10 @@ module MU
             @config['mu_name'] = @mu_name
 
           end
+
           @config['instance_secret'] ||= Password.random(50)
 
+          @groomer = MU::Groomer.new(self) unless MU.inGem?
         end
 
         @@userdata_semaphore = Mutex.new
@@ -938,11 +940,17 @@ module MU
 
           # See if this node already exists in our config management. If it does,
           # we're done.
-          if @groomer.haveBootstrapped?
+          if MU.inGem?
+            MU.log "Deploying from a gem, not grooming"
+
+            return true
+          elsif @groomer.haveBootstrapped?
             MU.log "Node #{node} has already been bootstrapped, skipping groomer setup.", MU::NOTICE
+
             if @config['groom'].nil? or @config['groom']
               @groomer.saveDeployData
             end
+
             MU::MommaCat.unlock(instance.instance_id+"-orchestrate")
             MU::MommaCat.unlock(instance.instance_id+"-groom")
             return true

data/modules/mu/clouds/aws/vpc.rb

@@ -1161,7 +1161,6 @@ MU.log "wtf", MU::ERR, details: peer if peer_obj.nil? or peer_obj.first.nil?
         # @param region [String]: The cloud provider region
         # @return [void]
         def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
-
           tagfilters = [
             {name: "tag:MU-ID", values: [MU.deploy_id]}
           ]
@@ -1172,7 +1171,7 @@ MU.log "wtf", MU::ERR, details: peer if peer_obj.nil? or peer_obj.first.nil?
           vpcs = []
           retries = 0
           begin
-            resp = MU::Cloud::AWS.ec2(region: region, credentials: credentials).describe_vpcs(filters: tagfilters).vpcs
+            resp = MU::Cloud::AWS.ec2(region: region, credentials: credentials).describe_vpcs(filters: tagfilters, max_results: 1000).vpcs
             vpcs = resp if !resp.empty?
           rescue Aws::EC2::Errors::InvalidVpcIDNotFound => e
             if retries < 5

data/modules/mu/config.rb

@@ -317,6 +317,13 @@ module MU
         }
       end
 
+      # A way of dynamically defining +attr_reader+ without leaking memory
+      def self.define_reader(name)
+        define_method(name) {
+          instance_variable_get("@#{name.to_s}")
+        }
+      end
+
       # @param cfg [Hash]: A Basket of Kittens configuration hash containing
       # lookup information for a cloud object
       def initialize(cfg)
@@ -327,7 +334,7 @@ module MU
           elsif !cfg[field.to_sym].nil?
             self.instance_variable_set("@#{field.to_s}".to_sym, cfg[field.to_sym])
           end
-          self.singleton_class.instance_eval { attr_reader field.to_sym }
+          MU::Config::Ref.define_reader(field)
         }
         if cfg['tag'] and cfg['tag']['key'] and
            !cfg['tag']['key'].empty? and cfg['tag']['value']
@@ -985,6 +992,15 @@ return
 
       @config['credentials'] ||= @default_credentials
 
+      if @config['cloud'] and !MU::Cloud.availableClouds.include?(@config['cloud'])
+        if MU::Cloud.supportedClouds.include?(@config['cloud'])
+          MU.log "Cloud provider #{@config['cloud']} declared, but no #{@config['cloud']} credentials available", MU::ERR
+        else
+          MU.log "Cloud provider #{@config['cloud']} is not supported", MU::ERR, details: MU::Cloud.supportedClouds
+        end
+        exit 1
+      end
+
       types = MU::Cloud.resource_types.values.map { |v| v[:cfg_plural] }
 
       MU::Cloud.resource_types.values.map { |v| v[:cfg_plural] }.each { |type|
@@ -1274,6 +1290,16 @@ $CONFIGURABLES
       }
       ok = true
 
+      if descriptor['cloud'] and
+         !MU::Cloud.availableClouds.include?(descriptor['cloud'])
+        if MU::Cloud.supportedClouds.include?(descriptor['cloud'])
+          MU.log "#{cfg_name} #{descriptor['name']} is configured with cloud #{descriptor['cloud']}, but no #{descriptor['cloud']} credentials available", MU::ERR
+        else
+          MU.log "#{cfg_name} #{descriptor['name']}: Cloud provider #{descriptor['cloud']} is not supported", MU::ERR, details: MU::Cloud.supportedClouds
+        end
+        return false
+      end
+
       descriptor["#MU_CLOUDCLASS"] = classname
 
       applyInheritedDefaults(descriptor, cfg_plural)
@@ -1975,7 +2001,7 @@ $CONFIGURABLES
         conf_chunk.map! { |item|
           # If we're working on a resource type, go get implementation-specific
           # schema information so that we set those defaults correctly.
-          realschema = if type and schema_chunk["items"] and schema_chunk["items"]["properties"] and item["cloud"]
+          realschema = if type and schema_chunk["items"] and schema_chunk["items"]["properties"] and item["cloud"] and MU::Cloud.supportedClouds.include?(item['cloud'])
 
             cloudclass = Object.const_get("MU").const_get("Cloud").const_get(item["cloud"]).const_get(type)
             toplevel_required, cloudschema = cloudclass.schema(self)
@@ -2126,6 +2152,10 @@ $CONFIGURABLES
       kitten['cloud'] ||= @config['cloud']
       kitten['cloud'] ||= MU::Config.defaultCloud
 
+      if !MU::Cloud.supportedClouds.include?(kitten['cloud'])
+        return
+      end
+
       cloudclass = Object.const_get("MU").const_get("Cloud").const_get(kitten['cloud'])
       shortclass, cfg_name, cfg_plural, classname = MU::Cloud.getResourceNames(type)
       resclass = Object.const_get("MU").const_get("Cloud").const_get(kitten['cloud']).const_get(shortclass)
@@ -2198,6 +2228,7 @@ $CONFIGURABLES
         count = count + @kittens[type].size
       }
 
+
       if count == 0
         MU.log "You must declare at least one resource to create", MU::ERR
         ok = false
@@ -2437,8 +2468,10 @@ $CONFIGURABLES
     # and turn into documentation.
     def self.printSchema(kitten_rb, class_hierarchy, schema, in_array = false, required = false, prefix: nil)
       return if schema.nil?
+
       if schema["type"] == "object"
-        printme = Array.new
+        printme = []
+
         if !schema["properties"].nil?
           # order sub-elements by whether they're required, so we can use YARD's
           # grouping tags on them
@@ -2453,6 +2486,53 @@ $CONFIGURABLES
           printme << "# @!group Optional parameters" if schema["required"].nil? or schema["required"].size == 0
           prop_list.each { |name|
             prop = schema["properties"][name]
+
+            if class_hierarchy.size == 1
+
+              _shortclass, cfg_name, cfg_plural, _classname = MU::Cloud.getResourceNames(name)
+              if cfg_name
+                example_path = MU.myRoot+"/modules/mu/config/"+cfg_name+".yml"
+                if File.exist?(example_path)
+                  example = "#\n# Examples:\n#\n"
+                  # XXX these variables are all parameters from the BoKs in
+                  # modules/tests. A really clever implementation would read
+                  # and parse them to get default values, perhaps, instead of
+                  # hard-coding them here.
+                  instance_type = "t2.medium"
+                  db_size = "db.t2.medium"
+                  vpc_name = "some_vpc"
+                  logs_name = "some_loggroup"
+                  queues_name = "some_queue"
+                  server_pools_name = "some_server_pool"
+                  ["simple", "complex"].each { |complexity|
+                    erb = ERB.new(File.read(example_path), nil, "<>")
+                    example += "#      !!!yaml\n"
+                    example += "#      ---\n"
+                    example += "#      appname: #{complexity}\n"
+                    example += "#      #{cfg_plural}:\n"
+                    firstline = true
+                    erb.result(binding).split(/\n/).each { |l|
+                      l.sub!(/#.*/, "")
+                      next if l.empty? or l.match(/^\s+$/)
+                      if firstline
+                        l = "- "+l
+                        firstline = false
+                      else
+                        l = "  "+l
+                      end
+                      example += "#      "+l+"    "+"\n"
+                    }
+                    example += "# &nbsp;\n#\n" if complexity == "simple"
+                  }
+                  schema["properties"][name]["items"]["description"] ||= ""
+                  if !schema["properties"][name]["items"]["description"].empty?
+                    schema["properties"][name]["items"]["description"] += "\n"
+                  end
+                  schema["properties"][name]["items"]["description"] += example
+                end
+              end
+            end
+
             if !schema["required"].nil? and schema["required"].include?(name)
               printme << "# @!group Required parameters" if !req
               req = true