cloud-mu 3.1.3 → 3.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c5db54ee8a4ea5ea9884c3f468e162e8b27d60de12a0ccc8cbb30006a30818eb
-  data.tar.gz: e6fee0a6d491ebb38a13754d41c9d4df4b7d360f820769f49a07a93cd94ee042
+  metadata.gz: 87070670dfd1950848c17f897f49f087ada7a3155e3a5ef8e24ffb2c62583e7f
+  data.tar.gz: 8cee858ebde553ee022e56db98e90c62a28eb4a0b0247f74bd48a940956fe1c5
 SHA512:
-  metadata.gz: 66ffb8861d10e000a7afda4d7047a32e89f7cd348a43c6058d7cee9fb4a218e7c0db7b97d25d38eb5e08a53ed9003d4916497bb103fec1edb8c45694eddd4add
-  data.tar.gz: d843093658cb63138abc07290274c599f1e084cf39fd905699516bf2b0785a4b7859561084300762f63152fb3d240762032b65acd86ebc0a072b5809166497ab
+  metadata.gz: 5ed8ad1fd6834f0b251079eb7d8b3f1805b0f88aa47486943f4bb7ad75196161c6c41a8e53364f755d46fe7fa6e51b8fb83ab518747369d07cdb0cb360689866
+  data.tar.gz: 53acff0bb1ea78fada8a9d8e6b3084ccb02209b42a1b19f99a80d792706cb55f1a2d5e97b48ee1fa54cc8b2ee8f7947feecd7a1465f24e18f9cdb86a499de8fa

data/Dockerfile

@@ -4,22 +4,34 @@ RUN mkdir -p /opt/mu/etc/ /home/mu /usr/local/ruby-current/lib/ruby/gems/2.5.0/g
 
 WORKDIR /home/mu
 
+RUN df -h
+
 RUN apt-get update
 
-RUN apt-get install -y ruby2.5-dev dnsutils ansible build-essential
+RUN apt-get install -y ruby2.5-dev dnsutils ansible build-essential python-pip curl openssh-client
 
 RUN apt-get upgrade -y
 
+RUN df -h
+
 COPY ./cloud-mu-*.gem /home/mu
 
-RUN gem install ./cloud-mu-*.gem thin -N
+RUN gem install ./cloud-mu-*.gem -N
+
+RUN df -h
 
-RUN rm cloud-mu-*.gem
+RUN ls -la
+
+#RUN rm --verbose -f cloud-mu-*.gem
+
+RUN pip install pywinrm
 
 RUN apt-get remove -y build-essential ruby2.5-dev
 
 RUN apt-get autoremove -y
 
+RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.17.4/bin/linux/amd64/kubectl && mv kubectl /usr/bin && chmod +x /usr/bin/kubectl
+
 EXPOSE 2260
 
 CMD /usr/sbin/init

data/ansible/roles/mu-windows/README.md

@@ -0,0 +1,33 @@
+Role Name
+=========
+
+Used during image builds for Windows 2012R2, 2016, and 2019. Runs all updates and installs chocolatey, git, openssh, and ruby.
+
+Requirements
+------------
+
+Windows host with internet connectivity and no other major services running.
+
+License
+-------
+
+Copyright:: Copyright (c) 2020 eGlobalTech, Inc., all rights reserved
+
+Licensed under the BSD-3 license (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License in the root of the project or at
+
+    http://egt-labs.com/mu/LICENSE.html
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Author Information
+------------------
+
+Current developers: John Stange, Robert Patt-Corner, Ryan Bolyard
+
+egt-labs-admins@egt-labs.com

data/ansible/roles/mu-windows/defaults/main.yml

@@ -0,0 +1,2 @@
+---
+# defaults file for mu-windows

data/ansible/roles/mu-windows/files/LaunchConfig.json

@@ -0,0 +1,9 @@
+{
+  "setComputerName": false,
+  "setMonitorAlwaysOn": true,
+  "setWallpaper": true,
+  "addDnsSuffixList": true,
+  "extendBootVolumeSize": true,
+  "handleUserData": true,
+  "adminPasswordType": "Random"
+}

data/ansible/roles/mu-windows/files/config.xml

@@ -0,0 +1,76 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Ec2ConfigurationSettings>
+  <Plugins>
+    <Plugin>
+      <Name>Ec2SetPassword</Name>
+      <State>Enabled</State>
+    </Plugin>
+    <Plugin>
+      <Name>Ec2SetComputerName</Name>
+      <State>Disabled</State>
+    </Plugin>
+    <Plugin>
+      <Name>Ec2InitializeDrives</Name>
+      <State>Enabled</State>
+    </Plugin>
+    <Plugin>
+      <Name>Ec2EventLog</Name>
+      <State>Disabled</State>
+    </Plugin>
+    <Plugin>
+      <Name>Ec2ConfigureRDP</Name>
+      <State>Disabled</State>
+    </Plugin>
+    <Plugin>
+      <Name>Ec2OutputRDPCert</Name>
+      <State>Enabled</State>
+    </Plugin>
+    <Plugin>
+      <Name>Ec2SetDriveLetter</Name>
+      <State>Enabled</State>
+    </Plugin>
+    <Plugin>
+      <Name>Ec2WindowsActivate</Name>
+      <State>Enabled</State>
+    </Plugin>
+    <Plugin>
+      <Name>Ec2DynamicBootVolumeSize</Name>
+      <State>Disabled</State>
+    </Plugin>
+    <Plugin>
+      <Name>Ec2SetHibernation</Name>
+      <State>Enabled</State>
+    </Plugin>
+    <Plugin>
+      <Name>Ec2SetMonitorAlwaysOn</Name>
+      <State>Disabled</State>
+    </Plugin>
+    <Plugin>
+      <Name>Ec2ElasticGpuSetup</Name>
+      <State>Enabled</State>
+    </Plugin>
+    <Plugin>
+      <Name>Ec2FeatureLogging</Name>
+      <State>Enabled</State>
+    </Plugin>
+    <Plugin>
+      <Name>Ec2SetENAConfig</Name>
+      <State>Enabled</State>
+    </Plugin>
+    <Plugin>
+      <Name>Ec2HandleUserData</Name>
+      <State>Enabled</State>
+    </Plugin>
+    <Plugin>
+      <Name>AWS.EC2.Windows.CloudWatch.PlugIn</Name>
+      <State>Disabled</State>
+    </Plugin>
+  </Plugins>
+  <GlobalSettings>
+    <ManageShutdown>true</ManageShutdown>
+    <SetDnsSuffixList>true</SetDnsSuffixList>
+    <WaitForMetaDataAvailable>true</WaitForMetaDataAvailable>
+    <ShouldAddRoutes>true</ShouldAddRoutes>
+    <RemoveCredentialsfromSysprepOnStartup>true</RemoveCredentialsfromSysprepOnStartup>
+  </GlobalSettings>
+</Ec2ConfigurationSettings>

data/ansible/roles/mu-windows/handlers/main.yml

@@ -0,0 +1,2 @@
+---
+# handlers file for mu-windows

data/ansible/roles/mu-windows/meta/main.yml

@@ -0,0 +1,53 @@
+galaxy_info:
+  author: your name
+  description: your description
+  company: your company (optional)
+
+  # If the issue tracker for your role is not on github, uncomment the
+  # next line and provide a value
+  # issue_tracker_url: http://example.com/issue/tracker
+
+  # Choose a valid license ID from https://spdx.org - some suggested licenses:
+  # - BSD-3-Clause (default)
+  # - MIT
+  # - GPL-2.0-or-later
+  # - GPL-3.0-only
+  # - Apache-2.0
+  # - CC-BY-4.0
+  license: license (GPL-2.0-or-later, MIT, etc)
+
+  min_ansible_version: 2.4
+
+  # If this a Container Enabled role, provide the minimum Ansible Container version.
+  # min_ansible_container_version:
+
+  #
+  # Provide a list of supported platforms, and for each platform a list of versions.
+  # If you don't wish to enumerate all versions for a particular platform, use 'all'.
+  # To view available platforms and versions (or releases), visit:
+  # https://galaxy.ansible.com/api/v1/platforms/
+  #
+  # platforms:
+  # - name: Fedora
+  #   versions:
+  #   - all
+  #   - 25
+  # - name: SomePlatform
+  #   versions:
+  #   - all
+  #   - 1.0
+  #   - 7
+  #   - 99.99
+
+  galaxy_tags: []
+    # List tags for your role here, one per line. A tag is a keyword that describes
+    # and categorizes the role. Users find roles by searching for tags. Be sure to
+    # remove the '[]' above, if you add tags to this list.
+    #
+    # NOTE: A tag is limited to a single word comprised of alphanumeric characters.
+    #       Maximum 20 tags per role.
+
+dependencies: []
+  # List your role dependencies here, one per line. Be sure to remove the '[]' above,
+  # if you add dependencies to this list.
+  

data/ansible/roles/mu-windows/tasks/main.yml

@@ -0,0 +1,36 @@
+---
+- name: Install all updates with automatic reboots
+  win_updates:
+    reboot: yes
+    reboot_timeout: 1800
+
+- name: Install git
+  win_chocolatey:
+    name: git
+    state: present
+
+- name: Install ruby
+  win_chocolatey:
+    name: ruby
+    state: present
+
+- name: Install openssh
+  win_chocolatey:
+    name: openssh
+    state: present
+
+- name: "Tell EC2Config to set a random password on next boot (Windows 2012)"
+  when: ((ansible_facts['distribution_major_version'] | int) < 10 and mu_build_image is defined and mu_build_image == True)
+  win_copy:
+    src: config.xml
+    dest: "c:/Program Files/Amazon/EC2ConfigService/Settings/config.xml"
+
+- name: "Tell EC2Launch to set a random password (Windows 2016+)"
+  when: ((ansible_facts['distribution_major_version'] | int) >= 10 and mu_build_image is defined and mu_build_image == True)
+  win_copy:
+    src: LaunchConfig.json
+    dest: "c:/ProgramData/Amazon/EC2-Windows/Launch/Config/LaunchConfig.json"
+
+- name: "Tell EC2Launch to run on next boot (Windows 2016+)"
+  when: ((ansible_facts['distribution_major_version'] | int) >= 10 and mu_build_image is defined and mu_build_image == True)
+  win_shell: C:\ProgramData\Amazon\EC2-Windows\Launch\Scripts\InitializeInstance.ps1 -Schedule

data/ansible/roles/mu-windows/tests/inventory

@@ -0,0 +1,2 @@
+localhost
+

data/ansible/roles/mu-windows/tests/test.yml

@@ -0,0 +1,5 @@
+---
+- hosts: localhost
+  remote_user: root
+  roles:
+    - mu-windows

data/ansible/roles/mu-windows/vars/main.yml

@@ -0,0 +1,2 @@
+---
+# vars file for mu-windows

data/bin/mu-adopt

@@ -21,12 +21,6 @@ require 'bundler/setup'
 require 'optimist'
 require 'mu'
 
-available_clouds = MU::Cloud.supportedClouds
-available_clouds.reject! { |cloud|
-  cloudclass = Object.const_get("MU").const_get("Cloud").const_get(cloud)
-  cloudclass.listCredentials.nil? or cloudclass.listCredentials.size == 0
-}
-
 available_types = MU::Cloud.resource_types.keys.map { |t| t.to_s }
 grouping_options = {
   "logical" => "Group resources in logical layers (folders and habitats together, users/roles/groups together, network resources together, etc)",
@@ -39,15 +33,19 @@ $opt = Optimist::options do
   EOS
   opt :appname, "The overarching name of the application stack we will generate", :required => false, :default => "mu", :type => :string
   opt :types, "The resource types to scan and import. Valid types: #{available_types.join(", ")}", :required => false, :type => :strings, :default => available_types
-  opt :clouds, "The cloud providers to scan and import.", :required => false, :type => :strings, :default => available_clouds
+  opt :clouds, "The cloud providers to scan and import.", :required => false, :type => :strings, :default => MU::Cloud.availableClouds
   opt :parent, "Where applicable, resources which reside in the root folder or organization are configured with the specified parent in our target BoK", :required => false, :type => :string
   opt :billing, "Force-set this billing entity on created resources, instead of copying from the live resources", :required => false, :type => :string
   opt :sources, "One or more sets of credentials to use when importing resources. By default we will search and import from all sets of available credentials for each cloud provider specified with --clouds", :required => false, :type => :strings
   opt :credentials, "Override the 'credentials' value in our generated Baskets of Kittens to target a single, specific account. Our default behavior is to set each resource to deploy into the account from which it was sourced.", :required => false, :type => :string
   opt :savedeploys, "Generate actual deployment metadata in #{MU.dataDir}/deployments, as though the resources we found were created with mu-deploy. If we are generating more than one configuration, and a resource needs to reference another resource (e.g. to declare a VPC in which to reside), this will allow us to reference them as virtual resource, rather than by raw cloud identifier.", :required => false, :type => :boolean, :default => false
   opt :diff, "List the differences between what we find and an existing, saved deploy from a previous run, if one exists.", :required => false, :type => :boolean
+  opt :merge_changes, "When using --diff, merge detected changes into the baseline deploy after reporting on them.", :required => false, :type => :boolean, :default => false
   opt :grouping, "Methods for grouping found resources into separate Baskets.\n\n"+MU::Adoption::GROUPMODES.keys.map { |g| "* "+g.to_s+": "+MU::Adoption::GROUPMODES[g] }.join("\n")+"\n\n", :required => false, :type => :string, :default => "logical"
   opt :habitats, "Limit scope of searches to the named accounts/projects/subscriptions, instead of search all habitats visible to our credentials.", :required => false, :type => :strings
+  opt :regions, "Restrict to operating on a subset of available regions, instead of all that we know about.", :require => false, :type => :strings
+  opt :scrub, "Whether to set scrub_mu_isms in the BoKs we generate", :default => $MU_CFG.has_key?('adopt_scrub_mu_isms') ? $MU_CFG['adopt_scrub_mu_isms'] : false
+  opt :pattern, "Only adopt resources whose resource name would match this pattern. Must be a valid regular expression. Alphabetical characters will be treated case-insensitively.", :required => false, :type => :string
 end
 
 ok = true
@@ -63,6 +61,16 @@ if $opt[:diff]
   $opt[:savedeploys] = false
 end
 
+pattern = nil
+if $opt[:pattern]
+  begin
+    pattern = Regexp.new($opt[:pattern], true)
+  rescue RegexpError => e
+    MU.log "Invalid --pattern option: #{e.message}", MU::ERR
+    exit 1
+  end
+end
+
 types = []
 $opt[:types].each { |t|
   t_name = t.gsub(/-/, "_")
@@ -101,9 +109,12 @@ if !ok
   exit 1
 end
 
-
-adoption = MU::Adoption.new(clouds: clouds, types: types, parent: $opt[:parent], billing: $opt[:billing], sources: $opt[:sources], credentials: $opt[:credentials], group_by: $opt[:grouping].to_sym, savedeploys: $opt[:savedeploys], diff: $opt[:diff], habitats: $opt[:habitats])
-adoption.scrapeClouds
+adoption = MU::Adoption.new(clouds: clouds, types: types, parent: $opt[:parent], billing: $opt[:billing], sources: $opt[:sources], credentials: $opt[:credentials], group_by: $opt[:grouping].to_sym, savedeploys: $opt[:savedeploys], diff: $opt[:diff], habitats: $opt[:habitats], scrub_mu_isms: $opt[:scrub], regions: $opt[:regions], merge: $opt[:merge_changes], pattern: pattern)
+found = adoption.scrapeClouds
+if found.nil? or found.empty?
+  MU.log "No resources found to adopt", MU::WARN, details: {"clouds" => clouds, "types" => types }
+  exit
+end
 MU.log "Generating baskets", MU::DEBUG
 boks = adoption.generateBaskets(prefix: $opt[:appname])
 
@@ -112,10 +123,7 @@ boks.each_pair { |appname, bok|
   File.open("#{appname}.yaml", "w") { |f|
     f.write JSON.parse(JSON.generate(bok)).to_yaml
   }
-  conf_engine = MU::Config.new("#{appname}.yaml")
-  stack_conf = conf_engine.config
 #  puts stack_conf.to_yaml
-  MU.log "#{appname}.yaml validated successfully", MU::NOTICE
   MU::Cloud.resource_types.each_pair { |type, cfg|
     if bok[cfg[:cfg_plural]]
       MU.log "#{bok[cfg[:cfg_plural]].size.to_s} #{cfg[:cfg_plural]}", MU::NOTICE

data/bin/mu-azure-tests

@@ -0,0 +1,57 @@
+#!/usr/local/ruby-current/bin/ruby
+# Copyright:: Copyright (c) 2014 eGlobalTech, Inc., all rights reserved
+#
+# Licensed under the BSD-3 license (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License in the root of the project or at
+#
+#     http://egt-labs.com/mu/LICENSE.html
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'rubygems'
+require 'bundler/setup'
+require 'json'
+require 'erb'
+require 'optimist'
+require 'json-schema'
+require File.realpath(File.expand_path(File.dirname(__FILE__)+"/mu-load-config.rb"))
+require 'mu'
+
+(0..100000).to_a.each { |n|
+retries = 0
+seed = nil
+#        begin
+#          raise MuError, "Failed to allocate an unused MU-ID after #{retries} tries!" if retries > 70
+#          seedsize = 1 + (retries/10).abs
+#          seed = (0...seedsize+1).map { ('a'..'z').to_a[rand(26)] }.join
+#        end while seed == "mu" or seed[0] == seed[1]
+seed = "nn"
+handle = MU::MommaCat.generateHandle(seed)
+puts handle
+}
+exit
+
+#pp MU::Cloud::Azure.listRegions
+#pp MU::Cloud::Azure::Habitat.testcalls
+#pp MU::Cloud::Azure::VPC.find(cloud_id: MU::Cloud::Azure::Id.new(resource_group: "mu", name: "mu-vnet"))
+#pp MU::Cloud::Azure.authorization.role_assignments.list_for_resource_group("AKS-DEV-2019062015-KA-EASTUS")
+#pp MU::Cloud::Azure::Role.find(role_name: "Azure Kubernetes Service Cluster Admin Role")
+#puts MU::Cloud::Azure.default_subscription
+#pp MU::Cloud::Azure.fetchPublicIP("MYVPC-DEV-2019061911-XI-EASTUS", "ip-addr-thingy")
+#pp MU::Cloud::Azure.ensureProvider("egtazure", "Microsoft.ContainerService", force: true)
+pp MU::Cloud::Azure::Server.find(cloud_id: "mu")
+exit
+pp MU::Cloud::Azure::Server.fetchImage("OpenLogic/CentOS/6")
+pp MU::Cloud::Azure::Server.fetchImage("OpenLogic/CentOS/7")
+pp MU::Cloud::Azure::Server.fetchImage("RedHat/RHEL/8")
+pp MU::Cloud::Azure::Server.fetchImage("RedHat/RHEL/7")
+pp MU::Cloud::Azure::Server.fetchImage("RedHat/RHEL/6")
+pp MU::Cloud::Azure::Server.fetchImage("Debian/debian-10/10")
+pp MU::Cloud::Azure::Server.fetchImage("MicrosoftWindowsServer/WindowsServer/2012-R2-Datacenter")
+pp MU::Cloud::Azure::Server.fetchImage("MicrosoftWindowsServer/WindowsServer/2016-Datacenter")
+pp MU::Cloud::Azure::Server.fetchImage("MicrosoftWindowsServer/WindowsServer/2019-Datacenter")

data/bin/mu-cleanup

@@ -24,10 +24,8 @@ require 'mu'
 Dir.chdir(MU.installDir)
 
 credentials = []
-MU::Cloud.supportedClouds.each { |cloud|
-  cloudclass = Object.const_get("MU").const_get("Cloud").const_get(cloud)
-  next if cloudclass.listCredentials.nil? or cloudclass.listCredentials.size == 0
-  credentials.concat(cloudclass.listCredentials)
+MU::Cloud.availableClouds.each { |cloud|
+  credentials.concat(MU::Cloud.cloudClass(cloud).listCredentials)
 }
 credentials.uniq!
 

data/bin/mu-configure

@@ -113,6 +113,44 @@ $CONFIGURABLES = {
     "desc" => "Disable the Momma Cat grooming daemon. Nodes which require asynchronous Ansible/Chef bootstraps will not function. This option is only honored in gem-based installations.",
     "boolean" => true
   },
+  "adopt_change_notify" => {
+    "title" => "Adoption Change Notifications",
+    "subtree" => {
+      "slack" => {
+        "title" => "Send to Slack",
+        "desc" => "Report modifications to adopted resources, detected by mu-adopt --diff, to the Slack webhook and channel configured under Slack Configuration.",
+        "boolean" => true
+      },
+      "slack_snippet_threshold" => {
+        "title" => "Attachment Threshold",
+        "desc" => "If a list of details about a modified resources is longer than this number of lines (in JSON), it will be sent as an \"attachment,\" which in Slack means a blockquote that displays a few lines with a \"Show more\" button. The internal default is 5 lines."
+      },
+#      "email" => {
+#        "title" => "Send Email",
+#        "desc" => "",
+#        "boolean" => true
+#      }
+    }
+  },
+  "adopt_scrub_mu_isms" => {
+    "title" => "Scrub Mu-isms from Baskets of Kittens",
+    "default" => false,
+    "desc" => "Ordinarily, Mu will automatically name, tag and generate auxiliary resources in a standard Mu-ish fashion that allows for deployment of multiple clones of a given stack. Toggling this flag will change the default behavior of mu-adopt, when it creates stack descriptors from found resources, to enable or disable this behavior (see also mu-adopt's --scrub option).",
+    "boolean" => true
+  },
+  "slack" => {
+    "title" => "Slack Configuration",
+    "subtree" => {
+      "webhook" => {
+        "title" => "Webhook",
+        "desc" => "The hooks.slack.com URL for the webook to which we'll send deploy notifications"
+      },
+      "channel" => {
+        "title" => "Channel",
+        "desc" => "The channel name (without leading #) to which alerts should be sent."
+      }
+    }
+  },
   "mommacat_port" => {
     "title" => "Momma Cat Listen Port",
     "pattern" => /^[0-9]+$/i,
@@ -241,11 +279,25 @@ $CONFIGURABLES = {
         "required" => false,
         "desc" => "For Google Cloud projects which are attached to a GSuite domain. GCP service accounts cannot view or manage GSuite resources (groups, users, etc) directly, but must instead masquerade as a GSuite user which has delegated authority to the service account. See also: https://developers.google.com/identity/protocols/OAuth2ServiceAccount#delegatingauthority"
       },
+      "org" => {
+        "title" => "Default Org/Domain",
+        "desc" => "For credential sets which have access to multiple GSuite or Cloud Identity orgs, you must specify a default organization (e.g. my.domain.com)."
+      },
       "customer_id" => {
         "title" => "GSuite Customer ID",
         "required" => false,
         "desc" => "For Google Cloud projects which are attached to a GSuite domain. Some API calls (groups, users, etc) require this identifier. From admin.google.com, choose Security, the Single Sign On, and look for the Entity ID field. The value after idpid= in the URL there should be the customer ID."
       },
+      "ignore_habitats" => {
+        "title" => "Ignore These Projects",
+        "desc" => "Optional list of projects to ignore, for credentials which have visibility into multiple projects",
+        "array" => true
+      },
+      "restrict_to_habitats" => {
+        "title" => "Operate On Only These Projects",
+        "desc" => "Optional list of projects to which we'll restrict all of our activities.",
+        "array" => true
+      },
       "default" => {
         "title" => "Is Default Account",
         "default" => false,