cloud-mu 3.1.3 → 3.3.0

data/modules/mu/config/database.yml

@@ -4,7 +4,7 @@ name: database-complex
 size: db.r4.large
 engine: postgres
 engine_version: 9.6.6
-storage: 5
+storage: 21
 add_cluster_node: true
 allow_major_version_upgrade: true
 auto_minor_version_upgrade: false
@@ -24,6 +24,5 @@ vpc:
   name: <%= vpc_name %>
 size: <%= db_size %>
 engine: mariadb
-storage: 5
 
 <% end %>

data/modules/mu/config/dnszone.rb

@@ -14,7 +14,7 @@
 
 module MU
   class Config
-    # Basket of Kittens config schema and parser logic. See modules/mu/clouds/*/dnszone.rb
+    # Basket of Kittens config schema and parser logic. See modules/mu/providers/*/dnszone.rb
     class DNSZone
 
       # Base configuration schema for a DNSZone
@@ -60,7 +60,7 @@ module MU
       # @param default_type [String]: The type of record to make default (e.g. An, CNAME, etc)
       # @param need_zone [Boolean]: Whether to explicitly require a zone be declared
       # @return [Hash]
-      def self.records_primitive(need_target: true, default_type: nil, need_zone: false)
+      def self.records_primitive(need_target: true, default_type: nil, need_zone: false, embedded_type: nil)
         dns_records_primitive = {
           "type" => "array",
           "maxItems" => 100,
@@ -107,8 +107,9 @@ module MU
                   },
                   "mu_type" => {
                     "type" => "string",
-                    "description" => "The Mu resource type to search the deployment for.",
-                      "enum" => ["loadbalancer", "server", "database", "cache_cluster"]
+                    "description" => "The mu type of a resource being targeted.",
+                    "enum" => embedded_type ? [embedded_type] : ["loadbalancer", "server", "database", "cache_cluster", "endpoint", "cdn"],
+                    "default" => embedded_type
                   },
                   "target_type" => {
                       "description" => "If the target is a public or a private resource. This only applies to servers/server_pools when using automatic DNS registration. If set to public but the target only has a private address, the private address will be used",
@@ -312,15 +313,15 @@ module MU
       end
 
       # Generic pre-processing of {MU::Config::BasketofKittens::dnszones}, bare and unvalidated.
-      # @param zone [Hash]: The resource to process and validate
-      # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
+      # @param _zone [Hash]: The resource to process and validate
+      # @param _configurator [MU::Config]: The overall deployment configurator of which this resource is a member
       # @return [Boolean]: True if validation succeeded, False otherwise
-      def self.validate(zone, configurator)
+      def self.validate(_zone, _configurator)
         ok = true
-        ok
-      end
 # TODO non-local VPCs are valid, but require an account field, which insertKitten doesn't know anything about
 # if !zone['account'].nil? and zone['account'] != MU.account_number
+        ok
+      end
 
     end
   end

data/modules/mu/config/doc_helpers.rb

@@ -0,0 +1,516 @@
+# Copyright:: Copyright (c) 2020 eGlobalTech, Inc., all rights reserved
+#
+# Licensed under the BSD-3 license (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License in the root of the project or at
+#
+#     http://egt-labs.com/mu/LICENSE.html
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module MU
+
+  # Methods and structures for parsing Mu's configuration files. See also {MU::Config::BasketofKittens}.
+  class Config
+
+    # Accessor for our Basket of Kittens schema definition, with various
+    # cloud-specific details merged so we can generate documentation for them.
+    def self.docSchema
+      docschema = Marshal.load(Marshal.dump(@@schema))
+      only_children = {}
+      MU::Cloud.resource_types.each_pair { |classname, attrs|
+        MU::Cloud.supportedClouds.each { |cloud|
+          begin
+            require "mu/providers/#{cloud.downcase}/#{attrs[:cfg_name]}"
+          rescue LoadError
+            next
+          end
+          _required, res_schema = MU::Cloud.resourceClass(cloud, classname).schema(self)
+          docschema["properties"][attrs[:cfg_plural]]["items"]["description"] ||= ""
+          docschema["properties"][attrs[:cfg_plural]]["items"]["description"] += "\n#\n# `#{cloud}`: "+MU::Cloud.resourceClass(cloud, classname).quality
+          res_schema.each { |key, cfg|
+            if !docschema["properties"][attrs[:cfg_plural]]["items"]["properties"][key]
+              only_children[attrs[:cfg_plural]] ||= {}
+              only_children[attrs[:cfg_plural]][key] ||= {}
+              only_children[attrs[:cfg_plural]][key][cloud] = cfg
+            end
+          }
+        }
+      }
+
+      # recursively chase down description fields in arrays and objects of our
+      # schema and prepend stuff to them for documentation
+      def self.prepend_descriptions(prefix, cfg)
+        cfg["prefix"] = prefix
+        if cfg["type"] == "array" and cfg["items"]
+          cfg["items"] = prepend_descriptions(prefix, cfg["items"])
+        elsif cfg["type"] == "object" and cfg["properties"]
+          cfg["properties"].keys.each { |key|
+            cfg["properties"][key] = prepend_descriptions(prefix, cfg["properties"][key])
+          }
+        end
+        cfg
+      end
+
+      MU::Cloud.resource_types.each_pair { |classname, attrs|
+        MU::Cloud.supportedClouds.each { |cloud|
+          res_class = nil
+          begin
+            res_class = MU::Cloud.resourceClass(cloud, classname)
+          rescue MU::Cloud::MuCloudResourceNotImplemented
+            next
+          end
+          required, res_schema = res_class.schema(self)
+          next if required.size == 0 and res_schema.size == 0
+          res_schema.each { |key, cfg|
+            cfg["description"] ||= ""
+            if !cfg["description"].empty?
+              cfg["description"] = "\n# +"+cloud.upcase+"+: "+cfg["description"]
+            end
+            if docschema["properties"][attrs[:cfg_plural]]["items"]["properties"][key]
+              schemaMerge(docschema["properties"][attrs[:cfg_plural]]["items"]["properties"][key], cfg, cloud)
+              docschema["properties"][attrs[:cfg_plural]]["items"]["properties"][key]["description"] ||= ""
+              docschema["properties"][attrs[:cfg_plural]]["items"]["properties"][key]["description"] += "\n"+(cfg["description"].match(/^#/) ? "" : "# ")+cfg["description"]
+              MU.log "Munging #{cloud}-specific #{classname.to_s} schema into BasketofKittens => #{attrs[:cfg_plural]} => #{key}", MU::DEBUG, details: docschema["properties"][attrs[:cfg_plural]]["items"]["properties"][key]
+            else
+              if only_children[attrs[:cfg_plural]][key]
+                prefix = only_children[attrs[:cfg_plural]][key].keys.map{ |x| x.upcase }.join(" & ")+" ONLY"
+                cfg["description"].gsub!(/^\n#/, '') # so we don't leave the description blank in the "optional parameters" section
+                cfg = prepend_descriptions(prefix, cfg)
+              end
+
+              docschema["properties"][attrs[:cfg_plural]]["items"]["properties"][key] = cfg
+            end
+            docschema["properties"][attrs[:cfg_plural]]["items"]["properties"][key]["clouds"] = {}
+            docschema["properties"][attrs[:cfg_plural]]["items"]["properties"][key]["clouds"][cloud] = cfg
+          }
+
+          docschema['required'].concat(required)
+          docschema['required'].uniq!
+        }
+      }
+
+      docschema
+    end
+
+    # Output the dependencies of this BoK stack as a directed acyclic graph.
+    # Very useful for debugging.
+    def visualizeDependencies
+      # GraphViz won't like MU::Config::Tail, pare down to plain Strings
+      config = MU::Config.stripConfig(@config)
+      begin
+        g = GraphViz.new(:G, :type => :digraph)
+        # Generate a GraphViz node for each resource in this stack
+        nodes = {}
+        MU::Cloud.resource_types.each_pair { |classname, attrs|
+          nodes[attrs[:cfg_name]] = {}
+          if config.has_key?(attrs[:cfg_plural]) and config[attrs[:cfg_plural]]
+            config[attrs[:cfg_plural]].each { |resource|
+              nodes[attrs[:cfg_name]][resource['name']] = g.add_nodes("#{classname}: #{resource['name']}")
+            }
+          end
+        }
+        # Now add edges corresponding to the dependencies they list
+        MU::Cloud.resource_types.values.each { |attrs|
+          if config.has_key?(attrs[:cfg_plural]) and config[attrs[:cfg_plural]]
+            config[attrs[:cfg_plural]].each { |resource|
+              if resource.has_key?("dependencies")
+                me = nodes[attrs[:cfg_name]][resource['name']]
+                resource["dependencies"].each { |dep|
+                  parent = nodes[dep['type']][dep['name']]
+                  g.add_edges(me, parent)
+                }
+              end
+            }
+          end
+        }
+        # Spew some output?
+        MU.log "Emitting dependency graph as /tmp/#{config['appname']}.jpg", MU::NOTICE
+        g.output(:jpg => "/tmp/#{config['appname']}.jpg")
+      rescue StandardError => e
+        MU.log "Failed to generate GraphViz dependency tree: #{e.inspect}. This should only matter to developers.", MU::WARN, details: e.backtrace
+      end
+    end
+
+    # Generate a documentation-friendly dummy Ruby class for our mu.yaml main
+    # config.
+    def self.emitConfigAsRuby
+      example = %Q{---
+public_address: 1.2.3.4
+mu_admin_email: egtlabs@eglobaltech.com
+mu_admin_name: Joe Schmoe
+mommacat_port: 2260
+banner: My Example Mu Master
+mu_repository: git://github.com/cloudamatic/mu.git
+repos:
+- https://github.com/cloudamatic/mu_demo_platform
+allow_invade_foreign_vpcs: true
+ansible_dir:
+aws:
+  egtdev:
+    region: us-east-1
+    log_bucket_name: egt-mu-log-bucket
+    default: true
+    name: egtdev
+  personal:
+    region: us-east-2
+    log_bucket_name: my-mu-log-bucket
+    name: personal
+  google:
+    egtlabs:
+      project: egt-labs-admin
+      credentials_file: /opt/mu/etc/google.json
+      region: us-east4
+      log_bucket_name: hexabucket-761234
+      default: true
+}
+      mu_yaml_schema = eval(%Q{
+$NOOP = true
+load "#{MU.myRoot}/bin/mu-configure"
+$CONFIGURABLES
+})
+      return if mu_yaml_schema.nil? or !mu_yaml_schema.is_a?(Hash)
+      muyamlpath = "#{MU.myRoot}/modules/mu/mu.yaml.rb"
+      MU.log "Converting mu.yaml schema to Ruby objects in #{muyamlpath}"
+      muyaml_rb = File.new(muyamlpath, File::CREAT|File::TRUNC|File::RDWR, 0644)
+      muyaml_rb.puts "# Configuration schema for mu.yaml. See also {https://github.com/cloudamatic/mu/wiki/Configuration the Mu wiki}."
+      muyaml_rb.puts "#"
+      muyaml_rb.puts "# Example:"
+      muyaml_rb.puts "#"
+      muyaml_rb.puts "# <pre>"
+      example.split(/\n/).each { |line|
+        muyaml_rb.puts "#      "+line+"    " # markdooooown
+      }
+      muyaml_rb.puts "# </pre>"
+      muyaml_rb.puts "module MuYAML"
+      muyaml_rb.puts "\t# The configuration file format for Mu's main config file."
+      MU::Config.printMuYamlSchema(muyaml_rb, [], { "subtree" => mu_yaml_schema })
+      muyaml_rb.puts "end"
+      muyaml_rb.close
+    end
+
+    # Take the schema we've defined and create a dummy Ruby class tree out of
+    # it, basically so we can leverage Yard to document it.
+    def self.emitSchemaAsRuby
+      kittenpath = "#{MU.myRoot}/modules/mu/kittens.rb"
+      MU.log "Converting Basket of Kittens schema to Ruby objects in #{kittenpath}"
+      kitten_rb = File.new(kittenpath, File::CREAT|File::TRUNC|File::RDWR, 0644)
+      kitten_rb.puts "### THIS FILE IS AUTOMATICALLY GENERATED, DO NOT EDIT ###"
+      kitten_rb.puts "#"
+      kitten_rb.puts "#"
+      kitten_rb.puts "#"
+      kitten_rb.puts "module MU"
+      kitten_rb.puts "class Config"
+      kitten_rb.puts "\t# The configuration file format for Mu application stacks."
+      self.printSchema(kitten_rb, ["BasketofKittens"], MU::Config.docSchema)
+      kitten_rb.puts "end"
+      kitten_rb.puts "end"
+      kitten_rb.close
+
+    end
+
+    # Emit our Basket of Kittens schema in a format that YARD can comprehend
+    # and turn into documentation.
+    def self.printSchema(kitten_rb, class_hierarchy, schema, in_array = false, required = false, prefix: nil)
+      return if schema.nil?
+
+      if schema["type"] == "object"
+        printme = []
+
+        if !schema["properties"].nil?
+          # order sub-elements by whether they're required, so we can use YARD's
+          # grouping tags on them
+          if !schema["required"].nil? and schema["required"].size > 0
+            prop_list = schema["properties"].keys.sort_by { |name|
+              schema["required"].include?(name) ? 0 : 1
+            }
+          else
+            prop_list = schema["properties"].keys
+          end
+          req = false
+          printme << "# @!group Optional parameters" if schema["required"].nil? or schema["required"].size == 0
+          prop_list.each { |name|
+            prop = schema["properties"][name]
+
+            if class_hierarchy.size == 1
+
+              _shortclass, cfg_name, cfg_plural, _classname = MU::Cloud.getResourceNames(name, false)
+              if cfg_name
+                example_path = MU.myRoot+"/modules/mu/config/"+cfg_name+".yml"
+                if File.exist?(example_path)
+                  example = "#\n# Examples:\n#\n"
+                  # XXX these variables are all parameters from the BoKs in
+                  # modules/tests. A really clever implementation would read
+                  # and parse them to get default values, perhaps, instead of
+                  # hard-coding them here.
+                  instance_type = "t2.medium"
+                  db_size = "db.t2.medium"
+                  vpc_name = "some_vpc"
+                  logs_name = "some_loggroup"
+                  queues_name = "some_queue"
+                  server_pools_name = "some_server_pool"
+                  ["simple", "complex"].each { |complexity|
+                    erb = ERB.new(File.read(example_path), nil, "<>")
+                    example += "#      !!!yaml\n"
+                    example += "#      ---\n"
+                    example += "#      appname: #{complexity}\n"
+                    example += "#      #{cfg_plural}:\n"
+                    firstline = true
+                    erb.result(binding).split(/\n/).each { |l|
+                      l.chomp!
+                      l.sub!(/#.*/, "") if !l.match(/#(?:INTERNET|NAT|DENY)/)
+                      next if l.empty? or l.match(/^\s+$/)
+                      if firstline
+                        l = "- "+l
+                        firstline = false
+                      else
+                        l = "  "+l
+                      end
+                      example += "#      "+l+"    "+"\n"
+                    }
+                    example += "# &nbsp;\n#\n" if complexity == "simple"
+                  }
+                  schema["properties"][name]["items"]["description"] ||= ""
+                  if !schema["properties"][name]["items"]["description"].empty?
+                    schema["properties"][name]["items"]["description"] += "\n"
+                  end
+                  schema["properties"][name]["items"]["description"] += example
+                end
+              end
+            end
+
+            if !schema["required"].nil? and schema["required"].include?(name)
+              printme << "# @!group Required parameters" if !req
+              req = true
+            else
+              if req
+                printme << "# @!endgroup"
+                printme << "# @!group Optional parameters"
+              end
+              req = false
+            end
+
+            printme << self.printSchema(kitten_rb, class_hierarchy+ [name], prop, false, req, prefix: schema["prefix"])
+          }
+          printme << "# @!endgroup"
+        end
+
+        tabs = 1
+        class_hierarchy.each { |classname|
+          if classname == class_hierarchy.last and !schema['description'].nil?
+            kitten_rb.puts ["\t"].cycle(tabs).to_a.join('') + "# #{schema['description']}\n"
+          end
+          kitten_rb.puts ["\t"].cycle(tabs).to_a.join('') + "class #{classname}"
+          tabs = tabs + 1
+        }
+        printme.each { |lines|
+          if !lines.nil? and lines.is_a?(String)
+            lines.lines.each { |line|
+              kitten_rb.puts ["\t"].cycle(tabs).to_a.join('') + line
+            }
+          end
+        }
+
+        i = class_hierarchy.size
+        until i == 0 do
+          tabs = tabs - 1
+          kitten_rb.puts ["\t"].cycle(tabs).to_a.join('') + "end"
+          i -= 1
+        end
+
+        # And now that we've dealt with our children, pass our own rendered
+        # commentary back up to our caller.
+        name = class_hierarchy.last
+        if in_array
+          type = "Array<#{class_hierarchy.join("::")}>"
+        else
+          type = class_hierarchy.join("::")
+        end
+
+        docstring = "\n"
+        docstring = docstring + "# **REQUIRED**\n" if required
+        docstring = docstring + "# **"+schema["prefix"]+"**\n" if schema["prefix"]
+        docstring = docstring + "# #{schema['description'].gsub(/\n/, "\n#")}\n" if !schema['description'].nil?
+        docstring = docstring + "#\n"
+        docstring = docstring + "# @return [#{type}]\n"
+        docstring = docstring + "# @see #{class_hierarchy.join("::")}\n"
+        docstring = docstring + "attr_accessor :#{name}"
+        return docstring
+
+      elsif schema["type"] == "array"
+        return self.printSchema(kitten_rb, class_hierarchy, schema['items'], true, required, prefix: prefix)
+      else
+        name = class_hierarchy.last
+        if schema['type'].nil?
+          MU.log "Couldn't determine schema type in #{class_hierarchy.join(" => ")}", MU::WARN, details: schema
+          return nil
+        end
+        if in_array
+          type = "Array<#{schema['type'].capitalize}>"
+        else
+          type = schema['type'].capitalize
+        end
+        docstring = "\n"
+
+        prefixes = []
+        prefixes << "# **REQUIRED**" if required and schema['default'].nil?
+        prefixes << "# **"+schema["prefix"]+"**" if schema["prefix"]
+        prefixes << "# **Default: `#{schema['default']}`**" if !schema['default'].nil?
+        if !schema['enum'].nil? and !schema["enum"].empty?
+          prefixes << "# **Must be one of: `#{schema['enum'].join(', ')}`**"
+        elsif !schema['pattern'].nil?
+          # XXX unquoted regex chars confuse the hell out of YARD. How do we
+          # quote {}[] etc in YARD-speak?
+          prefixes << "# **Must match pattern `#{schema['pattern'].gsub(/\n/, "\n#")}`**"
+        end
+
+        if prefixes.size > 0
+          docstring += prefixes.join(",\n")
+          if schema['description'] and schema['description'].size > 1
+            docstring += " - "
+          end
+          docstring += "\n"
+        end
+
+        docstring = docstring + "# #{schema['description'].gsub(/\n/, "\n#")}\n" if !schema['description'].nil?
+        docstring = docstring + "#\n"
+        docstring = docstring + "# @return [#{type}]\n"
+        docstring = docstring + "attr_accessor :#{name}"
+
+        return docstring
+      end
+    end
+
+    # Emit our mu.yaml schema in a format that YARD can comprehend and turn into
+    # documentation.
+    def self.printMuYamlSchema(muyaml_rb, class_hierarchy, schema, in_array = false, required = false)
+      return if schema.nil?
+      if schema["subtree"]
+        printme = Array.new
+        # order sub-elements by whether they're required, so we can use YARD's
+        # grouping tags on them
+        have_required = schema["subtree"].keys.any? { |k| schema["subtree"][k]["required"] }
+        prop_list = schema["subtree"].keys.sort { |a, b|
+          if schema["subtree"][a]["required"] and !schema["subtree"][b]["required"]
+            -1
+          elsif !schema["subtree"][a]["required"] and schema["subtree"][b]["required"]
+            1
+          else
+            a <=> b
+          end
+        }
+
+        req = false
+        printme << "# @!group Optional parameters" if !have_required
+        prop_list.each { |name|
+          prop = schema["subtree"][name]
+          if prop["required"]
+            printme << "# @!group Required parameters" if !req
+            req = true
+          else
+            if req
+              printme << "# @!endgroup"
+              printme << "# @!group Optional parameters"
+            end
+            req = false
+          end
+
+          printme << self.printMuYamlSchema(muyaml_rb, class_hierarchy+ [name], prop, false, req)
+        }
+        printme << "# @!endgroup"
+
+        desc = (schema['desc'] || schema['title'])
+
+        tabs = 1
+        class_hierarchy.each { |classname|
+          if classname == class_hierarchy.last and desc
+            muyaml_rb.puts ["\t"].cycle(tabs).to_a.join('') + "# #{desc}\n"
+          end
+          muyaml_rb.puts ["\t"].cycle(tabs).to_a.join('') + "class #{classname}"
+          tabs = tabs + 1
+        }
+        printme.each { |lines|
+          if !lines.nil? and lines.is_a?(String)
+            lines.lines.each { |line|
+              muyaml_rb.puts ["\t"].cycle(tabs).to_a.join('') + line
+            }
+          end
+        }
+
+#        class_hierarchy.each { |classname|
+#          tabs = tabs - 1
+#          muyaml_rb.puts ["\t"].cycle(tabs).to_a.join('') + "end"
+#        }
+        i = class_hierarchy.size
+        until i == 0 do
+          tabs = tabs - 1
+          muyaml_rb.puts ["\t"].cycle(tabs).to_a.join('') + "end"
+          i -= 1
+        end
+
+        # And now that we've dealt with our children, pass our own rendered
+        # commentary back up to our caller.
+        name = class_hierarchy.last
+        if in_array
+          type = "Array<#{class_hierarchy.join("::")}>"
+        else
+          type = class_hierarchy.join("::")
+        end
+
+        docstring = "\n"
+        docstring = docstring + "# **REQUIRED**\n" if required
+#        docstring = docstring + "# **"+schema["prefix"]+"**\n" if schema["prefix"]
+        docstring = docstring + "# #{desc.gsub(/\n/, "\n#")}\n" if desc
+        docstring = docstring + "#\n"
+        docstring = docstring + "# @return [#{type}]\n"
+        docstring = docstring + "# @see #{class_hierarchy.join("::")}\n"
+        docstring = docstring + "attr_accessor :#{name}"
+        return docstring
+
+      else
+        in_array = schema["array"]
+        name = class_hierarchy.last
+        type = if schema['boolean']
+          "Boolean"
+        else
+          "String"
+        end
+        if in_array
+          type = "Array<#{type}>"
+        end
+        docstring = "\n"
+
+        prefixes = []
+        prefixes << "# **REQUIRED**" if schema["required"] and schema['default'].nil?
+#        prefixes << "# **"+schema["prefix"]+"**" if schema["prefix"]
+        prefixes << "# **Default: `#{schema['default']}`**" if !schema['default'].nil?
+        if !schema['pattern'].nil?
+          # XXX unquoted regex chars confuse the hell out of YARD. How do we
+          # quote {}[] etc in YARD-speak?
+          prefixes << "# **Must match pattern `#{schema['pattern'].to_s.gsub(/\n/, "\n#")}`**"
+        end
+
+        desc = (schema['desc'] || schema['title'])
+        if prefixes.size > 0
+          docstring += prefixes.join(",\n")
+          if desc and desc.size > 1
+            docstring += " - "
+          end
+          docstring += "\n"
+        end
+
+        docstring = docstring + "# #{desc.gsub(/\n/, "\n#")}\n" if !desc.nil?
+        docstring = docstring + "#\n"
+        docstring = docstring + "# @return [#{type}]\n"
+        docstring = docstring + "attr_accessor :#{name}"
+
+        return docstring
+      end
+    end
+
+  end #class
+end #module