cloud-mu 3.0.0beta → 3.0.0

data/modules/mu/config.rb

@@ -212,23 +212,25 @@ module MU
     # layers that don't care about the metadata in Tails.
     # @param config [Hash]: The configuration tree to convert
     # @return [Hash]: The modified configuration
-    def self.manxify(config)
+    def self.manxify(config, remove_runtime_keys: false)
       if config.is_a?(Hash)
         newhash = {}
         config.each_pair { |key, val|
-          newhash[key] = self.manxify(val)
+          next if remove_runtime_keys and key.match(/^#MU_/)
+          next if val.is_a?(Array) and val.empty?
+          newhash[key] = self.manxify(val, remove_runtime_keys: remove_runtime_keys)
         }
         config = newhash
       elsif config.is_a?(Array)
         newarray = []
         config.each { |val|
-          newarray << self.manxify(val)
+          newarray << self.manxify(val, remove_runtime_keys: remove_runtime_keys)
         }
         config = newarray
       elsif config.is_a?(MU::Config::Tail)
         return config.to_s
       elsif config.is_a?(MU::Config::Ref)
-        return config.to_h
+        return self.manxify(config.to_h,  remove_runtime_keys: remove_runtime_keys)
       end
       return config
     end
@@ -238,7 +240,7 @@ module MU
     # @param config [Hash]
     # @return [Hash]
     def self.stripConfig(config)
-      MU::Config.manxify(Marshal.load(Marshal.dump(MU.structToHash(config.dup))))
+      MU::Config.manxify(Marshal.load(Marshal.dump(MU.structToHash(config.dup))), remove_runtime_keys: true)
     end
 
     # A wrapper class for resources to refer to other resources, whether they
@@ -334,7 +336,7 @@ module MU
         end
 
         if @deploy_id and !@mommacat
-          @mommacat = MU::MommaCat.new(@deploy_id, set_context_to_me: false, create: false)
+          @mommacat = MU::MommaCat.getLitter(@deploy_id, set_context_to_me: false)
         elsif @mommacat and !@deploy_id
           @deploy_id = @mommacat.deploy_id
         end
@@ -569,6 +571,7 @@ return
 
       def initialize(name, value, prettyname = nil, cloudtype = "String", valid_values = [], description = "", is_list_element = false, prefix: "", suffix: "", pseudo: false, runtimecode: nil, index: 0)
         @name = name
+        @bindings = {}
         @value = value
         @valid_values = valid_values
         @pseudo = pseudo
@@ -764,7 +767,7 @@ return
 
       # Make sure our parameter values are all available in the local namespace
       # that ERB will be using, minus any that conflict with existing variables
-      erb_binding = get_binding
+      erb_binding = get_binding(@@tails.keys.sort)
       @@tails.each_pair { |key, tail|
         next if !tail.is_a?(MU::Config::Tail) or tail.is_list_element
         # XXX figure out what to do with lists
@@ -801,7 +804,7 @@ return
 
       begin
         config = JSON.parse(raw_json)
-        if param_pass
+        if param_pass and config.is_a?(Hash)
           config.keys.each { |key|
             if key != "parameters"
               if key == "appname" and @@parameters["myAppName"].nil?
@@ -812,7 +815,7 @@ return
               config.delete(key)
             end
           }
-        else
+        elsif config.is_a?(Hash)
           config.delete("parameters")
         end
       rescue JSON::ParserError => e
@@ -833,6 +836,7 @@ return
 
     attr_reader :kittens
     attr_reader :updating
+    attr_reader :existing_deploy
     attr_reader :kittencfg_semaphore
 
     # Load, resolve, and validate a configuration file ("Basket of Kittens").
@@ -858,6 +862,9 @@ return
       @admin_firewall_rules = []
       @skipinitialupdates = skipinitialupdates
       @updating = updating
+      if @updating
+        @existing_deploy = MU::MommaCat.new(@updating)
+      end
       @default_credentials = default_credentials
 
       ok = true
@@ -1035,21 +1042,80 @@ return
       end
     end
 
+    # Generate a documentation-friendly dummy Ruby class for our mu.yaml main
+    # config.
+    def self.emitConfigAsRuby
+      example = %Q{---
+public_address: 1.2.3.4
+mu_admin_email: egtlabs@eglobaltech.com
+mu_admin_name: Joe Schmoe
+mommacat_port: 2260
+banner: My Example Mu Master
+mu_repository: git://github.com/cloudamatic/mu.git
+repos:
+- https://github.com/cloudamatic/mu_demo_platform
+allow_invade_foreign_vpcs: true
+ansible_dir:
+aws:
+  egtdev:
+    region: us-east-1
+    log_bucket_name: egt-mu-log-bucket
+    default: true
+    name: egtdev
+  personal:
+    region: us-east-2
+    log_bucket_name: my-mu-log-bucket
+    name: personal
+  google:
+    egtlabs:
+      project: egt-labs-admin
+      credentials_file: /opt/mu/etc/google.json
+      region: us-east4
+      log_bucket_name: hexabucket-761234
+      default: true
+}
+      mu_yaml_schema = eval(%Q{
+$NOOP = true
+load "#{MU.myRoot}/bin/mu-configure"
+$CONFIGURABLES
+})
+      return if mu_yaml_schema.nil? or !mu_yaml_schema.is_a?(Hash)
+      muyamlpath = "#{MU.myRoot}/modules/mu/mu.yaml.rb"
+      MU.log "Converting mu.yaml schema to Ruby objects in #{muyamlpath}"
+      muyaml_rb = File.new(muyamlpath, File::CREAT|File::TRUNC|File::RDWR, 0644)
+      muyaml_rb.puts "# Configuration schema for mu.yaml. See also {https://github.com/cloudamatic/mu/wiki/Configuration the Mu wiki}."
+      muyaml_rb.puts "#"
+      muyaml_rb.puts "# Example:"
+      muyaml_rb.puts "#"
+      muyaml_rb.puts "# <pre>"
+      example.split(/\n/).each { |line|
+        muyaml_rb.puts "#      "+line+"    " # markdooooown
+      }
+      muyaml_rb.puts "# </pre>"
+      muyaml_rb.puts "module MuYAML"
+      muyaml_rb.puts "\t# The configuration file format for Mu's main config file."
+      self.printMuYamlSchema(muyaml_rb, [], { "subtree" => mu_yaml_schema })
+      muyaml_rb.puts "end"
+      muyaml_rb.close
+    end
+
     # Take the schema we've defined and create a dummy Ruby class tree out of
     # it, basically so we can leverage Yard to document it.
     def self.emitSchemaAsRuby
       kittenpath = "#{MU.myRoot}/modules/mu/kittens.rb"
       MU.log "Converting Basket of Kittens schema to Ruby objects in #{kittenpath}"
-      dummy_kitten_class = File.new(kittenpath, File::CREAT|File::TRUNC|File::RDWR, 0644)
-      dummy_kitten_class.puts "### THIS FILE IS AUTOMATICALLY GENERATED, DO NOT EDIT ###"
-      dummy_kitten_class.puts ""
-      dummy_kitten_class.puts "module MU"
-      dummy_kitten_class.puts "class Config"
-      dummy_kitten_class.puts "\t# The configuration file format for Mu application stacks."
-      self.printSchema(dummy_kitten_class, ["BasketofKittens"], MU::Config.docSchema)
-      dummy_kitten_class.puts "end"
-      dummy_kitten_class.puts "end"
-      dummy_kitten_class.close
+      kitten_rb = File.new(kittenpath, File::CREAT|File::TRUNC|File::RDWR, 0644)
+      kitten_rb.puts "### THIS FILE IS AUTOMATICALLY GENERATED, DO NOT EDIT ###"
+      kitten_rb.puts "#"
+      kitten_rb.puts "#"
+      kitten_rb.puts "#"
+      kitten_rb.puts "module MU"
+      kitten_rb.puts "class Config"
+      kitten_rb.puts "\t# The configuration file format for Mu application stacks."
+      self.printSchema(kitten_rb, ["BasketofKittens"], MU::Config.docSchema)
+      kitten_rb.puts "end"
+      kitten_rb.puts "end"
+      kitten_rb.close
 
     end
 
@@ -1066,7 +1132,6 @@ return
       subnet_bits = cidr.netmask.prefix_len
       begin
         subnet_bits += 1
-
         if subnet_bits > max_mask
           MU.log "Can't subdivide #{cidr.to_s} into #{subnets_desired.to_s}", MU::ERR
           raise MuError, "Subnets smaller than /#{max_mask} not permitted"
@@ -1148,7 +1213,7 @@ return
     # an extra pass to make sure we get all intra-stack dependencies correct.
     # @param acl [Hash]: The configuration hash for the FirewallRule to check
     # @return [Hash]
-    def resolveIntraStackFirewallRefs(acl)
+    def resolveIntraStackFirewallRefs(acl, delay_validation = false)
       acl["rules"].each { |acl_include|
         if acl_include['sgs']
           acl_include['sgs'].each { |sg_ref|
@@ -1171,7 +1236,7 @@ return
               siblingfw = haveLitterMate?(sg_ref, "firewall_rules")
               if !siblingfw["#MU_VALIDATED"]
 # XXX raise failure somehow
-                insertKitten(siblingfw, "firewall_rules")
+                insertKitten(siblingfw, "firewall_rules", delay_validation: delay_validation)
               end
             end
           }
@@ -1185,10 +1250,15 @@ return
     # @param type [String]: The type of resource being added
     # @param delay_validation [Boolean]: Whether to hold off on calling the resource's validateConfig method
     # @param ignore_duplicates [Boolean]: Do not raise an exception if we attempt to insert a resource with a +name+ field that's already in use
-    def insertKitten(descriptor, type, delay_validation = false, ignore_duplicates: false)
+    def insertKitten(descriptor, type, delay_validation = false, ignore_duplicates: false, overwrite: false)
       append = false
       start = Time.now
       shortclass, cfg_name, cfg_plural, classname = MU::Cloud.getResourceNames(type)
+      MU.log "insertKitten on #{cfg_name} #{descriptor['name']} (delay_validation: #{delay_validation.to_s})", MU::DEBUG, details: caller[0]
+
+      if overwrite
+        removeKitten(descriptor['name'], type)
+      end
 
       if !ignore_duplicates and haveLitterMate?(descriptor['name'], cfg_name)
 #        raise DuplicateNameError, "A #{shortclass} named #{descriptor['name']} has already been inserted into this configuration"
@@ -1284,7 +1354,7 @@ return
           siblingvpc = haveLitterMate?(descriptor["vpc"]["name"], "vpcs")
 
           if siblingvpc and siblingvpc['bastion'] and
-             ["server", "server_pool"].include?(cfg_name) and
+             ["server", "server_pool", "container_cluster"].include?(cfg_name) and
              !descriptor['bastion']
             if descriptor['name'] != siblingvpc['bastion'].to_h['name']
               descriptor["dependencies"] << {
@@ -1298,7 +1368,7 @@ return
           # resolved before we can proceed
           if ["server", "server_pool", "loadbalancer", "database", "cache_cluster", "container_cluster", "storage_pool"].include?(cfg_name)
             if !siblingvpc["#MU_VALIDATED"]
-              ok = false if !insertKitten(siblingvpc, "vpcs")
+              ok = false if !insertKitten(siblingvpc, "vpcs", overwrite: overwrite)
             end
           end
           if !MU::Config::VPC.processReference(descriptor['vpc'],
@@ -1349,13 +1419,15 @@ return
       # Does it have generic ingress rules?
       fwname = cfg_name+descriptor['name']
 
-      if !haveLitterMate?(fwname, "firewall_rules") and
-         (descriptor['ingress_rules'] or
-         ["server", "server_pool", "database"].include?(cfg_name))
+      if (descriptor['ingress_rules'] or
+         ["server", "server_pool", "database", "cache_cluster"].include?(cfg_name))
         descriptor['ingress_rules'] ||= []
         fw_classobj = Object.const_get("MU").const_get("Cloud").const_get(descriptor["cloud"]).const_get("FirewallRule")
 
-        acl = {
+        acl = haveLitterMate?(fwname, "firewall_rules")
+        already_exists = !acl.nil?
+
+        acl ||= {
           "name" => fwname,
           "rules" => descriptor['ingress_rules'],
           "region" => descriptor['region'],
@@ -1375,10 +1447,11 @@ return
         ["optional_tags", "tags", "cloud", "project"].each { |param|
           acl[param] = descriptor[param] if descriptor[param]
         }
-        descriptor["add_firewall_rules"] = [] if descriptor["add_firewall_rules"].nil?
+        descriptor["add_firewall_rules"] ||= []
         descriptor["add_firewall_rules"] << {"rule_name" => fwname, "type" => "firewall_rules" } # XXX why the duck is there a type argument required here?
-        acl = resolveIntraStackFirewallRefs(acl)
-        ok = false if !insertKitten(acl, "firewall_rules", delay_validation)
+
+        acl = resolveIntraStackFirewallRefs(acl, delay_validation)
+        ok = false if !insertKitten(acl, "firewall_rules", delay_validation, overwrite: already_exists)
       end
 
       # Does it declare association with any sibling LoadBalancers?
@@ -1413,10 +1486,6 @@ return
               "type" => "firewall_rule",
               "name" => acl_include["rule_name"]
             }
-            siblingfw = haveLitterMate?(acl_include["rule_name"], "firewall_rules")
-            if !siblingfw["#MU_VALIDATED"]
-              ok = false if !insertKitten(siblingfw, "firewall_rules", delay_validation)
-            end
           elsif acl_include["rule_name"]
             MU.log shortclass.to_s+" #{descriptor['name']} depends on FirewallRule #{acl_include["rule_name"]}, but no such rule declared.", MU::ERR
             ok = false
@@ -1428,13 +1497,14 @@ return
       if descriptor["alarms"] && !descriptor["alarms"].empty?
         descriptor["alarms"].each { |alarm|
           alarm["name"] = "#{cfg_name}-#{descriptor["name"]}-#{alarm["name"]}"
-          alarm['dimensions'] = [] if !alarm['dimensions']
+          alarm['dimensions'] ||= []
+          alarm["namespace"] ||= descriptor['name']
           alarm["credentials"] = descriptor["credentials"]
           alarm["#TARGETCLASS"] = cfg_name
           alarm["#TARGETNAME"] = descriptor['name']
           alarm['cloud'] = descriptor['cloud']
 
-          ok = false if !insertKitten(alarm, "alarms", true)
+          ok = false if !insertKitten(alarm, "alarms", true, overwrite: overwrite)
         }
         descriptor.delete("alarms")
       end
@@ -1863,12 +1933,22 @@ return
 
     # (see #include)
     def include(file)
-      MU::Config.include(file, get_binding, param_pass = @param_pass)
+      MU::Config.include(file, get_binding(@@tails.keys.sort), param_pass = @param_pass)
+    end
+
+    @@bindings = {}
+    # Keep a cache of bindings we've created as sandbox contexts for ERB
+    # processing, so we don't keep reloading the entire Mu library inside new
+    # ones.
+    def self.global_bindings
+      @@bindings
     end
 
     # Namespace magic to pass to ERB's result method.
-    def get_binding
-      binding
+    def get_binding(keyset)
+#      return MU::Config.global_bindings[keyset] if MU::Config.global_bindings[keyset]
+      MU::Config.global_bindings[keyset] = binding
+      MU::Config.global_bindings[keyset]
     end
 
     def applySchemaDefaults(conf_chunk = config, schema_chunk = schema, depth = 0, siblings = nil, type: nil)
@@ -2231,10 +2311,131 @@ return
 #      end
     end
 
+    # Emit our mu.yaml schema in a format that YARD can comprehend and turn into
+    # documentation.
+    def self.printMuYamlSchema(muyaml_rb, class_hierarchy, schema, in_array = false, required = false, prefix: nil)
+      return if schema.nil?
+      if schema["subtree"]
+        printme = Array.new
+        # order sub-elements by whether they're required, so we can use YARD's
+        # grouping tags on them
+        have_required = schema["subtree"].keys.any? { |k| schema["subtree"][k]["required"] }
+        prop_list = schema["subtree"].keys.sort { |a, b|
+          if schema["subtree"][a]["required"] and !schema["subtree"][b]["required"]
+            -1
+          elsif !schema["subtree"][a]["required"] and schema["subtree"][b]["required"]
+            1
+          else
+            a <=> b
+          end
+        }
+
+        req = false
+        printme << "# @!group Optional parameters" if !have_required
+        prop_list.each { |name|
+          prop = schema["subtree"][name]
+          if prop["required"]
+            printme << "# @!group Required parameters" if !req
+            req = true
+          else
+            if req
+              printme << "# @!endgroup"
+              printme << "# @!group Optional parameters"
+            end
+            req = false
+          end
+
+          printme << self.printMuYamlSchema(muyaml_rb, class_hierarchy+ [name], prop, false, req)
+        }
+        printme << "# @!endgroup"
+
+        desc = (schema['desc'] || schema['title'])
+
+        tabs = 1
+        class_hierarchy.each { |classname|
+          if classname == class_hierarchy.last and desc
+            muyaml_rb.puts ["\t"].cycle(tabs).to_a.join('') + "# #{desc}\n"
+          end
+          muyaml_rb.puts ["\t"].cycle(tabs).to_a.join('') + "class #{classname}"
+          tabs = tabs + 1
+        }
+        printme.each { |lines|
+          if !lines.nil? and lines.is_a?(String)
+            lines.lines.each { |line|
+              muyaml_rb.puts ["\t"].cycle(tabs).to_a.join('') + line
+            }
+          end
+        }
+
+        class_hierarchy.each { |classname|
+          tabs = tabs - 1
+          muyaml_rb.puts ["\t"].cycle(tabs).to_a.join('') + "end"
+        }
+
+        # And now that we've dealt with our children, pass our own rendered
+        # commentary back up to our caller.
+        name = class_hierarchy.last
+        if in_array
+          type = "Array<#{class_hierarchy.join("::")}>"
+        else
+          type = class_hierarchy.join("::")
+        end
+
+        docstring = "\n"
+        docstring = docstring + "# **REQUIRED**\n" if required
+#        docstring = docstring + "# **"+schema["prefix"]+"**\n" if schema["prefix"]
+        docstring = docstring + "# #{desc.gsub(/\n/, "\n#")}\n" if desc
+        docstring = docstring + "#\n"
+        docstring = docstring + "# @return [#{type}]\n"
+        docstring = docstring + "# @see #{class_hierarchy.join("::")}\n"
+        docstring = docstring + "attr_accessor :#{name}"
+        return docstring
+
+      else
+        in_array = schema["array"]
+        name = class_hierarchy.last
+        type = if schema['boolean']
+          "Boolean"
+        else
+          "String"
+        end
+        if in_array
+          type = "Array<#{type}>"
+        end
+        docstring = "\n"
+
+        prefixes = []
+        prefixes << "# **REQUIRED**" if schema["required"] and schema['default'].nil?
+#        prefixes << "# **"+schema["prefix"]+"**" if schema["prefix"]
+        prefixes << "# **Default: `#{schema['default']}`**" if !schema['default'].nil?
+        if !schema['pattern'].nil?
+          # XXX unquoted regex chars confuse the hell out of YARD. How do we
+          # quote {}[] etc in YARD-speak?
+          prefixes << "# **Must match pattern `#{schema['pattern'].to_s.gsub(/\n/, "\n#")}`**"
+        end
+
+        desc = (schema['desc'] || schema['title'])
+        if prefixes.size > 0
+          docstring += prefixes.join(",\n")
+          if desc and desc.size > 1
+            docstring += " - "
+          end
+          docstring += "\n"
+        end
+
+        docstring = docstring + "# #{desc.gsub(/\n/, "\n#")}\n" if !desc.nil?
+        docstring = docstring + "#\n"
+        docstring = docstring + "# @return [#{type}]\n"
+        docstring = docstring + "attr_accessor :#{name}"
+
+        return docstring
+      end
+
+    end
 
-    # Emit our Basket of Kittesn schema in a format that YARD can comprehend
+    # Emit our Basket of Kittens schema in a format that YARD can comprehend
     # and turn into documentation.
-    def self.printSchema(dummy_kitten_class, class_hierarchy, schema, in_array = false, required = false, prefix: nil)
+    def self.printSchema(kitten_rb, class_hierarchy, schema, in_array = false, required = false, prefix: nil)
       return if schema.nil?
       if schema["type"] == "object"
         printme = Array.new
@@ -2263,7 +2464,7 @@ return
               req = false
             end
 
-            printme << self.printSchema(dummy_kitten_class, class_hierarchy+ [name], prop, false, req, prefix: schema["prefix"])
+            printme << self.printSchema(kitten_rb, class_hierarchy+ [name], prop, false, req, prefix: schema["prefix"])
           }
           printme << "# @!endgroup"
         end
@@ -2271,22 +2472,22 @@ return
         tabs = 1
         class_hierarchy.each { |classname|
           if classname == class_hierarchy.last and !schema['description'].nil?
-            dummy_kitten_class.puts ["\t"].cycle(tabs).to_a.join('') + "# #{schema['description']}\n"
+            kitten_rb.puts ["\t"].cycle(tabs).to_a.join('') + "# #{schema['description']}\n"
           end
-          dummy_kitten_class.puts ["\t"].cycle(tabs).to_a.join('') + "class #{classname}"
+          kitten_rb.puts ["\t"].cycle(tabs).to_a.join('') + "class #{classname}"
           tabs = tabs + 1
         }
         printme.each { |lines|
           if !lines.nil? and lines.is_a?(String)
             lines.lines.each { |line|
-              dummy_kitten_class.puts ["\t"].cycle(tabs).to_a.join('') + line
+              kitten_rb.puts ["\t"].cycle(tabs).to_a.join('') + line
             }
           end
         }
 
         class_hierarchy.each { |classname|
           tabs = tabs - 1
-          dummy_kitten_class.puts ["\t"].cycle(tabs).to_a.join('') + "end"
+          kitten_rb.puts ["\t"].cycle(tabs).to_a.join('') + "end"
         }
 
         # And now that we've dealt with our children, pass our own rendered
@@ -2309,7 +2510,7 @@ return
         return docstring
 
       elsif schema["type"] == "array"
-        return self.printSchema(dummy_kitten_class, class_hierarchy, schema['items'], true, required, prefix: prefix)
+        return self.printSchema(kitten_rb, class_hierarchy, schema['items'], true, required, prefix: prefix)
       else
         name = class_hierarchy.last
         if schema['type'].nil?