forj 0.0.36 → 0.0.37

data/lib/forj-config.rb

@@ -30,7 +30,7 @@ class ForjDefault
    # @yDefaults = defaults.yaml file data hash
 
    attr_reader   :yDefaults
-   
+
    def initialize()
       # Load yaml documents (defaults)
       # If config doesn't exist, it will be created, empty with 'defaults:' only
@@ -38,7 +38,7 @@ class ForjDefault
       if not $LIB_PATH
          Logging.fatal(1, 'Internal $LIB_PATH was not set.')
       end
-      
+
       Logging.info ('Reading default configuration...')
 
       @sDefaultsName=File.join($LIB_PATH,'defaults.yaml')
@@ -48,14 +48,14 @@ class ForjDefault
 
 end
 
-class ForjConfig 
+class ForjConfig
 
    # Internal variables:
    # @sConfigName='config.yaml'
    # @yRuntime = data in memory.
    # @yLocal = config.yaml file data hash.
    # @yConfig = defaults.yaml + local_config data hash
-   
+
    attr_reader   :yLocal
    attr_reader   :yConfig
    attr_reader   :sConfigName
@@ -64,7 +64,6 @@ class ForjConfig
       # Load yaml documents (defaults + config)
       # If config doesn't exist, it will be created, empty with 'defaults:' only
 
-
       if not $FORJ_DATA_PATH
          Logging.fatal(1, 'Internal $FORJ_DATA_PATH was not set.')
       end
@@ -74,42 +73,34 @@ class ForjConfig
       if sConfigName
          if File.dirname(sConfigName) == '.'
             sConfigName= File.join($FORJ_DATA_PATH,sConfigName)
-         end  
+         end
          sConfigName = File.expand_path(sConfigName)
          if not File.exists?(sConfigName)
             Logging.warning("Config file '%s' doesn't exists. Using default one." % [sConfigName] )
             @sConfigName=File.join($FORJ_DATA_PATH,sConfigDefaultName)
-         else   
+         else
             @sConfigName=sConfigName
          end
       else
          @sConfigName=File.join($FORJ_DATA_PATH,sConfigDefaultName)
       end
 
-
       @Default=ForjDefault.new
-      
-    
+
       if File.exists?(@sConfigName)
          @yLocal=YAML.load_file(@sConfigName)
       else
          @yLocal={ 'default' => nil }
-         if not File.exists?(@sConfigName)
-            # Write the empty file
-            if not File.exists?($FORJ_DATA_PATH)
-               Dir.mkdir($FORJ_DATA_PATH)
-            end
-            Logging.info ('Creating your default configuration file ...')
-            self.SaveConfig()
-         end
+         # Write the empty file
+         Logging.info ('Creating your default configuration file ...')
+         self.SaveConfig()
       end
-     
+
       BuildConfig()
-      
+
       @yRuntime={}
    end
 
-
    def SaveConfig()
      begin
        File.open(@sConfigName, 'w') do |out|
@@ -122,7 +113,31 @@ class ForjConfig
      Logging.info ('Configuration file "%s" updated.' % @sConfigName)
      return true
    end
-   
+
+   def ExtraSave(sFile, section, name)
+      hVal = rhGet(@Default.yDefaults, :extra_loaded, section, name)
+      if hVal
+         begin
+            File.open(sFile, 'w') do |out|
+               YAML.dump(hVal, out)
+            end
+         rescue => e
+            Logging.error("%s\n%s" % [e.message, e.backtrace.join("\n")])
+            return false
+         end
+         Logging.info ('Configuration file "%s" updated.' % sFile)
+         return true
+      end
+   end
+
+   def ExtraLoad(sFile, section, name)
+      if File.exists?(sFile)
+         hVal = YAML.load_file(sFile)
+         rhSet(@Default.yDefaults, hVal, :extra_loaded, section, name)
+      end
+      BuildConfig()
+   end
+
    def LocalSet(key, value, section = 'default')
      if not key or not value
         return false
@@ -134,70 +149,187 @@ class ForjConfig
         @yLocal[section].merge!({key => value})
      else
         @yLocal.merge!(section => {key => value})
-     end   
+     end
      BuildConfig()
      return true
    end
-   
-   def LocalDel(key,section = 'default')
+
+   def LocalGet(key, section = 'default', default = nil)
+      if @yLocal.has_key?(key)
+         return @yLocal[key]
+      end
+      default
+   end
+
+   def LocalDel(key, section = 'default')
      if not key
         return false
      end
      if not @yLocal.has_key?(section)
         return false
-     end   
+     end
      @yLocal[section].delete(key)
      BuildConfig()
      return true
    end
-   
-   def setDefault(key, value)
-     if not key 
-        return false
-     end
-     if not @yRuntime[key]
-        @yRuntime[key] = value
-     end   
-     return true
+
+   def ExtraExist?(section, name, key)
+      return nil if not section or not name
+      
+      return(rhExist?(@yConfig, :extra_loaded, section, name) == 3) if not key
+      return(rhExist?(@yConfig, :extra_loaded, section, name, key) == 4)
    end
-   
-   def set(key, value)
-     if not key 
-        return false
-     end
-     if value
-        @yRuntime[key] = value
-     else
-        @yRuntime.delete(key)
-     end   
-     return true
+
+   def ExtraGet(section, name, key = nil, default = nil)
+      return nil if not section or not name
+      
+      if key
+         return default unless ExtraExist?(section, name, key)
+         rhGet(@yConfig, :extra_loaded, section, name, key)
+      else
+         return default unless rhExist?(@yConfig, :extra_loaded, section, name) == 3
+         rhGet(@yConfig, :extra_loaded, section, name)
+      end   
    end
-  
-   def get(key, default = @yConfig['default'][key])
-     if @yRuntime.has_key?(key)
-        @yRuntime[key]
-     else
-        default
-     end
+
+   def ExtraSet(section, name, key, value)
+      rhSet(@yConfig, value, :extra_loaded, section, name, key)
+   end
+
+   def set(key, value, par = {})
+      # Function to set a config key, but remove it if value is nil.
+      if not key
+         return false
+      end
+      if par[:section] and par[:name]
+         # Set data in extra_loaded
+         ExtraSet(par[:section], par[:name], key, value)
+      elsif par[:section]
+         # To set key=value on config.yaml, use LocalSet
+         if value
+            rhSet(@yRuntime, value, par[:section], key)
+         else
+            hVal = rhGet(@yRuntime, par[:section])
+            hVal.delete(key)
+         end
+      else
+         if value
+            rhSet(@yRuntime, value, key)
+         else
+            @yRuntime.delete(key)
+         end
+      end
+      true
    end
-  
+
+   def get(key, par = {})
+      if par[:section] and par[:name]
+         # Get data only from extra_loaded
+         return ExtraGet(par[:section], par[:name], key, par[:default])
+      elsif par[:section]
+         # If section/key is in runtime
+         return rhGet(@yRuntime, par[:section], key) if rhExist?(@yRuntime, par[:section], key) == 2
+         # If section/key is in default config
+         return rhGet(@yConfig,  par[:section], key) if rhExist?(@yConfig,  par[:section], key) == 2
+      else
+         # If key is in runtime
+         return rhGet(@yRuntime, key) if rhExist?(@yRuntime, key) == 1
+      end
+      # else key in default config of default section.
+      return rhGet(@yConfig, 'default', key) if rhExist?(@yConfig, 'default', key) == 2
+      # else default
+      par[:default]
+   end
+
+   def exist?(key, par = {})
+      if par[:section] and par[:name]
+         # section/name/key exist in extra_loaded ?
+         return ExtraExist?(par[:section], par[:name], key)
+      elsif par[:section]
+         # section/key exist in runtime?
+         return "runtime" if rhExist?(@yRuntime, par[:section], key) == 2
+         # section/key exist in default config ?
+         return "default" if rhExist?(@yConfig, par[:section], key) == 2
+      else
+         return "runtime" if rhExist?(@yRuntime, key) == 1
+         return "default" if rhExist?(@yRuntime, 'default', key) == 2
+      end
+      false
+   end
+
    def BuildConfig()
-      # This function implements the logic to get defaults, superseed by local config. 
+      # This function implements the logic to get defaults, superseed by local config.
       # Take care of ports array, which is simply merged.
-      
-      @yConfig={ 'default' => @Default.yDefaults['default'].clone }
+
+      @yConfig = @Default.yDefaults.clone
       if @yLocal['default']
          @yConfig['default'].merge!(@yLocal['default']) { |key, oldval, newval| key == 'ports'? newval.clone.push(oldval.clone).flatten: newval }
-      end   
-      @yConfig.merge!(@yLocal) { |key, oldval, newval| key == 'default'? oldval: newval }
+      end
+      @yConfig.merge!(@yLocal) { |key, oldval, newval| (key == 'default' or key == :extra_load)? oldval: newval }
    end
-   
+
    def LocalDefaultExist?(key)
-      if @yLocal['default'][key]
-         true
-      else
-         false
-      end      
+      return true if @yLocal['default'][key]
+      false
    end
 
+   def fatal_if_inexistent(key)
+      # Function to return in fatal error if a config data is nil. Help to control function requirement.
+      Logging.fatal(1, "Internal error - %s: '%s' is missing" % [caller(), key]) if not self.get(key)
+   end
+end
+
+def rhExist?(yVal, *p)
+
+   if p.length() == 0
+      return 0
+   end
+   return 0 if yVal.class != Hash
+   p=p.flatten
+   if p.length() == 1
+      return 1 if yVal[p[0]]
+      return 0
+   end
+   return 0 if not yVal or not yVal[p[0]]
+   ret = rhExist?(yVal[p[0]], p.drop(1)) if yVal[p[0]]
+   return 1 + ret
+   0
+end
+
+def rhGet(yVal, *p)
+
+   if p.length() == 0 or not yVal
+      return nil
+   end
+   p=p.flatten
+   if p.length() == 1
+      return yVal[p[0]] if yVal[p[0]]
+      return nil
+   end
+   return nil if not yVal
+   return rhGet(yVal[p[0]], p.drop(1)) if yVal[p[0]]
+   nil
+end
+
+def rhSet(yVal, value, *p)
+   if p.length() == 0
+      return yVal
+   end
+   p=p.flatten
+   if p.length() == 1
+      if yVal
+         yVal[p[0]] = value
+         return yVal
+      end
+      ret = { p[0] => value }
+      return ret
+   end
+   if yVal
+      yVal[p[0]] = {} if not yVal[p[0]] or yVal[p[0]].class != Hash
+      ret=rhSet(yVal[p[0]], value, p.drop(1))
+      return yVal
+   else
+      ret = rhSet(nil, value, p.drop(1))
+      return { p[0] => ret }
+   end
 end

data/lib/repositories.rb

@@ -28,6 +28,10 @@ include YamlParse
 #
 # Repositories module
 #
+
+# Current version of the infra. Compatible with forj version or higher.
+INFRA_VERSION = "37"
+
 module Repositories
   def clone_repo(maestro_url)
       current_dir = Dir.pwd
@@ -50,20 +54,115 @@ module Repositories
       Dir.chdir(current_dir)
   end
 
-  def create_infra(maestro_repo)
-    home = File.expand_path('~')
-    path = home + '/.forj/'
-    infra = path + 'infra/'
 
-    if File.directory?(infra)
-      FileUtils.rm_r(infra)
-    end
-    Dir.mkdir(infra)
 
-    command = 'cp -rp ~/.forj/maestro/templates/infra/cloud-init ~/.forj/infra/'
-    Kernel.system(command)
+   def create_infra(maestro_repo)
+      # Build our own infra from maestro infra templates.
+      infra = File.join($FORJ_DATA_PATH, 'infra')
+      dest_cloud_init = File.join(infra, 'cloud-init')
+      template = File.join(maestro_repo, 'templates', 'infra')
+      cloud_init = File.join(template, 'cloud-init')
 
-    fill_template = 'python %s/build_tmpl/build-env.py -p ~/.forj/infra --maestro-path %s' % [$LIB_PATH, maestro_repo]
-    Kernel.system(fill_template)
-  end
+      if File.directory?(infra)
+         Logging.debug("Cleaning up '%s'" % [infra])
+         FileUtils.rm_r(infra)
+      end
+      Helpers.ensure_dir_exists(dest_cloud_init)
+      Logging.debug("Copying recursively '%s' to '%s'" % [cloud_init, infra])
+      FileUtils.copy_entry(cloud_init, dest_cloud_init)
+
+      build_env = File.join(template,'maestro.box.master.env')
+      Logging.debug("Copying '%s' to '%s'" % [build_env, infra])
+      FileUtils.copy(build_env, infra)
+      
+      file_ver = File.join(infra, 'forj-cli.ver')
+      File.write(file_ver, $INFRA_VERSION)
+   end
+  
+   def infra_rebuild_required?(oConfig, infra_dir)
+      # This function check if the current infra is compatible with current gem version.
+      
+      # prior 0.0.37 - Use a template file of build env file.
+      # 0.0.37 - Using a generic version of build env file, fully managed by forj cli.
+      return false if not File.exists?(infra_dir)
+      
+      if infra_dir != File.join($FORJ_DATA_PATH, 'infra') 
+         # Do not take care. We do not manage it, ourself.
+         return false
+      end
+
+      file_ver = File.join(infra_dir, 'forj-cli.ver')
+      forj_infra_version = nil
+      forj_infra_version = File.read(file_ver) if File.exist?(file_ver)
+
+      if forj_infra_version
+         case forj_infra_version
+            when $INFRA_VERSION
+               return false
+            else
+               old_infra_data_update(oConfig, forj_infra_version, infra_dir)
+         end
+      else # Prior version 37
+         old_infra_data_update(oConfig, 36, infra_dir)
+      end   
+      true
+   end
+   
+   def old_infra_data_update(oConfig, version, infra_dir)
+      Logging.info("Migrating your local infra repo to the latest version.")
+   # Supporting old version.
+      case version
+         when 36
+            # Moving from 0.0.36 or less to 0.0.37 or higher.
+            # SET_COMPUTE="{SET_COMPUTE!}" => Setting for Compute. ignored. Coming from HPC
+            # SET_TENANT_NAME="{SET_TENANT_NAME!}" => Setting for Compute. ignored. Need to query HPC from current Tenant ID
+
+            # SET_DNS_TENANTID="{SET_DNS_TENANTID!}" => Setting for DNS. meta = dns_tenantid
+            #  ==> :forj_accounts, sAccountName, :dns, :tenant_id
+
+            # SET_DNS_ZONE="{SET_DNS_ZONE!}" => Setting for DNS. meta = dns_zone
+            # ==> :forj_accounts, sAccountName, :dns, :service
+
+            # SET_DOMAIN="{SET_DOMAIN!}" => Setting for Maestro (required) and DNS if enabled. 
+            # ==> :forj_accounts, sAccountName, :dns, :domain_name
+            sAccountName = oConfig.get('account_name')
+
+            yDns = {}
+            yDns = oConfig.ExtraGet(:forj_accounts, sAccountName, :dns) if oConfig.ExtraExist?(:forj_accounts, sAccountName, :dns)
+            build_env = File.join(infra_dir, 'maestro.box.master.env')
+            Logging.debug("Reading data from '%s'" % build_env)
+            tags = {'SET_DNS_TENANTID' => :tenant_id,
+                    'SET_DNS_ZONE' => :service,
+                    'SET_DOMAIN' => :domain_name
+                   }
+            begin
+               bUpdate = nil
+
+               File.open(build_env) do |f|
+                  f.each_line do |line|
+                     mObj = line.match(/^(SET_[A-Z_]+)=["'](.*)["'].*$/)
+                     if mObj
+                        Logging.debug("Reviewing detected '%s' tag" % [mObj[1]])
+                        tag = (tags[mObj[1]]? tags[mObj[1]] : nil)
+                        if tag and mObj[2]
+                           if bUpdate == nil and rhGet(yDns, tag) and rhGet(yDns, tag) != mObj[2]
+                              Logging.message("Your account setup is different than build env.")
+                              Logging.message("We suggest you to update your account setup with data from your build env.")
+                              bUpdate = agree("Do you want to update your setup with those build environment data?")
+                           end
+                           if bUpdate != nil and bUpdate
+                              Logging.debug("Saved: '%s' = '%s'" % [mObj[1],mObj[2]])
+                              rhSet(yDns, mObj[2], tag)
+                           end
+                        end
+                     end
+                  end
+               end
+            rescue => e
+               Logging.fatal(1, "Unable to open the build environment file to migrate it\n%s" % e.backtrace.join('\n'))
+            end
+            oConfig.ExtraSet(:forj_accounts, sAccountName, :dns, yDns)
+            oConfig.ExtraSave(File.join($FORJ_ACCOUNTS_PATH, sAccountName), :forj_accounts, sAccountName)
+      end
+   end
 end

data/lib/security.rb

@@ -137,20 +137,42 @@ module SecurityGroup
     rule
   end
 
-  def hpc_import_pubkey(oConfig, account)
+  def hpc_import_key(oConfig, account)
 
     key_name = oConfig.get('keypair_name')
     key_path = oConfig.get('keypair_path')
 
+    mObj = key_path.match(/^(.*)(\.pem)?$/)
+
+    key_path = mObj[1]
+
     Logging.fatal(1, "'keypair_path' undefined. check your config.yaml file.") if not key_path
     Logging.fatal(1, "'keypair_name' undefined. check your config.yaml file.") if not key_name
 
     pubkey_path = key_path + '.pub'
     Logging.fatal(1, "keypair '%s' are missing. Please call 'forj setup %s' to create the missing key pair required." % [pubkey_path, account]) if not File.exists?(pubkey_path)
-    
-    Logging.info("Importing your forj keypair '%s' to hpcloud." % pubkey_path)
-    command = 'hpcloud keypairs:import %s %s -a %s' % [key_name, pubkey_path, account]
-    Logging.debug("Executing command '%s'" % command)
-    Kernel.system(command)
+    if not File.exists?(File.join($HPC_KEYPAIRS, key_name + '.pub'))
+       Logging.info("Importing your forj public key '%s' to hpcloud." % pubkey_path)
+       command = 'hpcloud keypairs:import %s %s -a %s' % [key_name, pubkey_path, account]
+       Logging.debug("Executing command '%s'" % command)
+       Kernel.system(command)
+    else
+       Logging.info("Using '%s' as public key." % pubkey_path)
+    end
+    private_key = nil
+    private_key = key_path if File.exists?(key_path)
+    private_key = key_path + '.pem' if File.exists?(key_path + '.pem')
+    if not File.exists?(File.join($HPC_KEYPAIRS, key_name + '.pem'))
+       if private_key
+          Logging.info("Importing your forj private key '%s' to hpcloud." % private_key)
+          command = 'hpcloud keypairs:private:add %s %s' % [key_name, private_key]
+          Logging.debug("Executing command '%s'" % command)
+          Kernel.system(command)
+       else   
+          Logging.warning('Unable to find the private key. This will be required to access with ssh to Maestro and any blueprint boxes.')
+       end
+    else
+       Logging.info("Using '%s' as private key." % key_path)
+    end
   end
 end