forj 0.0.36 → 0.0.37

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 56a2084ebba0322d7ddd4c282f4920d0200a7f80
-  data.tar.gz: 10f942c5e93c5efb94c62b51447cee6dc7c057ce
+  metadata.gz: f8a5a5593d87c677a7565b45db486a1b840bc2b0
+  data.tar.gz: a29f5b8d7bf6b452e9e6814a19c2f61081fc5bad
 SHA512:
-  metadata.gz: de3ed0c44cf1d360e823d2ccff9ac391b08d16d5944c01e49c96175e4035e1493ca9c967a3dfc15b29e4eb6a9c316431846e5787eda884c241894f38fb8d42c3
-  data.tar.gz: 938f9c93e3eaa1b86d917e66f89e8dd93902bd4a414347fb77074d08e4e176f51be87df7863a7f6d09a21662156e4250b70947197c9db4b08b716ad4ff18eefa
+  metadata.gz: cbf34451071e200ccb9cc509059ad304ee8ced2e10e490d4adbdec81e132d7a52340a8a1d83a1d0ab3fe47c8add5fa63f5c730aeb6e13286400340ed4339f576
+  data.tar.gz: 85d90986583e24d0bacb1ca2cf9a9e212d7d4b86a12fee94f084ded13a214c725c7fb5a4df382380d119af3d6fbcb5f021c3f3c1bedb6859032b0d1e63c812c3

data/bin/forj

@@ -23,7 +23,7 @@ require 'ansi'
 $APP_PATH = File.dirname(__FILE__)
 $LIB_PATH = File.expand_path(File.join(File.dirname($APP_PATH),'lib'))
 
-$LOAD_PATH << './lib'
+$LOAD_PATH << $LIB_PATH
 
 require 'boot.rb'
 require 'down.rb'
@@ -47,7 +47,7 @@ include Logging
 # Initialize forj paths
 ensure_forj_dirs_exists()
 
-# Initialize global Log object 
+# Initialize global Log object
 $FORJ_LOGGER=ForjLog.new()
 
 
@@ -62,7 +62,7 @@ class Forj < Thor
   def help(task = nil, subcommand = false)
     if task
        self.class.task_help(shell, task)
-    else   
+    else
        puts <<-LONGDESC
 Quick steps: How to create a forj?
 ----------------------------------
@@ -71,25 +71,25 @@ Quick steps: How to create a forj?
     `$ forj setup [Provider]`
 
     Ex: `forj setup hpcloud`. In this example, your account will be named 'hpcloud'.
-    
-IMPORTANT NOTE: Without any provider, forj setup will choose 'hpcloud' as provider. 
+
+IMPORTANT NOTE: Without any provider, forj setup will choose 'hpcloud' as provider.
                 Currently, forj supports only hpcloud. Openstack will be available soon.
 
 2.  Create your forge on your default account
     `$ forj boot <blueprint> on <Provider> as <InstanceName> `
 
-    Ex: `forj boot redstone on hpcloud as MyForge` 
-       
+    Ex: `forj boot redstone on hpcloud as MyForge`
+
 forj command line details:
 --------------------------
        LONGDESC
        self.class.help(shell, subcommand)
     end
-  end 
+  end
 
-  # BOOT 
+  # BOOT
   desc 'boot <Blueprint> on <Provider> as <InstanceName> [options]', 'boot a Maestro box and instruct it to provision the blueprint'
-  
+
   long_desc <<-LONGDESC
 This task boot a new forge with the following options
 \x5- blueprint : Is the name of the blueprint (currently cli only supports redstone)
@@ -105,67 +105,65 @@ The list of predefined values can be retrieved with forj show defaults
 
   method_option :account_name,     :aliases => '-a',   :desc => 'Set the forj account name to use. By default, it takes the provider name.'
   method_option :infra,            :aliases => '-i',   :desc => 'Defines your Infra directory to use while booting. You can also set FORJ_INFRA_DIR.'
+  method_option :maestro_repo,     :aliases => '-m',   :desc => 'To use a different Maestro repository already cloned. By default, Maestro is cloned to ~/.forj/maestro from github.'
   method_option :key_name,         :aliases => '-k',   :desc => "Keypair name to use."
   method_option :key_path,         :aliases => '-p',   :desc => "Private or Public key file. forj will determine if you provide a public key or a private, if respectively the extension '.pub' exist or not."
 
   method_option :boothook,         :aliases => '-H',   :desc => 'By default, boothook file used is build/bin/build-tools/boothook.sh. Use this option to set another one.'
   method_option :build,            :aliases => '-B',   :desc => 'Replace the default build.sh'
-  method_option :build_config,     :aliases => '-C',   :desc => 'The build config file to load <confdir>/<BoxName>.<Config>.env. By default, uses "master" as Config.'
-  method_option :maestro_repo,     :aliases => '-M',   :desc => 'To use a different Maestro repository already cloned. By default, Maestro is cloned to ~/.forj/maestro from github.'
   method_option :branch,           :aliases => '-B',   :desc => 'The build will extract from git branch name. It sets the configuration build <config> to the branch name <branch>.'
-  method_option :box_name,         :aliases => '-N',   :desc => 'Defines the name of the box or box image to build.'
   method_option :test_box,         :aliases => '-T',   :desc => 'Create test.rb-box meta from the repository path provided.'
- 
-  def boot(blueprint, on, cloud_provider, as, name, test = false)
+
+  def boot(blueprint, on, cloud_provider, as, name)
       Logging.set_level(Logger::INFO) if options[:verbose]
       Logging.set_level(Logger::DEBUG) if options[:debug]
       oConfig=ForjConfig.new(options[:config])
-      
+
       Logging.fatal(1, "instance name '%s' not supported. Support only lower case, numeric and dash caracters." % [name]) if not /^[\d[[:lower:]]-]+$/ =~ name
-      
+
       # Options are added if they are set. Otherwise, get will retrieve the default value.
       oConfig.set('account_name', options[:account_name])
       oConfig.set('infra_repo',   options[:infra])
       oConfig.set('keypair_name', options[:key_name])
       if options[:key_path]
          mFound = options[:key_path].match(/^(.*)(\.pub)?$/)
-         if mFound 
+         if mFound
             key_path = File.expand_path(mFound[1])
             if mFound[2] and not File.exists?(File.expand_path(mFound[1]+mFound[2]))
                Logging.fatal(1, "'%s' is not a valid keypair files. At least the public key (.pub) is have to exist.")
-            end   
+            end
             oConfig.set('keypair_path', key_path)
          else
             Logging.fatal(1, "'%s' is not a valid keypair files. At least the public key (.pub) is have to exist.")
-         end   
-      end   
+         end
+      end
 
       # TODO: Be able to support the default provider from config.yaml
       oConfig.set('provider', cloud_provider)
       Boot.boot(blueprint, name,
-                options[:build], options[:build_config_dir],
+                options[:build],
                 options[:build_config], options[:branch],
                 options[:maestro_repo], options[:boothook],
-                options[:box_name], oConfig, test)
+                options[:box_name], oConfig)
   end
 
   desc 'show defaults', 'Show list of predefined value you can update in your ~/.forj/config.yaml'
 
   def show(name)
      oConfig=ForjConfig.new()
-     
-     puts 'List of predefined values:
+
+     puts 'List of default values:
 --------------------------'
-     puts oConfig.yDefaults['default'].to_yaml()
+     puts rhGet(oConfig.yConfig,'default').to_yaml()
      puts '--------------------------'
-     puts "You can set any of those variable in your ~/.forj/config.yaml, under section 'default:'"
+     puts "You can change any of those variables by editing/adding them in your ~/.forj/config.yaml, under section 'default:'"
   end
   # DOWN
   desc 'down', 'delete the Maestro box and all systems installed by the blueprint'
   long_desc <<-LONGDESC
   Not yet implemented
   LONGDESC
-  
+
   def down(name)
       Logging.set_level(Logger::INFO) if options[:verbose]
     Logging.set_level(Logger::DEBUG) if options[:debug]
@@ -176,17 +174,17 @@ The list of predefined values can be retrieved with forj show defaults
   desc 'ssh', 'connect to your forge thru ssh'
   long_desc <<-LONGDESC
 Connect through ssh to an existing instance
-    
+
         not yet implemented
   LONGDESC
-  
+
   def ssh(name, server)
     Logging.set_level(Logger::INFO) if options[:verbose]
     Logging.set_level(Logger::DEBUG) if options[:debug]
     Ssh.connect(name, server)
   end
 
-  # SETUP 
+  # SETUP
   method_option :account_name,     :aliases => '-a',   :desc => 'Set the forj account name to use. By default, it takes the provider name.'
 
   desc 'setup [Provider] [options]', 'set the hpcloud credentials for forj cli'
@@ -204,7 +202,8 @@ Several data will be requested like:
     Logging.set_level(Logger::INFO)  if options[:verbose]
     Logging.set_level(Logger::DEBUG) if options[:debug]
     oConfig=ForjConfig.new(options[:config])
-    Setup.setup(sProvider, oConfig, options)
+    oConfig.set('provider', sProvider)
+    Setup.setup(oConfig, options)
   end
 
 end

data/lib/boot.rb

@@ -34,10 +34,9 @@ include Helpers
 # Boot module
 #
 module Boot
-  def boot(blueprint, name,
-      build, infra_dir, build_config,
+  def boot(blueprint, name, build, build_config,
       branch, maestro_repo, boothook, box_name,
-      oConfig, test = false)
+      oConfig)
     begin
 
       # Check options and set data
@@ -48,7 +47,7 @@ module Boot
          Logging.fatal(1, "forj setup support only hpcloud. '%s' is currently not supported." % cloud_provider)
       end
 
-      oConfig.setDefault('account_name', cloud_provider)
+      oConfig.set('account_name', cloud_provider)
 
       initial_msg = 'booting %s on %s (~/.forj/forj.log)' % [blueprint , cloud_provider]
       Logging.high_level_msg(initial_msg) #################
@@ -56,21 +55,31 @@ module Boot
       # Initialize defaults
       maestro_url =  oConfig.get('maestro_url')
 
-      infra_dir = oConfig.get('infra_repo') unless infra_dir
+      infra_dir = File.expand_path(oConfig.get('infra_repo'))
+      
+      # Load Forj account data
+      forjAccountFile = File.join($FORJ_ACCOUNTS_PATH, oConfig.get('account_name'))
+      oConfig.ExtraLoad(forjAccountFile, :forj_accounts, oConfig.get('account_name'))
 
+      # Check about infra repo compatibility with forj cli
+      bBuildInfra = Repositories.infra_rebuild_required?(oConfig, infra_dir)
+      
       # Ask information if needed.
-      bBuildInfra=false
       if not Dir.exist?(File.expand_path(infra_dir))
          sAsk = 'Your \'%s\' infra directory doesn\'t exist. Do you want to create a new one from Maestro(repo github)/templates/infra (yes/no)?' % [infra_dir]
          bBuildInfra=agree(sAsk)
       else
-         Logging.info('Re-using your infra... in \'%s\'' % [infra_dir])
+         Logging.info('Re-using your infra... in \'%s\'' % [infra_dir]) if not bBuildInfra
       end
       if not Dir.exist?(File.expand_path(infra_dir)) and not bBuildInfra
          Logging.info ('Exiting.')
          return
       end
 
+      # Get FORJ DNS setting
+      yDNS = oConfig.ExtraGet(:forj_accounts, oConfig.get('account_name'), :dns)
+      Logging.fatal(1, "DNS or domain name are missing. Please execute forj setup %s" % oConfig.get('account_name')) if not yDNS
+
       # Step Maestro Clone
       if not maestro_repo
          Logging.high_level_msg('cloning maestro repo ...' ) #################
@@ -105,7 +114,7 @@ module Boot
 
       Logging.state('Configuring keypair...') #################
       Logging.info('Configuring keypair \'%s\'' % [oConfig.get('keypair_name')])
-      SecurityGroup.hpc_import_pubkey(oConfig, oFC.sAccountName)
+      SecurityGroup.hpc_import_key(oConfig, oFC.sAccountName)
 
       Logging.state('Configuring security group...') #################
 
@@ -116,55 +125,65 @@ module Boot
       Logging.state('Configuring security group ports...') #################
       ports.each do |port|
         port = port.to_s if port.class != String
-        if not /^\d+(-\d+)?$/ =~ port 
+        if not /^\d+(-\d+)?$/ =~ port
            Logging.error("Port '%s' is not valid. Must be <Port> or <PortMin>-<PortMax>" % [port])
-        else 
+        else
            mPortFound = /^(\d+)(-(\d+))?$/.match(port)
            portmin = mPortFound[1]
            portmax = (mPortFound[3]) ? (mPortFound[3]) : (portmin)
            Network.get_or_create_rule(oFC, security_group.id, 'tcp', portmin, portmax)
-        end   
+        end
       end
 
-      ENV['FORJ_HPC'] = oFC.sAccountName
-      ENV['FORJ_HPC_NET'] = network.name
-      ENV['FORJ_SECURITY_GROUP'] = oConfig.get('security_group')
-      ENV['FORJ_KEYPAIR'] = oConfig.get('keypair_name')
-      ENV['FORJ_HPC_NOVA_KEYPUB'] = oConfig.get('keypair_path')
-      ENV['FORJ_BASE_IMG'] = oConfig.get('image')
+      oBuildEnv = BuildEnv.new(oConfig)
+      ENV['FORJ_CLI_ENV'] = oBuildEnv.sBuildEnvFile
+      oBuildEnv.set('FORJ_HPC',             oFC.sAccountName)
+      oBuildEnv.set('FORJ_HPC_NET',         network.name)
+      oBuildEnv.set('FORJ_SECURITY_GROUP',  oConfig.get('security_group'))
+      oBuildEnv.set('FORJ_KEYPAIR',         oConfig.get('keypair_name'))
+      oBuildEnv.set('FORJ_HPC_NOVA_KEYPUB', oConfig.get('keypair_path') + '.pub')
+      oBuildEnv.set('FORJ_BASE_IMG',        oConfig.get('image'))
+      oBuildEnv.set('FORJ_FLAVOR',          oConfig.get('flavor'))
+      oBuildEnv.set('FORJ_TENANT_NAME',     rhGet(oConfig.ExtraGet(:forj_accounts, oFC.sAccountName, :compute), :tenant_name))
+      
+
+      oBuildEnv.set('FORJ_DOMAIN', yDNS[:domain_name])
+
+      if yDNS[:tenant_id]
+         oBuildEnv.set('FORJ_DNS_TENANTID', yDNS[:tenant_id])
+         oBuildEnv.set('FORJ_DNS_ZONE',     yDNS[:service])
+      end
+      oBuildEnv.save()
 
       # run build.sh to boot maestro
       puts
 
       build = 'bin/build.sh' unless build
 
-      build_config = oConfig.get('build_config') unless build_config
-
+      build_config = oConfig.get('build_config')
       branch = oConfig.get('branch') unless branch
-
-      box_name = oConfig.get('box_name') unless box_name
+      box_name = oConfig.get('box_name')
 
       meta = '--meta blueprint=%s ' % [blueprint]
 
       command = '%s --build_ID %s --box-name %s --build-conf-dir %s --build-config %s --gitBranch %s --debug-box %s' % [build, name, box_name, infra_dir, build_config, branch, meta]
 
-      maestro_build_path = File.expand_path('~/.forj/maestro/build')
+      maestro_build_path = File.join(maestro_repo, 'build')
 
       current_dir = Dir.pwd
-
-      Dir.chdir(File.expand_path('~/.forj/maestro/build'))
+      Dir.chdir(maestro_build_path)
 
       Logging.info("Calling '%s' from '%s'" %  [build, Dir.pwd])
-      Logging.debug(command)
+      Logging.debug("%s=%s %s" % ['FORJ_CLI_ENV', ENV['FORJ_CLI_ENV'], command])
       Kernel.system(ENV, command)
       Dir.chdir(current_dir)
 
-      if test
-        Logging.debug 'test flag is on, deleting objects'
-        Network.delete_router_interface(subnet.id, router)
-        Network.delete_subnet(subnet.id)
-        Network.delete_network(network.name)
-      end
+#      if test
+#        Logging.debug 'test flag is on, deleting objects'
+#        Network.delete_router_interface(subnet.id, router)
+#        Network.delete_subnet(subnet.id)
+#        Network.delete_network(network.name)
+#      end
 
     rescue Interrupt
       Logging.message("\n'%s' boot from '%s' interrupted by user" % [name, blueprint])
@@ -172,4 +191,51 @@ module Boot
       Logging.error("%s\n%s" % [e.message, e.backtrace.join("\n")])
     end
   end
+
+end
+
+
+class BuildEnv
+
+   attr_reader :sBuildEnvFile
+   
+   def initialize(oConfig)
+
+      oConfig.fatal_if_inexistent('infra_repo')
+      oConfig.fatal_if_inexistent('account_name')
+
+      sBuildDir = File.expand_path(File.join(oConfig.get('infra_repo'),'build'))
+      @sBuildEnvFile = File.join(sBuildDir, oConfig.get('account_name')+'.build.env')
+      Helpers.ensure_dir_exists(sBuildDir)
+      @yBuildEnvVar = {}
+      @oConfig = oConfig
+   end
+
+   def set(key, value)
+      # This file creates the in-memory data required to configure specific project build_env.
+      # Name : ~/.forj/account/<Account>.build.env
+      if value
+         Logging.debug("Setting '%s' = '%s'" % [key, value])
+         @yBuildEnvVar[key] = value
+      else
+         Logging.debug("'%s' is not set" % [key])
+      end
+   end
+
+   def save()
+      begin
+         File.open(@sBuildEnvFile, 'w') do |out|
+            @yBuildEnvVar.each do | key, value |
+               if rhExist?(@oConfig.yConfig, 'description', key)
+                  out.write("# %s - %s\n" % [key, rhGet(@oConfig.yConfig, 'description', key)])
+               end
+               value = "" if not value
+               out.write("%s='%s'\n\n" % [key, value])
+            end
+         end
+         Logging.debug("'%s' written." % [@sBuildEnvFile])
+      rescue => e
+         Logging.error("%s\n%s" % [e.message, e.backtrace.join("\n")])
+      end
+   end
 end

data/lib/connection.rb

@@ -32,17 +32,29 @@ class ForjConnection
    attr_accessor :oNetwork
    attr_accessor :sAccountName
 
-   def initialize(oConfig)
+   def initialize(oConfig, bAutoConnect = true)
 
-     @sAccountName = oConfig.get('account_name')
+     Logging.fatal(1, 'Internal Error: Missing global $HPC_ACCOUNTS') if not $HPC_ACCOUNTS
+     
+     @oConfig = oConfig
+     @sAccountName = @oConfig.get('account_name')
      @provider='HP' # TODO: Support multiple provider. (Generic Provider object required)
-     @sAccountName = oConfig.get('provider') if not @sAccountName
+     @sAccountName = @oConfig.get('provider') if not @sAccountName
      @sAccountName = 'hpcloud' if not @sAccountName
 
-     @credentials = get_credentials(@sAccountName)
-     oSSLError=SSLErrorMgt.new
+     @credentials = get_credentials()
 
      # Trying to get Compute object
+     compute_connect if bAutoConnect
+
+     # Trying to get Network object
+     network_connect if bAutoConnect
+
+   end
+
+   def compute_connect
+
+     oSSLError=SSLErrorMgt.new # Retry object
 
      Logging.debug("compute: Connecting to '%s' - Project '%s'" % [@provider, @credentials['tenant_id']])
      begin
@@ -59,10 +71,13 @@ class ForjConnection
        if not oSSLError.ErrorDetected(e.message,e.backtrace)
           retry
        end
-       Logging.fatal(1, 'Unable to connect.')
+       Logging.fatal(1, 'Compute: Unable to connect.')
      end
-
+   end
+   
+   def network_connect
      # Trying to get Network object
+     oSSLError=SSLErrorMgt.new # Retry object
      Logging.debug("HP network: Connecting to '%s' - Project '%s'" % [@provider, @credentials['tenant_id']])
      begin
        @oNetwork=Fog::HP::Network.new({
@@ -76,34 +91,33 @@ class ForjConnection
        if not oSSLError.ErrorDetected(e.message,e.backtrace)
           retry
        end
-       Logging.fatal(1, 'Unable to connect.')
+       Logging.fatal(1, 'Network: Unable to connect.')
      end
 
    end
 
-end
+   def get_credentials()
+     # TODO: Should support forj credentials. not hpcloud credentials.
+
+     creds = File.join($HPC_ACCOUNTS, @sAccountName)
+     if not File.exists?(creds)
+        Logging.fatal(1, "'%s' was not configured. Did you executed 'forj setup %s'? Please do it and retry." % [@sAccountName, @sAccountName])
+     end
+     @oConfig.ExtraLoad(creds, :hpc_accounts, @sAccountName)
+
+     template = @oConfig.ExtraGet(:hpc_accounts, @sAccountName)
+     credentials = {}
+     begin
+       credentials['access_key'] = template[:credentials][:account_id]
+       credentials['secret_key'] = template[:credentials][:secret_key]
+       credentials['auth_uri'] = template[:credentials][:auth_uri]
+       credentials['tenant_id'] = template[:credentials][:tenant_id]
+       credentials['availability_zone'] = template[:regions][:compute]
+     rescue => e
+       Logging.error("%s\n%s" % [e.message, e.backtrace.join("\n")])
+       puts 'your credentials are not configured, delete the file %s and run forj setup again' % [creds]
+     end
+     credentials
+   end
 
-def get_credentials(sAccountName)
-  # TODO: Should support forj credentials. not hpcloud credentials.
-
-  Logging.fatal(1, 'Internal Error: Missing sAccountName') if not sAccountName
-
-  creds = File.expand_path('~/.hpcloud/accounts/%s' % [sAccountName])
-  if not File.exists?(creds)
-     Logging.fatal(1, "'%s' was not configured. Did you executed 'forj setup %s'? Please do it and retry." % [sAccountName, sAccountName])
-  end
-  template = YAML.load_file(creds)
-  credentials = Hash.new
-
-  begin
-    credentials['access_key'] = template[:credentials][:account_id]
-    credentials['secret_key'] = template[:credentials][:secret_key]
-    credentials['auth_uri'] = template[:credentials][:auth_uri]
-    credentials['tenant_id'] = template[:credentials][:tenant_id]
-    credentials['availability_zone'] = template[:regions][:compute]
-  rescue => e
-    Logging.error("%s\n%s" % [e.message, e.backtrace.join("\n")])
-    puts 'your credentials are not configured, delete the file %s and run forj setup again' % [creds]
-  end
-  credentials
 end

data/lib/defaults.yaml

@@ -39,3 +39,18 @@ default:
   build_config: box
   branch: master
   box_name: maestro
+description:
+  # Description of build.sh environment variable defined by forj cli for build.sh. (~/.forj/infra/build/<Account>.build.env)
+  FORJ_HPC:             "HPCloud cli Account used to build your Maestro box"
+  FORJ_HPC_COMPUTE:     "HPCloud Compute service (like region-b.geo-1) used to run your Maestro Box"
+  FORJ_TENANT_NAME:     "HPCloud Tenant name used build your <Blueprint> nodes"
+  FORJ_HPC_NET:         "HPCloud Network name to use, while booting all boxes."
+  FORJ_KEYPAIR:         "Keypair used to access boxes."
+  FORJ_SECURITY_GROUP:  "Security group associated to each box"
+  FORJ_HPC_NOVA_KEYPUB: "Public key used by build.sh to ensure his existence on HPCloud"
+  FORJ_BASE_IMG:        "Base image used to build all boxes"
+  FORJ_FLAVOR:          "Flavor used to build Maestro"
+  # DNS specific data
+  FORJ_DNS_TENANTID:    "HPCloud Project ID to use to create DNS entries for each boxes."
+  FORJ_DNS_ZONE:        "HPCloud Domain name service to use for each boxes DNS entries. (Ex: region-a.geo-1)"
+  FORJ_DNS_DOMAIN:      "Domain used for DNS. Each server will be attached to a public IP. An 'A' record in the DNS service will need to be added to your HPCloud DOMAIN."