forj 0.0.35 → 0.0.36

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b1995afd2ab24967e07aef8b4451acbff58dd367
-  data.tar.gz: 662ce3b7da524bf101ba45a3526ca1c04d1a336f
+  metadata.gz: 56a2084ebba0322d7ddd4c282f4920d0200a7f80
+  data.tar.gz: 10f942c5e93c5efb94c62b51447cee6dc7c057ce
 SHA512:
-  metadata.gz: bb8659a39b98f5aed5a0e26337428e8a96667d8f2c35482855e24b0db706492020a6d03564ded3d635d9f096a52b01e25f2d1a685bb4e824c5a93111ab464350
-  data.tar.gz: 60918b32d461c2ad7b5e43dda24172a036d6c22c6fef392d0ec4ce5092652d6e16bb250abc98ea038f9e8b581a0361b2112905487d43bd3c2d9784b0c455fe11
+  metadata.gz: de3ed0c44cf1d360e823d2ccff9ac391b08d16d5944c01e49c96175e4035e1493ca9c967a3dfc15b29e4eb6a9c316431846e5787eda884c241894f38fb8d42c3
+  data.tar.gz: 938f9c93e3eaa1b86d917e66f89e8dd93902bd4a414347fb77074d08e4e176f51be87df7863a7f6d09a21662156e4250b70947197c9db4b08b716ad4ff18eefa

data/README.md

@@ -4,24 +4,23 @@ Forj cli
 
 Installation
 ------------
-### For ruby 2.0
 
-**Fedora/CentOS/Redhat rpm like package system**
+###Fedora/CentOS/Redhat rpm like package system
+
+**For ruby 2.0**
 
     $ sudo yum install ruby-devel libxml2-devel libxslt-devel python-yaml gcc git -y
     $ sudo gem install forj
 
-**Ubuntu/Debian deb like package system (not tested)**
+###Ubuntu/Debian deb like package system
 
-    $ sudo apt-get install ruby-dev build-essential libopenssl-ruby1.9.1 libssl-dev zlib1g-dev git -y
-    $ sudo gem install forj
-
-### For ruby 1.9
+**For ruby 1.9**
 
-    $ sudo apt-get install ruby-dev build-essential libopenssl-ruby1.9.1 libssl-dev zlib1g-dev git -y
+    $ sudo apt-get install ruby-dev build-essential libopenssl-ruby1.9.1 libssl-dev zlib1g-dev libxml2-dev libxslt-dev git -y
     $ sudo gem install forj
 
-### For ruby 1.8
+
+**For ruby 1.8**
 
     $ sudo apt-get install ruby1.8-dev ruby1.8 ri1.8 rdoc1.8 irb1.8 -y
     $ sudo apt-get install libreadline-ruby1.8 libruby1.8 libopenssl-ruby -y

data/bin/forj

@@ -23,16 +23,16 @@ require 'ansi'
 $APP_PATH = File.dirname(__FILE__)
 $LIB_PATH = File.expand_path(File.join(File.dirname($APP_PATH),'lib'))
 
-$FORJ_DATA_PATH= File.expand_path('~/.forj')
 $LOAD_PATH << './lib'
 
-require_relative '../lib/boot.rb'
+require 'boot.rb'
+require 'down.rb'
+require 'setup.rb'
+require 'ssh.rb'
+
 include Boot
-require_relative '../lib/down.rb'
 include Down
-require_relative '../lib/setup.rb'
 include Setup
-require_relative '../lib/ssh.rb'
 include Ssh
 
 require 'forj-config.rb' # Load class ForjConfig
@@ -44,6 +44,9 @@ require 'connection.rb' # Load class ForjConnection
 
 include Logging
 
+# Initialize forj paths
+ensure_forj_dirs_exists()
+
 # Initialize global Log object 
 $FORJ_LOGGER=ForjLog.new()
 
@@ -102,8 +105,8 @@ The list of predefined values can be retrieved with forj show defaults
 
   method_option :account_name,     :aliases => '-a',   :desc => 'Set the forj account name to use. By default, it takes the provider name.'
   method_option :infra,            :aliases => '-i',   :desc => 'Defines your Infra directory to use while booting. You can also set FORJ_INFRA_DIR.'
-  method_option :key_name,         :aliases => '-k',   :desc => 'Import a key pair.'
-  method_option :key_path,         :aliases => '-p',   :desc => 'Public key data'
+  method_option :key_name,         :aliases => '-k',   :desc => "Keypair name to use."
+  method_option :key_path,         :aliases => '-p',   :desc => "Private or Public key file. forj will determine if you provide a public key or a private, if respectively the extension '.pub' exist or not."
 
   method_option :boothook,         :aliases => '-H',   :desc => 'By default, boothook file used is build/bin/build-tools/boothook.sh. Use this option to set another one.'
   method_option :build,            :aliases => '-B',   :desc => 'Replace the default build.sh'
@@ -124,7 +127,18 @@ The list of predefined values can be retrieved with forj show defaults
       oConfig.set('account_name', options[:account_name])
       oConfig.set('infra_repo',   options[:infra])
       oConfig.set('keypair_name', options[:key_name])
-      oConfig.set('keypair_path', options[:key_path])
+      if options[:key_path]
+         mFound = options[:key_path].match(/^(.*)(\.pub)?$/)
+         if mFound 
+            key_path = File.expand_path(mFound[1])
+            if mFound[2] and not File.exists?(File.expand_path(mFound[1]+mFound[2]))
+               Logging.fatal(1, "'%s' is not a valid keypair files. At least the public key (.pub) is have to exist.")
+            end   
+            oConfig.set('keypair_path', key_path)
+         else
+            Logging.fatal(1, "'%s' is not a valid keypair files. At least the public key (.pub) is have to exist.")
+         end   
+      end   
 
       # TODO: Be able to support the default provider from config.yaml
       oConfig.set('provider', cloud_provider)

data/lib/boot.rb

@@ -42,16 +42,16 @@ module Boot
 
       # Check options and set data
       cloud_provider=oConfig.get('provider')
-      raise 'No provider specified.' if not cloud_provider
+      Logging.fatal(1, 'No provider specified.') if not cloud_provider
 
       if cloud_provider != 'hpcloud'
-         raise "forj setup support only hpcloud. '%s' is currently not supported." % cloud_provider
+         Logging.fatal(1, "forj setup support only hpcloud. '%s' is currently not supported." % cloud_provider)
       end
 
       oConfig.setDefault('account_name', cloud_provider)
 
       initial_msg = 'booting %s on %s (~/.forj/forj.log)' % [blueprint , cloud_provider]
-      Logging.message(initial_msg)
+      Logging.high_level_msg(initial_msg) #################
 
       # Initialize defaults
       maestro_url =  oConfig.get('maestro_url')
@@ -73,6 +73,7 @@ module Boot
 
       # Step Maestro Clone
       if not maestro_repo
+         Logging.high_level_msg('cloning maestro repo ...' ) #################
          Logging.info('cloning maestro repo from \'%s\'...' % maestro_url)
          Repositories.clone_repo(maestro_url)
          maestro_repo=File.expand_path('~/.forj/maestro')
@@ -92,6 +93,7 @@ module Boot
       # Connect to services
       oFC=ForjConnection.new(oConfig)
 
+      Logging.high_level_msg('Configuring network...') #################
       Logging.info('Configuring network \'%s\'' % [oConfig.get('network')])
       begin
         network = Network.get_or_create_network(oFC, oConfig.get('network'))
@@ -101,16 +103,17 @@ module Boot
         Logging.fatal(1, "Network properly configured is required.\n%s\n%s" % [e.message, e.backtrace.join("\n")])
       end
 
-
+      Logging.state('Configuring keypair...') #################
       Logging.info('Configuring keypair \'%s\'' % [oConfig.get('keypair_name')])
-      key_name = oConfig.get('keypair_name')
-      key_path = oConfig.get('keypair_path')
-      SecurityGroup.upload_existing_key(key_name, key_path)
+      SecurityGroup.hpc_import_pubkey(oConfig, oFC.sAccountName)
+
+      Logging.state('Configuring security group...') #################
 
       Logging.info('Configuring Security Group \'%s\'' % [oConfig.get('security_group')])
       security_group = SecurityGroup.get_or_create_security_group(oFC, oConfig.get('security_group'))
       ports = oConfig.get('ports')
 
+      Logging.state('Configuring security group ports...') #################
       ports.each do |port|
         port = port.to_s if port.class != String
         if not /^\d+(-\d+)?$/ =~ port 
@@ -123,10 +126,11 @@ module Boot
         end   
       end
 
+      ENV['FORJ_HPC'] = oFC.sAccountName
       ENV['FORJ_HPC_NET'] = network.name
       ENV['FORJ_SECURITY_GROUP'] = oConfig.get('security_group')
-      ENV['FORJ_KEYPAIR'] = key_name
-      ENV['FORJ_HPC_NOVA_KEYPUB'] = key_path
+      ENV['FORJ_KEYPAIR'] = oConfig.get('keypair_name')
+      ENV['FORJ_HPC_NOVA_KEYPUB'] = oConfig.get('keypair_path')
       ENV['FORJ_BASE_IMG'] = oConfig.get('image')
 
       # run build.sh to boot maestro

data/lib/connection.rb

@@ -30,15 +30,16 @@ class ForjConnection
 
    attr_accessor :oCompute
    attr_accessor :oNetwork
+   attr_accessor :sAccountName
 
    def initialize(oConfig)
 
-     sAccountName = oConfig.get('account_name')
+     @sAccountName = oConfig.get('account_name')
      @provider='HP' # TODO: Support multiple provider. (Generic Provider object required)
-     sAccountName = oConfig.get('provider') if not sAccountName
-     sAccountName = 'hpcloud' if not sAccountName
+     @sAccountName = oConfig.get('provider') if not @sAccountName
+     @sAccountName = 'hpcloud' if not @sAccountName
 
-     @credentials = get_credentials(sAccountName)
+     @credentials = get_credentials(@sAccountName)
      oSSLError=SSLErrorMgt.new
 
      # Trying to get Compute object
@@ -85,7 +86,7 @@ end
 def get_credentials(sAccountName)
   # TODO: Should support forj credentials. not hpcloud credentials.
 
-  raise 'Internal Error: Missing sAccountName' if not sAccountName
+  Logging.fatal(1, 'Internal Error: Missing sAccountName') if not sAccountName
 
   creds = File.expand_path('~/.hpcloud/accounts/%s' % [sAccountName])
   if not File.exists?(creds)

data/lib/defaults.yaml

@@ -14,17 +14,28 @@
 
 default:
   maestro_url: https://github.com/forj-oss/maestro.git
+
+  # Default Infra repository to use. If missing, it will be proposed to be created.
   infra_repo: ~/.forj/infra
+  
   # You can set proto2b in your ~/.forj/config.yaml if you built it from maestro/build. Read the maestro/README.md to create it.
   image: Ubuntu Precise 12.04.4 LTS Server 64-bit 20140414 (Rescue Image)
+  
+  # Flavor to use for Maestro
   flavor: standard.xsmall
+  
+  # Ports to open for Maestro, added to the security group
+  security_group: forj
   ports: [22, 80, 443, 3000, 3131, 3132, 3233, 3134, 3135, 4505-4506, 5000, 5666, 8000, 8080-8081, 8083, 8125, 8139-8140, 8773-8776, 9292, 29418, 35357]
-  keypair_path: ~/.hpcloud/keypairs/nova
-  keypair_name: nova
+  
+  # Private key file path. Those files (private/public key) will be added to ~/.forj/keypairs/ as respectively 'keypair_name' and 'keypair_name'.pub
+  keypair_path: ~/.ssh/forj-id_rsa 
+  keypair_name: forj
+
   # Network: If network doesn't exist, forj cli will try to create it, and attach it a router.
   network: forj
-  security_group: forj
-  # at this point you have to clone the infra project manually
+
+  # build.sh internal variables.  
   build_config: box
   branch: master
   box_name: maestro

data/lib/forj-config.rb

@@ -36,7 +36,7 @@ class ForjDefault
       # If config doesn't exist, it will be created, empty with 'defaults:' only
 
       if not $LIB_PATH
-         raise 'Internal $LIB_PATH was not set.'
+         Logging.fatal(1, 'Internal $LIB_PATH was not set.')
       end
       
       Logging.info ('Reading default configuration...')
@@ -66,7 +66,7 @@ class ForjConfig
 
 
       if not $FORJ_DATA_PATH
-         raise 'Internal $FORJ_DATA_PATH was not set.'
+         Logging.fatal(1, 'Internal $FORJ_DATA_PATH was not set.')
       end
 
       sConfigDefaultName='config.yaml'
@@ -191,5 +191,13 @@ class ForjConfig
       end   
       @yConfig.merge!(@yLocal) { |key, oldval, newval| key == 'default'? oldval: newval }
    end
+   
+   def LocalDefaultExist?(key)
+      if @yLocal['default'][key]
+         true
+      else
+         false
+      end      
+   end
 
 end

data/lib/helpers.rb

@@ -15,6 +15,8 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 
+require 'fileutils' 
+
 module Helpers
   def get_home_path
     File.expand_path('~')
@@ -25,4 +27,34 @@ module Helpers
       Dir.mkdir path
     end
   end
+  
+  def dir_exists?(path)
+    if File.exists?(path)
+       if not File.directory?(path)
+          msg = "'%s' is not a directory. Please fix it." % path
+          if $FORJ_LOGGER
+             Logging.fatal(1, msg)
+          else
+             raise msg
+          end
+       end
+       if not File.readable?(path) or not File.writable?(path) or not File.executable?(path)
+          msg = "%s is not a valid directory. Check permissions and fix it." % path
+          if $FORJ_LOGGER
+             Logging.fatal(1, msg)
+          else
+             raise msg
+          end
+       end
+       return true
+    end
+    false
+  end
+  
+  def ensure_dir_exists(path)
+    if not dir_exists?(path)
+       FileUtils.mkpath(path) if not File.directory?(path)
+    end
+  end
+  
 end

data/lib/log.rb

@@ -61,11 +61,17 @@ module Logging
      # Class used to create 2 log object, in order to keep track of error in a log file and change log output to OUTPUT on needs (option flags).
      
      attr_reader :level
-  
+
      def initialize(sLogFile = 'forj.log', level = Logger::WARN)
-        if not $FORJ_DATA_PATH
+        
+        if not $FORJ_DATA_PATH 
            raise "Internal Error: Unable to initialize ForjLog - global FORJ_DATA_PATH not set"
         end
+
+        if not Helpers.dir_exists?($FORJ_DATA_PATH)
+           raise "Internal Error: Unable to initialize ForjLog - '%s' doesn't exist." % $FORJ_DATA_PATH
+        end
+
         @oFileLogger = Logger.new(File.join($FORJ_DATA_PATH, sLogFile), 'weekly')
         @oFileLogger.level = Logger::DEBUG
         @oFileLogger.formatter = proc do |severity, datetime, progname, msg| 
@@ -160,6 +166,9 @@ module Logging
      print("%s%s ...\r" % [message, ANSI.clear_line]) if $FORJ_LOGGER.level == Logger::INFO
   end
   
-
+  def high_level_msg(message)
+     # Not DEBUG and not INFO. Just printed to the output.
+     puts ("%s" % [message]) if $FORJ_LOGGER.level > 1
+  end
 
 end

data/lib/network.rb

@@ -227,7 +227,7 @@ module Network
   
   # Router interface to connect to the network
   def create_router_interface(subnet, router)
-    raise "Internal Error: subnet/router object not passed." if not subnet or not router
+    Logging.fatal(1, "Internal Error: subnet/router object not passed.") if not subnet or not router
 
     Logging.debug("Attaching subnet '%s' to router '%s'" % [subnet.name, router.name])
     begin

data/lib/security.rb

@@ -16,11 +16,13 @@
 #    limitations under the License.
 
 require 'rubygems'
-require 'require_relative'
+require 'highline/import'
 
 #
 # SecurityGroup module
 #
+
+# TODO: Introduce most of HPCloud task in an hpcloud object.
 module SecurityGroup
 
   def get_or_create_security_group(oFC, name)
@@ -135,8 +137,20 @@ module SecurityGroup
     rule
   end
 
-  def upload_existing_key(key_name, key_path)
-    command = 'hpcloud keypairs:import %s %s' % [key_name, key_path]
+  def hpc_import_pubkey(oConfig, account)
+
+    key_name = oConfig.get('keypair_name')
+    key_path = oConfig.get('keypair_path')
+
+    Logging.fatal(1, "'keypair_path' undefined. check your config.yaml file.") if not key_path
+    Logging.fatal(1, "'keypair_name' undefined. check your config.yaml file.") if not key_name
+
+    pubkey_path = key_path + '.pub'
+    Logging.fatal(1, "keypair '%s' are missing. Please call 'forj setup %s' to create the missing key pair required." % [pubkey_path, account]) if not File.exists?(pubkey_path)
+    
+    Logging.info("Importing your forj keypair '%s' to hpcloud." % pubkey_path)
+    command = 'hpcloud keypairs:import %s %s -a %s' % [key_name, pubkey_path, account]
+    Logging.debug("Executing command '%s'" % command)
     Kernel.system(command)
   end
 end

data/lib/setup.rb

@@ -31,13 +31,13 @@ module Setup
   def setup(sProvider, oConfig, options )
     begin
 
-      raise 'No provider specified.' if not sProvider
+      Logging.fatal(1, 'No provider specified.') if not sProvider
 
       sAccountName = sProvider # By default, the account name uses the same provider name.
       sAccountName = options[:account_name] if options[:account_name]
 
       if sProvider != 'hpcloud'
-         raise "forj setup support only hpcloud. '%s' is currently not supported." % sProvider
+         Logging.fatal(1, "forj setup support only hpcloud. '%s' is currently not supported." % sProvider)
       end
 
       # TODO: Support of multiple providers thanks to fog.
@@ -50,11 +50,14 @@ module Setup
          Kernel.system('hpcloud account:copy hp %s' % [sAccountName])
       end
 
-      case Kernel.system('hpcloud account:setup %s' % [sAccountName] )
+      Logging.info("Configuring hpcloud account '%s'" % [sAccountName] )
+      command = 'hpcloud account:setup %s' % [sAccountName]
+      Logging.debug("Executing : '%s'" % command)
+      case Kernel.system(command)
          when false
-           raise "Unable to setup your hpcloud account"
+           Logging.fatal(1, "Unable to setup your hpcloud account")
          when nil
-           raise "Unable to execute 'hpcloud' cli. Please check hpcloud installation."
+           Logging.fatal(1, "Unable to execute 'hpcloud' cli. Please check hpcloud installation.")
       end
 
       if not oConfig.yConfig['default'].has_key?('account')
@@ -64,6 +67,10 @@ module Setup
 
       # Implementation of simple credential encoding for build.sh/maestro
       save_maestro_creds(sAccountName)
+      
+      # Check/create keypair
+      keypair_setup(oConfig)
+      
     rescue RuntimeError => e
       Logging.fatal(1,e.message)
     rescue  => e
@@ -72,7 +79,65 @@ module Setup
   end
 end
 
+def ensure_forj_dirs_exists()
+  # Function to create FORJ paths if missing.
+
+  # Defining Global variables
+  $FORJ_DATA_PATH = File.expand_path(File.join('~', '.forj'))
+  $FORJ_ACCOUNT_PATH = File.join($FORJ_DATA_PATH, 'account') # Not currently used...
+  $FORJ_KEYPAIRS_PATH = File.join($FORJ_DATA_PATH, 'keypairs')
+  $FORJ_CREDS_PATH = File.expand_path(File.join('~', '.cache', 'forj'))
+  
+  # TODO: To move to an hpcloud object.
+  $HPC_KEYPAIRS = File.expand_path(File.join('~', '.hpcloud', 'keypairs'))
+
+  Helpers.ensure_dir_exists($FORJ_DATA_PATH)
+  Helpers.ensure_dir_exists($FORJ_ACCOUNT_PATH)
+  Helpers.ensure_dir_exists($FORJ_KEYPAIRS_PATH)
+  Helpers.ensure_dir_exists($FORJ_CREDS_PATH)
+end
+  
+def keypair_setup(oConfig)
+
+   key_path = oConfig.get('keypair_path')
+
+   Logging.info("Configuring forj keypair '%s'" % [key_path] )
+
+   if not File.exists?(key_path)
+      # Need to create a key. ask if we need so.
+      real_key_path = File.expand_path(ask("If your ssh keypair doesn't exist, forj will ask ssh-keygen to create one for you.\nPrivate key file path:") do |q|
+         q.validate = /\w+/
+         q.default = key_path
+      end)
+      if not File.exists?(real_key_path)
+         Helpers.ensure_dir_exists(File.dirname(real_key_path))
+         command = 'ssh-keygen -t rsa -f %s' % real_key_path
+         Logging.debug("Executing '%s'" % command)
+         system(command)
+      end
+      if not File.exists?(real_key_path)
+         Logging.fatal(1, "'%s' not found. Unable to add your keypair to hpcloud. Create it yourself and provide it with -p option. Then retry." % [real_key_path])
+      else
+         if real_key_path != key_path and not oConfig.LocalDefaultExist?('keypair_path')
+            Logging.debug("Saving forj keypair '%s' as default." % [real_key_path] )
+            oConfig.LocalSet('keypair_path', real_key_path)
+            oConfig.SaveConfig()
+         end   
+      end
+   end
+end
+
+
 def save_maestro_creds(sAccountName)
+  # Check required global data
+  if not $FORJ_CREDS_PATH
+     Logging.fatal(1, "Internal error: '$FORJ_CREDS_PATH' missing.")
+  end
+  if not Helpers.dir_exists?($FORJ_CREDS_PATH)
+     Logging.fatal(1, "Internal error: '%s' doesn't exist." % $FORJ_CREDS_PATH)
+  end
+
+  Logging.info("Completing hpcloud account '%s' information." % [sAccountName] )
 
   # TODO Be able to load the previous username if the g64 file exists.
   hpcloud_os_user = ask('Enter hpcloud username: ') do |q|
@@ -87,16 +152,12 @@ def save_maestro_creds(sAccountName)
 
   add_creds = {:credentials => {:hpcloud_os_user=> hpcloud_os_user, :hpcloud_os_key=> hpcloud_os_key}}
 
-  sForjCache=File.expand_path('~/.cache/forj/')
-  cloud_fog = '%s/%s.g64' % [sForjCache, sAccountName]
-
-
-  Helpers.create_directory(sForjCache) if not File.directory?(sForjCache)
+  cloud_fog = File.join($FORJ_CREDS_PATH, sAccountName+'.g64')
 
   # Security fix: Remove old temp file with clear password.
-  old_file = '%s/master.forj-13.5' % [sForjCache]
+  old_file = '%s/master.forj-13.5' % [$FORJ_CREDS_PATH]
   File.delete(old_file) if File.exists?(old_file)
-  old_file = '%s/creds' % [sForjCache]
+  old_file = '%s/creds' % [$FORJ_CREDS_PATH]
   File.delete(old_file) if File.exists?(old_file)
 
   hpcloud_creds = File.expand_path('~/.hpcloud/accounts/%s' % [sAccountName])

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: forj
 version: !ruby/object:Gem::Version
-  version: 0.0.35
+  version: 0.0.36
 platform: ruby
 authors:
 - forj team
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-06-12 00:00:00.000000000 Z
+date: 2014-08-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -122,7 +122,7 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: 1.4.3
-description: forj command line
+description: forj cli - See https://www.forj.io for documentation/information
 email:
 - forj@forj.io
 executables: