forj 1.0.1 → 1.0.2

data/forj.gemspec

@@ -0,0 +1,64 @@
+#!/usr/bin/env ruby
+# encoding: UTF-8
+
+# (c) Copyright 2014 Hewlett-Packard Development Company, L.P.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+Gem::Specification.new do |s|
+  s.name        = 'forj'
+  s.homepage = 'https://www.forj.io'
+
+  s.version     = '1.0.2'
+  s.date        = '2014-12-18'
+  s.summary     = 'forj command line'
+  s.description = 'forj cli - See https://www.forj.io for documentation/information'
+
+  s.authors     = ['forj team']
+  s.email       = %w(forj@forj.io)
+
+  s.files         = `git ls-files -z`.split("\x0")
+  s.executables   = s.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  s.test_files    = s.files.grep(%r{^(test|spec|features)/})
+  s.require_paths = ["lib"]
+
+  s.homepage    = 'https://forj.io'
+  s.license     = 'Apache License, Version 2.0.'
+  s.post_install_message = 'Go to docs.forj.io for more information on how to use forj cli'
+  s.required_ruby_version = '>= 1.8.5'
+
+  s.bindir = 'bin'
+
+  ## This gets added to the $LOAD_PATH so that 'lib/NAME.rb' can be required as
+  ## require 'NAME.rb' or'/lib/NAME/file.rb' can be as require 'NAME/file.rb'
+  s.require_paths = %w[lib]
+
+  s.add_runtime_dependency 'thor', '>=0.16.0'
+  s.add_runtime_dependency 'fog', '1.19.0'
+  s.add_runtime_dependency 'git', '>=1.2.7'
+  s.add_runtime_dependency 'highline', '>= 1.6.21'
+  s.add_runtime_dependency 'ansi', '>= 1.4.3'
+  s.add_runtime_dependency 'encryptor', '>=1.3.0'
+  s.add_runtime_dependency 'json', '1.7.5'
+  s.add_runtime_dependency 'bundler'
+  s.add_runtime_dependency 'nokogiri','1.5.11'
+  s.add_runtime_dependency 'lorj', '~> 0.2.0'
+
+  s.add_development_dependency "bundler"
+  s.add_development_dependency "rake", "~> 10.0"
+  s.add_development_dependency "rspec", "~> 3.1.0"
+  s.rdoc_options << '--title' << 'Lorj - The Process Controllers framework system' <<
+  '--main' << 'README.md'
+
+
+end

data/{lib → forj}/defaults.yaml

@@ -37,6 +37,9 @@
   # Network: If network doesn't exist, forj cli will try to create it, and attach it a router.
   :network: forj
 
+  # Users: Default user for ssh connection, if user doesn't exits, forj cli will try to get the user from the server image on it's name attribute
+  :users: ['ubuntu', 'fedora', 'cloud-user', 'cirros', 'centos', 'cloud', 'root']
+
   # build.sh internal variables.
   :build_config: box
   :branch: master
@@ -78,6 +81,7 @@
       -  :desc:    "Maestro and blueprint configuration:"
          :add:
             - :keypair_files
+            - :ssh_user
       -  :desc:    "DNS Configuration for Maestro:"
 :sections:
   # This section define updatable data available from config.yaml. But will never be added in an account file.
@@ -121,7 +125,7 @@
          The public key is used to configure your server to authorize you to access the box with your private key.
          This keypair files will be copied to '$HOME/.forj/keypairs/ under <keypair_name> files for 'forj' needs.
 
-         If the keypair does exists locally, it will be created for you.
+         If the keypair doesn't exist locally, it will be created for you.
       :desc:               "Base keypair file name"
       :default_value:      "~/.ssh/id_rsa-forj"
       :validate_function:  :forj_check_keypairs_files
@@ -252,6 +256,24 @@
          :value:        :name
          :validate:     :list_strict
       :ask_step:        2
+    :ssh_user:
+      :desc:        "User name for ssh connection of your selected image."
+      :explanation: |-
+        The image name you have selected has a unique SSH Account access.
+
+        Thanks to the name of the image, setup assume the account name to use.
+        If this name is incoherent with the image you choosed, please update it.
+
+        Checking image '<%= config[:image_name] %>'...
+      :account:     true
+      :ask_step:    2
+      :list_values:
+        :query_type:  :process_call # Will execute a query on image and return the list of images.
+        :query_call:  :setup_ssh_user
+        :query_params:
+          :image_name: lorj::config[:image_name]
+        :object:      :image
+        :value:       :name
     :ports:
       :desc:            "List of security group rules (1 port or range of ports) to open to the external network."
     :branch:

data/lib/appinit.rb

@@ -15,11 +15,11 @@
 #    limitations under the License.
 
 
+require 'lorj'
 
-# Base Logging system started and loaded.
-require 'log.rb' # Load default loggers
-include Logging
-
+# Module to initialize the application
+# TODO: Cleanup about Global variables used. Should be replaced by PrcLib
+#       or other kind of setting.
 module AppInit
 
    def AppInit.forj_initialize()
@@ -45,7 +45,7 @@ module AppInit
    end
 
    def AppInit.ensure_dir_exists(path)
-      if not dir_exists?(path)
+      if not PrcLib.dir_exists?(path)
        FileUtils.mkpath(path) if not File.directory?(path)
       end
    end

data/lib/build_tmpl/build-env.py

@@ -0,0 +1,293 @@
+#!/bin/env python
+#
+# (c) Copyright 2014 Hewlett-Packard Development Company, L.P.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+# This script is used to create a env file used by build.sh.
+# It is based on Maestro/templates/infra/maestro.box.GITBRANCH.env.tmpl
+
+# This script detects few tags, and ask to enter the appropriate vallue, 
+# then save the result to a directory given by the user.
+
+# As this script is very early in the boot process, we are running on end user worstation.
+# We will avoid adding non default module like:
+# - ptemplates, django for template interpreter. We use our own simple code to make it work with basics rules.
+#   It provides variable detection, with comments and default variables.
+
+import sys 
+import getopt
+#import urllib2
+#from urlparse import urlparse,ParseResult
+import re
+import logging
+import logging.handlers
+import yaml
+import os
+#import subprocess
+#import distutils.spawn
+#import string 
+#from datetime import date,datetime
+#import time
+#import tempfile
+import readline
+
+# Defining defaults
+
+GITBRANCH='master'
+MAESTRO_RPATH_TMPL="templates/infra/"
+MAESTRO_TMPL_FILE="maestro.box.GITBRANCH.env.tmpl"
+MAESTRO_TMPL_PROVIDER='hpcloud'
+
+# 
+# 
+   
+##############################
+def help(sMsg):
+   print 'build-env.py [-h|--help] --maestro-repo MaestroPath --env-path BuildEnvPath [--gitbranch BranchName] [-d|--debug] [-v|--verbose]'
+
+# TODO Support for re-build it.
+
+##############################
+# TODO: split this function in 3, for code readibility
+
+def BuildEnv(EnvPath, MaestroFile, MaestroPath, GitBranch):
+  """Build env file if doesn't exist."""
+
+  global MAESTRO_TMPL_FILE
+  oLogging=logging.getLogger('build-env')
+
+  # -----------------------------------------------
+  # Check step ------------------------------------
+
+  if not os.path.exists(EnvPath):
+     oLogging.error("'%s' do not exist. This directory is required. (--env-path)", EnvPath)
+     sys.exit(1)
+
+  if not os.path.isdir(EnvPath):
+     oLogging.error("'%s' is not a valid directory (--env_path)", EnvPath)
+     sys.exit(1)
+
+  if not os.path.isdir(MaestroPath):
+     oLogging.error("'%s' is not a valid directory. Should refer to Maestro repository root dir.(--maestro-path)", MaestroPath)
+     sys.exit(1)
+     
+
+  sInputFile=os.path.join(MaestroPath, MaestroFile)
+  if not os.path.exists(sInputFile):
+     oLogging.error("Unable to find template '%s' from '%s'. Are you sure about Maestro Repository root path given? Check and retry.", MaestroPath, MaestroFile)
+     sys.exit(1)
+
+  # Replace GITBRANCH Name by real git branch value.
+  sOutputFile=os.path.join(EnvPath,re.sub('GITBRANCH',GITBRANCH,MAESTRO_TMPL_FILE))
+  sOutputFile=re.sub('.tmpl$','',sOutputFile)
+  
+  if os.path.exists(sOutputFile):
+     # TODO Support for re-build it.
+     oLogging.info("'%s' exists. If you want to rewrite it, remove the file before.", sOutputFile)
+     sys.exit(0)
+  
+  # Load template variables and build list of variable to query
+  oLogging.debug("Opening template '%s'",sInputFile)
+  try:
+    fTmpl=open(sInputFile)
+  except IOError as e:
+    oLogging.error("Unable to open '%s'. Errno: %s (%s)", sInputFile, e.errno, e.strerror)
+    sys.exit(2)
+  
+  # -----------------------------------------------
+  # Reading template ------------------------------
+
+  print("Reading template '{0}'".format(sInputFile))
+  oComments={}
+  oVars={}
+
+  # Search for Variable. If the variable name ends with '!', this value could not be null.
+  reVar=re.compile('[^$]{([A-Z_-]+)(!)?(:.*?)?}')
+  # 1: Variable Name
+  # 2: Required if = '!' 
+  # 3: Default
+
+
+  # Search for comment
+  reComment=re.compile('^ *# +([A-Z_-]+): (.*)$')
+
+  for line in fTmpl:
+
+      # Detecting comment
+      oComment=reComment.search(line)
+      if oComment <> None:
+         oLogging.debug('Found "%s" = "%s"', oComment.group(1), oComment.group(2))
+         if oVars.has_key(oComment.group(1)):
+            oComments[oComment.group(1)]=oVars[oVar.group(1)]['comments']+'\n# '+oComment.group(2)
+         elif oComments.has_key(oComment.group(1)):
+            oComments[oComment.group(1)]=oComments[oComment.group(1)]+'\n# '+ oComment.group(2)
+         else:
+            oComments[oComment.group(1)]='# '+oComment.group(2)
+
+      oVar=reVar.search(line)   
+      if oVar <> None:
+         oLogging.debug('Found var "%s" from "%s"',oVar.group(1),line)
+
+         sComment=''
+         if oComments.has_key(oVar.group(1)):
+            sComment=oComments[oVar.group(1)]
+         else:
+            oComments.remove(oVar.group(1))
+
+         sDefault=''
+         bRequired=False
+         if oVar.group(2) <> None  :
+            bRequired=True
+            oLogging.debug("'%s' is required",oVar.group(1))
+         if oVar.group(3) <> None:
+            sDefault=oVar.group(3)[1:]
+            oLogging.debug("'%s' default value is '%s'",oVar.group(1),sDefault)
+            
+
+         oVars[oVar.group(1)]={ 'comments': sComment,
+                                'required': bRequired,
+                                'default': sDefault } 
+
+  oLogging.debug('template loaded.')
+
+  # -----------------------------------------------
+  # Ask section -----------------------------------
+
+  print "We need some information from you. Please ask the following question:\n"
+  # Time to ask information to the user.
+
+  for sElem in oVars:
+     print '{0}'.format(oVars[sElem]['comments'])
+     sPar="# "
+     if oVars[sElem]['default'] <> '':
+        sPar+='Default is "'+oVars[sElem]['default']+'".'
+     if oVars[sElem]['required'] :
+        if sPar <> "# " :
+           sPar+=' Required.'
+        else:
+           sPar='# Required.'
+
+     if sPar <> "":
+        print sPar
+     sValue=""
+
+     while sValue == "":
+       sValue=raw_input(sElem+'=')
+
+       if sValue == "":
+          sValue=oVars[sElem]['default']
+       
+       if oVars[sElem]['required']:
+          if sValue == "":
+             print "Value required. Please enter a value."
+       else:
+          break
+
+     oVars[sElem]['value']=sValue
+     print '{0}="{1}"\n'.format(sElem,sValue)
+
+
+  fTmpl.seek(0) # Read the file again to replace data
+  try:
+     fOutputFile=open(sOutputFile,"w+")
+  except IOError as e:
+    oLogging.error("Unable to open '%s' for write. Errno: %s (%s)", sOutputFile, e.errno, e.strerror)
+    sys.exit(2)
+
+  print "--------------------------------\nThank you\nWriting '{0}'".format(sOutputFile)
+
+  # -----------------------------------------------
+  # Time to save the template. --------------------
+  
+  reReplaced=re.compile('([^$])({([A-Z_-]+)(!)?(:.*?)?})')
+  for line in fTmpl:
+     oVar=reReplaced.search(line)
+
+     if oVar <> None:
+    
+        if oVar.group(1) <> None:
+           sValue=oVar.group(1)+oVars[oVar.group(3)]['value']
+        else:
+           sValue=oVars[oVar.group(3)]['value']
+        sNewLine=reReplaced.sub(sValue,line)
+     else:
+        sNewLine=line
+
+     sys.stdout.flush()
+     fOutputFile.write(sNewLine)
+  
+  fOutputFile.close()
+  fTmpl.close()
+  print 'Done'
+
+
+
+##############################
+def main(argv):
+  """Main function"""
+
+  global GITBRANCH
+  global MAESTRO_TMPL_PROVIDER
+  global MAESTRO_RPATH_TMPL
+
+  oLogging=logging.getLogger('build-env')
+  oLogging.setLevel(20)
+
+  try:
+     opts,args = getopt.getopt(argv,"hp:vd:r:P:", ["help", "--for-provider=", "env-path=" , "maestro-path=" ,"debug" ,"verbose" ])
+  except getopt.GetoptError, e:
+     oLogging.error('Error: '+e.msg)
+     help()
+     sys.exit(2)
+
+  for opt, arg in opts:
+     if opt in ('-h', '--help'):
+        help()
+        sys.exit()
+     elif opt in ('-v','--verbose'):
+        if oLogging.level >20:
+           oLogging.setLevel(oLogging.level-10)
+     elif opt in ('--debug','-d'):
+        logging.getLogger().setLevel(logging.DEBUG)
+        logging.debug("Setting debug mode")
+        oLogging.setLevel(logging.DEBUG)
+     elif opt in ('-p', '--env-path'):
+        ENV_PATH=arg
+     elif opt in ('--gitbranch'):
+        GITBRANCH=arg
+     elif opt in ('-r','--maestro-path'):
+        MAESTRO_PATH=arg
+     elif opt in ('-P','--for-provider'):
+        TMPL_PROVIDER=arg
+
+  # Start Main tasks - Testing required variables.
+  if not 'ENV_PATH' in locals() or not 'MAESTRO_PATH' in locals():
+     oLogging.error("--env-path or --maestro-path values missing. Please check command flags.")
+     sys.exit(1)
+
+
+  global MAESTRO_TMPL_FILE # template file to use.
+  
+  sMaestroFile=os.path.join(MAESTRO_RPATH_TMPL, MAESTRO_TMPL_PROVIDER+'-'+MAESTRO_TMPL_FILE)
+  BuildEnv(ENV_PATH, sMaestroFile, MAESTRO_PATH, GITBRANCH)
+
+  sys.exit(0)
+
+#####################################
+logging.basicConfig(format='%(asctime)s: %(levelname)s - %(message)s', datefmt='%m/%d/%Y %I:%M:%S %p')
+
+if __name__ == "__main__":
+   main(sys.argv[1:])
+
+

data/lib/cloud_test.rb

@@ -0,0 +1,121 @@
+#!/usr/bin/env ruby
+
+#require 'byebug'
+
+$APP_PATH = File.dirname(__FILE__)
+$LIB_PATH = File.expand_path(File.join(File.dirname($APP_PATH),'lib'))
+
+$LOAD_PATH << $LIB_PATH
+
+$LOAD_PATH << File.join($LIB_PATH, 'lib-forj', 'lib')
+
+require 'appinit.rb'
+
+# Initialize forj paths
+AppInit::forj_initialize()
+
+# Initialize global Log object
+$FORJ_LOGGER=ForjLog.new()
+
+require 'lib-forj.rb'
+
+Logging.set_level(Logger::DEBUG)
+
+# Load global Config
+oConfig = ForjConfig.new()
+
+aProcesses = []
+
+# Defines how to manage Maestro and forges
+# create a maestro box. Identify a forge instance, delete it,...
+aProcesses << File.join($LIB_PATH, 'forj', 'ForjCore.rb')
+
+# Defines how cli will control FORJ features
+# boot/down/ssh/...
+aProcesses << File.join($LIB_PATH, 'forj', 'ForjCli.rb')
+
+$LIB_FORJ_DEBUG = 3 # verbose
+
+infra_dir = File.expand_path(oConfig.get(:infra_repo))
+
+# Ask information if needed.
+if not Dir.exist?(File.expand_path(infra_dir))
+   Logging.warning(<<-END
+Your infra workspace directory is missing.
+
+Forj uses an infra workspace directory to store any kind of data that are private to you.
+We provides ways to send those data securily to your new Forge instance, as metadata.
+In production case, we suggest you to keep it safe in your SCM preferred database.
+
+If you already have an existing infra workspace, use 'forj set infra_repo=<PathToYourRepo>' to set it and restart.
+
+Otherwise, we will build a new one with some predefined data, you can review and update later.
+   END
+   )
+   sAsk = "Do you want to create a new one from Maestro (yes/no)?" % [infra_dir]
+   bBuildInfra=agree(sAsk)
+   if not bBuildInfra
+      puts 'Process aborted on your demand.'
+      exit 0
+   end
+end
+
+oCloud = ForjCloud.new(oConfig, 'hpcloud', aProcesses)
+
+#oConfig.set(:instance_name, "test")
+#oCloud.Create(:metadata)
+#oCloud.Create(:infra_repository)
+#oCloud.Create(:userdata)
+
+
+#oCloud.Setup(:server, 'hpcloud')
+#oCloud.Setup(:forge, 'hpcloud')
+
+#oCloud.Create(:forge)
+
+#oConfig.set(:instance_name, 'servertestluis')
+#oCloud.Create(:forge)
+
+oForge = oCloud.Get(:forge, "luistest")
+
+#Ask the user to get server(s) to destroy
+server_id_length = 0
+server_name_length = 0
+
+oForge[:server].each{ |server|
+  if server[:id].length() >  server_id_length
+    server_id_length = server[:id].length()
+  end
+
+  if server[:name].length() >  server_name_length
+    server_name_length = server[:name].length()
+  end
+}
+
+server_index = 1
+#Display headers
+puts "|%s |%s |%s |" % ["Index ".ljust(6), "Name".ljust(server_name_length), "ID".ljust(server_id_length) ]
+#puts "|%s |%s |%s |" % ["0", "all".ljust(server_name_length), " ".ljust(server_id_length) ]
+#Display Forge servers detail
+oForge[:server].each{ |server|
+  puts "|%s |%s |%s |" % [server_index.to_s().ljust(6), server[:name].to_s().ljust(server_name_length), server[:id].to_s().ljust(server_id_length) ]
+  server_index = server_index + 1
+}
+
+oHighLine = HighLine.new()
+
+index = oHighLine.ask("Select the index of the server to create the ssh connection", Integer)do |q|
+  q.below=oForge[:server].count + 1
+  q.above=0
+end
+
+
+oConfig.set(:instance_name, 'luistest')
+oConfig.set(:forge_server, oForge[:server][index - 1][:id])
+oConfig.set(:server_name, oForge[:server][index - 1][:name])
+#oConfig.set(:box, 'maestro')
+#oConfig.set(:instance_name, 'luistest')
+#oConfig.Create(:server)
+oCloud.Create(:ssh)
+
+#oCloud.Query(:server, 'maestro')