forj 0.0.48 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 889541f2f3a27b5291850a04d26e706d82fae15c
-  data.tar.gz: 45c2194d9130f2b7776fe6d5e0d24bb873f6c0a8
+  metadata.gz: 1fe45b075653728015aec4bace3b604d214c94e4
+  data.tar.gz: f00361d094599e50b214c8fecdb17e95b04c56a5
 SHA512:
-  metadata.gz: 6e83e7256e96eb4dab205210c2e0e12f40eaf06ba6d790f48dc2359571b2566b2013d6b4da78ed8c28494999b85d6ad872f426a222e363d84f6a890b8905abd3
-  data.tar.gz: 5bbfe230e6199ddadfaae0a7fed6384878f80455533d22fc5eb142c4a5875a323b7d2081c01638e9f85ff37c27b2972d86f9976ac0cf46c42d5ed834c6fcbc23
+  metadata.gz: 3429c0634f5dea3a1174c47277e0e188905e52cc61ef6a7b66a3d507463a7373166e750a79bb9d0999ea1e0751fe1c542138c224ab99cd680cc91fc7e47ec95f
+  data.tar.gz: 7ba23851e38c291693c8d24b1abc09022cad244eb56b4e03731145c576181e8f94d792872023eee0451f0f0e1eba78624c1897c7c4bad2e21723a0ef91515f3c

data/Gemfile

@@ -19,19 +19,20 @@ source 'https://rubygems.org'
 group(:development, :test) do
   gem 'rake'
   gem 'debugger'
+  gem 'byebug'
   gem 'rspec', "~> 3.1.0", :require => false
 end
 
 gem 'mime-types','1.25.1'
 gem 'excon','0.31.0'
-gem 'json','~>1.7.5'
+gem 'json','1.7.5'
 gem 'nokogiri','1.5.11'
 gem 'fog', '1.19.0'
 gem 'git', '>=1.2.7'
 gem 'rainbow'
 gem 'rbx-require-relative', '~> 0.0.7'
 gem 'thor', '>=0.16.0'
-gem 'hpcloud', '~>2.0.8'
+gem 'hpcloud', '~>2.0.9'
 gem 'highline','>=1.6.21'
 gem 'ansi','>=1.4.3'
 gem 'encryptor','>=1.3.0'

data/bin/forj

@@ -25,6 +25,7 @@ $APP_PATH = File.dirname(__FILE__)
 $LIB_PATH = File.expand_path(File.join(File.dirname($APP_PATH),'lib'))
 
 $LOAD_PATH << $LIB_PATH
+$LOAD_PATH << File.join($LIB_PATH, 'lib-forj', 'lib')
 
 require 'appinit.rb' # Load generic Application level function
 
@@ -38,20 +39,18 @@ require 'forj-config.rb' # Load class ForjConfig and Meta data class variables.
 require 'forj-account.rb' # Load class ForjAccount
 require 'connection.rb' # Load class ForjConnection
 
-require 'boot.rb'
 require 'down.rb'
-require 'setup.rb'
 require 'ssh.rb'
 
-include Boot
 include Down
-include Setup
 include Ssh
 require 'forj-settings.rb' # Settings features
 
 #require 'debugger' # Use to debug with Ruby < 2.0
 #require 'byebug'   # Use to debug with Ruby >= 2.0
+require 'lib-forj.rb'
 
+$LIB_FORJ_DEBUG = 1 # less verbose
 
 
 class ForjThor < Thor
@@ -59,6 +58,7 @@ class ForjThor < Thor
   class_option :debug,   :aliases => '-d', :desc => 'Set debug mode'
   class_option :verbose, :aliases => '-v', :desc => 'Set verbose mode'
   class_option :config,  :aliases => '-c', :desc => 'Path to a different forj config file. By default, use ~/.forj/config.yaml'
+  class_option :libforj_debug,             :desc => "Set lib-forj debug level verbosity. 1 to 5. Default is one."
 
 
   desc "help [action]", "Describe available FORJ actions or one specific action"
@@ -144,6 +144,11 @@ Maestro/infra bootstrap debugging:"
       Logging.set_level(Logger::INFO) if options[:verbose]
       Logging.set_level(Logger::DEBUG) if options[:debug]
 
+      unless options[:libforj_debug].nil?
+         $LIB_FORJ_DEBUG = options[:libforj_debug].to_i
+         Logging.set_level(Logger::DEBUG)
+      end
+
       oConfig = ForjConfig.new(options[:config])
 
 
@@ -179,6 +184,7 @@ Maestro/infra bootstrap debugging:"
       oConfig.set(:branch ,        options[:branch])
       oConfig.set(:test_box, File.expand_path(options[:test_box])) if options[:test_box] and File.directory?(File.expand_path(options[:test_box]))
 
+      Logging.warning("test_box is currently disabled in this version. It will be re-activated in newer version.") if options[:test_box]
 
       if options[:key_path]
          mFound = options[:key_path].match(/^(.*)(\.pub)?$/)
@@ -192,8 +198,28 @@ Maestro/infra bootstrap debugging:"
             Logging.fatal(1, "'%s' is not a valid keypair files. At least the public key (.pub) is have to exist.")
          end
       end
+      aProcesses = []
+
+      # Defines how to manage Maestro and forges
+      # create a maestro box. Identify a forge instance, delete it,...
+      aProcesses << File.join($LIB_PATH, 'forj', 'ForjCore.rb')
+
+      # Defines how cli will control FORJ features
+      # boot/down/ssh/...
+      aProcesses << File.join($LIB_PATH, 'forj', 'ForjCli.rb')
+
+      oCloud = ForjCloud.new(oForjAccount, oConfig[:account_name], aProcesses)
 
-      Boot.boot(blueprint, name, options[:build], options[:boothook], options[:box_name], oForjAccount)
+      oConfig[:instance_name] = name
+
+      if blueprint == 'maestro'
+         Logging.info("Starting boot process of '%s'. No blueprint requested." % oConfig[:instance_name])
+      else
+         oConfig[:blueprint] = blueprint
+         Logging.info("Starting boot process of '%s' with blueprint '%s'." % [oConfig[:instance_name], oConfig[:blueprint]])
+      end
+      oCloud.Create(:forge)
+      #Boot.boot(blueprint, name, options[:build], options[:boothook], options[:box_name], oForjAccount)
   end
 
 ################################# Show defaults
@@ -367,12 +393,9 @@ ex: forj ssh myforge review
   end
 
 ################################# SETUP
-  method_option :keypair_name,         :aliases => '-k',   :desc => "config keypair_name : Keypair name attached as default to your FORJ account."
-  method_option :keypair_path,         :aliases => '-p',   :desc => "config keypair_path : SSH key file (private or public) to attach as default to your FORJ account.
-                                       It will attach any detected private/public key thanks to the file name
-                                       (without extension/.pem = private, .pub = public)"
 
   desc 'setup [AccountName [Provider]] [options]', "Setup FORJ cloud account credentials and information."
+
   long_desc <<-LONGDESC
 This setup will configure a FORJ account used to connect to your cloud system.
 \x5It will ask for your cloud provider credentials and services.
@@ -389,14 +412,33 @@ Several data will be requested like:
 \x5- domain name to add to each boxes hostname
   LONGDESC
   def setup(sAccountName = 'hpcloud', sProvider = "hpcloud")
-    Logging.set_level(Logger::INFO)  if options[:verbose]
-    Logging.set_level(Logger::DEBUG) if options[:debug]
-    oConfig=ForjConfig.new(options[:config])
-    oConfig.set(:provider, sProvider)
-    oConfig.set(:account_name, sAccountName)
-    oConfig.set(:keypair_path, options[:keypair_path]) if options[:keypair_path]
-    oConfig.set(:keypair_name, options[:keypair_name]) if options[:keypair_name]
-    Setup.setup(oConfig)
+      Logging.set_level(Logger::INFO)  if options[:verbose]
+      Logging.set_level(Logger::DEBUG) if options[:debug]
+
+      unless options[:libforj_debug].nil?
+         $LIB_FORJ_DEBUG = options[:libforj_debug].to_i
+         Logging.set_level(Logger::DEBUG)
+      end
+
+      oConfig=ForjConfig.new(options[:config])
+      oConfig.set(:provider_name, sProvider)
+      oConfig.set(:account_name, sAccountName)
+
+      aProcesses = []
+
+      # Defines how to manage Maestro and forges
+      # create a maestro box. Identify a forge instance, delete it,...
+      aProcesses << File.join($LIB_PATH, 'forj', 'ForjCore.rb')
+
+      # Defines how cli will control FORJ features
+      # boot/down/ssh/...
+      aProcesses << File.join($LIB_PATH, 'forj', 'ForjCli.rb')
+
+      oCloud = ForjCloud.new(oConfig, sAccountName, aProcesses)
+
+      oCloud.Setup(:forge, sAccountName)
+      oCloud.config.ac_save()
+
   end
 
 end

data/lib/appinit.rb

@@ -29,6 +29,7 @@ module AppInit
       $FORJ_DATA_PATH = File.expand_path(File.join('~', '.forj'))
       $FORJ_ACCOUNTS_PATH = File.join($FORJ_DATA_PATH, 'accounts')
       $FORJ_KEYPAIRS_PATH = File.join($FORJ_DATA_PATH, 'keypairs')
+      $FORJ_BUILD_PATH = File.join($FORJ_DATA_PATH, '.build')
       $FORJ_CREDS_PATH = File.expand_path(File.join('~', '.cache', 'forj'))
 
       # TODO: To move to an hpcloud object.
@@ -37,6 +38,7 @@ module AppInit
 
       AppInit.ensure_dir_exists($FORJ_DATA_PATH)
       AppInit.ensure_dir_exists($FORJ_ACCOUNTS_PATH)
+      AppInit.ensure_dir_exists($FORJ_BUILD_PATH)
       AppInit.ensure_dir_exists($FORJ_KEYPAIRS_PATH)
       FileUtils.chmod(0700, $FORJ_KEYPAIRS_PATH)
       AppInit.ensure_dir_exists($FORJ_CREDS_PATH)

data/lib/build_tmpl/bootstrap_build.sh

@@ -0,0 +1,145 @@
+#!/bin/bash
+
+# (c) Copyright 2014 Hewlett-Packard Development Company, L.P.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+
+APP_NAME="maestro"
+APPPATH=.
+BUILD_DIR="$1/.build"
+MAESTRO_REPO="$2"
+BOOTSTRAP_DIR="$3"
+BOOTSTRAP_EXTRA="$4"
+META_JSON="$5"
+MIME_SCRIPT="$6"
+USER_DATA="$7"
+
+mkdir -p $BUILD_DIR
+
+BOOTHOOK="$MAESTRO_REPO/build/bin/build-tools/boothook.sh"
+
+function Info
+{
+   echo "INFO: $1"
+}
+
+function Warning
+{
+   echo "WARN: $1"
+}
+
+function bootstrap_build
+{
+Info "Preparing cloudinit mime."
+
+cd $MAESTRO_REPO/build/$APP_NAME
+
+# Maestro BootHook - Set hostname and workaround missing meta.js
+Info "Boothook used: $BOOTHOOK"
+sed 's!${metadata-json}!'"$META_JSON"'!g' $BOOTHOOK > $BUILD_DIR/boothook.sh
+TOMIME="$BUILD_DIR/boothook.sh:text/cloud-boothook"
+
+# Add cloudconfig.yaml - Inform cloud-init to no update hostname & install basic packages.
+if [ -f cloudconfig.yaml ]
+then
+   TOMIME="$TOMIME cloudconfig.yaml"
+fi
+
+# Maestro boot. Use maestro/{#}-*.sh to build a shell script added to the cloud-init mime file.
+BOOT_BOX=$BUILD_DIR/boot_${APP_NAME}.sh
+
+rm -f $BOOT_BOX
+
+if [ "$BOOTSTRAP_DIR" != "" ]
+then
+   if [ "$(find $BOOTSTRAP_DIR -maxdepth 1 \( -type f -o -type l \) -name \*.sh -exec basename {} \; | wc -l)" -eq 0 ]
+   then
+      Warning "'$BOOTSTRAP_DIR' contains no bootstrap files. Please check it."
+   else
+      Info "Completing Box basic cloud-init with '$BOOTSTRAP_DIR'"
+   fi
+fi
+if [ -d "$BOOTSTRAP_EXTRA" ]
+then
+   Info "Completing Box basic cloud-init with '$BOOTSTRAP_EXTRA'"
+fi
+
+BOOT_FILES="$(find bootstrap $BOOTSTRAP_EXTRA $BOOTSTRAP_DIR -maxdepth 1 \( -type f -o -type l \) -name \*.sh -exec basename {} \; | sort -u)"
+
+echo "Read boot script: "
+for BOOT_FILE in $BOOT_FILES
+do
+   for DIR in bootstrap $BOOTSTRAP_EXTRA $BOOTSTRAP_DIR
+   do
+      if [ ! -f $BOOT_BOX ]
+      then
+         printf "#!/bin/bash\n# This script was automatically generated by ${0}. Do not edit it.\n" > $BOOT_BOX
+      fi
+      if [ -r $DIR/$BOOT_FILE ]
+      then
+         printf "\n#############\n# bootstrap builder: include $DIR/$BOOT_FILE\n#############\n\n" >> $BOOT_BOX
+         cat $DIR/$BOOT_FILE >> $BOOT_BOX
+         echo "Included : $DIR/$BOOT_FILE"
+      fi
+   done
+done
+printf "\n"
+if [ -r $BOOT_BOX ]
+then
+   Info "$BOOT_BOX added to the mime."
+   TOMIME="$TOMIME $BOOT_BOX"
+fi
+if [ -d "$BOOTSTRAP_EXTRA" ]
+then
+   rm -fr "$BOOTSTRAP_EXTRA"
+fi
+
+for FILE in $TOMIME
+do
+   if [ "$(echo $FILE| grep ':')" = "" ]
+   then
+      TYPE=""
+   else
+      TYPE=$(echo $FILE| sed 's/^.*:/:/g')
+   fi
+   FILE=$(echo $FILE| sed 's/:.*//g')
+   if [ "$(echo $FILE | cut -c1)" = / ]
+   then # Metadata file built to use
+       REAL_FILE="$(echo "$BUILD_DIR/$(basename $FILE)")"
+   else # Source file to use
+      REAL_FILE="$(echo "$FILE")"
+   fi
+   if [ ! -f "$REAL_FILE" ]
+   then
+      printf "\ncloudinit.conf: $REAL_FILE not found.\n"
+      exit 1
+   fi
+   FILES="$FILES $REAL_FILE$TYPE"
+   printf "$FILE\e[32m$TYPE\e[0m "
+done
+
+$MIME_SCRIPT $FILES -o "$USER_DATA"
+if [ $? -ne 0 ]
+then
+   echo "cloudinit.conf: Error while building $BUILD_DIR/userdata.mime with 'write-mime-multipart.py'. Please check."
+   exit 1
+fi
+echo " > $USER_DATA"
+
+
+
+}
+
+
+bootstrap_build

data/lib/build_tmpl/write-mime-multipart.py

@@ -0,0 +1,128 @@
+#!/usr/bin/python
+
+# (c) Copyright 2014 Hewlett-Packard Development Company, L.P.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+# largely taken from python examples
+# http://docs.python.org/library/email-examples.html
+
+import os
+import sys
+
+
+from email import encoders
+from email.mime.base import MIMEBase
+from email.mime.multipart import MIMEMultipart
+from email.mime.text import MIMEText
+from optparse import OptionParser
+import gzip
+
+from base64 import b64encode
+
+COMMASPACE = ', '
+
+starts_with_mappings = {
+    '#include': 'text/x-include-url',
+    '#!': 'text/x-shellscript',
+    '#cloud-config': 'text/cloud-config',
+    '#upstart-job': 'text/upstart-job',
+    '#part-handler': 'text/part-handler',
+    '#cloud-boothook': 'text/cloud-boothook'
+}
+
+
+def get_type(fname, deftype):
+    f = file(fname, "rb")
+    line = f.readline()
+    f.close()
+    rtype = deftype
+    for _str, mtype in starts_with_mappings.items():
+        if line.startswith(_str):
+            rtype = mtype
+            break
+    return rtype
+
+
+def main():
+    outer = MIMEMultipart()
+
+    parser = OptionParser()
+
+    parser.add_option("-o", "--output", dest="output",
+                      help="write output to FILE [default %default]",
+                      metavar="FILE", default="-")
+    parser.add_option("-z", "--gzip", dest="compress", action="store_true",
+                      help="compress output", default=False)
+    parser.add_option("-d", "--default", dest="deftype",
+                      help="default mime type [default %default]",
+                      default="text/plain")
+    parser.add_option("--delim", dest="delim",
+                      help="delimiter [default %default]", default=":")
+    parser.add_option("-b", "--base64", dest="base64", action="store_true",
+                      help="encode content base64", default=False)
+
+    (options, args) = parser.parse_args()
+
+    if (len(args)) < 1:
+        parser.error("Must give file list see '--help'")
+
+    for arg in args:
+        t = arg.split(options.delim, 1)
+        path = t[0]
+        if len(t) > 1:
+            mtype = t[1]
+        else:
+            mtype = get_type(path, options.deftype)
+
+        maintype, subtype = mtype.split('/', 1)
+        if maintype == 'text':
+            fp = open(path)
+            # Note: we should handle calculating the charset
+            msg = MIMEText(fp.read(), _subtype=subtype)
+            fp.close()
+        else:
+            fp = open(path, 'rb')
+            msg = MIMEBase(maintype, subtype)
+            msg.set_payload(fp.read())
+            fp.close()
+            # Encode the payload using Base64
+            encoders.encode_base64(msg)
+
+        # Set the filename parameter
+        msg.add_header('Content-Disposition', 'attachment',
+                       filename=os.path.basename(path))
+
+        outer.attach(msg)
+
+    if options.output is "-":
+        ofile = sys.stdout
+    else:
+        ofile = file(options.output, "wb")
+
+    if options.base64:
+        output = b64encode(outer.as_string())
+    else:
+        output = outer.as_string()
+
+    if options.compress:
+        gfile = gzip.GzipFile(fileobj=ofile, filename=options.output)
+        gfile.write(output)
+        gfile.close()
+    else:
+        ofile.write(output)
+
+    ofile.close()
+
+if __name__ == '__main__':
+    main()

data/lib/connection.rb

@@ -17,8 +17,6 @@
 
 require 'rubygems'
 require 'fog'
-require 'yaml_parse.rb'
-include YamlParse
 
 #
 # Connection module
@@ -29,19 +27,19 @@ class SSLErrorMgt
    def initialize()
       @iRetry=0
    end
- 
+
    def ErrorDetected(message,backtrace)
-      if message.match('SSLv2/v3 read server hello A: unknown protocol') 
+      if message.match('SSLv2/v3 read server hello A: unknown protocol')
          if @iRetry <5
             sleep(2)
             @iRetry+=1
             print "%s/5 try...\r" % @iRetry if $FORJ_LOGGER.level == 0
             return false
-         else   
+         else
             Logging.error('Too many retry. %s' % message)
             return true
          end
-      else   
+      else
          Logging.error("%s\n%s" % [message,backtrace.join("\n")])
          return true
       end
@@ -58,7 +56,7 @@ class ForjConnection
    def initialize(oConfig, bAutoConnect = true)
 
      Logging.fatal(1, 'Internal Error: Missing global $HPC_ACCOUNTS') if not $HPC_ACCOUNTS
-     
+
      @oConfig = oConfig
      @sAccountName = @oConfig.get(:account_name)
      @provider='HP' # TODO: Support multiple provider. (Generic Provider object required)
@@ -97,7 +95,7 @@ class ForjConnection
        Logging.fatal(1, 'Compute: Unable to connect.', e)
      end
    end
-   
+
    def network_connect
      # Trying to get Network object
      oSSLError=SSLErrorMgt.new # Retry object