vmc_knife 0.0.01

data/LICENSE

@@ -0,0 +1,24 @@
+Copyright (c) 2010-2011 Intalio Pte, All Rights Reserved
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+
+This software downloads additional open source software components upon install
+that are distributed under separate terms and conditions.  Please see the license
+information provided in the individual software components for more information.
+

data/README.md

@@ -0,0 +1,14 @@
+# VMC Knife
+Extensions for vmc: the VMware Cloud CLI; the command line interface to VMware's Application Platform
+
+MIT license
+
+## Usecase
+Prepare a file that describes the list of applications to deploy, their settings, their environment variables, required data-services and names, build file for the apps.
+Run the deployment file on the command: will either install update, modify the apps on the cloudfoundry.
+Run the deployment file via a web-interface.
+
+    prepare a vmc deployment file.
+    vmc_knife vmc_deployment_file.json
+
+

data/bin/vmc_knife

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+require File.expand_path('../../lib/vmc_knife', __FILE__)
+
+VMC::Cli::KnifeRunner.run(ARGV.dup)
+

data/lib/restclient/restclient_add_timeout.rb

@@ -0,0 +1,40 @@
+require 'rest_client'
+
+# Make sure the rest-client won't time out too quickly:
+# vmc uses the rest-client for its rest calls but does not let us configure the timeouts.
+# We monkey patch here.
+module RestClient
+
+  class << self
+    attr_accessor :timeout
+    attr_accessor :open_timeout
+  end
+
+  class Request
+    
+    def self.execute(args, &block)
+      #puts "Calling overriden RestClient::Request execute"
+      timeouts = {:timeout=>90000000, :open_timeout=>90000000}
+      args.merge!(timeouts)
+      #puts "Req args #{args}"
+      new(args).execute(& block)
+    end
+    
+  end
+  
+  def self.post(url, payload, headers={}, &block)
+    Request.execute(:method => :post,
+                    :url => url,
+                    :payload => payload,
+                    :headers => headers,
+                    :timeout=>@timeout,
+                    :open_timeout=>@open_timeout,
+                    &block)
+  end
+  
+  
+end
+
+#RestClient.open_timeout = 90000000
+#RestClient.timeout = 90000000
+ 

data/lib/vmc_knife/cli_extensions.rb

@@ -0,0 +1,138 @@
+require 'rubygems'
+require 'cli' #this is the cli from vmc.
+
+# Adds some new commands to the vmc's cli.
+#
+# Reconfigure applications according to a saas recipe.
+# The SaaS recipe is a json object that contains the manifest of each application.
+# as well as a short declaration of the services expected and their nature.
+# Usage: edit the json recipe.
+# vmc_knife configure-applications
+# 
+# Also bundles utilities to reconfigure the hostname of the cloud_controller and the gateways accordingly:
+# vmc_knife configure-vcap
+# and publish the urls in the deployed apps with zeroconf on ubuntu (avahi)
+# vmc configure-vcap-mdns
+class VMC::Cli::KnifeRunner < VMC::Cli::Runner
+  
+  def parse_command!
+    # just return if already set, happends with -v, -h
+    return if @namespace && @action
+    
+    verb = @args.first
+    case verb
+
+    when 'expand-manifest'
+      usage('vmc_knife expand-manifest <path_to_json_manifest> <path_to_destination_expanded_manifest>')
+      @args.shift # consumes the argument.
+      if @args.size == 1
+        set_cmd(:knife, :expand_manifest, 1)
+      elsif @args.size == 2
+        set_cmd(:knife, :expand_manifest, 2)
+      else
+        set_cmd(:knife, :expand_manifest)
+      end
+    when 'login', 'target'
+      usage('vmc_knife login [<path_to_json_manifest>]')
+      @args.shift # consumes the argument.
+      if @args.size == 1
+        set_cmd(:knifemisc, :login, 1)
+      else
+        set_cmd(:knifemisc, :login)
+      end
+    when 'configure-all'
+      usage('vmc_knife configure-all [<path_to_cloud_controller_config_yml>] [<path_to_json_manifest_or_uri>]')
+      @args.shift # consumes the argument.
+      if @args.size <= 2
+        set_cmd(:knife, :configure_all, @args.size)
+      else
+        set_cmd(:knife, :configure_all, @args.size) # too many
+      end
+    when 'configure-vcap'
+      usage('vmc_knife configure-vcap [<path_to_cloud_controller_config_yml>] [<path_to_json_manifest_or_uri>]')
+      @args.shift # consumes the argument.
+      if @args.size <= 2
+        set_cmd(:knife, :configure_cloud_controller, @args.size)
+      else
+        set_cmd(:knife, :configure_cloud_controller, @args.size) # too many
+      end
+    when 'configure-vcap-etc-hosts'
+      usage('vmc_knife configure-vap-etc-hosts [<path_to_etc_hosts>] [<path_to_json_manifest_or_uri>]')
+      @args.shift # consumes the argument.
+      if @args.size <= 2
+        set_cmd(:knife, :configure_etc_hosts, @args.size)
+      else
+        set_cmd(:knife, :configure_etc_hosts, @args.size) # too many
+      end
+    when 'configure-vcap-mdns'
+      usage('vmc_knife configure-vap-mdns [<path_to_aliases>]')
+      @args.shift # consumes the argument.
+      if @args.size <= 2
+        set_cmd(:knife, :configure_etc_avahi_aliases, @args.size)
+      else
+        set_cmd(:knife, :configure_etc_avahi_aliases, @args.size) # too many
+      end
+    when 'configure-applications'
+      usage('vmc_knife configure-applications [<applications_regexp>]')
+      @args.shift # consumes the argument.
+      if @args.size <= 2
+        set_cmd(:knifeapps, :configure_applications, @args.size)
+      else
+        set_cmd(:knifeapps, :configure_applications, @args.size) # too many
+      end
+    when 'configure-services'
+      usage('vmc_knife configure-services [<services_regexp>]')
+      @args.shift # consumes the argument.
+      if @args.size <= 2
+        set_cmd(:knifeapps, :configure_services, @args.size)
+      else
+        set_cmd(:knifeapps, :configure_services, @args.size) # too many
+      end
+    when 'configure-recipes'
+      usage('vmc_knife configure-recipes [<recipes_regexp>]')
+      @args.shift # consumes the argument.
+      if @args.size <= 2
+        set_cmd(:knifeapps, :configure_recipes, @args.size)
+      else
+        set_cmd(:knifeapps, :configure_recipes, @args.size) # too many
+      end
+    when 'upload-applications'
+      usage('vmc_knife upload-applications [<applications_regexp>]')
+      @args.shift # consumes the argument.
+      if @args.size <= 2
+        set_cmd(:knifeapps, :upload_applications, @args.size)
+      else
+        set_cmd(:knifeapps, :upload_applications, @args.size) # too many
+      end
+    when 'start-applications'
+      usage('vmc_knife start-applications [<applications_regexp>]')
+      @args.shift # consumes the argument.
+      if @args.size <= 2
+        set_cmd(:knifeapps, :start_applications, @args.size)
+      else
+        set_cmd(:knifeapps, :start_applications, @args.size) # too many
+      end
+    when 'stop-applications'
+      usage('vmc_knife stop-applications [<applications_regexp>]')
+      @args.shift # consumes the argument.
+      if @args.size <= 2
+        set_cmd(:knifeapps, :stop_applications, @args.size)
+      else
+        set_cmd(:knifeapps, :stop_applications, @args.size) # too many
+      end
+    when 'restart-applications'
+      usage('vmc_knife restart-applications [<applications_regexp>]')
+      @args.shift # consumes the argument.
+      if @args.size <= 2
+        set_cmd(:knifeapps, :restart_applications, @args.size)
+      else
+        set_cmd(:knifeapps, :restart_applications, @args.size) # too many
+      end
+    when 'help'
+      display "vmc_knife expand-manifest|login|start/stop/restart-applications|upload-applications|configure|configure-recipes|configure-applications|configure-services|configure-vcap|configure-vcap-mdns|configure-vcap-etc-hosts [<manifest_path>]"
+    else
+      super
+    end
+  end
+  
+end

data/lib/vmc_knife/commands/knife_cmds.rb

@@ -0,0 +1,186 @@
+# Commands for vmc_knife.
+
+module VMC::KNIFE::Cli
+  
+  #loads the manifest file.
+  #when the path is not specified, look in the current directory.
+  #when the path is a directory, look for the first json file it can find.
+  #the json file actually loaded is set as the attribute @manifest_path
+  def load_manifest(manifest_file_path=nil)
+    manifest_file_path = Dir.pwd if manifest_file_path.nil?
+    if File.directory? manifest_file_path
+      #look for the first .json file if possible that is not an expanded.json
+      Dir.glob(File.join(manifest_file_path,"*.json")).each do |file|
+        @manifest_path = file
+        if VMC::Cli::Config.trace
+          display "Using the manifest #{@manifest_path}"
+        end
+        return VMC::KNIFE::JSON_EXPANDER.expand_json @manifest_path
+      end
+      raise "Unable to find a *.json file in #{manifest_file_path}"
+    else
+      @manifest_path = manifest_file_path
+      return VMC::KNIFE::JSON_EXPANDER.expand_json @manifest_path
+    end
+  end
+end
+
+module VMC::Cli::Command
+
+  class Knife < Base
+    include VMC::KNIFE::Cli
+    
+    # expands the json manifest. outputs it in the destination path.
+    def expand_manifest(manifest_file_path, destination=nil)
+      res = VMC::KNIFE::JSON_EXPANDER.expand_json manifest_file_path
+      if destination.nil?
+        noextension = File.basename(manifest_file_path, File.extname(manifest_file_path))
+        destination = File.join File.dirname(manifest_file_path), "#{noextension}.expanded.json"
+      end
+      display "Expanding the manifest #{manifest_file_path} into #{destination}"
+      if VMC::Cli::Config.trace
+        display JSON.pretty_generate(res)
+      end
+      File.open(destination, 'w') {|f| f.write(JSON.pretty_generate(res)) }
+    end
+    
+    # updates the cloud_controller
+    def configure_cloud_controller(cloud_controller_yml=nil,manifest_file_path_or_uri=nil)
+      __update(manifest_file_path_or_uri,cloud_controller_yml,VMC::KNIFE::VCAPUpdateCloudControllerConfig,"cloud_controller")
+    end
+    # updates /etc/hosts
+    def configure_etc_hosts(etc_hosts=nil,manifest_file_path_or_uri=nil)
+      __update(manifest_file_path_or_uri,etc_hosts,VMC::KNIFE::VCAPUpdateEtcHosts,"/etc/hosts")
+    end
+    # updates /etc/avahi/aliases
+    def configure_etc_avahi_aliases(etc_avahi_aliases=nil,manifest_file_path=nil)
+      man = load_manifest(manifest_file_path)
+      update_aliases = VMC::KNIFE::VCAPUpdateAvahiAliases.new(etc_avahi_aliases,man,client)
+      update_aliases.do_exec = true
+      update_aliases.execute
+    end
+    
+    def configure_all(manifest_file_path_or_uri=nil)
+      display "Stop applications ..."
+      VMC::Cli::Command::Knifeapps.new(@options).stop_applications(nil,manifest_file_path_or_uri)
+      display "Configure_cloud_controller ..."
+      configure_cloud_controller(nil,manifest_file_path_or_uri)
+      display "Configure_etc_hosts ..."
+      configure_etc_hosts(nil,manifest_file_path_or_uri)
+      display "Login again ..."
+      VMC::Cli::Command::Knifemisc.new(@options).login(manifest_file_path_or_uri)
+      display "Configure_applications ..."
+      VMC::Cli::Command::Knifeapps.new(@options).configure_applications(nil,manifest_file_path_or_uri)
+      display "Configure_etc_avahi_aliases ..."
+      configure_etc_avahi_aliases(nil,manifest_file_path_or_uri)
+      display "Start applications ..."
+      VMC::Cli::Command::Knifeapps.new(@options).start_applications(nil,manifest_file_path_or_uri)
+    end
+
+    private
+    def __update(manifest_file_path_or_uri,config,_class,msg_label)
+      unless manifest_file_path_or_uri.nil?
+        if File.exists? manifest_file_path_or_uri
+          man = load_manifest(manifest_file_path_or_uri)
+          uri = man['target']
+        else
+          uri = manifest_file_path_or_uri
+        end
+      else
+        man = load_manifest(nil)
+        uri = man['target']
+      end
+      raise "No uri defined" unless uri
+      update_cc = _class.new(uri,config)
+      if update_cc.update_pending()
+        display "Configuring #{msg_label} with uri: #{uri}" if VMC::Cli::Config.trace
+        update_cc.execute()
+      end
+    end
+
+  end
+  
+  class Knifemisc < Misc
+    include VMC::KNIFE::Cli
+    
+    # configures the target and login according to the info in the manifest.
+    def login(manifest_file_path=nil)
+      man  = load_manifest(manifest_file_path)
+      target_url = man['target']
+      raise "No target defined in the manifest #{@manifest_path}" if target_url.nil? 
+      set_target(target_url)
+      
+      email = man['email']
+      password = man['password']
+      @options[:email] = email if email
+      @options[:password] = password if password
+      
+      tries ||= 0
+      # login_and_save_token:
+      token = client.login(email, password)
+      VMC::Cli::Config.store_token(token)
+      
+    rescue VMC::Client::TargetError
+      display "Problem with login, invalid account or password.".red
+      retry if (tries += 1) < 3 && prompt_ok && !@options[:password]
+      exit 1
+    rescue => e
+      display "Problem with login, #{e}, try again or register for an account.".red
+      display e.backtrace
+      exit 1
+      
+    end
+    
+  end
+  
+  class Knifeapps < Apps
+    include VMC::KNIFE::Cli
+
+    def configure_applications(app_names_regexp=nil,manifest_file_path=nil)
+      configure(nil,nil,app_names_regexp,manifest_file_path)
+    end
+    def configure_services(services_names_regexp=nil,manifest_file_path=nil)
+      configure(nil,nil,services_names_regexp,manifest_file_path)
+    end
+    def configure_recipes(recipe_names_regexp=nil,manifest_file_path=nil)
+      configure(recipe_names_regexp,nil,nil,manifest_file_path)
+    end
+              
+    # Configure the applications according to their manifest.
+    # The parameters are related to selecting a subset of the applications to configure.
+    # nil means all apps for all recipes found in the manifest are configured.
+    # @param recipes The list of recipes: nil: search the apps in all recipes
+    # @param app_role_names The names of the apps in each recipe; nil: configure all apps found.
+    def configure(recipes_regexp=nil,app_names_regexp=nil,service_names_regexp=nil,manifest_file_path=nil)
+      man = load_manifest(manifest_file_path)
+      configurer = VMC::KNIFE::RecipesConfigurationApplier.new(man,client,recipes_regexp,app_names_regexp,service_names_regexp)
+      if VMC::Cli::Config.trace
+        display "Pending updates"
+        display JSON.pretty_generate(configurer.updates_pending)
+      end
+      configurer.execute
+    end
+    
+    def upload_applications(app_names_regexp=nil,manifest_file_path=nil)
+      recipe_configuror(:upload,nil,nil,app_names_regexp,manifest_file_path)
+    end
+    def start_applications(app_names_regexp=nil,manifest_file_path=nil)
+      recipe_configuror(:start,nil,nil,app_names_regexp,manifest_file_path)
+    end
+    def stop_applications(app_names_regexp=nil,manifest_file_path=nil)
+      recipe_configuror(:stop,nil,nil,app_names_regexp,manifest_file_path)
+    end
+    def restart_applications(app_names_regexp=nil,manifest_file_path=nil)
+      recipe_configuror(:restart,nil,nil,app_names_regexp,manifest_file_path)
+    end
+    
+    def recipe_configuror(method_sym_name,recipes_regexp=nil,app_names_regexp=nil,service_names_regexp=nil,manifest_file_path=nil)
+      man = load_manifest(manifest_file_path)
+      configurer = VMC::KNIFE::RecipesConfigurationApplier.new(man,client,recipes_regexp,app_names_regexp,service_names_regexp)
+      method_object = configurer.method(method_sym_name)
+      method_object.call
+    end
+    
+  end
+  
+end

data/lib/vmc_knife/json_diff.rb

@@ -0,0 +1,83 @@
+require 'json'
+module VMC
+  module KNIFE
+    module JSON_DIFF
+    
+    DIFF_NODE_NAME = "__diff__"
+    TYPE_NODE_NAME = "__type__"
+    PLUS="+"
+    MINUS="-"
+    CHANGED="=>"
+    DISPLAY_TYPE_ONLY_WHEN_CHANGE=true
+    
+    def self.compare(a,b,pretty=true)
+      if a.kind_of?(String) && b.kind_of?(String) && File.exists?(a) && File.exists?(b)
+        a = JSON.parse File.open(a).read
+        b = JSON.parse File.open(b).read
+      end
+      if pretty
+        JSON.pretty_generate compare_tree(a, b)
+      else
+        compare_tree(a, b).to_json
+      end
+    end
+    
+    def self.compare_tree(a, b)
+      typeA = a.class.name
+      typeB = b.class.name
+      
+      aString = a.to_s unless is_array_or_hash?(a)
+      bString = b.to_s unless is_array_or_hash?(b)
+            
+      node = Hash.new
+      if a.nil?
+        node[DIFF_NODE_NAME]=PLUS
+        node[TYPE_NODE_NAME]=typeB unless DISPLAY_TYPE_ONLY_WHEN_CHANGE
+        node['value']=bString if bString
+      elsif b.nil?
+        node[DIFF_NODE_NAME]=MINUS
+        node[TYPE_NODE_NAME]=typeA unless DISPLAY_TYPE_ONLY_WHEN_CHANGE
+        node['value']=aString if aString
+      elsif (typeA != typeB) || (!aString.nil? && a != b)
+        node[DIFF_NODE_NAME]=CHANGED
+        if typeA != typeB
+          node[TYPE_NODE_NAME]="#{typeA} => #{typeB}"
+        else
+          node[TYPE_NODE_NAME]=typeA unless DISPLAY_TYPE_ONLY_WHEN_CHANGE
+        end
+        node['value']="#{aString} => #{bString}" if aString
+      else
+        node[TYPE_NODE_NAME]=typeA unless DISPLAY_TYPE_ONLY_WHEN_CHANGE
+        node['value']=aString if aString
+      end
+      
+      if aString
+        return node
+      end
+      child_node=node
+#      child_node = Hash.new
+#      node['child']=child_node
+      keys = Array.new
+      keys = collect_keys(a,keys)
+      keys = collect_keys(b,keys)
+      keys.sort!
+      for i in 0..(keys.length-1)
+        if (keys[i] != keys[i-1])
+          value = compare_tree(a && a[keys[i]], b && b[keys[i]]);
+          child_node[keys[i]]=value
+        end
+      end
+      node
+    end
+    private
+    def self.is_array_or_hash?(obj)
+      obj.kind_of?(Array) || obj.kind_of?(Hash)
+    end
+    def self.collect_keys(obj, collector)
+      return Array.new unless obj.kind_of? Hash
+      collector + obj.keys
+    end
+    
+    end # end of JSON_DIFF
+  end # end of KNIFE
+end

data/lib/vmc_knife/json_expander.rb

@@ -0,0 +1,95 @@
+require 'vmc/client'
+require 'json'
+
+module VMC
+  module KNIFE
+    module JSON_EXPANDER
+    
+      # Loads a json file.
+      # Makes up to 10 passes evaluating ruby in the values that contain #{}.
+      def self.expand_json(file_path)
+        raise "The file #{file_path} does not exist" unless File.exists? file_path
+        data = JSON.parse File.open(file_path).read
+        #puts "got data #{data.to_json}"
+        passes = 0
+        while passes < 100
+          #puts "pass #{passes}"
+          break unless expand_data(data,data)
+          passes += 1
+        end
+        puts data.to_json unless passes < 100
+        raise "More than 100 passes evaluating the ruby template in the json file" unless passes < 100
+        #puts "got data #{data.to_json}"
+        data
+      end
+      
+      # Traverses the JSON object
+      # Eval the values that are strings and contain a #{}
+      # Does not do it recursively
+      # data The root data passed as 'this' in the binding to the eval function
+      # @return true if there was a change.
+      def self.expand_data(data,current)
+        at_least_one_eval = false
+        if current.kind_of? Hash
+          current.each_pair do | k, v |
+            if v.kind_of? String
+              if /\#{.+}/ =~ v
+                at_least_one_eval = true
+                begin
+                  evalled = eval_v(v,data,current)
+                  current[k] = evalled unless evalled.nil?
+                rescue => e
+                  raise "Error thrown evaluating #{v}: #{e.inspect}"
+                end
+              end
+            else
+              at_least_one_eval ||= expand_data(data,v)
+            end
+          end
+        elsif current.kind_of? Array
+          index = 0
+          current.each do | v |
+            if v.kind_of? String
+              if /\#{.+}/ =~ v
+                at_least_one_eval = true
+                begin
+                  evalled = eval_v(v,data,current)
+                  current[index] = evalled unless evalled.nil?
+                rescue => e
+                  raise "Error thrown evaluating #{v}: #{e.inspect}"
+                end
+              end
+            else
+              at_least_one_eval ||= expand_data(data,v)
+            end
+            index+=1
+          end
+        end
+        at_least_one_eval
+      end
+      
+      # internal eval a reference.
+      # the reference is always wrapped in a json string.
+      # however if it is purely a ruby script ("#{ruby here}" ) we unwrap it
+      # to avoid casting the result into a string.
+      def self.eval_v(v,data,current)
+        #puts "evalling #{v}"
+        if /^\#{([^}]*)}$/ =~ v
+          val = $1
+        else
+          val = '"'+v+'"'
+        end
+        evalled = eval(val,get_binding(data,current))
+        #puts "evaluating #{v} => #{evalled} class #{evalled.class.name}"
+        evalled
+      end
+      
+      def self.get_binding(this,current)
+        binding
+      end
+      
+    end #end of JSON_EXPANDER
+    
+  end # end of KNIFE
+  
+end

data/lib/vmc_knife/version.rb

@@ -0,0 +1,8 @@
+module VMC
+  module KNIFE
+    module Cli
+      # This version number is used as the RubyGem release version.
+      VERSION = '0.0.01'
+    end
+  end
+end