vmc 0.3.13 → 0.3.14.beta.3

data/README.md

@@ -62,6 +62,8 @@ MIT license, please see the LICENSE file.  All rights reserved._
       bind-service <servicename> <appname>         Bind a service to an application
       unbind-service <servicename> <appname>       Unbind service from the application
       clone-services <src-app> <dest-app>          Clone service bindings from <src-app> application to <dest-app>
+      tunnel <servicename> [--port]                Create a local tunnel to a service
+      tunnel <servicename> <clientcmd>             Create a local tunnel to a service and start a local client
 
     Administration
       user                                         Display user account information

data/caldecott_helper/Gemfile

@@ -0,0 +1,10 @@
+source "http://rubygems.org"
+
+gem 'rack', '~> 1.2.0'
+gem 'caldecott', '= 0.0.3'
+gem 'bundler'
+gem 'em-websocket'
+gem 'async_sinatra'
+gem 'thin'
+gem 'json'
+gem 'uuidtools'

data/caldecott_helper/Gemfile.lock

@@ -0,0 +1,48 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    addressable (2.2.6)
+    async_sinatra (0.5.0)
+      rack (>= 1.2.1)
+      sinatra (>= 1.0)
+    caldecott (0.0.3)
+      addressable (= 2.2.6)
+      async_sinatra (= 0.5.0)
+      em-http-request (= 0.3.0)
+      em-websocket (= 0.3.1)
+      json (= 1.6.1)
+      uuidtools (= 2.1.2)
+    daemons (1.1.4)
+    em-http-request (0.3.0)
+      addressable (>= 2.0.0)
+      escape_utils
+      eventmachine (>= 0.12.9)
+    em-websocket (0.3.1)
+      addressable (>= 2.1.1)
+      eventmachine (>= 0.12.9)
+    escape_utils (0.2.4)
+    eventmachine (0.12.10)
+    json (1.6.1)
+    rack (1.2.4)
+    sinatra (1.2.7)
+      rack (~> 1.1)
+      tilt (>= 1.2.2, < 2.0)
+    thin (1.2.11)
+      daemons (>= 1.0.9)
+      eventmachine (>= 0.12.6)
+      rack (>= 1.0.0)
+    tilt (1.3.3)
+    uuidtools (2.1.2)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  async_sinatra
+  bundler
+  caldecott (= 0.0.3)
+  em-websocket
+  json
+  rack (~> 1.2.0)
+  thin
+  uuidtools

data/caldecott_helper/server.rb

@@ -0,0 +1,43 @@
+#!/usr/bin/env ruby
+# Copyright (c) 2009-2011 VMware, Inc.
+$:.unshift(File.dirname(__FILE__) + '/lib')
+
+require 'rubygems'
+require 'bundler/setup'
+
+require 'caldecott'
+require 'sinatra'
+require 'json'
+require 'eventmachine'
+
+port = ENV['VMC_APP_PORT']
+port ||= 8081
+
+# add vcap specific stuff to Caldecott
+class VcapHttpTunnel < Caldecott::Server::HttpTunnel
+  get '/info' do
+    { "version" => '0.0.4' }.to_json
+  end
+
+  def self.get_tunnels
+    super
+  end
+
+  get '/services' do
+    services_env = ENV['VMC_SERVICES']
+    return "no services env" if services_env.nil? or services_env.empty?
+    services_env
+  end
+
+  get '/services/:service' do |service_name|
+    services_env = ENV['VMC_SERVICES']
+    not_found if services_env.nil?
+
+    services = JSON.parse(services_env)
+    service = services.find { |s| s["name"] == service_name }
+    not_found if service.nil?
+    service["options"].to_json
+  end
+end
+
+VcapHttpTunnel.run!(:port => port, :auth_token => ENV["CALDECOTT_AUTH"])

data/config/clients.yml

@@ -0,0 +1,11 @@
+redis:
+  redis-cli: redis-cli -h ${host} -p ${port} -a ${password}
+
+mysql:
+  mysql: mysql --protocol=TCP --host=${host} --port=${port} --user=${user} --password=${password} ${name}
+
+mongodb:
+  mongo: mongo --host ${host} --port ${port} -u ${user} -p ${password} ${name}
+
+postgresql:
+  psql: PGPASSWORD='${password}' psql -h ${host} -p ${port} -d ${name} -U ${user} -w

data/lib/cli.rb

@@ -11,6 +11,7 @@ module VMC
     autoload :Runner,         "#{ROOT}/cli/runner"
     autoload :ZipUtil,        "#{ROOT}/cli/zip_util"
     autoload :ServicesHelper, "#{ROOT}/cli/services_helper"
+    autoload :TunnelHelper,   "#{ROOT}/cli/tunnel_helper"
 
     module Command
       autoload :Base,         "#{ROOT}/cli/commands/base"

data/lib/cli/commands/apps.rb

@@ -185,7 +185,7 @@ module VMC::Cli::Command
       uris << url
       app[:uris] = uris
       client.update_app(appname, app)
-      display "Succesfully mapped url".green
+      display "Successfully mapped url".green
     end
 
     def unmap(appname, url)
@@ -196,7 +196,7 @@ module VMC::Cli::Command
       err "Invalid url" unless deleted
       app[:uris] = uris
       client.update_app(appname, app)
-      display "Succesfully unmapped url".green
+      display "Successfully unmapped url".green
 
     end
 
@@ -268,6 +268,8 @@ module VMC::Cli::Command
     end
 
     def logs(appname)
+      # Check if we have an app before progressing further
+      client.app_info(appname)
       return grab_all_logs(appname) if @options[:all] && !@options[:instance]
       instance = @options[:instance] || '0'
       grab_logs(appname, instance)
@@ -686,7 +688,7 @@ module VMC::Cli::Command
 
       display "The following provisioned services are available"
       name = ask(
-        "Please select one you which to prevision",
+        "Please select one you wish to use",
         { :indexed => true,
           :choices => user_services.collect { |s| s[:name] }
         }

data/lib/cli/commands/misc.rb

@@ -38,7 +38,7 @@ module VMC::Cli::Command
         exit(false)
       else
         VMC::Cli::Config.store_target(target_url)
-        say "Succesfully targeted to [#{target_url}]".green
+        say "Successfully targeted to [#{target_url}]".green
       end
     end
 

data/lib/cli/commands/services.rb

@@ -2,6 +2,7 @@ module VMC::Cli::Command
 
   class Services < Base
     include VMC::Cli::ServicesHelper
+    include VMC::Cli::TunnelHelper
 
     def services
       ss = client.services_info
@@ -81,5 +82,96 @@ module VMC::Cli::Command
       check_app_for_restart(dest_app)
     end
 
+    def tunnel(service=nil, client_name=nil)
+      unless defined? Caldecott
+        display "To use `vmc tunnel', you must first install Caldecott:"
+        display ""
+        display "\tgem install caldecott"
+        display ""
+        display "Note that you'll need a C compiler. If you're on OS X, Xcode"
+        display "will provide one. If you're on Windows, try DevKit."
+        display ""
+        display "This manual step will be removed in the future."
+        display ""
+        err "Caldecott is not installed."
+      end
+
+      ps = client.services
+      err "No services available to tunnel to" if ps.empty?
+
+      unless service
+        choices = ps.collect { |s| s[:name] }.sort
+        service = ask(
+          "Which service to tunnel to?",
+          :choices => choices,
+          :indexed => true
+        )
+      end
+
+      info = ps.select { |s| s[:name] == service }.first
+
+      err "Unknown service '#{service}'" unless info
+
+      # TODO: rather than default to a particular port, we should get
+      # the defaults based on the service name.. i.e. known services should
+      # have known local default ports for this side of the tunnel.
+      port = pick_tunnel_port(@options[:port] || 10000)
+
+      raise VMC::Client::AuthError unless client.logged_in?
+
+      if not tunnel_pushed?
+        display "Deploying tunnel application '#{tunnel_appname}'."
+        auth = ask("Create a password", :echo => "*")
+        push_caldecott(auth)
+        bind_service_banner(service, tunnel_appname, false)
+        start_caldecott
+      else
+        auth = ask("Password", :echo => "*")
+      end
+
+      if not tunnel_healthy?(auth)
+        display "Redeploying tunnel application '#{tunnel_appname}'."
+
+        # We don't expect caldecott not to be running, so take the
+        # most aggressive restart method.. delete/re-push
+        client.delete_app(tunnel_appname)
+        invalidate_tunnel_app_info
+
+        push_caldecott(auth)
+        bind_service_banner(service, tunnel_appname, false)
+        start_caldecott
+      end
+
+      if not tunnel_bound?(service)
+        bind_service_banner(service, tunnel_appname)
+      end
+
+      conn_info = tunnel_connection_info info[:vendor], service, auth
+      display_tunnel_connection_info(conn_info)
+      start_tunnel(service, port, conn_info, auth)
+
+      clients = get_clients_for(info[:vendor])
+      if clients.empty?
+        which = "none"
+      else
+        which = client_name || ask(
+          "Which client would you like to start?",
+          :choices => ["none"] + clients.keys,
+          :indexed => true
+        )
+      end
+
+      if which == "none"
+        wait_for_tunnel_end
+      else
+        wait_for_tunnel_start(port)
+        start_local_prog(which, local_prog_cmdline(clients[which], port, conn_info))
+      end
+    end
+
+    def get_clients_for(type)
+      conf = VMC::Cli::Config.clients
+      conf[type] || {}
+    end
   end
 end

data/lib/cli/commands/user.rb

@@ -15,6 +15,7 @@ module VMC::Cli::Command
       tries ||= 0
 
       unless no_prompt
+        display "Attempting login to [#{target_url}]" if target_url
         email ||= ask("Email")
         password ||= ask("Password", :echo => "*")
       end

data/lib/cli/config.rb

@@ -14,6 +14,9 @@ module VMC::Cli
     TOKEN_FILE     = '~/.vmc_token'
     INSTANCES_FILE = '~/.vmc_instances'
     ALIASES_FILE   = '~/.vmc_aliases'
+    CLIENTS_FILE   = '~/.vmc_clients'
+
+    STOCK_CLIENTS = File.expand_path("../../../config/clients.yml", __FILE__)
 
     class << self
       attr_accessor :colorize
@@ -102,6 +105,30 @@ module VMC::Cli
         File.open(aliases_file, 'wb') {|f| f.write(aliases.to_yaml)}
       end
 
+      def deep_merge(a, b)
+        merge = proc do |_, old, new|
+          if new.is_a?(Hash) and old.is_a?(Hash)
+            old.merge(new, &merge)
+          else
+            new
+          end
+        end
+
+        a.merge(b, &merge)
+      end
+
+      def clients
+        return @clients if @clients
+
+        stock = YAML.load_file(STOCK_CLIENTS)
+        if File.exists? CLIENTS_FILE
+          user = YAML.load_file(CLIENTS_FILE)
+          @clients = deep_merge(stock, user)
+        else
+          @clients = stock
+        end
+      end
+
       def lock_and_read(file)
         File.open(file, "r") {|f|
           f.flock(File::LOCK_EX)

data/lib/cli/runner.rb

@@ -77,6 +77,8 @@ class VMC::Cli::Runner
       opts.on('-v', '--version')   {         set_cmd(:misc, :version) }
       opts.on('-h', '--help')      {         puts "#{command_usage}\n"; exit }
 
+      opts.on('--port PORT')       { |port|  @options[:port] = port }
+
       opts.on('--runtime RUNTIME') { |rt|    @options[:runtime] = rt }
 
       # deprecated
@@ -367,6 +369,12 @@ class VMC::Cli::Runner
       usage('vmc unalias <alias>')
       set_cmd(:misc, :unalias, 1)
 
+    when 'tunnel'
+      usage('vmc tunnel [servicename] [clientcmd] [--port port]')
+      set_cmd(:services, :tunnel, 0) if @args.size == 0
+      set_cmd(:services, :tunnel, 1) if @args.size == 1
+      set_cmd(:services, :tunnel, 2) if @args.size == 2
+
     when 'help'
       display_help if @args.size == 0
       @help_only = true

data/lib/cli/tunnel_helper.rb

@@ -0,0 +1,289 @@
+# Copyright (c) 2009-2011 VMware, Inc.
+
+require 'addressable/uri'
+
+begin
+  require 'caldecott'
+rescue LoadError
+end
+
+module VMC::Cli
+  module TunnelHelper
+    PORT_RANGE = 10
+
+    HELPER_APP = File.expand_path("../../../caldecott_helper", __FILE__)
+
+    # bump this AND the version info reported by HELPER_APP/server.rb
+    # this is to keep the helper in sync with any updates here
+    HELPER_VERSION = '0.0.4'
+
+    def tunnel_uniquename
+      random_service_name(tunnel_appname)
+    end
+
+    def tunnel_appname
+      "caldecott"
+    end
+
+    def tunnel_app_info
+      return @tun_app_info if @tunnel_app_info
+      begin
+        @tun_app_info = client.app_info(tunnel_appname)
+      rescue => e
+        @tun_app_info = nil
+      end
+    end
+
+    def tunnel_url
+      return @tunnel_url if @tunnel_url
+
+      tun_url = tunnel_app_info[:uris][0]
+
+      ["https", "http"].each do |scheme|
+        url = "#{scheme}://#{tun_url}"
+        begin
+          RestClient.get(url)
+
+        # https failed
+        rescue Errno::ECONNREFUSED
+
+        # we expect a 404 since this request isn't auth'd
+        rescue RestClient::ResourceNotFound
+          return @tunnel_url = url
+        end
+      end
+
+      err "Cannot determine URL for #{tun_url}"
+    end
+
+    def invalidate_tunnel_app_info
+      @tunnel_url = nil
+      @tunnel_app_info = nil
+    end
+
+    def tunnel_pushed?
+      not tunnel_app_info.nil?
+    end
+
+    def tunnel_healthy?(token)
+      return false unless tunnel_app_info[:state] == 'STARTED'
+
+      begin
+        response = RestClient.get(
+          "#{tunnel_url}/info",
+          "Auth-Token" => token
+        )
+
+        info = JSON.parse(response)
+        if info["version"] == HELPER_VERSION
+          true
+        else
+          stop_caldecott
+          false
+        end
+      rescue RestClient::Exception
+        stop_caldecott
+        false
+      end
+    end
+
+    def tunnel_bound?(service)
+      tunnel_app_info[:services].include?(service)
+    end
+
+    def tunnel_connection_info(type, service, token)
+      display "Getting tunnel connection info: ", false
+      response = nil
+      10.times do
+        begin
+          response = RestClient.get("#{tunnel_url}/services/#{service}", "Auth-Token" => token)
+          break
+        rescue RestClient::Exception
+          sleep 1
+        end
+
+        display ".", false
+      end
+
+      unless response
+        err "Expected remote tunnel to know about #{service}, but it doesn't"
+      end
+
+      display "OK".green
+
+      info = JSON.parse(response)
+      case type
+      when "rabbitmq"
+        uri = Addressable::URI.parse info["url"]
+        info["hostname"] = uri.host
+        info["port"] = uri.port
+        info["vhost"] = uri.path[1..-1]
+        info["user"] = uri.user
+        info["password"] = uri.password
+        info.delete "url"
+
+      # we use "db" as the "name" for mongo
+      # existing "name" is junk
+      when "mongodb"
+        info["name"] = info["db"]
+        info.delete "db"
+
+      # our "name" is irrelevant for redis
+      when "redis"
+        info.delete "name"
+      end
+
+      ['hostname', 'port', 'password'].each do |k|
+        err "Could not determine #{k} for #{service}" if info[k].nil?
+      end
+
+      info
+    end
+
+    def display_tunnel_connection_info(info)
+      display ''
+      display "Service connection info: "
+
+      to_show = [nil, nil, nil] # reserved for user, pass, db name
+      info.keys.each do |k|
+        case k
+        when "host", "hostname", "port", "node_id"
+          # skip
+        when "user", "username"
+          # prefer "username" over "user"
+          to_show[0] = k unless to_show[0] == "username"
+        when "password"
+          to_show[1] = k
+        when "name"
+          to_show[2] = k
+        else
+          to_show << k
+        end
+      end
+      to_show.compact!
+
+      align_len = to_show.collect(&:size).max + 1
+
+      to_show.each do |k|
+        # TODO: modify the server services rest call to have explicit knowledge
+        # about the items to return.  It should return all of them if
+        # the service is unknown so that we don't have to do this weird
+        # filtering.
+        display "  #{k.ljust align_len}: ", false
+        display "#{info[k]}".yellow
+      end
+      display ''
+    end
+
+    def start_tunnel(service, local_port, conn_info, auth)
+      display "Starting tunnel to #{service.bold} on port #{local_port.to_s.bold}."
+
+      @local_tunnel_thread = Thread.new do
+        Caldecott::Client.start({
+          :local_port => local_port,
+          :tun_url => tunnel_url,
+          :dst_host => conn_info['hostname'],
+          :dst_port => conn_info['port'],
+          :log_file => STDOUT,
+          :log_level => "ERROR",
+          :auth_token => auth,
+          :quiet => true
+        })
+      end
+
+      at_exit { @local_tunnel_thread.kill }
+    end
+
+    def pick_tunnel_port(port)
+      original = port
+
+      PORT_RANGE.times do |n|
+        begin
+          TCPSocket.open('localhost', port)
+          port += 1
+        rescue => e
+          return port
+        end
+      end
+
+      err "Could not pick a port between #{original} and #{original + PORT_RANGE - 1}"
+    end
+
+    def wait_for_tunnel_start(port)
+      10.times do |n|
+        begin
+          TCPSocket.open('localhost', port)
+          display '' if n > 0
+          return true
+        rescue => e
+          display "Waiting for local tunnel to become available", false if n == 0
+          display '.', false
+          sleep 1
+        end
+      end
+      err "Could not connect to local tunnel."
+    end
+
+    def wait_for_tunnel_end
+      display "Open another shell to run command-line clients or"
+      display "use a UI tool to connect using the displayed information."
+      display "Press Ctrl-C to exit..."
+      @local_tunnel_thread.join
+    end
+
+    def local_prog_cmdline(command, local_port, tunnel_info)
+      cmd = command.dup
+      cmd.gsub!(/\$\{\s*([^\}]+)\s*\}/) do
+        case $1
+        when "host"
+          # TODO: determine proper host
+          "localhost"
+        when "port"
+          local_port
+        when "user", "username"
+          tunnel_info["username"]
+        else
+          tunnel_info[$1] || err("Unknown symbol '#{$1}'")
+        end
+      end
+      cmd
+    end
+
+    def start_local_prog(which, cmdline)
+      display "Launching '#{cmdline}'"
+      display ''
+      unless system(cmdline)
+        err "Failed to start '#{which}' client; is it in your $PATH?"
+      end
+    end
+
+    def push_caldecott(token)
+      client.create_app(
+        tunnel_appname,
+        { :name => tunnel_appname,
+          :staging => {:framework => "sinatra"},
+          :uris => ["#{tunnel_uniquename}.#{VMC::Cli::Config.suggest_url}"],
+          :instances => 1,
+          :resources => {:memory => 64},
+          :env => ["CALDECOTT_AUTH=#{token}"]
+        }
+      )
+
+      Command::Apps.new({}).send(:upload_app_bits, tunnel_appname, HELPER_APP)
+
+      invalidate_tunnel_app_info
+    end
+
+    def stop_caldecott
+      Command::Apps.new({}).stop(tunnel_appname)
+
+      invalidate_tunnel_app_info
+    end
+
+    def start_caldecott
+      Command::Apps.new({}).start(tunnel_appname)
+
+      invalidate_tunnel_app_info
+    end
+  end
+end

data/lib/cli/usage.rb

@@ -78,6 +78,8 @@ Currently available vmc commands are:
     bind-service <servicename> <appname>         Bind a service to an application
     unbind-service <servicename> <appname>       Unbind service from the application
     clone-services <src-app> <dest-app>          Clone service bindings from <src-app> application to <dest-app>
+    tunnel <servicename> [--port]                Create a local tunnel to a service
+    tunnel <servicename> <clientcmd>             Create a local tunnel to a service and start a local client
 
   Administration
     user                                         Display user account information

data/lib/cli/version.rb

@@ -2,6 +2,6 @@ module VMC
   module Cli
     # This version number is used as the RubyGem release version.
     # The internal VMC version number is VMC::VERSION.
-    VERSION = '0.3.13'
+    VERSION = '0.3.14.beta.3'
   end
 end

metadata

@@ -1,8 +1,8 @@
 --- !ruby/object:Gem::Specification 
 name: vmc
 version: !ruby/object:Gem::Version 
-  prerelease: 
-  version: 0.3.13
+  prerelease: 7
+  version: 0.3.14.beta.3
 platform: ruby
 authors: 
 - VMware
@@ -10,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-11-02 00:00:00 -07:00
+date: 2011-11-14 00:00:00 -08:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -68,42 +68,53 @@ dependencies:
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
-        version: 0.2.0
+        version: 0.3.0
   type: :runtime
   version_requirements: *id005
 - !ruby/object:Gem::Dependency 
-  name: rake
+  name: addressable
   prerelease: false
   requirement: &id006 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        version: 2.2.6
+  type: :runtime
+  version_requirements: *id006
+- !ruby/object:Gem::Dependency 
+  name: rake
+  prerelease: false
+  requirement: &id007 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
         version: "0"
   type: :development
-  version_requirements: *id006
+  version_requirements: *id007
 - !ruby/object:Gem::Dependency 
   name: rspec
   prerelease: false
-  requirement: &id007 !ruby/object:Gem::Requirement 
+  requirement: &id008 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
         version: 1.3.0
   type: :development
-  version_requirements: *id007
+  version_requirements: *id008
 - !ruby/object:Gem::Dependency 
   name: webmock
   prerelease: false
-  requirement: &id008 !ruby/object:Gem::Requirement 
+  requirement: &id009 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - "="
       - !ruby/object:Gem::Version 
         version: 1.5.0
   type: :development
-  version_requirements: *id008
+  version_requirements: *id009
 description: Client library and CLI that provides access to the VMware Cloud Application Platform.
 email: support@vmware.com
 executables: 
@@ -117,6 +128,7 @@ files:
 - LICENSE
 - README.md
 - Rakefile
+- config/clients.yml
 - lib/cli/commands/admin.rb
 - lib/cli/commands/apps.rb
 - lib/cli/commands/base.rb
@@ -129,6 +141,7 @@ files:
 - lib/cli/frameworks.rb
 - lib/cli/runner.rb
 - lib/cli/services_helper.rb
+- lib/cli/tunnel_helper.rb
 - lib/cli/usage.rb
 - lib/cli/version.rb
 - lib/cli/zip_util.rb
@@ -136,6 +149,9 @@ files:
 - lib/vmc/client.rb
 - lib/vmc/const.rb
 - lib/vmc.rb
+- caldecott_helper/Gemfile
+- caldecott_helper/Gemfile.lock
+- caldecott_helper/server.rb
 - bin/vmc
 has_rdoc: true
 homepage: http://vmware.com
@@ -155,9 +171,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
 required_rubygems_version: !ruby/object:Gem::Requirement 
   none: false
   requirements: 
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version 
-      version: "0"
+      version: 1.3.1
 requirements: []
 
 rubyforge_project: