tunnel-vmc-plugin 0.0.1

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/config/clients.yml

@@ -0,0 +1,17 @@
+redis:
+  redis-cli: -h ${host} -p ${port} -a ${password}
+
+mysql:
+  mysql: --protocol=TCP --host=${host} --port=${port} --user=${user} --password=${password} ${name}
+  mysqldump: --protocol=TCP --host=${host} --port=${port} --user=${user} --password=${password} ${name} > ${ask Output file}
+
+mongodb:
+  mongo: --host ${host} --port ${port} -u ${user} -p ${password} ${name}
+  mongodump: --host ${host} --port ${port} -u ${user} -p ${password} --db ${name}
+  mongorestore: --host ${host} --port ${port} -u ${user} -p ${password} --db ${name} ${ask Directory or filename to restore from}
+
+postgresql:
+  psql:
+    command: -h ${host} -p ${port} -d ${name} -U ${user} -w
+    environment:
+      - PGPASSWORD='${password}'

data/helper-app/Gemfile

@@ -0,0 +1,10 @@
+source "http://rubygems.org"
+
+gem 'rack', '~> 1.2.0'
+gem 'caldecott', '= 0.0.3'
+gem 'bundler'
+gem 'em-websocket'
+gem 'async_sinatra'
+gem 'thin'
+gem 'json'
+gem 'uuidtools'

data/helper-app/Gemfile.lock

@@ -0,0 +1,48 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    addressable (2.2.6)
+    async_sinatra (0.5.0)
+      rack (>= 1.2.1)
+      sinatra (>= 1.0)
+    caldecott (0.0.3)
+      addressable (= 2.2.6)
+      async_sinatra (= 0.5.0)
+      em-http-request (= 0.3.0)
+      em-websocket (= 0.3.1)
+      json (= 1.6.1)
+      uuidtools (= 2.1.2)
+    daemons (1.1.4)
+    em-http-request (0.3.0)
+      addressable (>= 2.0.0)
+      escape_utils
+      eventmachine (>= 0.12.9)
+    em-websocket (0.3.1)
+      addressable (>= 2.1.1)
+      eventmachine (>= 0.12.9)
+    escape_utils (0.2.4)
+    eventmachine (0.12.10)
+    json (1.6.1)
+    rack (1.2.4)
+    sinatra (1.2.7)
+      rack (~> 1.1)
+      tilt (>= 1.2.2, < 2.0)
+    thin (1.2.11)
+      daemons (>= 1.0.9)
+      eventmachine (>= 0.12.6)
+      rack (>= 1.0.0)
+    tilt (1.3.3)
+    uuidtools (2.1.2)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  async_sinatra
+  bundler
+  caldecott (= 0.0.3)
+  em-websocket
+  json
+  rack (~> 1.2.0)
+  thin
+  uuidtools

data/helper-app/server.rb

@@ -0,0 +1,43 @@
+#!/usr/bin/env ruby
+# Copyright (c) 2009-2011 VMware, Inc.
+$:.unshift(File.dirname(__FILE__) + '/lib')
+
+require 'rubygems'
+require 'bundler/setup'
+
+require 'caldecott'
+require 'sinatra'
+require 'json'
+require 'eventmachine'
+
+port = ENV['VMC_APP_PORT']
+port ||= 8081
+
+# add vcap specific stuff to Caldecott
+class VcapHttpTunnel < Caldecott::Server::HttpTunnel
+  get '/info' do
+    { "version" => '0.0.4' }.to_json
+  end
+
+  def self.get_tunnels
+    super
+  end
+
+  get '/services' do
+    services_env = ENV['VMC_SERVICES']
+    return "no services env" if services_env.nil? or services_env.empty?
+    services_env
+  end
+
+  get '/services/:service' do |service_name|
+    services_env = ENV['VMC_SERVICES']
+    not_found if services_env.nil?
+
+    services = JSON.parse(services_env)
+    service = services.find { |s| s["name"] == service_name }
+    not_found if service.nil?
+    service["options"].to_json
+  end
+end
+
+VcapHttpTunnel.run!(:port => port, :auth_token => ENV["CALDECOTT_AUTH"])

data/lib/tunnel-vmc-plugin/plugin.rb

@@ -0,0 +1,87 @@
+require "vmc/plugin"
+require File.expand_path("../tunnel", __FILE__)
+
+VMC.Plugin(VMC::Service) do
+  include VMCTunnel
+
+  desc "tunnel SERVICE [CLIENT]", "create a local tunnel to a service"
+  flag(:service) { |choices|
+    ask("Which service?", :choices => choices)
+  }
+  flag(:client)
+  flag(:port, :default => 10000)
+  def tunnel(service = nil, client_name = nil)
+    unless defined? Caldecott
+      $stderr.puts "To use `vmc tunnel', you must first install Caldecott:"
+      $stderr.puts ""
+      $stderr.puts "\tgem install caldecott"
+      $stderr.puts ""
+      $stderr.puts "Note that you'll need a C compiler. If you're on OS X, Xcode"
+      $stderr.puts "will provide one. If you're on Windows, try DevKit."
+      $stderr.puts ""
+      $stderr.puts "This manual step will be removed in the future."
+      $stderr.puts ""
+      err "Caldecott is not installed."
+      return
+    end
+
+    client_name ||= input(:client)
+
+    services = client.services
+    if services.empty?
+      err "No services available to tunnel to"
+      return
+    end
+
+    service ||= input(:service, services.collect(&:name).sort)
+
+    info = services.find { |s| s.name == service }
+
+    unless info
+      err "Unknown service '#{service}'"
+      return
+    end
+
+    clients = tunnel_clients[info.vendor] || {}
+
+    unless client_name
+      if clients.empty?
+        client_name = "none"
+      else
+        client_name = ask(
+          "Which client would you like to start?",
+          :choices => ["none"] + clients.keys)
+      end
+    end
+
+    tunnel = CFTunnel.new(client, info)
+    port = tunnel.pick_port!(input(:port))
+
+    conn_info =
+      with_progress("Opening tunnel on port #{c(port, :blue)}") do
+        tunnel.open!
+      end
+
+    if client_name == "none"
+      unless simple_output?
+        puts ""
+        display_tunnel_connection_info(conn_info)
+
+        puts ""
+        puts "Open another shell to run command-line clients or"
+        puts "use a UI tool to connect using the displayed information."
+        puts "Press Ctrl-C to exit..."
+      end
+
+      tunnel.wait_for_end
+    else
+      with_progress("Waiting for local tunnel to become available") do
+        tunnel.wait_for_start
+      end
+
+      unless start_local_prog(clients, client_name, conn_info, port)
+        err "'#{client_name}' execution failed; is it in your $PATH?"
+      end
+    end
+  end
+end

data/lib/tunnel-vmc-plugin/tunnel.rb

@@ -0,0 +1,406 @@
+require "addressable/uri"
+
+begin
+  require "caldecott"
+rescue LoadError
+end
+
+class CFTunnel
+  HELPER_NAME = "caldecott"
+  HELPER_APP = File.expand_path("../../../helper-app", __FILE__)
+
+  # bump this AND the version info reported by HELPER_APP/server.rb
+  # this is to keep the helper in sync with any updates here
+  HELPER_VERSION = "0.0.4"
+
+  def initialize(client, service, port = 10000)
+    @client = client
+    @service = service
+    @port = port
+  end
+
+  def open!
+    if helper.exists?
+      auth = helper_auth
+
+      unless helper_healthy?(auth)
+        delete_helper
+        auth = create_helper
+      end
+    else
+      auth = create_helper
+    end
+
+    bind_to_helper unless helper_already_binds?
+
+    info = get_connection_info(auth)
+
+    start_tunnel(info, auth)
+
+    info
+  end
+
+  def wait_for_start
+    10.times do |n|
+      begin
+        TCPSocket.open("localhost", @port).close
+        return true
+      rescue => e
+        sleep 1
+      end
+    end
+
+    raise "Could not connect to local tunnel."
+  end
+
+  def wait_for_end
+    if @local_tunnel_thread
+      @local_tunnel_thread.join
+    else
+      raise "Tunnel wasn't started!"
+    end
+  end
+
+  PORT_RANGE = 10
+  def pick_port!(port = @port)
+    original = port
+
+    PORT_RANGE.times do |n|
+      begin
+        TCPSocket.open("localhost", port)
+        port += 1
+      rescue
+        return @port = port
+      end
+    end
+
+    @port = grab_ephemeral_port
+  end
+
+  private
+
+  def helper
+    @helper ||= @client.app(HELPER_NAME)
+  end
+
+  def create_helper
+    auth = UUIDTools::UUID.random_create.to_s
+    push_helper(auth)
+    start_helper
+    auth
+  end
+
+  def helper_auth
+    helper.env.each do |e|
+      name, val = e.split("=", 2)
+      return val if name == "CALDECOTT_AUTH"
+    end
+
+    nil
+  end
+
+  def helper_healthy?(token)
+    return false unless helper.healthy?
+
+    begin
+      response = RestClient.get(
+        "#{helper_url}/info",
+        "Auth-Token" => token
+      )
+
+      info = JSON.parse(response)
+      if info["version"] == HELPER_VERSION
+        true
+      else
+        stop_helper
+        false
+      end
+    rescue RestClient::Exception
+      stop_helper
+      false
+    end
+  end
+
+  def helper_already_binds?
+    helper.services.include? @service.name
+  end
+
+  def push_helper(token)
+    target_base = @client.target.sub(/^[^\.]+\./, "")
+
+    app = @client.app(HELPER_NAME)
+    app.framework = "sinatra"
+    app.url = "#{random_helper_url}.#{target_base}"
+    app.total_instances = 1
+    app.memory = 64
+    app.env = ["CALDECOTT_AUTH=#{token}"]
+    app.services = [@service.name]
+    app.create!
+
+    begin
+      app.upload(HELPER_APP)
+      invalidate_tunnel_app_info
+    rescue
+      app.delete!
+      raise
+    end
+  end
+
+  def delete_helper
+    helper.delete!
+    invalidate_tunnel_app_info
+  end
+
+  def stop_helper
+    helper.stop!
+    invalidate_tunnel_app_info
+  end
+
+  TUNNEL_CHECK_LIMIT = 60
+  def start_helper
+    helper.start!
+
+    seconds = 0
+    until helper.healthy?
+      sleep 1
+      seconds += 1
+      if seconds == TUNNEL_CHECK_LIMIT
+        raise "Helper application failed to start."
+      end
+    end
+
+    invalidate_tunnel_app_info
+  end
+
+  def bind_to_helper
+    helper.bind(@service.name)
+    helper.restart!
+  end
+
+  def invalidate_tunnel_app_info
+    @helper_url = nil
+    @helper = nil
+  end
+
+  def helper_url
+    return @helper_url if @helper_url
+
+    tun_url = helper.url
+
+    ["https", "http"].each do |scheme|
+      url = "#{scheme}://#{tun_url}"
+      begin
+        RestClient.get(url)
+
+      # https failed
+      rescue Errno::ECONNREFUSED
+
+      # we expect a 404 since this request isn't auth'd
+      rescue RestClient::ResourceNotFound
+        return @helper_url = url
+      end
+    end
+
+    raise "Cannot determine URL for #{tun_url}"
+  end
+
+  def get_connection_info(token)
+    response = nil
+    10.times do
+      begin
+        response =
+          RestClient.get(
+            helper_url + "/" + safe_path("services", @service.name),
+            "Auth-Token" => token)
+
+        break
+      rescue RestClient::Exception => e
+        p [e, e.to_s]
+        sleep 1
+      end
+    end
+
+    unless response
+      raise "Remote tunnel helper is unaware of #{@service.name}!"
+    end
+
+    info = JSON.parse(response)
+    case @service.vendor
+    when "rabbitmq"
+      uri = Addressable::URI.parse info["url"]
+      info["hostname"] = uri.host
+      info["port"] = uri.port
+      info["vhost"] = uri.path[1..-1]
+      info["user"] = uri.user
+      info["password"] = uri.password
+      info.delete "url"
+
+    # we use "db" as the "name" for mongo
+    # existing "name" is junk
+    when "mongodb"
+      info["name"] = info["db"]
+      info.delete "db"
+
+    # our "name" is irrelevant for redis
+    when "redis"
+      info.delete "name"
+    end
+
+    ["hostname", "port", "password"].each do |k|
+      raise "Could not determine #{k} for #{@service.name}" if info[k].nil?
+    end
+
+    info
+  end
+
+  def start_tunnel(conn_info, auth)
+    @local_tunnel_thread = Thread.new do
+      Caldecott::Client.start({
+        :local_port => @port,
+        :tun_url => helper_url,
+        :dst_host => conn_info["hostname"],
+        :dst_port => conn_info["port"],
+        :log_file => STDOUT,
+        :log_level => ENV["VMC_TUNNEL_DEBUG"] || "ERROR",
+        :auth_token => auth,
+        :quiet => true
+      })
+    end
+
+    at_exit { @local_tunnel_thread.kill }
+  end
+
+  def random_helper_url
+    random = sprintf("%x", rand(1000000))
+    "caldecott-#{random}"
+  end
+
+  def safe_path(*segments)
+    segments.flatten.collect { |x|
+      URI.encode x.to_s, Regexp.new("[^#{URI::PATTERN::UNRESERVED}]")
+    }.join("/")
+  end
+
+  def grab_ephemeral_port
+    socket = TCPServer.new("0.0.0.0", 0)
+    socket.setsockopt(Socket::SOL_SOCKET, Socket::SO_REUSEADDR, true)
+    Socket.do_not_reverse_lookup = true
+    socket.addr[1]
+  ensure
+    socket.close
+  end
+end
+
+module VMCTunnel
+  CLIENTS_FILE = "#{VMC::CONFIG_DIR}/tunnel-clients.yml"
+  STOCK_CLIENTS = File.expand_path("../../../config/clients.yml", __FILE__)
+
+  def display_tunnel_connection_info(info)
+    puts "Service connection info:"
+
+    to_show = [nil, nil, nil] # reserved for user, pass, db name
+    info.keys.each do |k|
+      case k
+      when "host", "hostname", "port", "node_id"
+        # skip
+      when "user", "username"
+        # prefer "username" over "user"
+        to_show[0] = k unless to_show[0] == "username"
+      when "password"
+        to_show[1] = k
+      when "name"
+        to_show[2] = k
+      else
+        to_show << k
+      end
+    end
+    to_show.compact!
+
+    align_len = to_show.collect(&:size).max + 1
+
+    to_show.each do |k|
+      # TODO: modify the server services rest call to have explicit knowledge
+      # about the items to return.  It should return all of them if
+      # the service is unknown so that we don't have to do this weird
+      # filtering.
+      print "  #{k.ljust align_len}: "
+      puts c("#{info[k]}", :yellow)
+    end
+
+    puts ""
+  end
+
+  def start_local_prog(clients, command, info, port)
+    client = clients[File.basename(command)]
+
+    cmdline = "#{command} "
+
+    case client
+    when Hash
+      cmdline << resolve_symbols(client["command"], info, port)
+      client["environment"].each do |e|
+        if e =~ /([^=]+)=(["']?)([^"']*)\2/
+          ENV[$1] = resolve_symbols($3, info, port)
+        else
+          raise "Invalid environment variable: #{e}"
+        end
+      end
+    when String
+      cmdline << resolve_symbols(client, info, port)
+    else
+      raise "Unknown client info: #{client.inspect}."
+    end
+
+    if verbose?
+      puts ""
+      puts "Launching '#{cmdline}'"
+    end
+
+    system(cmdline)
+  end
+
+  def tunnel_clients
+    return @tunnel_clients if @tunnel_clients
+
+    stock = YAML.load_file(STOCK_CLIENTS)
+    clients = File.expand_path CLIENTS_FILE
+    if File.exists? clients
+      user = YAML.load_file(clients)
+      @tunnel_clients = deep_merge(stock, user)
+    else
+      @tunnel_clients = stock
+    end
+  end
+
+  def resolve_symbols(str, info, local_port)
+    str.gsub(/\$\{\s*([^\}]+)\s*\}/) do
+      sym = $1
+
+      case sym
+      when "host"
+        # TODO: determine proper host
+        "localhost"
+      when "port"
+        local_port
+      when "user", "username"
+        info["username"]
+      when /^ask (.+)/
+        ask($1)
+      else
+        info[sym] || raise("Unknown symbol in config: #{sym}")
+      end
+    end
+  end
+
+  def deep_merge(a, b)
+    merge = proc { |old, new|
+      if old === Hash && new === Hash
+        old.merge(new, &merge)
+      else
+        new
+      end
+    }
+
+    a.merge(b, &merge)
+  end
+end

data/lib/tunnel-vmc-plugin/version.rb

@@ -0,0 +1,3 @@
+module VMCTunnel
+  VERSION = "0.0.1"
+end

metadata

@@ -0,0 +1,101 @@
+--- !ruby/object:Gem::Specification
+name: tunnel-vmc-plugin
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Alex Suraci
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-05-23 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: addressable
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.2.6
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.2.6
+- !ruby/object:Gem::Dependency
+  name: eventmachine
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.0.0.beta
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.0.0.beta
+- !ruby/object:Gem::Dependency
+  name: caldecott
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.0.5
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.0.5
+description: 
+email:
+- asuraci@vmware.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Rakefile
+- lib/tunnel-vmc-plugin/tunnel.rb
+- lib/tunnel-vmc-plugin/plugin.rb
+- lib/tunnel-vmc-plugin/version.rb
+- helper-app/Gemfile.lock
+- helper-app/server.rb
+- helper-app/Gemfile
+- config/clients.yml
+homepage: http://cloudfoundry.com/
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: tunnel-vmc-plugin
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: External access to your services on Cloud Foundry via a Caldecott HTTP tunnel.
+test_files: []