tunnel-cf-plugin 0.2.2

data/Rakefile

@@ -0,0 +1,5 @@
+require "rake"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+task :default => :spec

data/config/clients.yml

@@ -0,0 +1,17 @@
+redis:
+  redis-cli: -h ${host} -p ${port} -a ${password}
+
+mysql:
+  mysql: --protocol=TCP --host=${host} --port=${port} --user=${user} --password=${password} ${name}
+  mysqldump: --protocol=TCP --host=${host} --port=${port} --user=${user} --password=${password} ${name} > ${ask Output file}
+
+mongodb:
+  mongo: --host ${host} --port ${port} -u ${user} -p ${password} ${name}
+  mongodump: --host ${host} --port ${port} -u ${user} -p ${password} --db ${name}
+  mongorestore: --host ${host} --port ${port} -u ${user} -p ${password} --db ${name} ${ask Directory or filename to restore from}
+
+postgresql:
+  psql:
+    command: -h ${host} -p ${port} -d ${name} -U ${user} -w
+    environment:
+      - PGPASSWORD='${password}'

data/helper-app/Gemfile

@@ -0,0 +1,10 @@
+source "http://rubygems.org"
+
+gem 'rack', '~> 1.2.0'
+gem 'caldecott', '= 0.0.3'
+gem 'bundler'
+gem 'em-websocket'
+gem 'async_sinatra'
+gem 'thin'
+gem 'json'
+gem 'uuidtools'

data/helper-app/Gemfile.lock

@@ -0,0 +1,48 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    addressable (2.2.6)
+    async_sinatra (0.5.0)
+      rack (>= 1.2.1)
+      sinatra (>= 1.0)
+    caldecott (0.0.3)
+      addressable (= 2.2.6)
+      async_sinatra (= 0.5.0)
+      em-http-request (= 0.3.0)
+      em-websocket (= 0.3.1)
+      json (= 1.6.1)
+      uuidtools (= 2.1.2)
+    daemons (1.1.8)
+    em-http-request (0.3.0)
+      addressable (>= 2.0.0)
+      escape_utils
+      eventmachine (>= 0.12.9)
+    em-websocket (0.3.1)
+      addressable (>= 2.1.1)
+      eventmachine (>= 0.12.9)
+    escape_utils (0.2.4)
+    eventmachine (0.12.10)
+    json (1.6.1)
+    rack (1.2.5)
+    sinatra (1.2.8)
+      rack (~> 1.1)
+      tilt (>= 1.2.2, < 2.0)
+    thin (1.4.1)
+      daemons (>= 1.0.9)
+      eventmachine (>= 0.12.6)
+      rack (>= 1.0.0)
+    tilt (1.3.3)
+    uuidtools (2.1.2)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  async_sinatra
+  bundler
+  caldecott (= 0.0.3)
+  em-websocket
+  json
+  rack (~> 1.2.0)
+  thin
+  uuidtools

data/helper-app/server.rb

@@ -0,0 +1,43 @@
+#!/usr/bin/env ruby
+# Copyright (c) 2009-2011 VMware, Inc.
+$:.unshift(File.dirname(__FILE__) + '/lib')
+
+require 'rubygems'
+require 'bundler/setup'
+
+require 'caldecott'
+require 'sinatra'
+require 'json'
+require 'eventmachine'
+
+port = ENV['CF_APP_PORT']
+port ||= 8081
+
+# add vcap specific stuff to Caldecott
+class VcapHttpTunnel < Caldecott::Server::HttpTunnel
+  get '/info' do
+    { "version" => '0.0.4' }.to_json
+  end
+
+  def self.get_tunnels
+    super
+  end
+
+  get '/services' do
+    services_env = ENV['CF_SERVICES']
+    return "no services env" if services_env.nil? or services_env.empty?
+    services_env
+  end
+
+  get '/services/:service' do |service_name|
+    services_env = ENV['CF_SERVICES']
+    not_found if services_env.nil?
+
+    services = JSON.parse(services_env)
+    service = services.find { |s| s["name"] == service_name }
+    not_found if service.nil?
+    service["options"].to_json
+  end
+end
+
+VcapHttpTunnel.run!(:port => port, :auth_token => ENV["CALDECOTT_AUTH"])

data/lib/tunnel-cf-plugin/plugin.rb

@@ -0,0 +1,179 @@
+require "cf/cli"
+require "tunnel-cf-plugin/tunnel"
+
+module CFTunnelPlugin
+  class Tunnel < CF::CLI
+    CLIENTS_FILE = "tunnel-clients.yml"
+    STOCK_CLIENTS = File.expand_path("../../../config/clients.yml", __FILE__)
+
+    desc "Create a local tunnel to a service."
+    group :services, :manage
+    input(:instance, :argument => :optional,
+          :from_given => find_by_name("service instance"),
+          :desc => "Service instance to tunnel to") { |instances|
+      ask("Which service instance?", :choices => instances,
+          :display => proc(&:name))
+    }
+    input(:client, :argument => :optional,
+          :desc => "Client to automatically launch") { |clients|
+      if clients.empty?
+        "none"
+      else
+        ask("Which client would you like to start?",
+            :choices => clients.keys.unshift("none"))
+      end
+    }
+    input(:port, :default => 10000, :desc => "Port to bind the tunnel to")
+    def tunnel
+      instances = client.service_instances
+      fail "No services available for tunneling." if instances.empty?
+
+      instance = input[:instance, instances.sort_by(&:name)]
+      vendor = v2? ? instance.service_plan.service.label : instance.vendor
+      clients = tunnel_clients[vendor] || {}
+      client_name = input[:client, clients]
+
+      tunnel = CFTunnel.new(client, instance)
+      port = tunnel.pick_port!(input[:port])
+
+      conn_info =
+        with_progress("Opening tunnel on port #{c(port, :name)}") do
+          tunnel.open!
+        end
+
+      if client_name == "none"
+        unless quiet?
+          line
+          display_tunnel_connection_info(conn_info)
+
+          line
+          line "Open another shell to run command-line clients or"
+          line "use a UI tool to connect using the displayed information."
+          line "Press Ctrl-C to exit..."
+        end
+
+        tunnel.wait_for_end
+      else
+        with_progress("Waiting for local tunnel to become available") do
+          tunnel.wait_for_start
+        end
+
+        unless start_local_prog(clients, client_name, conn_info, port)
+          fail "'#{client_name}' execution failed; is it in your $PATH?"
+        end
+      end
+    end
+
+    def tunnel_clients
+      return @tunnel_clients if @tunnel_clients
+      stock_config = YAML.load_file(STOCK_CLIENTS)
+      custom_config_file = File.expand_path("#{CF::CONFIG_DIR}/.cf/#{CLIENTS_FILE}")
+
+      if File.exists?(custom_config_file)
+        custom_config = YAML.load_file(custom_config_file)
+        @tunnel_clients = deep_merge(stock_config, custom_config)
+      else
+        @tunnel_clients = stock_config
+      end
+    end
+
+    private
+
+    def display_tunnel_connection_info(info)
+      line "Service connection info:"
+
+      to_show = [nil, nil, nil] # reserved for user, pass, db name
+      info.keys.each do |k|
+        case k
+        when "host", "hostname", "port", "node_id"
+          # skip
+        when "user", "username"
+          # prefer "username" over "user"
+          to_show[0] = k unless to_show[0] == "username"
+        when "password"
+          to_show[1] = k
+        when "name"
+          to_show[2] = k
+        else
+          to_show << k
+        end
+      end
+      to_show.compact!
+
+      align_len = to_show.collect(&:size).max + 1
+
+      indented do
+        to_show.each do |k|
+          # TODO: modify the server services rest call to have explicit knowledge
+          # about the items to return.  It should return all of them if
+          # the service is unknown so that we don't have to do this weird
+          # filtering.
+          line "#{k.ljust align_len}: #{b(info[k])}"
+        end
+      end
+
+      line
+    end
+
+    def start_local_prog(clients, command, info, port)
+      client = clients[File.basename(command)]
+
+      cmdline = "#{command} "
+
+      case client
+      when Hash
+        cmdline << resolve_symbols(client["command"], info, port)
+        client["environment"].each do |e|
+          if e =~ /([^=]+)=(["']?)([^"']*)\2/
+            ENV[$1] = resolve_symbols($3, info, port)
+          else
+            fail "Invalid environment variable: #{e}"
+          end
+        end
+      when String
+        cmdline << resolve_symbols(client, info, port)
+      else
+        raise "Unknown client info: #{client.inspect}."
+      end
+
+      if verbose?
+        line
+        line "Launching '#{cmdline}'"
+      end
+
+      system(cmdline)
+    end
+
+    def resolve_symbols(str, info, local_port)
+      str.gsub(/\$\{\s*([^\}]+)\s*\}/) do
+        sym = $1
+
+        case sym
+        when "host"
+          # TODO: determine proper host
+          "localhost"
+        when "port"
+          local_port
+        when "user", "username"
+          info["username"]
+        when /^ask (.+)/
+          ask($1)
+        else
+          info[sym] || raise("Unknown symbol in config: #{sym}")
+        end
+      end
+    end
+
+    def deep_merge(a, b)
+      merge = proc { |_, old, new|
+        if old.is_a?(Hash) && new.is_a?(Hash)
+          old.merge(new, &merge)
+        else
+          new
+        end
+      }
+
+      a.merge(b, &merge)
+    end
+  end
+end

data/lib/tunnel-cf-plugin/tunnel.rb

@@ -0,0 +1,308 @@
+require "addressable/uri"
+require "restclient"
+require "uuidtools"
+
+require "caldecott-client"
+
+
+class CFTunnel
+  HELPER_NAME = "caldecott"
+  HELPER_APP = File.expand_path("../../../helper-app", __FILE__)
+
+  # bump this AND the version info reported by HELPER_APP/server.rb
+  # this is to keep the helper in sync with any updates here
+  HELPER_VERSION = "0.0.4"
+
+  def initialize(client, service, port = 10000)
+    @client = client
+    @service = service
+    @port = port
+  end
+
+  def open!
+    if helper
+      auth = helper_auth
+
+      unless helper_healthy?(auth)
+        delete_helper
+        auth = create_helper
+      end
+    else
+      auth = create_helper
+    end
+
+    bind_to_helper if @service && !helper_already_binds?
+
+    info = get_connection_info(auth)
+
+    start_tunnel(info, auth)
+
+    info
+  end
+
+  def wait_for_start
+    10.times do |n|
+      begin
+        TCPSocket.open("localhost", @port).close
+        return true
+      rescue => e
+        sleep 1
+      end
+    end
+
+    raise "Could not connect to local tunnel."
+  end
+
+  def wait_for_end
+    if @local_tunnel_thread
+      @local_tunnel_thread.join
+    else
+      raise "Tunnel wasn't started!"
+    end
+  end
+
+  PORT_RANGE = 10
+  def pick_port!(port = @port)
+    original = port
+
+    PORT_RANGE.times do |n|
+      begin
+        TCPSocket.open("localhost", port)
+        port += 1
+      rescue
+        return @port = port
+      end
+    end
+
+    @port = grab_ephemeral_port
+  end
+
+  private
+
+  def helper
+    @helper ||= @client.app_by_name(HELPER_NAME)
+  end
+
+  def create_helper
+    auth = UUIDTools::UUID.random_create.to_s
+    push_helper(auth)
+    start_helper
+    auth
+  end
+
+  def helper_auth
+    helper.env["CALDECOTT_AUTH"]
+  end
+
+  def helper_healthy?(token)
+    return false unless helper.healthy?
+
+    begin
+      response = RestClient.get(
+        "#{helper_url}/info",
+        "Auth-Token" => token
+      )
+
+      info = JSON.parse(response)
+      if info["version"] == HELPER_VERSION
+        true
+      else
+        stop_helper
+        false
+      end
+    rescue RestClient::Exception
+      stop_helper
+      false
+    end
+  end
+
+  def helper_already_binds?
+    helper.binds? @service
+  end
+
+  def push_helper(token)
+    target_base = @client.target.sub(/^[^\.]+\./, "")
+
+    url = "#{random_helper_url}.#{target_base}"
+    is_v2 = @client.is_a?(CFoundry::V2::Client)
+
+    app = @client.app
+    app.name = HELPER_NAME
+    app.framework = @client.framework_by_name("sinatra")
+    app.runtime = @client.runtime_by_name("ruby19")
+    app.command = "bundle exec ruby server.rb -p $VCAP_APP_PORT"
+    app.total_instances = 1
+    app.memory = 64
+    app.env = { "CALDECOTT_AUTH" => token }
+
+    if is_v2
+      app.space = @client.current_space
+    else
+      app.services = [@service] if @service
+      app.url = url
+    end
+
+    app.create!
+
+    if is_v2
+      app.bind(@service) if @service
+      app.create_route(url)
+    end
+
+    begin
+      app.upload(HELPER_APP)
+      invalidate_tunnel_app_info
+    rescue
+      app.delete!
+      raise
+    end
+  end
+
+  def delete_helper
+    helper.delete!
+    invalidate_tunnel_app_info
+  end
+
+  def stop_helper
+    helper.stop!
+    invalidate_tunnel_app_info
+  end
+
+  TUNNEL_CHECK_LIMIT = 60
+  def start_helper
+    helper.start!
+
+    seconds = 0
+    until helper.healthy?
+      sleep 1
+      seconds += 1
+      if seconds == TUNNEL_CHECK_LIMIT
+        raise "Helper application failed to start."
+      end
+    end
+
+    invalidate_tunnel_app_info
+  end
+
+  def bind_to_helper
+    helper.bind(@service)
+    helper.restart!
+  end
+
+  def invalidate_tunnel_app_info
+    @helper_url = nil
+    @helper = nil
+  end
+
+  def helper_url
+    return @helper_url if @helper_url
+
+    tun_url = helper.url
+
+    ["https", "http"].each do |scheme|
+      url = "#{scheme}://#{tun_url}"
+      begin
+        RestClient.get(url)
+
+      # https failed
+      rescue Errno::ECONNREFUSED
+
+      # we expect a 404 since this request isn't auth'd
+      rescue RestClient::ResourceNotFound
+        return @helper_url = url
+      end
+    end
+
+    raise "Cannot determine URL for #{tun_url}"
+  end
+
+  def get_connection_info(token)
+    response = nil
+    10.times do
+      begin
+        response =
+          RestClient.get(
+            helper_url + "/" + safe_path("services", @service.name),
+            "Auth-Token" => token)
+
+        break
+      rescue RestClient::Exception => e
+        sleep 1
+      end
+    end
+
+    unless response
+      raise "Remote tunnel helper is unaware of #{@service.name}!"
+    end
+
+    is_v2 = @client.is_a?(CFoundry::V2::Client)
+
+    info = JSON.parse(response)
+    case (is_v2 ? @service.service_plan.service.label : @service.vendor)
+    when "rabbitmq"
+      uri = Addressable::URI.parse info["url"]
+      info["hostname"] = uri.host
+      info["port"] = uri.port
+      info["vhost"] = uri.path[1..-1]
+      info["user"] = uri.user
+      info["password"] = uri.password
+      info.delete "url"
+
+    # we use "db" as the "name" for mongo
+    # existing "name" is junk
+    when "mongodb"
+      info["name"] = info["db"]
+      info.delete "db"
+
+    # our "name" is irrelevant for redis
+    when "redis"
+      info.delete "name"
+
+    when "filesystem"
+      raise "Tunneling is not supported for this type of service"
+    end
+
+    ["hostname", "port", "password"].each do |k|
+      raise "Could not determine #{k} for #{@service.name}" if info[k].nil?
+    end
+
+    info
+  end
+
+  def start_tunnel(conn_info, auth)
+    @local_tunnel_thread = Thread.new do
+      Caldecott::Client.start({
+        :local_port => @port,
+        :tun_url => helper_url,
+        :dst_host => conn_info["hostname"],
+        :dst_port => conn_info["port"],
+        :log_file => STDOUT,
+        :log_level => ENV["CF_TUNNEL_DEBUG"] || "ERROR",
+        :auth_token => auth,
+        :quiet => true
+      })
+    end
+
+    at_exit { @local_tunnel_thread.kill }
+  end
+
+  def random_helper_url
+    random = sprintf("%x", rand(1000000))
+    "caldecott-#{random}"
+  end
+
+  def safe_path(*segments)
+    segments.flatten.collect { |x|
+      URI.encode x.to_s, Regexp.new("[^#{URI::PATTERN::UNRESERVED}]")
+    }.join("/")
+  end
+
+  def grab_ephemeral_port
+    socket = TCPServer.new("0.0.0.0", 0)
+    socket.setsockopt(Socket::SOL_SOCKET, Socket::SO_REUSEADDR, true)
+    Socket.do_not_reverse_lookup = true
+    socket.addr[1]
+  ensure
+    socket.close
+  end
+end

data/lib/tunnel-cf-plugin/version.rb

@@ -0,0 +1,3 @@
+module CFTunnelPlugin
+  VERSION = "0.2.2".freeze
+end

data/spec/plugin_spec.rb

@@ -0,0 +1,31 @@
+require "spec_helper"
+
+describe CFTunnelPlugin::Tunnel do
+  describe "#tunnel_clients" do
+    context "when the user has a custom clients.yml in their cf directory" do
+      it "overrides the default client config with the user's customizations" do
+        stub_const('CF::CONFIG_DIR', "#{SPEC_ROOT}/fixtures/fake_home_dirs/with_custom_clients")
+
+        expect(subject.tunnel_clients["postgresql"]).to eq({
+          "psql" => {
+            "command"=>"-h ${host} -p ${port} -d ${name} -U ${user} -w",
+            "environment" => ["PGPASSWORD='dont_ask_password'"]
+          }
+        })
+      end
+    end
+
+    context "when the user does not have a custom clients.yml" do
+      it "returns the default client config" do
+        stub_const('CF::CONFIG_DIR', '.')
+
+        expect(subject.tunnel_clients["postgresql"]).to eq({
+          "psql" => {
+            "command"=>"-h ${host} -p ${port} -d ${name} -U ${user} -w",
+            "environment" => ["PGPASSWORD='${password}'"]
+          }
+        })
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,14 @@
+SPEC_ROOT = File.dirname(__FILE__).freeze
+
+require "rspec"
+require "cfoundry"
+require "cfoundry/test_support"
+require "cf"
+require "cf/test_support"
+
+require "#{SPEC_ROOT}/../lib/tunnel-cf-plugin/plugin"
+
+RSpec.configure do |c|
+  c.include Fake::FakeMethods
+  c.include ::FakeHomeDir
+end

metadata

@@ -0,0 +1,221 @@
+--- !ruby/object:Gem::Specification 
+name: tunnel-cf-plugin
+version: !ruby/object:Gem::Version 
+  hash: 19
+  prerelease: 
+  segments: 
+  - 0
+  - 2
+  - 2
+  version: 0.2.2
+platform: ruby
+authors: 
+- Alex Suraci
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2013-03-25 00:00:00 Z
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: cfoundry
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 312055405
+        segments: 
+        - 0
+        - 5
+        - 3
+        - rc
+        - 5
+        version: 0.5.3.rc5
+    - - <
+      - !ruby/object:Gem::Version 
+        hash: 7
+        segments: 
+        - 0
+        - 6
+        version: "0.6"
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: addressable
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 7
+        segments: 
+        - 2
+        - 2
+        version: "2.2"
+  type: :runtime
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: caldecott-client
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 27
+        segments: 
+        - 0
+        - 0
+        - 2
+        version: 0.0.2
+  type: :runtime
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: rest-client
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 1
+        - 6
+        version: "1.6"
+  type: :runtime
+  version_requirements: *id004
+- !ruby/object:Gem::Dependency 
+  name: uuidtools
+  prerelease: false
+  requirement: &id005 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 1
+        segments: 
+        - 2
+        - 1
+        version: "2.1"
+  type: :runtime
+  version_requirements: *id005
+- !ruby/object:Gem::Dependency 
+  name: rake
+  prerelease: false
+  requirement: &id006 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 25
+        segments: 
+        - 0
+        - 9
+        version: "0.9"
+  type: :development
+  version_requirements: *id006
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  prerelease: false
+  requirement: &id007 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 21
+        segments: 
+        - 2
+        - 11
+        version: "2.11"
+  type: :development
+  version_requirements: *id007
+- !ruby/object:Gem::Dependency 
+  name: webmock
+  prerelease: false
+  requirement: &id008 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 29
+        segments: 
+        - 1
+        - 9
+        version: "1.9"
+  type: :development
+  version_requirements: *id008
+- !ruby/object:Gem::Dependency 
+  name: rr
+  prerelease: false
+  requirement: &id009 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 15
+        segments: 
+        - 1
+        - 0
+        version: "1.0"
+  type: :development
+  version_requirements: *id009
+description: 
+email: 
+- asuraci@vmware.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- Rakefile
+- lib/tunnel-cf-plugin/plugin.rb
+- lib/tunnel-cf-plugin/tunnel.rb
+- lib/tunnel-cf-plugin/version.rb
+- helper-app/Gemfile
+- helper-app/Gemfile.lock
+- helper-app/server.rb
+- config/clients.yml
+- spec/plugin_spec.rb
+- spec/spec_helper.rb
+homepage: http://cloudfoundry.com/
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: tunnel-cf-plugin
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: External access to your services on Cloud Foundry via a Caldecott HTTP tunnel.
+test_files: 
+- spec/plugin_spec.rb
+- spec/spec_helper.rb