tools-cf-plugin 1.0.0 → 1.1.0

data/lib/tools-cf-plugin/plugin.rb

@@ -1 +1,3 @@
-require "tools-cf-plugin/watch"
+require "tools-cf-plugin/watch"
+require "tools-cf-plugin/tunnel/watch"
+require "tools-cf-plugin/tunnel/watch_logs"

data/lib/tools-cf-plugin/tunnel/base.rb

@@ -0,0 +1,133 @@
+require "yaml"
+require "cli"
+require "net/ssh/gateway"
+
+require "cf/cli"
+
+module CFTools
+  module Tunnel
+    class Base < CF::CLI
+      BOSH_CONFIG = "~/.bosh_config"
+      
+      def precondition
+      end
+
+      def director(director_host, gateway)
+        if address_reachable?(director_host, 25555)
+          director_for(25555, director_host)
+        else
+          dport =
+            with_progress("Opening local tunnel to director") do
+              tunnel_to(director_host, 25555, gateway)
+            end
+
+          director_for(dport)
+        end
+      end
+
+      def connected_director(director_host, gateway)
+        director = director(director_host, gateway)
+
+        authenticate_with_director(
+          director,
+          "https://#{director_host}:25555",
+          director_credentials(director_host))
+
+        director
+      end
+
+      def authenticate_with_director(director, remote_director, auth)
+        if auth && login_to_director(director, auth["username"], auth["password"])
+          return true
+        end
+
+        while true
+          line unless quiet?
+          user = ask("Director Username")
+          pass = ask("Director Password", :echo => "*", :forget => true)
+          break if login_to_director(director, user, pass)
+        end
+
+        save_auth(remote_director, "username" => user, "password" => pass)
+
+        true
+      end
+
+      def login_to_director(director, user, pass)
+        director.user = user
+        director.password = pass
+
+        with_progress("Authenticating as #{c(user, :name)}") do |s|
+          director.authenticated? || s.fail
+        end
+      end
+
+      def tunnel_to(address, remote_port, gateway)
+        user, host = gateway.split("@", 2)
+        Net::SSH::Gateway.new(host, user).open(address, remote_port)
+      end
+
+      private
+
+      def address_reachable?(host, port)
+        Timeout.timeout(1) do
+          TCPSocket.new(host, port).close
+          true
+        end
+      rescue Timeout::Error, Errno::ECONNREFUSED
+        false
+      end
+
+      def director_for(port, host = "127.0.0.1")
+        Bosh::Cli::Director.new("https://#{host}:#{port}")
+      end
+
+      def director_credentials(director)
+        return unless cfg = bosh_config
+
+        _, auth = cfg["auth"].find do |d, _|
+          d.include?(director)
+        end
+
+        auth
+      end
+
+      def current_deployment(director)
+        deployments = director.list_deployments
+
+        deployments.find do |d|
+          d["releases"].any? { |r| r["name"] == "cf-release" }
+        end
+      end
+
+      def current_deployment_manifest(director)
+        deployment = current_deployment(director)
+        YAML.load(director.get_deployment(deployment["name"])["manifest"])
+      end
+
+      def save_auth(director, auth)
+        cfg = bosh_config || { "auth" => {} }
+
+        cfg["auth"][director] = auth
+
+        save_bosh_config(cfg)
+      end
+
+      def save_bosh_config(config)
+        File.open(bosh_config_file, "w") do |io|
+          io.write(YAML.dump(config))
+        end
+      end
+
+      def bosh_config
+        return unless File.exists?(bosh_config_file)
+
+        YAML.load_file(bosh_config_file)
+      end
+
+      def bosh_config_file
+        File.expand_path(BOSH_CONFIG)
+      end
+    end
+  end
+end

data/lib/tools-cf-plugin/tunnel/log_entry.rb

@@ -0,0 +1,46 @@
+require "json"
+require "time"
+
+module CFTools
+  module Tunnel
+    class LogEntry
+      attr_reader :label, :line, :stream
+
+      def initialize(label, line, stream)
+        @label = label
+        @line = line
+        @stream = stream
+        @fallback_timestamp = Time.now
+      end
+
+      def message
+        json = JSON.parse(@line)
+        json["message"]
+      rescue JSON::ParserError
+        @line
+      end
+
+      def log_level
+        json = JSON.parse(@line)
+        json["log_level"]
+      rescue JSON::ParserError
+      end
+
+      def timestamp
+        json = JSON.parse(@line)
+
+        timestamp = json["timestamp"]
+        case timestamp
+        when String
+          Time.parse(timestamp)
+        when Numeric
+          Time.at(timestamp)
+        else
+          @fallback_timestamp
+        end
+      rescue JSON::ParserError
+        @fallback_timestamp
+      end
+    end
+  end
+end

data/lib/tools-cf-plugin/tunnel/multi_line_stream.rb

@@ -0,0 +1,100 @@
+require "base64"
+require "json"
+require "net/ssh/gateway"
+require "thread"
+
+module CFTools
+  module Tunnel
+    class MultiLineStream
+      include Interact::Pretty
+
+      def initialize(director, deployment, user, host)
+        @director = director
+        @deployment = deployment
+        @gateway_user = user
+        @gateway_host = host
+      end
+
+      def stream(locations)
+        entries = entry_queue
+
+        Thread.abort_on_exception = true
+
+        locations.each do |(name, index), locs|
+          Thread.new do
+            stream_location(name, index, locs, entries)
+          end
+        end
+
+        while entry = entries.pop
+          yield entry
+        end
+      end
+
+      private
+
+      def gateway
+        @gateway ||= Net::SSH::Gateway.new(@gateway_host, @gateway_user)
+      end
+
+      def entry_queue
+        Queue.new
+      end
+
+      def public_key
+        Net::SSH::Authentication::KeyManager.new(nil).each_identity do |i|
+          return sane_public_key(i)
+        end
+      end
+
+      def sane_public_key(pkey)
+        "#{pkey.ssh_type} #{Base64.encode64(pkey.to_blob).split.join} #{pkey.comment}"
+      end
+
+      def generate_user
+        "bosh_cf_watch_logs_#{rand(36**9).to_s(36)}"
+      end
+
+      def create_ssh_user(job, index, entries)
+        user = generate_user
+
+        entries << LogEntry.new(
+          "#{job}/#{index}", c("creating user...", :warning), :stdout)
+
+        status, task_id = @director.setup_ssh(
+          @deployment, job, index, user,
+          public_key, nil, :use_cache => false)
+
+        raise "SSH setup failed." unless status == :done
+
+        entries << LogEntry.new("#{job}/#{index}", c("created!", :good), :stdout)
+
+        sessions = JSON.parse(@director.get_task_result_log(task_id))
+
+        session = sessions.first
+
+        raise "No session?" unless session
+
+        [session["ip"], user]
+      end
+
+      def stream_location(job, index, locations, entries)
+        ip, user = create_ssh_user(job, index, entries)
+
+        entries << LogEntry.new(
+          "#{job}/#{index}", c("connecting", :warning), :stdout)
+
+        gateway.ssh(ip, user) do |ssh|
+          entries << LogEntry.new(
+            "#{job}/#{index}", c("connected!", :good), :stdout)
+
+          locations.each do |loc|
+            loc.stream_lines(ssh) do |entry|
+              entries << entry
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/tools-cf-plugin/tunnel/stream_location.rb

@@ -0,0 +1,37 @@
+require "tools-cf-plugin/tunnel/log_entry"
+
+module CFTools
+  module Tunnel
+    class StreamLocation
+      attr_reader :path, :label
+
+      def initialize(path, label)
+        @path = path
+        @label = label
+      end
+
+      def stream_lines(ssh)
+        ssh.exec("tail -f /var/vcap/sys/log/#@path") do |ch, stream, chunk|
+          if pending = ch[:pending]
+            chunk = pending + chunk
+            ch[:pending] = nil
+          end
+
+          chunk.each_line do |line|
+            if line.end_with?("\n")
+              yield log_entry(line, stream)
+            else
+              ch[:pending] = line
+            end
+          end
+        end
+      end
+
+      private
+
+      def log_entry(line, stream)
+        LogEntry.new(@label, line, stream)
+      end
+    end
+  end
+end

data/lib/tools-cf-plugin/tunnel/watch.rb

@@ -0,0 +1,52 @@
+require "yaml"
+require "cli"
+require "net/ssh"
+
+require "cf/cli"
+
+require "tools-cf-plugin/tunnel/base"
+
+module CFTools::Tunnel
+  class Watch < Base
+    desc "Watch, by grabbing the connection info from your BOSH deployment."
+    input :director, :argument => :required, :desc => "BOSH director address"
+    input :gateway, :argument => :optional, 
+          :default => proc { "vcap@#{input[:director]}" },
+          :desc => "SSH connection string (default: vcap@director)"
+    def tunnel_watch
+      director_host = input[:director]
+      gateway = input[:gateway]
+
+      director = connected_director(director_host, gateway)
+
+      manifest =
+        with_progress("Downloading deployment manifest") do
+          current_deployment_manifest(director)
+        end
+
+      nats = manifest["properties"]["nats"]
+
+      nport =
+        with_progress("Opening local tunnel to NATS") do
+          tunnel_to(nats["address"], nats["port"], gateway) 
+        end
+
+      with_progress("Logging in as admin user") do
+        login_as_admin(manifest)
+      end
+
+      invoke :watch, :port => nport,
+             :user => nats["user"], :password => nats["password"]
+    end
+
+    private
+
+    def login_as_admin(manifest)
+      admin = manifest["properties"]["uaa"]["scim"]["users"].grep(/cloud_controller\.admin/).first
+      admin_user, admin_pass, _ = admin.split("|", 3)
+
+      @@client = CFoundry::V2::Client.new(manifest["properties"]["cc"]["srv_api_uri"])
+      client.login(admin_user, admin_pass)
+    end
+  end
+end

data/lib/tools-cf-plugin/tunnel/watch_logs.rb

@@ -0,0 +1,122 @@
+require "yaml"
+require "thread"
+
+require "tools-cf-plugin/tunnel/base"
+require "tools-cf-plugin/tunnel/multi_line_stream"
+require "tools-cf-plugin/tunnel/stream_location"
+
+module CFTools::Tunnel
+  class WatchLogs < Base
+    LOGS = {
+      "cloud_controller" => ["cloud_controller_ng/cloud_controller_ng.log"],
+      "dea_next" => ["dea_next/dea_next.log"],
+      "health_manager" => ["health_manager_next/health_manager_next.log"],
+      "router" => ["gorouter/gorouter.log"]
+    }
+
+    desc "Stream logs from the jobs of a deployment"
+    input :director, :argument => :required, :desc => "BOSH director address"
+    input :gateway, :argument => :optional, 
+          :default => proc { "vcap@#{input[:director]}" },
+          :desc => "SSH connection string (default: vcap@director)"
+    def watch_logs
+      director_host = input[:director]
+      gateway = input[:gateway]
+
+      director = connected_director(director_host, gateway)
+
+      deployment =
+        with_progress("Getting deployment info") do
+          current_deployment(director)
+        end
+
+      locations =
+        with_progress("Finding logs for #{c(deployment["name"], :name)}") do
+          locs = stream_locations(director, deployment["name"])
+
+          if locs.empty?
+            fail "No locations found."
+          else
+            locs
+          end
+        end
+
+      stream = stream_for(director, deployment["name"], gateway)
+
+      stream.stream(locations) do |entry|
+        line pretty_print_entry(entry)
+      end
+    end
+
+    private
+
+    def stream_for(director, deployment, gateway)
+      user, host = gateway.split("@", 2)
+      MultiLineStream.new(director, deployment, user, host)
+    end
+
+    def max_label_size
+      LOGS.keys.collect(&:size).sort.last + 3
+    end
+
+    def pretty_print_entry(entry)
+      log_level = entry.log_level || ""
+      level_padding = " " * (6 - log_level.size)
+      [ c(entry.label.ljust(max_label_size), :name),
+        entry.timestamp.strftime("%r"),
+        "#{pretty_log_level(log_level)}#{level_padding}",
+        level_colored_message(entry)
+      ].join("  ")
+    end
+
+    def stream_locations(director, deployment)
+      locations = Hash.new { |h, k| h[k] = [] }
+
+      director.fetch_vm_state(deployment, :use_cache => false).each do |vm|
+        name = vm["job_name"]
+        index = vm["index"]
+        next unless LOGS.key?(name)
+
+        vm["ips"].each do |ip|
+          LOGS[name].each do |file|
+            locations[[name, index]] << StreamLocation.new(file, "#{name}/#{index}")
+          end
+        end
+      end
+
+      locations
+    end
+
+    def level_colored_message(entry)
+      msg = entry.message
+
+      case entry.log_level
+      when "warn"
+        c(msg, :warning)
+      when "error"
+        c(msg, :bad)
+      when "fatal"
+        c(msg, :error)
+      else
+        msg
+      end
+    end
+
+    def pretty_log_level(level)
+      case level
+      when "info"
+        d(level)
+      when "debug", "debug1", "debug2", "all"
+        c(level, :good)
+      when "warn"
+        c(level, :warning)
+      when "error"
+        c(level, :bad)
+      when "fatal"
+        c(level, :error)
+      else
+        level
+      end
+    end
+  end
+end

data/lib/tools-cf-plugin/version.rb

@@ -1,3 +1,3 @@
 module CFTools
-  VERSION = "1.0.0".freeze
+  VERSION = "1.1.0".freeze
 end

data/lib/tools-cf-plugin/watch.rb

@@ -14,7 +14,7 @@ module CFTools
     group :admin
     input :app, :argument => :optional, :from_given => by_name(:app),
           :desc => "Application to watch"
-    input :host, :alias => "-h", :default => "localhost",
+    input :host, :alias => "-h", :default => "127.0.0.1",
           :desc => "NATS server address"
     input :port, :alias => "-P", :default => 4222, :type => :integer,
           :desc => "NATS server port"
@@ -30,6 +30,7 @@ module CFTools
       pass = input[:password]
 
       @requests = {}
+      @seen_apps = {}
       @request_ticker = 0
 
       $stdout.sync = true
@@ -99,8 +100,12 @@ module CFTools
         sub, msg = pretty_dea_shutdown(sub, msg)
       when /^cloudcontrollers\.hm\.requests\.\w+$/
         sub, msg = process_cloudcontrollers_hm_request(sub, msg)
-      when /([^\.]+)\.announce$/
-        sub, msg = process_service_announcement(sub, msg)
+      when /^([^.]+)\.announce$/
+        sub, msg = pretty_service_announcement(sub, msg)
+      when "vcap.component.announce"
+        sub, msg = pretty_component_announcement(sub, msg)
+      when "vcap.component.discover"
+        sub, msg = pretty_component_discover(sub, msg)
       end
 
       if reply
@@ -122,6 +127,8 @@ module CFTools
         sub, msg = pretty_healthmanager_status_response(sub, msg)
       when "healthmanager.health"
         sub, msg = pretty_healthmanager_health_response(sub, msg)
+      when "vcap.component.discover"
+        sub, msg = pretty_component_discover_response(sub, msg)
       end
 
       line "#{timestamp}\t#{REPLY_PREFIX}#{sub} (#{c(id, :error)})\t#{msg}"
@@ -264,8 +271,8 @@ module CFTools
 
     def pretty_healthmanager_health(sub, msg)
       payload = JSON.parse(msg)
-      apps = payload["droplets"].collect { |d| client.app(d["droplet"]) }
-      [d(sub), "querying health for: #{name_list(apps)}"]
+      apps = payload["droplets"].collect { |d| pretty_app(d["droplet"]) }
+      [d(sub), "querying health for: #{list(apps)}"]
     end
 
     def pretty_healthmanager_health_response(sub, msg)
@@ -284,13 +291,7 @@ module CFTools
       dea, _ = payload["id"].split("-", 2)
 
       apps = payload["app_id_to_count"].collect do |guid, count|
-        app = client.app(guid)
-
-        if app.exists?
-          "#{count} x #{app.name} (#{guid})"
-        else
-          "#{count} x unknown (#{guid})"
-        end
+        "#{count} x #{pretty_app(guid)}"
       end
 
       [c(sub, :error), "dea: #{dea}, apps: #{list(apps)}"]
@@ -314,7 +315,7 @@ module CFTools
       [c("hm.request", :warning), message]
     end
 
-    def process_service_announcement(sub, msg)
+    def pretty_service_announcement(sub, msg)
       payload = JSON.parse(msg)
       id = payload["id"]
       plan = payload["plan"]
@@ -326,6 +327,33 @@ module CFTools
       [d(sub), "id: #{id}, plan: #{plan}, supported versions: #{list(s_versions)}, capacity: (available: #{c_avail}, max: #{c_max}, unit: #{c_unit})"]
     end
 
+    def pretty_component_announcement(sub, msg)
+      payload = JSON.parse(msg)
+      type = payload["type"]
+      index = payload["index"]
+      uuid = payload["uuid"]
+      time = payload["start"]
+
+      [d(sub), "type: #{type}, index: #{index}, uuid: #{uuid}, start time: #{time}"]
+    end
+
+    def pretty_component_discover(sub, msg)
+      [d(sub), msg]
+    end
+
+    def pretty_component_discover_response(sub, msg)
+      payload = JSON.parse(msg)
+      type = payload["type"]
+      index = payload["index"]
+      host = payload["host"]
+      uptime = payload["uptime"]
+
+      message = "type: #{type}, index: #{index}, host: #{host}"
+      message << ", uptime: #{uptime}" if uptime
+
+      [d(sub), message]
+    end
+
     def pretty_hm_op(op)
       case op
       when "STOP"
@@ -338,12 +366,20 @@ module CFTools
     end
 
     def pretty_app(guid)
-      app = client.app(guid)
+      existing_app =
+        if @seen_apps.key?(guid)
+          @seen_apps[guid]
+        else
+          app = client.app(guid)
+          app if app.exists?
+        end
 
-      if app.exists?
-        c(app.name, :name)
+      if existing_app
+        @seen_apps[guid] = existing_app
+        c(existing_app.name, :name)
       else
-        d("unknown")
+        @seen_apps[guid] = nil
+        d("unknown (#{guid})")
       end
     end