tools-cf-plugin 1.0.0 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1 +1,3 @@
1
- require "tools-cf-plugin/watch"
1
+ require "tools-cf-plugin/watch"
2
+ require "tools-cf-plugin/tunnel/watch"
3
+ require "tools-cf-plugin/tunnel/watch_logs"
@@ -0,0 +1,133 @@
1
+ require "yaml"
2
+ require "cli"
3
+ require "net/ssh/gateway"
4
+
5
+ require "cf/cli"
6
+
7
+ module CFTools
8
+ module Tunnel
9
+ class Base < CF::CLI
10
+ BOSH_CONFIG = "~/.bosh_config"
11
+
12
+ def precondition
13
+ end
14
+
15
+ def director(director_host, gateway)
16
+ if address_reachable?(director_host, 25555)
17
+ director_for(25555, director_host)
18
+ else
19
+ dport =
20
+ with_progress("Opening local tunnel to director") do
21
+ tunnel_to(director_host, 25555, gateway)
22
+ end
23
+
24
+ director_for(dport)
25
+ end
26
+ end
27
+
28
+ def connected_director(director_host, gateway)
29
+ director = director(director_host, gateway)
30
+
31
+ authenticate_with_director(
32
+ director,
33
+ "https://#{director_host}:25555",
34
+ director_credentials(director_host))
35
+
36
+ director
37
+ end
38
+
39
+ def authenticate_with_director(director, remote_director, auth)
40
+ if auth && login_to_director(director, auth["username"], auth["password"])
41
+ return true
42
+ end
43
+
44
+ while true
45
+ line unless quiet?
46
+ user = ask("Director Username")
47
+ pass = ask("Director Password", :echo => "*", :forget => true)
48
+ break if login_to_director(director, user, pass)
49
+ end
50
+
51
+ save_auth(remote_director, "username" => user, "password" => pass)
52
+
53
+ true
54
+ end
55
+
56
+ def login_to_director(director, user, pass)
57
+ director.user = user
58
+ director.password = pass
59
+
60
+ with_progress("Authenticating as #{c(user, :name)}") do |s|
61
+ director.authenticated? || s.fail
62
+ end
63
+ end
64
+
65
+ def tunnel_to(address, remote_port, gateway)
66
+ user, host = gateway.split("@", 2)
67
+ Net::SSH::Gateway.new(host, user).open(address, remote_port)
68
+ end
69
+
70
+ private
71
+
72
+ def address_reachable?(host, port)
73
+ Timeout.timeout(1) do
74
+ TCPSocket.new(host, port).close
75
+ true
76
+ end
77
+ rescue Timeout::Error, Errno::ECONNREFUSED
78
+ false
79
+ end
80
+
81
+ def director_for(port, host = "127.0.0.1")
82
+ Bosh::Cli::Director.new("https://#{host}:#{port}")
83
+ end
84
+
85
+ def director_credentials(director)
86
+ return unless cfg = bosh_config
87
+
88
+ _, auth = cfg["auth"].find do |d, _|
89
+ d.include?(director)
90
+ end
91
+
92
+ auth
93
+ end
94
+
95
+ def current_deployment(director)
96
+ deployments = director.list_deployments
97
+
98
+ deployments.find do |d|
99
+ d["releases"].any? { |r| r["name"] == "cf-release" }
100
+ end
101
+ end
102
+
103
+ def current_deployment_manifest(director)
104
+ deployment = current_deployment(director)
105
+ YAML.load(director.get_deployment(deployment["name"])["manifest"])
106
+ end
107
+
108
+ def save_auth(director, auth)
109
+ cfg = bosh_config || { "auth" => {} }
110
+
111
+ cfg["auth"][director] = auth
112
+
113
+ save_bosh_config(cfg)
114
+ end
115
+
116
+ def save_bosh_config(config)
117
+ File.open(bosh_config_file, "w") do |io|
118
+ io.write(YAML.dump(config))
119
+ end
120
+ end
121
+
122
+ def bosh_config
123
+ return unless File.exists?(bosh_config_file)
124
+
125
+ YAML.load_file(bosh_config_file)
126
+ end
127
+
128
+ def bosh_config_file
129
+ File.expand_path(BOSH_CONFIG)
130
+ end
131
+ end
132
+ end
133
+ end
@@ -0,0 +1,46 @@
1
+ require "json"
2
+ require "time"
3
+
4
+ module CFTools
5
+ module Tunnel
6
+ class LogEntry
7
+ attr_reader :label, :line, :stream
8
+
9
+ def initialize(label, line, stream)
10
+ @label = label
11
+ @line = line
12
+ @stream = stream
13
+ @fallback_timestamp = Time.now
14
+ end
15
+
16
+ def message
17
+ json = JSON.parse(@line)
18
+ json["message"]
19
+ rescue JSON::ParserError
20
+ @line
21
+ end
22
+
23
+ def log_level
24
+ json = JSON.parse(@line)
25
+ json["log_level"]
26
+ rescue JSON::ParserError
27
+ end
28
+
29
+ def timestamp
30
+ json = JSON.parse(@line)
31
+
32
+ timestamp = json["timestamp"]
33
+ case timestamp
34
+ when String
35
+ Time.parse(timestamp)
36
+ when Numeric
37
+ Time.at(timestamp)
38
+ else
39
+ @fallback_timestamp
40
+ end
41
+ rescue JSON::ParserError
42
+ @fallback_timestamp
43
+ end
44
+ end
45
+ end
46
+ end
@@ -0,0 +1,100 @@
1
+ require "base64"
2
+ require "json"
3
+ require "net/ssh/gateway"
4
+ require "thread"
5
+
6
+ module CFTools
7
+ module Tunnel
8
+ class MultiLineStream
9
+ include Interact::Pretty
10
+
11
+ def initialize(director, deployment, user, host)
12
+ @director = director
13
+ @deployment = deployment
14
+ @gateway_user = user
15
+ @gateway_host = host
16
+ end
17
+
18
+ def stream(locations)
19
+ entries = entry_queue
20
+
21
+ Thread.abort_on_exception = true
22
+
23
+ locations.each do |(name, index), locs|
24
+ Thread.new do
25
+ stream_location(name, index, locs, entries)
26
+ end
27
+ end
28
+
29
+ while entry = entries.pop
30
+ yield entry
31
+ end
32
+ end
33
+
34
+ private
35
+
36
+ def gateway
37
+ @gateway ||= Net::SSH::Gateway.new(@gateway_host, @gateway_user)
38
+ end
39
+
40
+ def entry_queue
41
+ Queue.new
42
+ end
43
+
44
+ def public_key
45
+ Net::SSH::Authentication::KeyManager.new(nil).each_identity do |i|
46
+ return sane_public_key(i)
47
+ end
48
+ end
49
+
50
+ def sane_public_key(pkey)
51
+ "#{pkey.ssh_type} #{Base64.encode64(pkey.to_blob).split.join} #{pkey.comment}"
52
+ end
53
+
54
+ def generate_user
55
+ "bosh_cf_watch_logs_#{rand(36**9).to_s(36)}"
56
+ end
57
+
58
+ def create_ssh_user(job, index, entries)
59
+ user = generate_user
60
+
61
+ entries << LogEntry.new(
62
+ "#{job}/#{index}", c("creating user...", :warning), :stdout)
63
+
64
+ status, task_id = @director.setup_ssh(
65
+ @deployment, job, index, user,
66
+ public_key, nil, :use_cache => false)
67
+
68
+ raise "SSH setup failed." unless status == :done
69
+
70
+ entries << LogEntry.new("#{job}/#{index}", c("created!", :good), :stdout)
71
+
72
+ sessions = JSON.parse(@director.get_task_result_log(task_id))
73
+
74
+ session = sessions.first
75
+
76
+ raise "No session?" unless session
77
+
78
+ [session["ip"], user]
79
+ end
80
+
81
+ def stream_location(job, index, locations, entries)
82
+ ip, user = create_ssh_user(job, index, entries)
83
+
84
+ entries << LogEntry.new(
85
+ "#{job}/#{index}", c("connecting", :warning), :stdout)
86
+
87
+ gateway.ssh(ip, user) do |ssh|
88
+ entries << LogEntry.new(
89
+ "#{job}/#{index}", c("connected!", :good), :stdout)
90
+
91
+ locations.each do |loc|
92
+ loc.stream_lines(ssh) do |entry|
93
+ entries << entry
94
+ end
95
+ end
96
+ end
97
+ end
98
+ end
99
+ end
100
+ end
@@ -0,0 +1,37 @@
1
+ require "tools-cf-plugin/tunnel/log_entry"
2
+
3
+ module CFTools
4
+ module Tunnel
5
+ class StreamLocation
6
+ attr_reader :path, :label
7
+
8
+ def initialize(path, label)
9
+ @path = path
10
+ @label = label
11
+ end
12
+
13
+ def stream_lines(ssh)
14
+ ssh.exec("tail -f /var/vcap/sys/log/#@path") do |ch, stream, chunk|
15
+ if pending = ch[:pending]
16
+ chunk = pending + chunk
17
+ ch[:pending] = nil
18
+ end
19
+
20
+ chunk.each_line do |line|
21
+ if line.end_with?("\n")
22
+ yield log_entry(line, stream)
23
+ else
24
+ ch[:pending] = line
25
+ end
26
+ end
27
+ end
28
+ end
29
+
30
+ private
31
+
32
+ def log_entry(line, stream)
33
+ LogEntry.new(@label, line, stream)
34
+ end
35
+ end
36
+ end
37
+ end
@@ -0,0 +1,52 @@
1
+ require "yaml"
2
+ require "cli"
3
+ require "net/ssh"
4
+
5
+ require "cf/cli"
6
+
7
+ require "tools-cf-plugin/tunnel/base"
8
+
9
+ module CFTools::Tunnel
10
+ class Watch < Base
11
+ desc "Watch, by grabbing the connection info from your BOSH deployment."
12
+ input :director, :argument => :required, :desc => "BOSH director address"
13
+ input :gateway, :argument => :optional,
14
+ :default => proc { "vcap@#{input[:director]}" },
15
+ :desc => "SSH connection string (default: vcap@director)"
16
+ def tunnel_watch
17
+ director_host = input[:director]
18
+ gateway = input[:gateway]
19
+
20
+ director = connected_director(director_host, gateway)
21
+
22
+ manifest =
23
+ with_progress("Downloading deployment manifest") do
24
+ current_deployment_manifest(director)
25
+ end
26
+
27
+ nats = manifest["properties"]["nats"]
28
+
29
+ nport =
30
+ with_progress("Opening local tunnel to NATS") do
31
+ tunnel_to(nats["address"], nats["port"], gateway)
32
+ end
33
+
34
+ with_progress("Logging in as admin user") do
35
+ login_as_admin(manifest)
36
+ end
37
+
38
+ invoke :watch, :port => nport,
39
+ :user => nats["user"], :password => nats["password"]
40
+ end
41
+
42
+ private
43
+
44
+ def login_as_admin(manifest)
45
+ admin = manifest["properties"]["uaa"]["scim"]["users"].grep(/cloud_controller\.admin/).first
46
+ admin_user, admin_pass, _ = admin.split("|", 3)
47
+
48
+ @@client = CFoundry::V2::Client.new(manifest["properties"]["cc"]["srv_api_uri"])
49
+ client.login(admin_user, admin_pass)
50
+ end
51
+ end
52
+ end
@@ -0,0 +1,122 @@
1
+ require "yaml"
2
+ require "thread"
3
+
4
+ require "tools-cf-plugin/tunnel/base"
5
+ require "tools-cf-plugin/tunnel/multi_line_stream"
6
+ require "tools-cf-plugin/tunnel/stream_location"
7
+
8
+ module CFTools::Tunnel
9
+ class WatchLogs < Base
10
+ LOGS = {
11
+ "cloud_controller" => ["cloud_controller_ng/cloud_controller_ng.log"],
12
+ "dea_next" => ["dea_next/dea_next.log"],
13
+ "health_manager" => ["health_manager_next/health_manager_next.log"],
14
+ "router" => ["gorouter/gorouter.log"]
15
+ }
16
+
17
+ desc "Stream logs from the jobs of a deployment"
18
+ input :director, :argument => :required, :desc => "BOSH director address"
19
+ input :gateway, :argument => :optional,
20
+ :default => proc { "vcap@#{input[:director]}" },
21
+ :desc => "SSH connection string (default: vcap@director)"
22
+ def watch_logs
23
+ director_host = input[:director]
24
+ gateway = input[:gateway]
25
+
26
+ director = connected_director(director_host, gateway)
27
+
28
+ deployment =
29
+ with_progress("Getting deployment info") do
30
+ current_deployment(director)
31
+ end
32
+
33
+ locations =
34
+ with_progress("Finding logs for #{c(deployment["name"], :name)}") do
35
+ locs = stream_locations(director, deployment["name"])
36
+
37
+ if locs.empty?
38
+ fail "No locations found."
39
+ else
40
+ locs
41
+ end
42
+ end
43
+
44
+ stream = stream_for(director, deployment["name"], gateway)
45
+
46
+ stream.stream(locations) do |entry|
47
+ line pretty_print_entry(entry)
48
+ end
49
+ end
50
+
51
+ private
52
+
53
+ def stream_for(director, deployment, gateway)
54
+ user, host = gateway.split("@", 2)
55
+ MultiLineStream.new(director, deployment, user, host)
56
+ end
57
+
58
+ def max_label_size
59
+ LOGS.keys.collect(&:size).sort.last + 3
60
+ end
61
+
62
+ def pretty_print_entry(entry)
63
+ log_level = entry.log_level || ""
64
+ level_padding = " " * (6 - log_level.size)
65
+ [ c(entry.label.ljust(max_label_size), :name),
66
+ entry.timestamp.strftime("%r"),
67
+ "#{pretty_log_level(log_level)}#{level_padding}",
68
+ level_colored_message(entry)
69
+ ].join(" ")
70
+ end
71
+
72
+ def stream_locations(director, deployment)
73
+ locations = Hash.new { |h, k| h[k] = [] }
74
+
75
+ director.fetch_vm_state(deployment, :use_cache => false).each do |vm|
76
+ name = vm["job_name"]
77
+ index = vm["index"]
78
+ next unless LOGS.key?(name)
79
+
80
+ vm["ips"].each do |ip|
81
+ LOGS[name].each do |file|
82
+ locations[[name, index]] << StreamLocation.new(file, "#{name}/#{index}")
83
+ end
84
+ end
85
+ end
86
+
87
+ locations
88
+ end
89
+
90
+ def level_colored_message(entry)
91
+ msg = entry.message
92
+
93
+ case entry.log_level
94
+ when "warn"
95
+ c(msg, :warning)
96
+ when "error"
97
+ c(msg, :bad)
98
+ when "fatal"
99
+ c(msg, :error)
100
+ else
101
+ msg
102
+ end
103
+ end
104
+
105
+ def pretty_log_level(level)
106
+ case level
107
+ when "info"
108
+ d(level)
109
+ when "debug", "debug1", "debug2", "all"
110
+ c(level, :good)
111
+ when "warn"
112
+ c(level, :warning)
113
+ when "error"
114
+ c(level, :bad)
115
+ when "fatal"
116
+ c(level, :error)
117
+ else
118
+ level
119
+ end
120
+ end
121
+ end
122
+ end
@@ -1,3 +1,3 @@
1
1
  module CFTools
2
- VERSION = "1.0.0".freeze
2
+ VERSION = "1.1.0".freeze
3
3
  end
@@ -14,7 +14,7 @@ module CFTools
14
14
  group :admin
15
15
  input :app, :argument => :optional, :from_given => by_name(:app),
16
16
  :desc => "Application to watch"
17
- input :host, :alias => "-h", :default => "localhost",
17
+ input :host, :alias => "-h", :default => "127.0.0.1",
18
18
  :desc => "NATS server address"
19
19
  input :port, :alias => "-P", :default => 4222, :type => :integer,
20
20
  :desc => "NATS server port"
@@ -30,6 +30,7 @@ module CFTools
30
30
  pass = input[:password]
31
31
 
32
32
  @requests = {}
33
+ @seen_apps = {}
33
34
  @request_ticker = 0
34
35
 
35
36
  $stdout.sync = true
@@ -99,8 +100,12 @@ module CFTools
99
100
  sub, msg = pretty_dea_shutdown(sub, msg)
100
101
  when /^cloudcontrollers\.hm\.requests\.\w+$/
101
102
  sub, msg = process_cloudcontrollers_hm_request(sub, msg)
102
- when /([^\.]+)\.announce$/
103
- sub, msg = process_service_announcement(sub, msg)
103
+ when /^([^.]+)\.announce$/
104
+ sub, msg = pretty_service_announcement(sub, msg)
105
+ when "vcap.component.announce"
106
+ sub, msg = pretty_component_announcement(sub, msg)
107
+ when "vcap.component.discover"
108
+ sub, msg = pretty_component_discover(sub, msg)
104
109
  end
105
110
 
106
111
  if reply
@@ -122,6 +127,8 @@ module CFTools
122
127
  sub, msg = pretty_healthmanager_status_response(sub, msg)
123
128
  when "healthmanager.health"
124
129
  sub, msg = pretty_healthmanager_health_response(sub, msg)
130
+ when "vcap.component.discover"
131
+ sub, msg = pretty_component_discover_response(sub, msg)
125
132
  end
126
133
 
127
134
  line "#{timestamp}\t#{REPLY_PREFIX}#{sub} (#{c(id, :error)})\t#{msg}"
@@ -264,8 +271,8 @@ module CFTools
264
271
 
265
272
  def pretty_healthmanager_health(sub, msg)
266
273
  payload = JSON.parse(msg)
267
- apps = payload["droplets"].collect { |d| client.app(d["droplet"]) }
268
- [d(sub), "querying health for: #{name_list(apps)}"]
274
+ apps = payload["droplets"].collect { |d| pretty_app(d["droplet"]) }
275
+ [d(sub), "querying health for: #{list(apps)}"]
269
276
  end
270
277
 
271
278
  def pretty_healthmanager_health_response(sub, msg)
@@ -284,13 +291,7 @@ module CFTools
284
291
  dea, _ = payload["id"].split("-", 2)
285
292
 
286
293
  apps = payload["app_id_to_count"].collect do |guid, count|
287
- app = client.app(guid)
288
-
289
- if app.exists?
290
- "#{count} x #{app.name} (#{guid})"
291
- else
292
- "#{count} x unknown (#{guid})"
293
- end
294
+ "#{count} x #{pretty_app(guid)}"
294
295
  end
295
296
 
296
297
  [c(sub, :error), "dea: #{dea}, apps: #{list(apps)}"]
@@ -314,7 +315,7 @@ module CFTools
314
315
  [c("hm.request", :warning), message]
315
316
  end
316
317
 
317
- def process_service_announcement(sub, msg)
318
+ def pretty_service_announcement(sub, msg)
318
319
  payload = JSON.parse(msg)
319
320
  id = payload["id"]
320
321
  plan = payload["plan"]
@@ -326,6 +327,33 @@ module CFTools
326
327
  [d(sub), "id: #{id}, plan: #{plan}, supported versions: #{list(s_versions)}, capacity: (available: #{c_avail}, max: #{c_max}, unit: #{c_unit})"]
327
328
  end
328
329
 
330
+ def pretty_component_announcement(sub, msg)
331
+ payload = JSON.parse(msg)
332
+ type = payload["type"]
333
+ index = payload["index"]
334
+ uuid = payload["uuid"]
335
+ time = payload["start"]
336
+
337
+ [d(sub), "type: #{type}, index: #{index}, uuid: #{uuid}, start time: #{time}"]
338
+ end
339
+
340
+ def pretty_component_discover(sub, msg)
341
+ [d(sub), msg]
342
+ end
343
+
344
+ def pretty_component_discover_response(sub, msg)
345
+ payload = JSON.parse(msg)
346
+ type = payload["type"]
347
+ index = payload["index"]
348
+ host = payload["host"]
349
+ uptime = payload["uptime"]
350
+
351
+ message = "type: #{type}, index: #{index}, host: #{host}"
352
+ message << ", uptime: #{uptime}" if uptime
353
+
354
+ [d(sub), message]
355
+ end
356
+
329
357
  def pretty_hm_op(op)
330
358
  case op
331
359
  when "STOP"
@@ -338,12 +366,20 @@ module CFTools
338
366
  end
339
367
 
340
368
  def pretty_app(guid)
341
- app = client.app(guid)
369
+ existing_app =
370
+ if @seen_apps.key?(guid)
371
+ @seen_apps[guid]
372
+ else
373
+ app = client.app(guid)
374
+ app if app.exists?
375
+ end
342
376
 
343
- if app.exists?
344
- c(app.name, :name)
377
+ if existing_app
378
+ @seen_apps[guid] = existing_app
379
+ c(existing_app.name, :name)
345
380
  else
346
- d("unknown")
381
+ @seen_apps[guid] = nil
382
+ d("unknown (#{guid})")
347
383
  end
348
384
  end
349
385