bosh-director 1.3215.4.0 → 1.3232.0

data/lib/bosh/director/agent_client.rb

@@ -100,6 +100,10 @@ module Bosh::Director
       send_message(:unmount_disk, *args)
     end
 
+    def delete_arp_entries(*args)
+      fire_and_forget(:delete_arp_entries, *args)
+    end
+
     def update_settings(certs)
       begin
         send_message(:update_settings, {"trusted_certs" => certs})
@@ -166,13 +170,7 @@ module Bosh::Director
       end
     end
 
-    def handle_method(method_name, args)
-      result = {}
-      result.extend(MonitorMixin)
-
-      cond = result.new_cond
-      timeout_time = Time.now.to_f + @timeout
-
+    def send_nats_request(method_name, args, &callback)
       request = { :protocol => PROTOCOL_VERSION, :method => method_name, :arguments => args }
 
       if @encryption_handler
@@ -182,8 +180,17 @@ module Bosh::Director
       end
 
       recipient = "#{@service_name}.#{@client_id}"
+      @nats_rpc.send_request(recipient, request, &callback)
+    end
 
-      request_id = @nats_rpc.send_request(recipient, request) do |response|
+    def handle_method(method_name, args)
+      result = {}
+      result.extend(MonitorMixin)
+
+      cond = result.new_cond
+      timeout_time = Time.now.to_f + @timeout
+
+      request_id = send_nats_request(method_name, args) do |response|
         if @encryption_handler
           begin
             response = @encryption_handler.decrypt(response["encrypted_data"])
@@ -206,7 +213,7 @@ module Bosh::Director
           unless timeout > 0
             @nats_rpc.cancel_request(request_id)
             raise RpcTimeout,
-              "Timed out sending `#{method_name}' to #{@client_id} " +
+              "Timed out sending '#{method_name}' to #{@client_id} " +
                 "after #{@timeout} seconds"
           end
           cond.wait(timeout)
@@ -279,6 +286,12 @@ module Bosh::Director
       end
     end
 
+    def fire_and_forget(message_name, *args)
+      send_nats_request(message_name, args)
+    rescue => e
+      @logger.warn("Ignoring '#{e.message}' error from the agent: #{e.inspect}. Received while trying to run: #{message_name} on client: '#{@client_id}'")
+    end
+
     def send_message(method_name, *args, &blk)
       task = start_task(method_name, *args)
       if task['agent_task_id']

data/lib/bosh/director/api.rb

@@ -10,7 +10,6 @@ require 'bosh/director/api/http_constants'
 require 'bosh/director/api/api_helper'
 require 'bosh/director/api/task_helper'
 
-require 'bosh/director/api/backup_manager'
 require 'bosh/director/api/deployment_manager'
 require 'bosh/director/api/instance_manager'
 require 'bosh/director/api/problem_manager'

data/lib/bosh/director/api/api_helper.rb

@@ -99,7 +99,7 @@ module Bosh::Director
 
         begin
           Psych.parse(yml_string)
-        rescue Psych::SyntaxError => e
+        rescue Exception => e
           raise BadManifest, "Incorrect YAML structure of the uploaded manifest: #{e.inspect}"
         end
       end

data/lib/bosh/director/api/controllers/base_controller.rb

@@ -20,9 +20,13 @@ module Bosh::Director
           @task_manager = TaskManager.new
           @dns_manager = DnsManagerProvider.create
           @disk_manager = DiskManager.new(nil, @logger)
+          @event_manager = EventManager.new(config.record_events)
         end
 
-        register Bosh::Director::Api::Extensions::Scoping
+        register(Bosh::Director::Api::Extensions::SyslogRequestLogger)
+        log_request_to_syslog
+
+        register(Bosh::Director::Api::Extensions::Scoping)
 
         mime_type :tgz,       'application/x-compressed'
         mime_type :multipart, 'multipart/form-data'

data/lib/bosh/director/api/controllers/cloud_configs_controller.rb

@@ -5,9 +5,15 @@ module Bosh::Director
     class CloudConfigsController < BaseController
       post '/', :consumes => :yaml do
         manifest_text = request.body.read
-        validate_manifest_yml(manifest_text)
+        begin
+          validate_manifest_yml(manifest_text)
+          Bosh::Director::Api::CloudConfigManager.new.update(manifest_text)
+          create_event
+        rescue => e
+          create_event e
+          raise e
+        end
 
-        Bosh::Director::Api::CloudConfigManager.new.update(manifest_text)
         status(201)
       end
 
@@ -36,6 +42,16 @@ module Bosh::Director
         end
         )
       end
+
+      private
+      def create_event(error = nil)
+        @event_manager.create_event({
+            user:        current_user,
+            action:      "update",
+            object_type: "cloud-config",
+            error:       error
+        })
+      end
     end
   end
 end

data/lib/bosh/director/api/controllers/deployments_controller.rb

@@ -371,15 +371,22 @@ module Bosh::Director
 
         redact =  params['redact'] != 'false'
 
-        diff = before_manifest.diff(after_manifest, redact)
+        result = {
+          'context' => {
+            'cloud_config_id' => after_cloud_config ? after_cloud_config.id : nil,
+            'runtime_config_id' => after_runtime_config ? after_runtime_config.id : nil
+          }
+        }
+
+        begin
+          diff = before_manifest.diff(after_manifest, redact)
+          result['diff'] = diff.map { |l| [l.to_s, l.status] }
+        rescue => error
+          result['diff'] = []
+          result['error'] = "Unable to diff manifest: #{error.inspect}\n#{error.backtrace.join("\n")}"
+        end
 
-        json_encode({
-            'context' => {
-              'cloud_config_id' => after_cloud_config ? after_cloud_config.id : nil,
-              'runtime_config_id' => after_runtime_config ? after_runtime_config.id : nil
-            },
-            'diff' => diff.map { |l| [l.to_s, l.status] }
-          })
+        json_encode(result)
       end
 
       post '/:deployment/errands/:errand_name/runs' do
@@ -436,7 +443,7 @@ module Bosh::Director
           Integer(str)
         rescue ArgumentError
           if str !~ /^[A-Fa-f0-9]{8}-[A-Fa-f0-9-]{27}$/
-            raise InstanceInvalidIndex, "Invalid instance index or id `#{str}'"
+            raise InstanceInvalidIndex, "Invalid instance index or id '#{str}'"
           end
         end
       end

data/lib/bosh/director/api/controllers/events_controller.rb

@@ -0,0 +1,37 @@
+require 'bosh/director/api/controllers/base_controller'
+
+module Bosh::Director
+  module Api::Controllers
+    class EventsController < BaseController
+      EVENT_LIMIT = 200
+
+      get '/' do
+        content_type(:json)
+
+        events = Models::Event.order_by(Sequel.desc(:id))
+
+        if params['before_id']
+          before_id = params['before_id'].to_i
+          events = events.filter("id < ?", before_id)
+        end
+
+        if params['task']
+          events = events.where(task: params['task'])
+        end
+
+        if params['deployment']
+          events = events.where(deployment: params['deployment'])
+        end
+
+        if params['instance']
+          events = events.where(instance: params['instance'])
+        end
+
+        events = events.limit(EVENT_LIMIT).map do |event|
+          @event_manager.event_to_hash(event)
+        end
+        json_encode(events)
+      end
+    end
+  end
+end

data/lib/bosh/director/api/controllers/locks_controller.rb

@@ -2,19 +2,15 @@ module Bosh::Director
   module Api::Controllers
     class LocksController < BaseController
       get '/', scope: :read do
-        redis = Config.redis
+        wait_until = Time.now - 6 * 10.seconds # 6 * lock_renewal_interval
+        Models::Lock.where{expired_at <= wait_until}.destroy
 
         locks = []
-        lock_keys = redis.keys('lock:*')
-        # Deliberately not using redis futures here as we expect that the number of lock keys will be very small
-        lock_keys.each do |lock_key|
-          lock_value = redis.get(lock_key)
-          unless lock_value.nil?
-            lock_type     = lock_key.split(':')[1]
-            lock_resource = lock_key.split(':')[2..-1]
-            lock_timeout  = lock_value.split(':')[0]
-            locks << { type: lock_type, resource: lock_resource, timeout: lock_timeout }
-          end
+        Models::Lock.all.each do |lock_record|
+          lock_type     = lock_record.name.split(':')[1]
+          lock_resource = lock_record.name.split(':')[2..-1]
+          lock_timeout  = lock_record.expired_at.strftime('%s.%6N')
+          locks << { type: lock_type, resource: lock_resource, timeout: lock_timeout }
         end
 
         content_type(:json)

data/lib/bosh/director/api/controllers/packages_controller.rb

@@ -16,7 +16,7 @@ module Bosh::Director
           fingerprint_list << package['fingerprint'] if package['fingerprint']
         end
 
-        matching_packages = Models::Package.where(fingerprint: fingerprint_list, ~:sha1 => nil, ~:blobstore_id => nil).all
+        matching_packages = Models::Package.where(Sequel.~(:sha1 => nil), Sequel.~(:blobstore_id => nil), fingerprint: fingerprint_list).all
 
         json_encode(matching_packages.map(&:fingerprint).compact.uniq)
       end

data/lib/bosh/director/api/controllers/runtime_configs_controller.rb

@@ -5,9 +5,15 @@ module Bosh::Director
     class RuntimeConfigsController < BaseController
       post '/', :consumes => :yaml do
         manifest_text = request.body.read
-        validate_manifest_yml(manifest_text)
+        begin
+          validate_manifest_yml(manifest_text)
+          Bosh::Director::Api::RuntimeConfigManager.new.update(manifest_text)
+          create_event
+        rescue => e
+          create_event e
+          raise e
+        end
 
-        Bosh::Director::Api::RuntimeConfigManager.new.update(manifest_text)
         status(201)
       end
 
@@ -36,6 +42,16 @@ module Bosh::Director
         end
         )
       end
+
+      private
+      def create_event(error = nil)
+        @event_manager.create_event({
+            user:        current_user,
+            action:      "update",
+            object_type: "runtime-config",
+            error:       error
+        })
+      end
     end
   end
 end

data/lib/bosh/director/api/controllers/tasks_controller.rb

@@ -17,7 +17,7 @@ module Bosh::Director
           task = @task_manager.find_task(params[:id])
           if type == 'debug' || type == 'cpi' || !type
             check_access_to_task(task, :admin)
-          elsif type == 'event' || type == 'result'
+          elsif type == 'event' || type == 'result' || type == 'none'
             check_access_to_task(task, :read)
           else
             raise UnauthorizedToAccessDeployment, "Unknown type #{type}"
@@ -63,7 +63,7 @@ module Bosh::Director
           @permission_authorizer.granted_or_raise(deployment, :read, token_scopes)
         end
 
-        tasks = dataset.order_by(:timestamp.desc).map
+        tasks = dataset.order_by(Sequel.desc(:timestamp)).map
 
         unless @permission_authorizer.is_granted?(:director, :read, token_scopes)
           permitted_deployments = @deployment_manager.all_by_name_asc.select { |deployment|
@@ -112,6 +112,11 @@ module Bosh::Director
       # at /var/vcap/store/director/tasks/5/event
       get '/:id/output', authorization: :task_output, scope: :authorization do
         log_type = params[:type] || 'debug'
+
+        if log_type == "none"
+          halt(204)
+        end
+
         task = @task_manager.find_task(params[:id])
 
         if task.output.nil?

data/lib/bosh/director/api/deployment_lookup.rb

@@ -4,7 +4,7 @@ module Bosh::Director
       def by_name(name)
         deployment = Models::Deployment[name: name]
         if deployment.nil?
-          raise DeploymentNotFound, "Deployment `#{name}' doesn't exist"
+          raise DeploymentNotFound, "Deployment '#{name}' doesn't exist"
         end
         deployment
       end

data/lib/bosh/director/api/deployment_manager.rb

@@ -12,7 +12,7 @@ module Bosh::Director
       end
 
       def all_by_name_asc
-        Bosh::Director::Models::Deployment.order_by(:name.asc).all
+        Bosh::Director::Models::Deployment.order_by(Sequel.asc(:name)).all
       end
 
       def create_deployment(username, deployment_manifest_file_path, cloud_config, runtime_config, deployment_name, options = {})

data/lib/bosh/director/api/event_manager.rb

@@ -0,0 +1,68 @@
+module Bosh::Director
+  module Api
+    class EventManager
+      def initialize(record_events)
+        @record_events = record_events
+      end
+
+      def event_to_hash(event)
+        {
+            'id' => event.id.to_s,
+            'parent_id' => event.parent_id.to_s,
+            'timestamp' => event.timestamp.to_i,
+            'user' => event.user,
+            'action' => event.action,
+            'object_type' => event.object_type,
+            'object_name' => event.object_name,
+            'error' => event.error,
+            'task' => event.task,
+            'deployment' => event.deployment,
+            'instance' => event.instance,
+            'context' => event.context
+        }.reject { |k, v| v.nil? || v == '' }
+      end
+
+      def create_event(options)
+        unless @record_events
+          return Models::Event.new
+        end
+
+        parent_id   = options.fetch(:parent_id, nil)
+        user        = options[:user]
+        action      = options[:action]
+        object_type = options[:object_type]
+        object_name = options.fetch(:object_name, nil)
+        task        = options.fetch(:task, nil)
+        error       = options.fetch(:error){ |error| error ? error.to_s: nil }
+        deployment  = options.fetch(:deployment, nil)
+        instance    = options.fetch(:instance, nil)
+        context     = options.fetch(:context, {})
+
+        Models::Event.create(
+            parent_id:   parent_id,
+            timestamp:   Time.now,
+            user:        user,
+            action:      action,
+            object_type: object_type,
+            object_name: object_name,
+            error:       error,
+            task:        task,
+            deployment:  deployment,
+            instance:    instance,
+            context:     context)
+      end
+
+      def remove_old_events (max_events = 10000)
+        if Bosh::Director::Models::Event.count > max_events
+          last_id = Bosh::Director::Models::Event.
+              order { Sequel.desc(:id) }.limit(1, max_events).first.id
+          last_parent_id = Bosh::Director::Models::Event.
+              order { Sequel.desc(:id) }.limit(max_events).min(:parent_id)
+          start_id_to_remove = (last_parent_id.nil? || (last_parent_id > last_id)) ? last_id+1: last_parent_id
+
+          Bosh::Director::Models::Event.filter("id < ?", start_id_to_remove).delete if start_id_to_remove != 0
+        end
+      end
+    end
+  end
+end

data/lib/bosh/director/api/extensions/scoping.rb

@@ -4,8 +4,8 @@ module Bosh::Director
       module Scoping
         module Helpers
           def current_user
-            @user.username if @user
-            end
+            @user.username_or_client if @user
+          end
 
           def token_scopes
             @user.scopes if @user

data/lib/bosh/director/api/extensions/syslog_request_logger.rb

@@ -0,0 +1,75 @@
+#feature not supported for ruby < 2.0.0
+begin
+  require 'syslog/logger'
+rescue LoadError
+  puts "Failed to load Syslog::Logger. Ruby version #{RUBY_VERSION} not supported. Use RUBY_VERSION >= 2.0.0"
+end
+
+require 'socket'
+
+module Bosh::Director
+  module Api
+    module Extensions
+      module SyslogRequestLogger
+
+        DESIRED_HEADERS = %w(
+          HTTP_HOST
+          HTTP_X_REAL_IP
+          HTTP_X_FORWARDED_FOR
+          HTTP_X_FORWARDED_PROTO
+          HTTP_USER_AGENT
+          HTTP_X_BOSH_UPLOAD_REQUEST_TIME
+        )
+
+        def log_request_to_syslog
+          after do
+            if @config.log_access_events_to_syslog && RUBY_VERSION.to_i > 1
+              header_string = ''
+              filtered_headers = request.env.select { |key, _| DESIRED_HEADERS.include?(key) }
+                                     .collect { |key, value| [key.sub(/^HTTP_/, ''), value] }
+              filtered_headers.each do |header_set|
+                header_string += header_set[0] + "\=" + header_set[1] + "&"
+              end
+              header_string = header_string[0..-2] if !header_string.empty?
+
+              filtered_ip_list = Socket.ip_address_list
+                        .reject { |addr| !addr.ip? || addr.ipv4_loopback? || addr.ipv6_loopback? }
+                        .map { |addr| addr.ip_address }
+
+              cef_version = 0
+              device_vendor = 'CloudFoundry'
+              device_product = 'BOSH'
+              device_version = Bosh::Director::VERSION
+              signature_id = 'director_api'
+              name = "#{request.path}"
+              severity = response.status >= 400 ?  7 : 1
+
+              extension = ''
+              if @user
+                extension += "duser=#{@user.username} " if @user.username
+                extension += "requestClientApplication=#{@user.client} " if @user.client
+              end
+
+              extension += "requestMethod=#{request.request_method} src=#{request.ip} spt=#{@config.port}" +
+                  " shost=#{Socket.gethostname}" +
+                  " cs1=#{filtered_ip_list.join(',')} cs1Label=ips" +
+                  " cs2=#{header_string} cs2Label=httpHeaders" +
+                  " cs3=#{current_user.nil? ? 'none' : identity_provider.client_info['type']} cs3Label=authType" +
+                  " cs4=#{response.status} cs4Label=responseStatus"
+              if response.status >= 400
+                extension += " cs5=#{response.body.join('')[0...500].strip} cs5Label=statusReason"
+              end
+
+              cef_log = 'CEF:%i|%s|%s|%s|%s|%s|%s|%s' % [cef_version, device_vendor, device_product,
+                                                         device_version, signature_id, name, severity, extension]
+              cef_log_encoded = cef_log.force_encoding(Encoding::UTF_8)
+
+              Syslog::Logger.new('vcap.bosh.director').info(cef_log_encoded)
+            end
+          end
+        end
+      end
+    end
+  end
+end
+