nucleus 0.1.0 → 0.2.0

data/lib/nucleus/adapters/v1/cloud_foundry_v2/logs.rb

@@ -1,303 +1,303 @@
-module Nucleus
-  module Adapters
-    module V1
-      class CloudFoundryV2 < Stub
-        module Logs
-          LOGGREGATOR_TYPES = [Enums::ApplicationLogfileType::API,
-                               Enums::ApplicationLogfileType::APPLICATION,
-                               Enums::ApplicationLogfileType::REQUEST,
-                               Enums::ApplicationLogfileType::SYSTEM]
-          # Carriage return (newline in Mac OS) + line feed (newline in Unix) == CRLF (newline in Windows)
-          CRLF = "\r\n"
-          WSP  = "\s"
-
-          # @see Stub#logs
-          def logs(application_name_or_id)
-            app_guid = app_guid(application_name_or_id)
-            # retrieve app for timestamps only :/
-            app_created = get("/v2/apps/#{app_guid}").body[:metadata][:created_at]
-            logs = []
-
-            begin
-              log_files_list = download_file(app_guid, 'logs')
-              # parse raw response to array
-              log_files_list.split(CRLF).each do |logfile_line|
-                filename = logfile_line.rpartition(' ').first.strip
-                if filename == 'staging_task.log'
-                  filename = 'build'
-                  log_type = Enums::ApplicationLogfileType::BUILD
-                else
-                  log_type = Enums::ApplicationLogfileType::OTHER
-                end
-                # TODO: right now, we always assume the log has recently been updated
-                logs.push(id: filename, name: filename, type: log_type, created_at: app_created,
-                          updated_at: Time.now.utc.iso8601)
-              end
-            rescue Errors::AdapterError
-              log.debug('no logs directory found for cf application')
-            end
-
-            # add the default logtypes, available according to:
-            # http://docs.cloudfoundry.org/devguide/deploy-apps/streaming-logs.html#format
-            LOGGREGATOR_TYPES.each do |type|
-              logs.push(id: type, name: type, type: type, created_at: app_created, updated_at: Time.now.utc.iso8601)
-            end
-            # TODO: 'all' is probably not perfect, since the build log wont be included
-            logs.push(id: 'all', name: 'all', type: Enums::ApplicationLogfileType::OTHER,
-                      created_at: app_created, updated_at: Time.now.utc.iso8601)
-            logs
-          end
-
-          # @see Stub#log?
-          def log?(application_name_or_id, log_id)
-            app_guid = app_guid(application_name_or_id)
-            # test file existence
-            log_id = 'staging_task.log' if log_id.to_sym == Enums::ApplicationLogfileType::BUILD
-            # checks also if application is even valid
-            response = get("/v2/apps/#{app_guid}/instances/0/files/logs/#{log_id}",
-                           follow_redirects: false, expects: [200, 302, 400])
-            return true if response == 200 || log_stream?(log_id)
-            return false if response == 400
-            # if 302 (only remaining option), followup...
-
-            # download log file
-            download_file(app_guid, "logs/#{log_id}")
-            # no error, file exists
-            true
-          rescue Errors::AdapterResourceNotFoundError, Errors::UnknownAdapterCallError,
-                 Excon::Errors::NotFound, Excon::Errors::BadRequest
-            false
-          end
-
-          # @see Stub#tail
-          def tail(application_name_or_id, log_id, stream)
-            app_guid = app_guid(application_name_or_id)
-            return tail_stream(app_guid, log_id, stream) if log_stream?(log_id)
-            tail_file(app_guid, log_id, stream)
-          end
-
-          # @see Stub#log_entries
-          def log_entries(application_name_or_id, log_id)
-            app_guid = app_guid(application_name_or_id)
-            # first check if this log is a file or must be fetched from the loggregator
-            if log_stream?(log_id)
-              # fetch recent data from loggregator and return an array of log entries
-              recent_decoded = recent_log_messages(app_guid, loggregator_filter(log_id))
-              recent_decoded.collect { |log_msg| construct_log_entry(log_msg) }
-            elsif log_id.to_sym == Enums::ApplicationLogfileType::BUILD
-              # handle special staging log
-              build_log_entries(app_guid)
-            else
-              download_logfile_entries(app_guid, log_id)
-            end
-          end
-
-          private
-
-          def build_log_entries(app_guid)
-            log_id = 'staging_task.log'
-            download_logfile_entries(app_guid, log_id)
-          rescue Errors::AdapterResourceNotFoundError
-            # if there was no build yet, return no entries instead of the 404 error
-            []
-          end
-
-          def loggregator_filter(log_id)
-            case log_id.to_sym
-            when Enums::ApplicationLogfileType::API
-              filter = ['API']
-            when Enums::ApplicationLogfileType::APPLICATION
-              filter = ['APP']
-            when Enums::ApplicationLogfileType::REQUEST
-              filter = ['RTR']
-            when Enums::ApplicationLogfileType::SYSTEM
-              filter = %w(STG LGR DEA)
-            when :all
-              # no filter, show all
-              filter = nil
-            else
-              # invalid log requests --> 404
-              fail Errors::AdapterResourceNotFoundError,
-                   "Invalid log file '#{log_id}', not available for application '#{app_guid}'"
-            end
-            filter
-          end
-
-          def construct_log_entry(decoded_message)
-            # 2015-03-22T15:28:55.83+0100 [RTR/0]      OUT message...
-            "#{Time.at(decoded_message.timestamp / 1_000_000_000.0).iso8601} "\
-              "[#{decoded_message.source_name}/#{decoded_message.source_id}] "\
-              "#{decoded_message.message_type == 1 ? 'OUT' : 'ERR'} #{decoded_message.message}"
-          end
-
-          def download_logfile_entries(app_guid, log_id, headers_to_use = nil)
-            # download log file
-            logfile_contents = download_file(app_guid, "logs/#{log_id}", headers_to_use)
-            # split file into entries by line breaks and return an array of log entries
-            logfile_contents.split("\n")
-          end
-
-          def download_file(app_guid, file_path, headers_to_use = nil)
-            expected_statuses = [200, 302, 400, 404]
-            # Hack, do not create fresh headers (which would fail) when in a deferred action
-            headers_to_use = headers unless headers_to_use
-
-            # log list consists of 2 parts, loggregator and files
-            log_files = get("/v2/apps/#{app_guid}/instances/0/files/#{file_path}",
-                            follow_redirects: false, expects: expected_statuses, headers: headers_to_use)
-            if log_files.status == 400 || log_files.status == 404
-              fail Errors::AdapterResourceNotFoundError,
-                   "Invalid log file: '#{file_path}' not available for application '#{app_guid}'"
-            end
-            return log_files.body if log_files.status == 200
-
-            # status must be 302, follow to the Location
-            download_location = log_files.headers[:Location]
-            # if IBM f*cked with the download URL, fix the address
-            download_location.gsub!(/objectstorage.service.networklayer.com/, 'objectstorage.softlayer.net')
-            Excon.defaults[:ssl_verify_peer] = false unless @check_certificates
-
-            connection_params = { ssl_verify_peer: @check_certificates }
-            connection = Excon.new(download_location, connection_params)
-            downloaded_logfile_response = connection.request(method: :get, expects: expected_statuses)
-
-            if downloaded_logfile_response.status == 404
-              fail Errors::AdapterResourceNotFoundError,
-                   "Invalid log file: '#{file_path}' not available for application '#{app_guid}'"
-            end
-            downloaded_logfile_response.body
-          end
-
-          def recent_log_messages(app_guid, filter = nil)
-            loggregator_recent_uri = "https://#{loggregator_endpoint}:443/recent?app=#{app_guid}"
-            # current log state before tailing, multipart message of protobuf objects
-            current_log_response = get(loggregator_recent_uri)
-            current_log_boundary = /boundary=(\w+)/.match(current_log_response.headers['Content-Type'])[1]
-            current_log = current_log_response.body
-
-            boundary_regexp = /--#{Regexp.quote(current_log_boundary)}(--)?#{CRLF}/
-            parts = current_log.split(boundary_regexp).collect do |chunk|
-              header_part = chunk.split(/#{CRLF}#{WSP}*#{CRLF}/m, 2)[0]
-              if header_part
-                headers = header_part.split(/\r\n/).map { |kv| kv }
-                headers.length > 1 ? headers[1] : nil
-              end
-            end.compact
-            # decode log messages
-            decoded_messages = parts.collect do |proto_message|
-              Message.decode(proto_message)
-            end.compact
-            return decoded_messages unless filter
-            # return filtered messages
-            decoded_messages.find_all do |msg|
-              filter.include?(msg.source_name)
-            end
-          end
-
-          def log_stream?(log_id)
-            LOGGREGATOR_TYPES.include?(log_id.to_sym) || log_id.to_sym == :all
-          end
-
-          def loggregator_endpoint
-            @endpoint_url.gsub(%r{^(\w*://)?(api)([-\.\w]+)$}i, 'loggregator\3')
-          end
-
-          def tail_file(app_guid, log_id, stream)
-            log.debug 'Tailing CF log file'
-            log_id = 'staging_task.log' if log_id.to_sym == Enums::ApplicationLogfileType::BUILD
-
-            # cache headers as they are bound to a request and could be lost with the next tick
-            headers_to_use = headers
-            latest_pushed_line = -1
-
-            # update every 3 seconds
-            @tail_file_timer = EM.add_periodic_timer(3) do
-              log.debug('Poll updated file tail...')
-              begin
-                latest_pushed_line = push_file_tail(app_guid, log_id, stream, latest_pushed_line, headers_to_use)
-              rescue Errors::AdapterResourceNotFoundError
-                log.debug('Logfile not found, finished tailing')
-                # file lost, close stream
-                @tail_file_timer.cancel if @tail_file_timer
-                stream.close
-              end
-            end
-            # listener to stop polling
-            StopListener.new(@tail_file_timer, :cancel)
-          end
-
-          def push_file_tail(app_guid, log_id, stream, pushed_line_idx, headers_to_use)
-            log.debug('Fetching file for tail response...')
-            entries = download_logfile_entries(app_guid, log_id, headers_to_use)
-            # file was shortened, close stream since we do not know where to continue
-            if entries.length < pushed_line_idx
-              log.debug('File was modified and shortened, stop tailing the file...')
-              stream.close
-            else
-              entries.each_with_index do |entry, index|
-                next if index <= pushed_line_idx
-                pushed_line_idx = index
-                stream.send_message(entry)
-              end
-              pushed_line_idx
-            end
-          end
-
-          def tail_stream(app_guid, log_id, stream)
-            filter = loggregator_filter(log_id)
-
-            # push current state
-            recent_log_messages(app_guid, filter).each { |entry| stream.send_message(construct_log_entry(entry)) }
-
-            # Now register websocket to receive the latest updates
-            ws = Faye::WebSocket::Client.new("wss://#{loggregator_endpoint}:443/tail/?app=#{app_guid}",
-                                             nil, headers: headers.slice('Authorization'))
-
-            ws.on :message do |event|
-              log.debug "CF loggregator message received: #{event}"
-              begin
-                msg = Message.decode(event.data.pack('C*'))
-                # notify stream to print new log line if msg type matches the applied filter
-                stream.send_message(construct_log_entry(msg)) if filter.nil? || filter.include?(msg.source_name)
-              rescue StandardError => e
-                log.error "Cloud Foundry log message de-serialization failed: #{e}"
-              end
-            end
-
-            ws.on :close do |event|
-              log.debug "Closing CF loggregator websocket: code=#{event.code}, reason=#{event.reason}"
-              ws = nil
-              # notify stream that no more update are to arrive and stream shall be closed
-              stream.close
-            end
-            # return listener to stop websocket
-            TailStopper.new(ws, :close)
-          end
-
-          # Message class definition, matching the Protocol Buffer definition of the Cloud Foundry loggregator.
-          # see also: https://github.com/cloudfoundry/loggregatorlib/blob/master/logmessage/log_message.proto
-          class Message < ::Protobuf::Message
-            class MessageType < ::Protobuf::Enum
-              define :OUT, 1
-              define :ERR, 2
-            end
-
-            required :bytes, :message, 1
-            required Logs::Message::MessageType, :message_type, 2
-            required :sint64, :timestamp, 3
-            required :string, :app_id, 4
-            optional :string, :source_id, 6
-            repeated :string, :drain_urls, 7
-            optional :string, :source_name, 8
-          end
-
-          class Envelope < ::Protobuf::Message
-            required :string, :routing_key, 1
-            required :bytes, :signature, 2
-            required Logs::Message, :log_message, 3
-          end
-        end
-      end
-    end
-  end
-end
+module Nucleus
+  module Adapters
+    module V1
+      class CloudFoundryV2 < Stub
+        module Logs
+          LOGGREGATOR_TYPES = [Enums::ApplicationLogfileType::API,
+                               Enums::ApplicationLogfileType::APPLICATION,
+                               Enums::ApplicationLogfileType::REQUEST,
+                               Enums::ApplicationLogfileType::SYSTEM].freeze
+          # Carriage return (newline in Mac OS) + line feed (newline in Unix) == CRLF (newline in Windows)
+          CRLF = "\r\n".freeze
+          WSP  = ' '.freeze
+
+          # @see Stub#logs
+          def logs(application_name_or_id)
+            app_guid = app_guid(application_name_or_id)
+            # retrieve app for timestamps only :/
+            app_created = get("/v2/apps/#{app_guid}").body[:metadata][:created_at]
+            logs = []
+
+            begin
+              log_files_list = download_file(app_guid, 'logs')
+              # parse raw response to array
+              log_files_list.split(CRLF).each do |logfile_line|
+                filename = logfile_line.rpartition(' ').first.strip
+                if filename == 'staging_task.log'
+                  filename = 'build'
+                  log_type = Enums::ApplicationLogfileType::BUILD
+                else
+                  log_type = Enums::ApplicationLogfileType::OTHER
+                end
+                # TODO: right now, we always assume the log has recently been updated
+                logs.push(id: filename, name: filename, type: log_type, created_at: app_created,
+                          updated_at: Time.now.utc.iso8601)
+              end
+            rescue Errors::AdapterError
+              log.debug('no logs directory found for cf application')
+            end
+
+            # add the default logtypes, available according to:
+            # http://docs.cloudfoundry.org/devguide/deploy-apps/streaming-logs.html#format
+            LOGGREGATOR_TYPES.each do |type|
+              logs.push(id: type, name: type, type: type, created_at: app_created, updated_at: Time.now.utc.iso8601)
+            end
+            # TODO: 'all' is probably not perfect, since the build log wont be included
+            logs.push(id: 'all', name: 'all', type: Enums::ApplicationLogfileType::OTHER,
+                      created_at: app_created, updated_at: Time.now.utc.iso8601)
+            logs
+          end
+
+          # @see Stub#log?
+          def log?(application_name_or_id, log_id)
+            app_guid = app_guid(application_name_or_id)
+            # test file existence
+            log_id = 'staging_task.log' if log_id.to_sym == Enums::ApplicationLogfileType::BUILD
+            # checks also if application is even valid
+            response = get("/v2/apps/#{app_guid}/instances/0/files/logs/#{log_id}",
+                           follow_redirects: false, expects: [200, 302, 400])
+            return true if response == 200 || log_stream?(log_id)
+            return false if response == 400
+            # if 302 (only remaining option), followup...
+
+            # download log file
+            download_file(app_guid, "logs/#{log_id}")
+            # no error, file exists
+            true
+          rescue Errors::AdapterResourceNotFoundError, Errors::UnknownAdapterCallError,
+                 Excon::Errors::NotFound, Excon::Errors::BadRequest
+            false
+          end
+
+          # @see Stub#tail
+          def tail(application_name_or_id, log_id, stream)
+            app_guid = app_guid(application_name_or_id)
+            return tail_stream(app_guid, log_id, stream) if log_stream?(log_id)
+            tail_file(app_guid, log_id, stream)
+          end
+
+          # @see Stub#log_entries
+          def log_entries(application_name_or_id, log_id)
+            app_guid = app_guid(application_name_or_id)
+            # first check if this log is a file or must be fetched from the loggregator
+            if log_stream?(log_id)
+              # fetch recent data from loggregator and return an array of log entries
+              recent_decoded = recent_log_messages(app_guid, loggregator_filter(log_id))
+              recent_decoded.collect { |log_msg| construct_log_entry(log_msg) }
+            elsif log_id.to_sym == Enums::ApplicationLogfileType::BUILD
+              # handle special staging log
+              build_log_entries(app_guid)
+            else
+              download_logfile_entries(app_guid, log_id)
+            end
+          end
+
+          private
+
+          def build_log_entries(app_guid)
+            log_id = 'staging_task.log'
+            download_logfile_entries(app_guid, log_id)
+          rescue Errors::AdapterResourceNotFoundError
+            # if there was no build yet, return no entries instead of the 404 error
+            []
+          end
+
+          def loggregator_filter(log_id)
+            case log_id.to_sym
+            when Enums::ApplicationLogfileType::API
+              filter = ['API']
+            when Enums::ApplicationLogfileType::APPLICATION
+              filter = ['APP']
+            when Enums::ApplicationLogfileType::REQUEST
+              filter = ['RTR']
+            when Enums::ApplicationLogfileType::SYSTEM
+              filter = %w(STG LGR DEA)
+            when :all
+              # no filter, show all
+              filter = nil
+            else
+              # invalid log requests --> 404
+              fail Errors::AdapterResourceNotFoundError,
+                   "Invalid log file '#{log_id}', not available for application '#{app_guid}'"
+            end
+            filter
+          end
+
+          def construct_log_entry(decoded_message)
+            # 2015-03-22T15:28:55.83+0100 [RTR/0]      OUT message...
+            "#{Time.at(decoded_message.timestamp / 1_000_000_000.0).iso8601} "\
+              "[#{decoded_message.source_name}/#{decoded_message.source_id}] "\
+              "#{decoded_message.message_type == 1 ? 'OUT' : 'ERR'} #{decoded_message.message}"
+          end
+
+          def download_logfile_entries(app_guid, log_id, headers_to_use = nil)
+            # download log file
+            logfile_contents = download_file(app_guid, "logs/#{log_id}", headers_to_use)
+            # split file into entries by line breaks and return an array of log entries
+            logfile_contents.split("\n")
+          end
+
+          def download_file(app_guid, file_path, headers_to_use = nil)
+            expected_statuses = [200, 302, 400, 404]
+            # Hack, do not create fresh headers (which would fail) when in a deferred action
+            headers_to_use = headers unless headers_to_use
+
+            # log list consists of 2 parts, loggregator and files
+            log_files = get("/v2/apps/#{app_guid}/instances/0/files/#{file_path}",
+                            follow_redirects: false, expects: expected_statuses, headers: headers_to_use)
+            if log_files.status == 400 || log_files.status == 404
+              fail Errors::AdapterResourceNotFoundError,
+                   "Invalid log file: '#{file_path}' not available for application '#{app_guid}'"
+            end
+            return log_files.body if log_files.status == 200
+
+            # status must be 302, follow to the Location
+            download_location = log_files.headers[:Location]
+            # if IBM f*cked with the download URL, fix the address
+            download_location.gsub!(/objectstorage.service.networklayer.com/, 'objectstorage.softlayer.net')
+            Excon.defaults[:ssl_verify_peer] = false unless @check_certificates
+
+            connection_params = { ssl_verify_peer: @check_certificates }
+            connection = Excon.new(download_location, connection_params)
+            downloaded_logfile_response = connection.request(method: :get, expects: expected_statuses)
+
+            if downloaded_logfile_response.status == 404
+              fail Errors::AdapterResourceNotFoundError,
+                   "Invalid log file: '#{file_path}' not available for application '#{app_guid}'"
+            end
+            downloaded_logfile_response.body
+          end
+
+          def recent_log_messages(app_guid, filter = nil)
+            loggregator_recent_uri = "https://#{loggregator_endpoint}:443/recent?app=#{app_guid}"
+            # current log state before tailing, multipart message of protobuf objects
+            current_log_response = get(loggregator_recent_uri)
+            current_log_boundary = /boundary=(\w+)/.match(current_log_response.headers['Content-Type'])[1]
+            current_log = current_log_response.body
+
+            boundary_regexp = /--#{Regexp.quote(current_log_boundary)}(--)?#{CRLF}/
+            parts = current_log.split(boundary_regexp).collect do |chunk|
+              header_part = chunk.split(/#{CRLF}#{WSP}*#{CRLF}/m, 2)[0]
+              if header_part
+                headers = header_part.split(/\r\n/).map { |kv| kv }
+                headers.length > 1 ? headers[1] : nil
+              end
+            end.compact
+            # decode log messages
+            decoded_messages = parts.collect do |proto_message|
+              Message.decode(proto_message)
+            end.compact
+            return decoded_messages unless filter
+            # return filtered messages
+            decoded_messages.find_all do |msg|
+              filter.include?(msg.source_name)
+            end
+          end
+
+          def log_stream?(log_id)
+            LOGGREGATOR_TYPES.include?(log_id.to_sym) || log_id.to_sym == :all
+          end
+
+          def loggregator_endpoint
+            @endpoint_url.gsub(%r{^(\w*://)?(api)([-\.\w]+)$}i, 'loggregator\3')
+          end
+
+          def tail_file(app_guid, log_id, stream)
+            log.debug 'Tailing CF log file'
+            log_id = 'staging_task.log' if log_id.to_sym == Enums::ApplicationLogfileType::BUILD
+
+            # cache headers as they are bound to a request and could be lost with the next tick
+            headers_to_use = headers
+            latest_pushed_line = -1
+
+            # update every 3 seconds
+            @tail_file_timer = EM.add_periodic_timer(3) do
+              log.debug('Poll updated file tail...')
+              begin
+                latest_pushed_line = push_file_tail(app_guid, log_id, stream, latest_pushed_line, headers_to_use)
+              rescue Errors::AdapterResourceNotFoundError
+                log.debug('Logfile not found, finished tailing')
+                # file lost, close stream
+                @tail_file_timer.cancel if @tail_file_timer
+                stream.close
+              end
+            end
+            # listener to stop polling
+            StopListener.new(@tail_file_timer, :cancel)
+          end
+
+          def push_file_tail(app_guid, log_id, stream, pushed_line_idx, headers_to_use)
+            log.debug('Fetching file for tail response...')
+            entries = download_logfile_entries(app_guid, log_id, headers_to_use)
+            # file was shortened, close stream since we do not know where to continue
+            if entries.length < pushed_line_idx
+              log.debug('File was modified and shortened, stop tailing the file...')
+              stream.close
+            else
+              entries.each_with_index do |entry, index|
+                next if index <= pushed_line_idx
+                pushed_line_idx = index
+                stream.send_message(entry)
+              end
+              pushed_line_idx
+            end
+          end
+
+          def tail_stream(app_guid, log_id, stream)
+            filter = loggregator_filter(log_id)
+
+            # push current state
+            recent_log_messages(app_guid, filter).each { |entry| stream.send_message(construct_log_entry(entry)) }
+
+            # Now register websocket to receive the latest updates
+            ws = Faye::WebSocket::Client.new("wss://#{loggregator_endpoint}:443/tail/?app=#{app_guid}",
+                                             nil, headers: headers.slice('Authorization'))
+
+            ws.on :message do |event|
+              log.debug "CF loggregator message received: #{event}"
+              begin
+                msg = Message.decode(event.data.pack('C*'))
+                # notify stream to print new log line if msg type matches the applied filter
+                stream.send_message(construct_log_entry(msg)) if filter.nil? || filter.include?(msg.source_name)
+              rescue StandardError => e
+                log.error "Cloud Foundry log message de-serialization failed: #{e}"
+              end
+            end
+
+            ws.on :close do |event|
+              log.debug "Closing CF loggregator websocket: code=#{event.code}, reason=#{event.reason}"
+              ws = nil
+              # notify stream that no more update are to arrive and stream shall be closed
+              stream.close
+            end
+            # return listener to stop websocket
+            TailStopper.new(ws, :close)
+          end
+
+          # Message class definition, matching the Protocol Buffer definition of the Cloud Foundry loggregator.
+          # see also: https://github.com/cloudfoundry/loggregatorlib/blob/master/logmessage/log_message.proto
+          class Message < ::Protobuf::Message
+            class MessageType < ::Protobuf::Enum
+              define :OUT, 1
+              define :ERR, 2
+            end
+
+            required :bytes, :message, 1
+            required Logs::Message::MessageType, :message_type, 2
+            required :sint64, :timestamp, 3
+            required :string, :app_id, 4
+            optional :string, :source_id, 6
+            repeated :string, :drain_urls, 7
+            optional :string, :source_name, 8
+          end
+
+          class Envelope < ::Protobuf::Message
+            required :string, :routing_key, 1
+            required :bytes, :signature, 2
+            required Logs::Message, :log_message, 3
+          end
+        end
+      end
+    end
+  end
+end