nucleus 0.1.0

data/lib/nucleus/adapters/v1/cloud_foundry_v2/domains.rb

@@ -0,0 +1,149 @@
+module Nucleus
+  module Adapters
+    module V1
+      class CloudFoundryV2 < Stub
+        # Application domain / route functionality to support the Cloud Foundry API.<br>
+        module Domains
+          # @see Stub#domains
+          def domains(domain_id)
+            app_guid = app_guid(domain_id)
+            assigned_routes = get("/v2/apps/#{app_guid}/routes").body
+            domains = []
+            assigned_routes[:resources].each do |assigned_route|
+              nucleus_domain = route_to_nucleus_domain(assigned_route)
+              domains.push(nucleus_domain) unless nucleus_domain[:name] == app_web_url(app_guid)
+            end
+            domains
+          end
+
+          # @see Stub#domain
+          def domain(application_name_or_id, domain_id)
+            app_guid = app_guid(application_name_or_id)
+            assigned_routes = get("/v2/apps/#{app_guid}/routes").body
+            assigned_routes[:resources].each do |assigned_route|
+              return route_to_nucleus_domain(assigned_route) if assigned_route[:metadata][:guid] == domain_id
+            end
+          end
+
+          # @see Stub#create_domain
+          def create_domain(application_name_or_id, domain)
+            domains(application_name_or_id).each do |existing_domain|
+              if existing_domain[:name] == domain[:name]
+                fail Errors::SemanticAdapterRequestError,
+                     "Domain '#{domain[:name]}' is already assigned to the application"
+              end
+            end
+
+            app_guid = app_guid(application_name_or_id)
+            # extract the hostname and the domain name from the FQDN
+            /(?<domain_host>([-\w]+\.)*)(?<domain_name>([-\w]+\.[-\w]+))/ =~ domain[:name]
+            domain_host.chomp!('.') unless domain_host.nil?
+
+            # finally build the response
+            route_to_nucleus_domain(create_cf_domain(app_guid, domain_name, domain_host))
+          end
+
+          # @see Stub#delete_domain
+          def delete_domain(application_name_or_id, route_id)
+            app_guid = app_guid(application_name_or_id)
+            # remove route from the app
+            delete_response = delete("/v2/apps/#{app_guid}/routes/#{route_id}", expects: [201, 400])
+            if delete_response.status == 400
+              cf_error = delete_response.body[:code]
+              if cf_error == 1002
+                fail Errors::AdapterResourceNotFoundError, 'Domain not found. '\
+                  'CF context specific: Route does not exist or is not assigned with this application'
+              else
+                # delete failed with 400, but not due to invalid domain
+                fail Errors::AdapterRequestError,
+                     "#{delete_response.body[:description]} (#{cf_error} - #{delete_response.body[:error_code]})"
+              end
+            end
+
+            # check route usage
+            route_in_apps = get("/v2/routes/#{route_id}/apps").body
+            return unless route_in_apps[:total_results] == 0
+
+            # route is no longer needed, delete
+            delete("/v2/routes/#{route_id}")
+          end
+
+          private
+
+          def domain?(application_name_or_id, domain_name)
+            app_guid = app_guid(application_name_or_id)
+            domain_without_protocol = %r{([a-zA-Z]+://)?([-\.\w]*)}.match(domain_name)[2]
+            assigned_routes = get("/v2/apps/#{app_guid}/routes").body
+            assigned_routes[:resources].each do |route|
+              route_domain = get(route[:entity][:domain_url]).body
+              return true if "#{route[:entity][:host]}.#{route_domain[:entity][:name]}" == domain_without_protocol
+            end
+            false
+          end
+
+          def route_to_nucleus_domain(route_resource)
+            route_entity = route_resource[:entity]
+            route_metadata = route_resource[:metadata]
+            assigned_domain = get(route_entity[:domain_url]).body
+            domain = { id: route_metadata[:guid], created_at: route_metadata[:created_at] }
+            if route_metadata[:updated_at].to_s == ''
+              domain[:updated_at] = route_metadata[:created_at]
+            else
+              domain[:updated_at] = route_metadata[:updated_at]
+            end
+
+            if route_entity[:host].to_s == ''
+              domain[:name] = assigned_domain[:entity][:name]
+            else
+              domain[:name] = "#{route_entity[:host]}.#{assigned_domain[:entity][:name]}"
+            end
+            domain
+          end
+
+          def cf_domain(domain_name)
+            %w(private shared).each do |domain_type|
+              response = get("/v2/#{domain_type}_domains").body
+              response[:resources].each do |domain|
+                return domain if domain[:entity][:name] == domain_name
+              end
+            end
+            nil
+          end
+
+          def cf_route(domain_guid, domain_host)
+            # There is no way to check if a root domain (empty hostname) is already taken.
+            # Therefore we must iterate through all routes and find matches...
+            all_routes = get('/v2/routes').body[:resources]
+            all_routes.each do |route|
+              return route if route[:entity][:domain_guid] == domain_guid && route[:entity][:host] == domain_host
+            end
+            nil
+          end
+
+          def create_cf_domain(app_guid, domain_name, domain_host)
+            created_domain = cf_domain(domain_name)
+            unless created_domain
+              # domain does not exist, create!
+              domain_request_body = { name: domain_name, owning_organization_guid: default_organization_guid }
+              created_domain = post('/v2/private_domains', body: domain_request_body).body
+            end
+
+            created_route = cf_route(created_domain[:metadata][:guid], domain_host)
+            unless created_route
+              # route does not exist, create!
+              route_request_body = { domain_guid: created_domain[:metadata][:guid],
+                                     host: domain_host, space_guid: user_space_guid }
+              created_route = post('/v2/routes', body: route_request_body).body
+            end
+
+            # assign the route to the application
+            put("/v2/apps/#{app_guid}/routes/#{created_route[:metadata][:guid]}").body
+
+            # return the actual route, not the association response
+            created_route
+          end
+        end
+      end
+    end
+  end
+end

data/lib/nucleus/adapters/v1/cloud_foundry_v2/lifecycle.rb

@@ -0,0 +1,41 @@
+module Nucleus
+  module Adapters
+    module V1
+      class CloudFoundryV2 < Stub
+        module Lifecycle
+          # @see Stub#start
+          def start(application_name_or_id)
+            app_guid = app_guid(application_name_or_id)
+            # fail if there is no deployment
+            unless deployed?(app_guid)
+              fail Errors::SemanticAdapterRequestError, 'Application must be deployed before it can be started'
+            end
+
+            # start by name or id
+            start_response = put("/v2/apps/#{app_guid}", body: { state: 'STARTED' })
+            to_nucleus_app(start_response.body)
+          end
+
+          # @see Stub#stop
+          def stop(application_name_or_id)
+            app_guid = app_guid(application_name_or_id)
+            # fail if there is no deployment
+            unless deployed?(app_guid)
+              fail Errors::SemanticAdapterRequestError, 'Application must be deployed before it can be stopped'
+            end
+
+            # stop by name or id
+            stop_response = put("/v2/apps/#{app_guid}", body: { state: 'STOPPED' })
+            to_nucleus_app(stop_response.body)
+          end
+
+          # @see Stub#restart
+          def restart(application_name_or_id)
+            stop(application_name_or_id)
+            start(application_name_or_id)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/nucleus/adapters/v1/cloud_foundry_v2/logs.rb

@@ -0,0 +1,303 @@
+module Nucleus
+  module Adapters
+    module V1
+      class CloudFoundryV2 < Stub
+        module Logs
+          LOGGREGATOR_TYPES = [Enums::ApplicationLogfileType::API,
+                               Enums::ApplicationLogfileType::APPLICATION,
+                               Enums::ApplicationLogfileType::REQUEST,
+                               Enums::ApplicationLogfileType::SYSTEM]
+          # Carriage return (newline in Mac OS) + line feed (newline in Unix) == CRLF (newline in Windows)
+          CRLF = "\r\n"
+          WSP  = "\s"
+
+          # @see Stub#logs
+          def logs(application_name_or_id)
+            app_guid = app_guid(application_name_or_id)
+            # retrieve app for timestamps only :/
+            app_created = get("/v2/apps/#{app_guid}").body[:metadata][:created_at]
+            logs = []
+
+            begin
+              log_files_list = download_file(app_guid, 'logs')
+              # parse raw response to array
+              log_files_list.split(CRLF).each do |logfile_line|
+                filename = logfile_line.rpartition(' ').first.strip
+                if filename == 'staging_task.log'
+                  filename = 'build'
+                  log_type = Enums::ApplicationLogfileType::BUILD
+                else
+                  log_type = Enums::ApplicationLogfileType::OTHER
+                end
+                # TODO: right now, we always assume the log has recently been updated
+                logs.push(id: filename, name: filename, type: log_type, created_at: app_created,
+                          updated_at: Time.now.utc.iso8601)
+              end
+            rescue Errors::AdapterError
+              log.debug('no logs directory found for cf application')
+            end
+
+            # add the default logtypes, available according to:
+            # http://docs.cloudfoundry.org/devguide/deploy-apps/streaming-logs.html#format
+            LOGGREGATOR_TYPES.each do |type|
+              logs.push(id: type, name: type, type: type, created_at: app_created, updated_at: Time.now.utc.iso8601)
+            end
+            # TODO: 'all' is probably not perfect, since the build log wont be included
+            logs.push(id: 'all', name: 'all', type: Enums::ApplicationLogfileType::OTHER,
+                      created_at: app_created, updated_at: Time.now.utc.iso8601)
+            logs
+          end
+
+          # @see Stub#log?
+          def log?(application_name_or_id, log_id)
+            app_guid = app_guid(application_name_or_id)
+            # test file existence
+            log_id = 'staging_task.log' if log_id.to_sym == Enums::ApplicationLogfileType::BUILD
+            # checks also if application is even valid
+            response = get("/v2/apps/#{app_guid}/instances/0/files/logs/#{log_id}",
+                           follow_redirects: false, expects: [200, 302, 400])
+            return true if response == 200 || log_stream?(log_id)
+            return false if response == 400
+            # if 302 (only remaining option), followup...
+
+            # download log file
+            download_file(app_guid, "logs/#{log_id}")
+            # no error, file exists
+            true
+          rescue Errors::AdapterResourceNotFoundError, Errors::UnknownAdapterCallError,
+                 Excon::Errors::NotFound, Excon::Errors::BadRequest
+            false
+          end
+
+          # @see Stub#tail
+          def tail(application_name_or_id, log_id, stream)
+            app_guid = app_guid(application_name_or_id)
+            return tail_stream(app_guid, log_id, stream) if log_stream?(log_id)
+            tail_file(app_guid, log_id, stream)
+          end
+
+          # @see Stub#log_entries
+          def log_entries(application_name_or_id, log_id)
+            app_guid = app_guid(application_name_or_id)
+            # first check if this log is a file or must be fetched from the loggregator
+            if log_stream?(log_id)
+              # fetch recent data from loggregator and return an array of log entries
+              recent_decoded = recent_log_messages(app_guid, loggregator_filter(log_id))
+              recent_decoded.collect { |log_msg| construct_log_entry(log_msg) }
+            elsif log_id.to_sym == Enums::ApplicationLogfileType::BUILD
+              # handle special staging log
+              build_log_entries(app_guid)
+            else
+              download_logfile_entries(app_guid, log_id)
+            end
+          end
+
+          private
+
+          def build_log_entries(app_guid)
+            log_id = 'staging_task.log'
+            download_logfile_entries(app_guid, log_id)
+          rescue Errors::AdapterResourceNotFoundError
+            # if there was no build yet, return no entries instead of the 404 error
+            []
+          end
+
+          def loggregator_filter(log_id)
+            case log_id.to_sym
+            when Enums::ApplicationLogfileType::API
+              filter = ['API']
+            when Enums::ApplicationLogfileType::APPLICATION
+              filter = ['APP']
+            when Enums::ApplicationLogfileType::REQUEST
+              filter = ['RTR']
+            when Enums::ApplicationLogfileType::SYSTEM
+              filter = %w(STG LGR DEA)
+            when :all
+              # no filter, show all
+              filter = nil
+            else
+              # invalid log requests --> 404
+              fail Errors::AdapterResourceNotFoundError,
+                   "Invalid log file '#{log_id}', not available for application '#{app_guid}'"
+            end
+            filter
+          end
+
+          def construct_log_entry(decoded_message)
+            # 2015-03-22T15:28:55.83+0100 [RTR/0]      OUT message...
+            "#{Time.at(decoded_message.timestamp / 1_000_000_000.0).iso8601} "\
+              "[#{decoded_message.source_name}/#{decoded_message.source_id}] "\
+              "#{decoded_message.message_type == 1 ? 'OUT' : 'ERR'} #{decoded_message.message}"
+          end
+
+          def download_logfile_entries(app_guid, log_id, headers_to_use = nil)
+            # download log file
+            logfile_contents = download_file(app_guid, "logs/#{log_id}", headers_to_use)
+            # split file into entries by line breaks and return an array of log entries
+            logfile_contents.split("\n")
+          end
+
+          def download_file(app_guid, file_path, headers_to_use = nil)
+            expected_statuses = [200, 302, 400, 404]
+            # Hack, do not create fresh headers (which would fail) when in a deferred action
+            headers_to_use = headers unless headers_to_use
+
+            # log list consists of 2 parts, loggregator and files
+            log_files = get("/v2/apps/#{app_guid}/instances/0/files/#{file_path}",
+                            follow_redirects: false, expects: expected_statuses, headers: headers_to_use)
+            if log_files.status == 400 || log_files.status == 404
+              fail Errors::AdapterResourceNotFoundError,
+                   "Invalid log file: '#{file_path}' not available for application '#{app_guid}'"
+            end
+            return log_files.body if log_files.status == 200
+
+            # status must be 302, follow to the Location
+            download_location = log_files.headers[:Location]
+            # if IBM f*cked with the download URL, fix the address
+            download_location.gsub!(/objectstorage.service.networklayer.com/, 'objectstorage.softlayer.net')
+            Excon.defaults[:ssl_verify_peer] = false unless @check_certificates
+
+            connection_params = { ssl_verify_peer: @check_certificates }
+            connection = Excon.new(download_location, connection_params)
+            downloaded_logfile_response = connection.request(method: :get, expects: expected_statuses)
+
+            if downloaded_logfile_response.status == 404
+              fail Errors::AdapterResourceNotFoundError,
+                   "Invalid log file: '#{file_path}' not available for application '#{app_guid}'"
+            end
+            downloaded_logfile_response.body
+          end
+
+          def recent_log_messages(app_guid, filter = nil)
+            loggregator_recent_uri = "https://#{loggregator_endpoint}:443/recent?app=#{app_guid}"
+            # current log state before tailing, multipart message of protobuf objects
+            current_log_response = get(loggregator_recent_uri)
+            current_log_boundary = /boundary=(\w+)/.match(current_log_response.headers['Content-Type'])[1]
+            current_log = current_log_response.body
+
+            boundary_regexp = /--#{Regexp.quote(current_log_boundary)}(--)?#{CRLF}/
+            parts = current_log.split(boundary_regexp).collect do |chunk|
+              header_part = chunk.split(/#{CRLF}#{WSP}*#{CRLF}/m, 2)[0]
+              if header_part
+                headers = header_part.split(/\r\n/).map { |kv| kv }
+                headers.length > 1 ? headers[1] : nil
+              end
+            end.compact
+            # decode log messages
+            decoded_messages = parts.collect do |proto_message|
+              Message.decode(proto_message)
+            end.compact
+            return decoded_messages unless filter
+            # return filtered messages
+            decoded_messages.find_all do |msg|
+              filter.include?(msg.source_name)
+            end
+          end
+
+          def log_stream?(log_id)
+            LOGGREGATOR_TYPES.include?(log_id.to_sym) || log_id.to_sym == :all
+          end
+
+          def loggregator_endpoint
+            @endpoint_url.gsub(%r{^(\w*://)?(api)([-\.\w]+)$}i, 'loggregator\3')
+          end
+
+          def tail_file(app_guid, log_id, stream)
+            log.debug 'Tailing CF log file'
+            log_id = 'staging_task.log' if log_id.to_sym == Enums::ApplicationLogfileType::BUILD
+
+            # cache headers as they are bound to a request and could be lost with the next tick
+            headers_to_use = headers
+            latest_pushed_line = -1
+
+            # update every 3 seconds
+            @tail_file_timer = EM.add_periodic_timer(3) do
+              log.debug('Poll updated file tail...')
+              begin
+                latest_pushed_line = push_file_tail(app_guid, log_id, stream, latest_pushed_line, headers_to_use)
+              rescue Errors::AdapterResourceNotFoundError
+                log.debug('Logfile not found, finished tailing')
+                # file lost, close stream
+                @tail_file_timer.cancel if @tail_file_timer
+                stream.close
+              end
+            end
+            # listener to stop polling
+            StopListener.new(@tail_file_timer, :cancel)
+          end
+
+          def push_file_tail(app_guid, log_id, stream, pushed_line_idx, headers_to_use)
+            log.debug('Fetching file for tail response...')
+            entries = download_logfile_entries(app_guid, log_id, headers_to_use)
+            # file was shortened, close stream since we do not know where to continue
+            if entries.length < pushed_line_idx
+              log.debug('File was modified and shortened, stop tailing the file...')
+              stream.close
+            else
+              entries.each_with_index do |entry, index|
+                next if index <= pushed_line_idx
+                pushed_line_idx = index
+                stream.send_message(entry)
+              end
+              pushed_line_idx
+            end
+          end
+
+          def tail_stream(app_guid, log_id, stream)
+            filter = loggregator_filter(log_id)
+
+            # push current state
+            recent_log_messages(app_guid, filter).each { |entry| stream.send_message(construct_log_entry(entry)) }
+
+            # Now register websocket to receive the latest updates
+            ws = Faye::WebSocket::Client.new("wss://#{loggregator_endpoint}:443/tail/?app=#{app_guid}",
+                                             nil, headers: headers.slice('Authorization'))
+
+            ws.on :message do |event|
+              log.debug "CF loggregator message received: #{event}"
+              begin
+                msg = Message.decode(event.data.pack('C*'))
+                # notify stream to print new log line if msg type matches the applied filter
+                stream.send_message(construct_log_entry(msg)) if filter.nil? || filter.include?(msg.source_name)
+              rescue StandardError => e
+                log.error "Cloud Foundry log message de-serialization failed: #{e}"
+              end
+            end
+
+            ws.on :close do |event|
+              log.debug "Closing CF loggregator websocket: code=#{event.code}, reason=#{event.reason}"
+              ws = nil
+              # notify stream that no more update are to arrive and stream shall be closed
+              stream.close
+            end
+            # return listener to stop websocket
+            TailStopper.new(ws, :close)
+          end
+
+          # Message class definition, matching the Protocol Buffer definition of the Cloud Foundry loggregator.
+          # see also: https://github.com/cloudfoundry/loggregatorlib/blob/master/logmessage/log_message.proto
+          class Message < ::Protobuf::Message
+            class MessageType < ::Protobuf::Enum
+              define :OUT, 1
+              define :ERR, 2
+            end
+
+            required :bytes, :message, 1
+            required Logs::Message::MessageType, :message_type, 2
+            required :sint64, :timestamp, 3
+            required :string, :app_id, 4
+            optional :string, :source_id, 6
+            repeated :string, :drain_urls, 7
+            optional :string, :source_name, 8
+          end
+
+          class Envelope < ::Protobuf::Message
+            required :string, :routing_key, 1
+            required :bytes, :signature, 2
+            required Logs::Message, :log_message, 3
+          end
+        end
+      end
+    end
+  end
+end