cloulu 0.0.0 → 0.1.1

data/lib/cfoundry/rest_client.rb

@@ -0,0 +1,299 @@
+require "cfoundry/trace_helpers"
+require "net/https"
+require "net/http/post/multipart"
+require "multi_json"
+require "fileutils"
+
+module CFoundry
+  class RestClient
+    include CFoundry::TraceHelpers
+
+    LOG_LENGTH = 10
+
+    HTTP_METHODS = {
+      "GET" => Net::HTTP::Get,
+      "PUT" => Net::HTTP::Put,
+      "POST" => Net::HTTP::Post,
+      "DELETE" => Net::HTTP::Delete,
+      "HEAD" => Net::HTTP::Head,
+    }
+
+    DEFAULT_OPTIONS = {
+      :follow_redirects => true
+    }
+
+    attr_reader :target
+
+    attr_accessor :trace, :backtrace, :log, :request_id, :token, :target, :proxy
+
+    def initialize(target, token = nil)
+      @target = target
+      @token = token
+      @trace = false
+      @backtrace = false
+      @log = false
+    end
+
+    def request(method, path, options = {})
+      request_uri(method, construct_url(path), DEFAULT_OPTIONS.merge(options))
+    end
+
+    def generate_headers(payload, options)
+      headers = {}
+
+      if payload.is_a?(String)
+        headers["Content-Length"] = payload.size
+      elsif !payload
+        headers["Content-Length"] = 0
+      end
+
+      headers["X-Request-Id"] = @request_id if @request_id
+      headers["Authorization"] = @token.auth_header if @token
+      headers["Proxy-User"] = @proxy if @proxy
+
+      if accept_type = mimetype(options[:accept])
+        headers["Accept"] = accept_type
+      end
+
+      if content_type = mimetype(options[:content])
+        headers["Content-Type"] = content_type
+      end
+
+      headers.merge!(options[:headers]) if options[:headers]
+      headers
+    end
+
+    private
+
+    def request_uri(method, uri, options = {})
+      uri = URI.parse(uri)
+
+      unless uri.is_a?(URI::HTTP)
+        raise InvalidTarget.new(@target)
+      end
+
+      # keep original options in case there's a redirect to follow
+      original_options = options.dup
+      payload = options[:payload]
+
+      if params = options[:params]
+        if uri.query
+          uri.query += "&" + encode_params(params)
+        else
+          uri.query = encode_params(params)
+        end
+      end
+
+      unless payload.is_a?(String)
+        case options[:content]
+          when :json
+            payload = MultiJson.dump(payload)
+          when :form
+            payload = encode_params(payload)
+        end
+      end
+
+      method_class = get_method_class(method)
+      if payload.is_a?(Hash)
+        multipart = method_class.const_get(:Multipart)
+        request = multipart.new(uri.request_uri, payload)
+      else
+        request = method_class.new(uri.request_uri)
+        request.body = payload if payload
+      end
+
+      headers = generate_headers(payload, options)
+
+      request_hash = {
+        :url => uri.to_s,
+        :method => method,
+        :headers => headers,
+        :body => payload
+      }
+
+      print_request(request_hash) if @trace
+
+      add_headers(request, headers)
+
+      # TODO: test http proxies
+      http = Net::HTTP.new(uri.host, uri.port)
+
+      # TODO remove this when staging returns streaming responses
+      http.read_timeout = 300
+
+      if uri.is_a?(URI::HTTPS)
+        http.use_ssl = true
+        http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+      end
+
+      before = Time.now
+      http.start do
+        response = http.request(request)
+        time = Time.now - before
+
+        response_hash = {
+          :headers => sane_headers(response),
+          :status => response.code,
+          :body => response.body
+        }
+
+        print_response(response_hash) if @trace
+        print_backtrace(caller) if @trace
+
+        log_request(time, request, response)
+
+        if response.is_a?(Net::HTTPRedirection) && options[:follow_redirects]
+          request_uri("GET", response["location"], original_options)
+        else
+          return request_hash, response_hash
+        end
+      end
+    rescue ::Timeout::Error => e
+      raise Timeout.new(method, uri, e)
+    rescue SocketError, Errno::ECONNREFUSED => e
+      raise TargetRefused, e.message
+    rescue URI::InvalidURIError
+      raise InvalidTarget.new(@target)
+    end
+
+    def construct_url(path)
+      uri = URI.parse(path)
+      return path if uri.scheme
+
+      path = "/#{path}" unless path[0] == ?\/
+      target + path
+    end
+
+    def get_method_class(method_string)
+      HTTP_METHODS[method_string.upcase]
+    end
+
+    def add_headers(request, headers)
+      headers.each { |key, value| request[key] = value }
+    end
+
+    def mimetype(content)
+      case content
+      when String
+        content
+      when :json
+        "application/json"
+      when :form
+        "application/x-www-form-urlencoded"
+      when nil
+        nil
+      # return request headers (not really Accept)
+      else
+        raise CFoundry::Error, "Unknown mimetype '#{content.inspect}'"
+      end
+    end
+
+    def encode_params(hash, escape = true)
+      hash.keys.map do |k|
+        v = hash[k]
+        v = MultiJson.dump(v) if v.is_a?(Hash)
+        v = URI.escape(v.to_s, /[^#{URI::PATTERN::UNRESERVED}]/) if escape
+        "#{k}=#{v}"
+      end.join("&")
+    end
+
+    def log_data(time, request, response)
+      { :time => time,
+        :request => {
+          :method => request.method,
+          :url => request.path,
+          :headers => sane_headers(request)
+        },
+        :response => {
+          :code => response.code,
+          :headers => sane_headers(response)
+        }
+      }
+    end
+
+    def log_line(io, data)
+      io.printf(
+        "[%s]  %0.3fs  %6s -> %d  %s\n",
+        Time.now.strftime("%F %T"),
+        data[:time],
+        data[:request][:method].to_s.upcase,
+        data[:response][:code],
+        data[:request][:url])
+    end
+
+    def log_request(time, request, response)
+      return unless @log
+
+      data = log_data(time, request, response)
+
+      case @log
+      when IO
+        log_line(@log, data)
+        return
+      when String
+        if File.exists?(@log)
+          log = File.readlines(@log).last(LOG_LENGTH - 1)
+        elsif !File.exists?(File.dirname(@log))
+          FileUtils.mkdir_p(File.dirname(@log))
+        end
+
+        File.open(@log, "w") do |io|
+          log.each { |l| io.print l } if log
+          log_line(io, data)
+        end
+
+        return
+      end
+
+      if @log.respond_to?(:call)
+        @log.call(data)
+        return
+      end
+
+      if @log.respond_to?(:<<)
+        @log << data
+        return
+      end
+    end
+
+    def print_request(request)
+      $stderr.puts ">>>"
+      $stderr.puts request_trace(request)
+    end
+
+    def print_response(response)
+      $stderr.puts response_trace(response)
+      $stderr.puts "<<<"
+    end
+
+    def print_backtrace(locs)
+      return unless @backtrace
+
+      interesting_locs = locs.drop_while { |loc|
+        loc =~ /\/(cfoundry\/|restclient\/|net\/http)/
+      }
+
+      $stderr.puts "--- backtrace:"
+
+      $stderr.puts "... (boring)" unless locs == interesting_locs
+
+      trimmed_locs = interesting_locs[0..5]
+
+      trimmed_locs.each do |loc|
+        $stderr.puts "=== #{loc}"
+      end
+
+      $stderr.puts "... (trimmed)" unless trimmed_locs == interesting_locs
+    end
+
+    def sane_headers(obj)
+      hds = {}
+
+      obj.each_header do |k, v|
+        hds[k] = v
+      end
+
+      hds
+    end
+  end
+end

data/lib/cfoundry/test_support.rb

@@ -0,0 +1,3 @@
+Dir[File.expand_path('../../../spec/{support,fakes}/**/*.rb', __FILE__)].each do |file|
+  require file
+end

data/lib/cfoundry/trace_helpers.rb

@@ -0,0 +1,40 @@
+require "net/https"
+require "multi_json"
+
+module CFoundry
+  module TraceHelpers
+
+    def request_trace(request)
+      return nil unless request
+      info = ["REQUEST: #{request[:method]} #{request[:url]}"]
+      info << "REQUEST_HEADERS:"
+      info << header_trace(request[:headers])
+      info << "REQUEST_BODY: #{request[:body]}" if request[:body]
+      info.join("\n")
+    end
+
+
+    def response_trace(response)
+      return nil unless response
+      info = ["RESPONSE: [#{response[:status]}]"]
+      info << "RESPONSE_HEADERS:"
+      info << header_trace(response[:headers])
+      info << "RESPONSE_BODY:"
+      begin
+        parsed_body = MultiJson.load(response[:body])
+        info << MultiJson.dump(parsed_body, :pretty => true)
+      rescue
+        info << "#{response[:body]}"
+      end
+      info.join("\n")
+    end
+
+    private
+
+    def header_trace(headers)
+      headers.sort.map do |key, value|
+        "  #{key} : #{value}"
+      end
+    end
+  end
+end

data/lib/cfoundry/uaaclient.rb

@@ -0,0 +1,112 @@
+require "cfoundry/baseclient"
+require 'uaa'
+
+module CFoundry
+  class UAAClient
+    attr_accessor :target, :client_id, :token, :trace
+
+    def initialize(target = "https://uaa.cloudfoundry.com", client_id = "vmc")
+      @target = target
+      @client_id = client_id
+      CF::UAA::Misc.symbolize_keys = true
+    end
+
+    def prompts
+      wrap_uaa_errors do
+        CF::UAA::Misc.server(target)[:prompts]
+      end
+    end
+
+    def authorize(username, password)
+      wrap_uaa_errors do
+        begin
+          token_issuer.owner_password_grant(username, password)
+        rescue CF::UAA::BadResponse => e
+          status_code = e.message[/\d+/] || 400
+          raise CFoundry::Denied.new("Authorization failed", status_code)
+        rescue CF::UAA::TargetError
+          token_issuer.implicit_grant_with_creds(:username => username, :password => password)
+        end
+      end
+    end
+
+    def users
+      wrap_uaa_errors do
+        scim.query(:user)
+      end
+    end
+
+    def change_password(guid, new, old)
+      wrap_uaa_errors do
+        scim.change_password(guid, new, old)
+      end
+    end
+
+    def password_score(password)
+      wrap_uaa_errors do
+        response = CF::UAA::Misc.password_strength(target, password)
+
+        required_score = response[:requiredScore] || 0
+        case (response[:score] || 0)
+          when 10 then
+            :strong
+          when required_score..9 then
+            :good
+          else
+            :weak
+        end
+      end
+    end
+
+    def add_user(email, password)
+      wrap_uaa_errors do
+        scim.add(
+          :user,
+          {:userName => email,
+            :emails => [{:value => email}],
+            :password => password,
+            :name => {:givenName => email, :familyName => email}
+          }
+        )
+      end
+    end
+
+    def try_to_refresh_token!
+      wrap_uaa_errors do
+        begin
+          token_info = token_issuer.refresh_token_grant(token.refresh_token)
+          self.token = AuthToken.from_uaa_token_info(token_info)
+        rescue CF::UAA::TargetError
+          self.token
+        end
+      end
+    end
+
+    private
+
+    def token_issuer
+      @token_issuer ||= CF::UAA::TokenIssuer.new(target, client_id, nil, :symbolize_keys => true)
+      @token_issuer.logger.level = @trace ? Logger::Severity::TRACE : 1
+      @token_issuer
+    end
+
+    def scim
+      auth_header = token && token.auth_header
+      scim = CF::UAA::Scim.new(target, auth_header)
+      scim.logger.level = @trace ? Logger::Severity::TRACE : 1
+      scim
+    end
+
+    def wrap_uaa_errors
+      yield
+    rescue CF::UAA::BadResponse
+      raise CFoundry::BadResponse
+    rescue CF::UAA::NotFound
+      raise CFoundry::NotFound
+    rescue CF::UAA::InvalidToken
+      raise CFoundry::Denied
+    rescue CF::UAA::TargetError => e
+      raise CFoundry::UAAError.new(e.info[:error_description], e.info[:error])
+    end
+  end
+end

data/lib/cfoundry/upload_helpers.rb

@@ -0,0 +1,187 @@
+require "tmpdir"
+require "fileutils"
+require "pathname"
+require "digest/sha1"
+
+require "cfoundry/zip"
+
+module CFoundry
+  module UploadHelpers
+    # Default paths to exclude from upload payload.
+    UPLOAD_EXCLUDE = %w{.git _darcs .svn}
+
+    # Minimum size for an application payload to bother checking resources.
+    #RESOURCE_CHECK_LIMIT = 64 * 1024
+    RESOURCE_CHECK_LIMIT = 1024 * 1024 * 1024
+
+    # Upload application's code to target. Do this after #create! and before
+    # #start!
+    #
+    # [path]
+    #   A path pointing to either a directory, or a .jar, .war, or .zip
+    #   file.
+    #
+    #   If a .vmcignore file is detected under the given path, it will be used
+    #   to exclude paths from the payload, similar to a .gitignore.
+    #
+    # [check_resources]
+    #   If set to `false`, the entire payload will be uploaded
+    #   without checking the resource cache.
+    #
+    #   Only do this if you know what you're doing.
+    def upload(path, check_resources = true)
+      unless File.exist? path
+        raise CFoundry::Error, "Invalid application path '#{path}'"
+      end
+
+      zipfile = "#{Dir.tmpdir}/#{@guid}.zip"
+      tmpdir = "#{Dir.tmpdir}/.vmc_#{@guid}_files"
+
+      FileUtils.rm_f(zipfile)
+      FileUtils.rm_rf(tmpdir)
+
+      prepare_package(path, tmpdir)
+
+      resources = determine_resources(tmpdir) if check_resources
+
+      packed = CFoundry::Zip.pack(tmpdir, zipfile)
+
+      @client.base.upload_app(@guid, packed && zipfile, resources || [])
+    ensure
+      FileUtils.rm_f(zipfile) if zipfile
+      FileUtils.rm_rf(tmpdir) if tmpdir
+    end
+
+    private
+
+    def prepare_package(path, to)
+      if path =~ /\.(jar|war|zip)$/
+        CFoundry::Zip.unpack(path, to)
+      elsif war_file = Dir.glob("#{path}/*.war").first
+        CFoundry::Zip.unpack(war_file, to)
+      else
+        check_unreachable_links(path)
+
+        FileUtils.mkdir(to)
+
+        files = Dir.glob("#{path}/{*,.[^\.]*}")
+
+        exclude = UPLOAD_EXCLUDE
+        if File.exists?("#{path}/.vmcignore")
+          exclude += File.read("#{path}/.vmcignore").split(/\n+/)
+        end
+
+        # prevent initial copying if we can, remove sub-files later
+        files.reject! do |f|
+          exclude.any? do |e|
+            File.fnmatch(f.sub(path + "/", ""), e)
+          end
+        end
+
+        FileUtils.cp_r(files, to)
+
+        find_sockets(to).each do |s|
+          File.delete s
+        end
+
+        # remove ignored globs more thoroughly
+        #
+        # note that the above file list only includes toplevel
+        # files/directories for cp_r, so this is where sub-files/etc. are
+        # removed
+        exclude.each do |e|
+          Dir.glob("#{to}/#{e}").each do |f|
+            FileUtils.rm_rf(f)
+          end
+        end
+      end
+    end
+
+    def check_unreachable_links(path)
+      files = Dir.glob("#{path}/**/*", File::FNM_DOTMATCH)
+
+      # only used for friendlier error message
+      pwd = Pathname.pwd
+
+      abspath = File.expand_path(path)
+      unreachable = []
+      files.each do |f|
+        file = Pathname.new(f)
+        if file.symlink? && !file.realpath.to_s.start_with?(abspath)
+          unreachable << file.relative_path_from(pwd)
+        end
+      end
+
+      unless unreachable.empty?
+        root = Pathname.new(path).relative_path_from(pwd)
+        raise CFoundry::Error,
+          "Path contains links '#{unreachable}' that point outside '#{root}'"
+      end
+    end
+
+    def find_sockets(path)
+      files = Dir.glob("#{path}/**/*", File::FNM_DOTMATCH)
+      files && files.select { |f| File.socket? f }
+    end
+
+    def determine_resources(path)
+      fingerprints, total_size = make_fingerprints(path)
+
+      return if total_size <= RESOURCE_CHECK_LIMIT
+
+      resources = @client.base.resource_match(fingerprints)
+
+      resources.each do |resource|
+        FileUtils.rm_f resource[:fn]
+        resource[:fn].sub!("#{path}/", "")
+      end
+
+      prune_empty_directories(path)
+
+      resources
+    end
+
+    # OK, HERES THE PLAN...
+    #
+    # 1. Get all the directories in the entire file tree.
+    # 2. Sort them by the length of their absolute path.
+    # 3. Go through the list, longest paths first, and remove
+    #    the directories that are empty.
+    #
+    # This ensures that directories containing empty directories
+    # are also pruned.
+    def prune_empty_directories(path)
+      all_files = Dir["#{path}/**/{*,.*}"]
+      all_files.reject! { |fn| fn =~ /\/\.+$/ }
+
+      directories = all_files.select { |x| File.directory?(x) }
+      directories.sort! { |a, b| b.size <=> a.size }
+
+      directories.each do |directory|
+        entries = Dir.entries(directory) - %w{. ..}
+        FileUtils.rmdir(directory) if entries.empty?
+      end
+    end
+
+    def make_fingerprints(path)
+      fingerprints = []
+      total_size = 0
+
+      Dir.glob("#{path}/**/*", File::FNM_DOTMATCH) do |filename|
+        next if File.directory?(filename)
+
+        size = File.size(filename)
+
+        total_size += size
+
+        fingerprints << {
+          :size => size,
+          :sha1 => Digest::SHA1.file(filename).hexdigest,
+          :fn => filename
+        }
+      end
+
+      [fingerprints, total_size]
+    end
+  end
+end