jfoundry 0.1.0.pre

data/lib/jfoundry/rest_client.rb

@@ -0,0 +1,331 @@
+require "jfoundry/trace_helpers"
+require "net/https"
+require "net/http/post/multipart"
+require "multi_json"
+require "fileutils"
+require "jfoundry/timer"
+require "jfoundry/signature/version"
+
+module JFoundry
+  class RestClient
+    class HTTPFactory
+      def self.create(uri, proxy_options = [])
+        http = Net::HTTP.new(uri.host, uri.port, *proxy_options)
+
+        if uri.is_a?(URI::HTTPS)
+          http.use_ssl = true
+          http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+        end
+
+        return http
+      end
+    end
+
+    include JFoundry::TraceHelpers
+    include JFoundry::ProxyOptions
+    include JFoundry::Timer
+    include JFoundry::Signature::Version
+
+    LOG_LENGTH = 10
+
+    HTTP_METHODS = {
+      "GET" => Net::HTTP::Get,
+      "PUT" => Net::HTTP::Put,
+      "POST" => Net::HTTP::Post,
+      "DELETE" => Net::HTTP::Delete,
+      "HEAD" => Net::HTTP::Head,
+    }
+
+    DEFAULT_OPTIONS = {
+      :follow_redirects => true
+    }
+
+    attr_reader :target
+
+    attr_accessor :trace, :backtrace, :log,
+      :request_id, :access_key, :secret_key, :version, :http_proxy, :https_proxy
+
+    def initialize(target, access_key, secret_key, version)
+      @target = target
+      #@token = token
+      @access_key = access_key
+      @secret_key = secret_key
+      @version = version
+      @trace = false
+      @backtrace = false
+      @log = false
+    end
+
+    def target=(target)
+      return if target == @target
+
+      @target = target
+      #@token = nil
+    end
+
+    def request(method, path, options = {})
+      request_uri(method, construct_url(path), DEFAULT_OPTIONS.merge(options))
+    end
+
+    def generate_headers(uri, payload, options)
+      headers = {}
+
+      if payload.is_a?(String)
+        headers["Content-Length"] = payload.size
+      elsif !payload
+        headers["Content-Length"] = 0
+      end
+
+      headers["X-Request-Id"] = @request_id if @request_id
+      #headers["Authorization"] = @token.auth_header if @token
+      headers['Version'] = @version 
+      headers['Date'] = get_time
+      headers['ACCESS-KEY'] = @access_key
+      headers['Path'] = URI.parse(uri.to_s).path
+
+      if accept_type = mimetype(options[:accept])
+        headers["Accept"] = accept_type
+      end
+
+      if content_type = mimetype(options[:content])
+        headers["Content-Type"] = content_type
+      end
+
+      headers.merge!(options[:headers]) if options[:headers]
+      headers
+    end
+
+    private
+
+    def request_uri(method, uri, options = {})
+      uri = URI.parse(uri)
+
+      unless uri.is_a?(URI::HTTP)
+        raise InvalidTarget.new(@target)
+      end
+
+      # keep original options in case there's a redirect to follow
+      original_options = options.dup
+      payload = options[:payload]
+
+      if options[:params]
+        encoded_params = encode_params(options[:params])
+        if encoded_params.respond_to?(:empty?) ? !encoded_params.empty? : encoded_params
+          if uri.query
+            uri.query += "&" + encoded_params
+          else
+            uri.query = encoded_params
+          end
+        end
+      end
+
+      unless payload.is_a?(String)
+        case options[:content]
+          when :json
+            payload = MultiJson.dump(payload)
+          when :form
+            payload = encode_params(payload)
+        end
+      end
+
+      method_class = get_method_class(method)
+      if payload.is_a?(Hash)
+        multipart = method_class.const_get(:Multipart)
+        request = multipart.new(uri.request_uri, payload)
+      else
+        request = method_class.new(uri.request_uri)
+        request.body = payload if payload
+      end
+
+      headers = generate_headers(uri, payload, options)
+
+      request_hash = {
+        :url => uri.to_s,
+        :method => method,
+        :headers => headers,
+        :body => payload
+      }
+
+      print_request(request_hash) if @trace
+
+      add_headers(request, headers)
+
+      signature = generate_signature(@secret_key, method, request.to_hash())
+      request['signature'] = signature
+
+      http = HTTPFactory.create(uri, proxy_options_for(uri))
+
+      # TODO remove this when staging returns streaming responses
+      http.read_timeout = 300
+
+      before = Time.now
+      http.start do
+        response = http.request(request)
+        time = Time.now - before
+
+        response_hash = {
+          :headers => sane_headers(response),
+          :status => response.code,
+          :body => response.body
+        }
+
+        print_response(response_hash) if @trace
+        print_backtrace(caller) if @trace
+
+        log_request(time, request, response)
+
+        if response.is_a?(Net::HTTPRedirection) && options[:follow_redirects]
+          request_uri("GET", response["location"], original_options)
+        else
+          return request_hash, response_hash
+        end
+      end
+    rescue ::Timeout::Error => e
+      raise Timeout.new(method, uri, e)
+    rescue SocketError, Errno::ECONNREFUSED => e
+      raise TargetRefused, e.message
+    rescue URI::InvalidURIError
+      raise InvalidTarget.new(@target)
+    end
+
+    def construct_url(path)
+      uri = URI.parse(path)
+      return path if uri.scheme
+
+      path = "/#{path}" unless path[0] == ?\/
+      target + path
+    end
+
+    def get_method_class(method_string)
+      HTTP_METHODS[method_string.upcase]
+    end
+
+    def add_headers(request, headers)
+      headers.each { |key, value| request[key] = value }
+    end
+
+    def mimetype(content)
+      case content
+      when String
+        content
+      when :json
+        "application/json"
+      when :form
+        "application/x-www-form-urlencoded"
+      when nil
+        nil
+      # return request headers (not really Accept)
+      else
+        raise JFoundry::Error, "Unknown mimetype '#{content.inspect}'"
+      end
+    end
+
+    def encode_params(hash, escape = true)
+      hash.keys.map do |k|
+        v = hash[k]
+        v = MultiJson.dump(v) if v.is_a?(Hash)
+        v = URI.escape(v.to_s, /[^#{URI::PATTERN::UNRESERVED}]/) if escape
+        "#{k}=#{v}"
+      end.join("&")
+    end
+
+    def log_data(time, request, response)
+      { :time => time,
+        :request => {
+          :method => request.method,
+          :url => request.path,
+          :headers => sane_headers(request)
+        },
+        :response => {
+          :code => response.code,
+          :headers => sane_headers(response)
+        }
+      }
+    end
+
+    def log_line(io, data)
+      io.printf(
+        "[%s]  %0.3fs  %6s -> %d  %s\n",
+        Time.now.strftime("%F %T"),
+        data[:time],
+        data[:request][:method].to_s.upcase,
+        data[:response][:code],
+        data[:request][:url])
+    end
+
+    def log_request(time, request, response)
+      return unless @log
+
+      data = log_data(time, request, response)
+
+      case @log
+      when IO
+        log_line(@log, data)
+        return
+      when String
+        if File.exists?(@log)
+          log = File.readlines(@log).last(LOG_LENGTH - 1)
+        elsif !File.exists?(File.dirname(@log))
+          FileUtils.mkdir_p(File.dirname(@log))
+        end
+
+        File.open(@log, "w") do |io|
+          log.each { |l| io.print l } if log
+          log_line(io, data)
+        end
+
+        return
+      end
+
+      if @log.respond_to?(:call)
+        @log.call(data)
+        return
+      end
+
+      if @log.respond_to?(:<<)
+        @log << data
+        return
+      end
+    end
+
+    def print_request(request)
+      $stderr.puts ">>>"
+      $stderr.puts request_trace(request)
+    end
+
+    def print_response(response)
+      $stderr.puts response_trace(response)
+      $stderr.puts "<<<"
+    end
+
+    def print_backtrace(locs)
+      return unless @backtrace
+
+      interesting_locs = locs.drop_while { |loc|
+        loc =~ /\/(jfoundry\/|restclient\/|net\/http)/
+      }
+
+      $stderr.puts "--- backtrace:"
+
+      $stderr.puts "... (boring)" unless locs == interesting_locs
+
+      trimmed_locs = interesting_locs[0..5]
+
+      trimmed_locs.each do |loc|
+        $stderr.puts "=== #{loc}"
+      end
+
+      $stderr.puts "... (trimmed)" unless trimmed_locs == interesting_locs
+    end
+
+    def sane_headers(obj)
+      hds = {}
+
+      obj.each_header do |k, v|
+        hds[k] = v
+      end
+
+      hds
+    end
+  end
+end

data/lib/jfoundry/signature/version.rb

@@ -0,0 +1,27 @@
+#!/usr/bin/ruby
+#encoding=utf-8
+
+require 'jfoundry/signer'
+
+module JFoundry 
+    module Signature
+        module Version
+
+            include JFoundry::Signer
+
+            def generate_signature secret_key, method, req 
+                return sign(secret_key, string_to_sign(method, req))
+            end 
+
+            def string_to_sign method, req 
+                str = [ 
+                    method.downcase,
+                    req['content-md5'],
+                    req['content-type'],
+                    req['date'],
+                    req['path']
+                ].join("\n")
+            end 
+        end 
+    end 
+end

data/lib/jfoundry/signer.rb

@@ -0,0 +1,13 @@
+#!/usr/bin/ruby
+#encoding=utf-8
+
+require 'base64'
+require 'openssl'
+
+module JFoundry 
+    module Signer
+        def sign secret_key, string_to_sign, digest = 'sha1'
+            return Base64.encode64(OpenSSL::HMAC.digest(digest, secret_key, string_to_sign)).strip
+        end 
+    end 
+end

data/lib/jfoundry/test_support.rb

@@ -0,0 +1,3 @@
+Dir[File.expand_path('../../../spec/{support}/**/*.rb', __FILE__)].each do |file|
+  require file unless file =~ /factory_girl/
+end

data/lib/jfoundry/timer.rb

@@ -0,0 +1,13 @@
+#!/usr/bin/ruby
+#encoding=utf-8
+
+require "time"
+
+module JFoundry
+    module Timer
+        def get_time
+            return Time.now.utc.strftime("%Y%m%dT%H%M%SZ")
+        end
+    end
+end
+

data/lib/jfoundry/trace_helpers.rb

@@ -0,0 +1,64 @@
+require "net/https"
+require "multi_json"
+
+module JFoundry
+  module TraceHelpers
+    PROTECTED_ATTRIBUTES = ['Authorization', 'credentials']
+
+    def request_trace(request)
+      return nil unless request
+      info = ["REQUEST: #{request[:method]} #{request[:url]}"]
+      info << "REQUEST_HEADERS:"
+      info << header_trace(request[:headers])
+      info << "REQUEST_BODY: #{request[:body]}" if request[:body]
+      info.join("\n")
+    end
+
+
+    def response_trace(response)
+      return nil unless response
+      info = ["RESPONSE: [#{response[:status]}]"]
+      info << "RESPONSE_HEADERS:"
+      info << header_trace(response[:headers])
+      info << "RESPONSE_BODY:"
+      begin
+        parsed_body = MultiJson.load(response[:body])
+        filter_protected_attributes(parsed_body)
+        info << MultiJson.dump(parsed_body, :pretty => true)
+      rescue
+        info << "#{response[:body]}"
+      end
+      info.join("\n")
+    end
+
+    private
+
+    def header_trace(headers)
+      headers.sort.map do |key, value|
+        unless PROTECTED_ATTRIBUTES.include?(key)
+          "  #{key} : #{value}"
+        else
+          "  #{key} : [PRIVATE DATA HIDDEN]"
+        end
+      end
+    end
+
+    def filter_protected_attributes(hash_or_array)
+      if hash_or_array.is_a? Array
+        hash_or_array.each do |value|
+          filter_protected_attributes(value)
+        end
+      else
+        hash_or_array.each do |key, value|
+          if PROTECTED_ATTRIBUTES.include? key
+            hash_or_array[key] = "[PRIVATE DATA HIDDEN]"
+          else
+            if value.is_a?(Hash) || value.is_a?(Array)
+              filter_protected_attributes(value)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/jfoundry/upload_helpers.rb

@@ -0,0 +1,222 @@
+require "tmpdir"
+require "fileutils"
+require "pathname"
+require "digest/sha1"
+
+require "jfoundry/zip"
+
+module JFoundry
+  module UploadHelpers
+    # Default paths to exclude from upload payload.
+    UPLOAD_EXCLUDE = %w{.git _darcs .svn}
+
+    # Minimum size for an application payload to bother checking resources.
+    RESOURCE_CHECK_LIMIT = 64 * 1024
+
+    # Upload application's code to target. Do this after #create! and before
+    # #start!
+    #
+    # [path]
+    #   A path pointing to either a directory, or a .jar, .war, or .zip
+    #   file.
+    #
+    #   If a .jfignore file is detected under the given path, it will be used
+    #   to exclude paths from the payload, similar to a .gitignore.
+    #
+    # [check_resources]
+    #   If set to `false`, the entire payload will be uploaded
+    #   without checking the resource cache.
+    #
+    #   Only do this if you know what you're doing.
+    def upload(path, check_resources = true)
+      unless File.exist? path
+        raise JFoundry::Error, "Invalid application path '#{path}'"
+      end
+
+      zipfile = "#{Dir.tmpdir}/#{@guid}.zip"
+      tmpdir = "#{Dir.tmpdir}/.jf_#{@guid}_files"
+
+      FileUtils.rm_f(zipfile)
+      FileUtils.rm_rf(tmpdir)
+
+      prepare_package(path, tmpdir)
+
+      resources = determine_resources(tmpdir) if check_resources
+
+      packed = JFoundry::Zip.pack(tmpdir, zipfile)
+
+      @client.base.upload_app(@guid, packed && zipfile, resources || [])
+    ensure
+      FileUtils.rm_f(zipfile) if zipfile
+      FileUtils.rm_rf(tmpdir) if tmpdir
+    end
+
+    private
+
+    def prepare_package(path, to)
+      if path =~ /\.(jar|war|zip)$/
+        JFoundry::Zip.unpack(path, to)
+      elsif (war_file = Dir.glob("#{path}/*.war").first)
+        JFoundry::Zip.unpack(war_file, to)
+      else
+        FileUtils.mkdir(to)
+
+        exclude = UPLOAD_EXCLUDE
+        if File.exists?("#{path}/.jfignore")
+          exclude += File.read("#{path}/.jfignore").split(/\n+/)
+        end
+
+        files = files_to_consider(path, exclude)
+
+        check_unreachable_links(files, path)
+
+        copy_tree(files, path, to)
+
+        find_sockets(to).each do |s|
+          File.delete s
+        end
+      end
+    end
+
+    def check_unreachable_links(files, path)
+      # only used for friendlier error message
+      pwd = Pathname.pwd
+
+      abspath = File.expand_path(path)
+      unreachable = []
+      files.each do |f|
+        file = Pathname.new(f)
+        if file.symlink? && !file.realpath.to_s.start_with?(abspath)
+          unreachable << file.relative_path_from(pwd)
+        end
+      end
+
+      unless unreachable.empty?
+        root = Pathname.new(path).relative_path_from(pwd)
+        raise JFoundry::Error,
+          "Path contains links '#{unreachable}' that point outside '#{root}'"
+      end
+    end
+
+    def files_to_consider(path, exclusions)
+      entries = all_files(path)
+
+      exclusions.each do |exclusion|
+        ignore_pattern = exclusion.start_with?("/") ? File.join(path, exclusion) : File.join(path, "**", exclusion)
+        dir_glob = Dir.glob(ignore_pattern, File::FNM_DOTMATCH)
+        entries -= dir_glob
+
+        ignore_pattern = File.join(path, "**", exclusion, "**", "*")
+        dir_glob = Dir.glob(ignore_pattern, File::FNM_DOTMATCH)
+        entries -= dir_glob
+      end
+
+      entries
+    end
+
+    def glob_matches?(file, path, pattern)
+      name = file.sub("#{path}/", "/")
+      flags = File::FNM_DOTMATCH
+
+      # when pattern ends with /, match only directories
+      if pattern.end_with?("/") && File.directory?(file)
+        name = "#{name}/"
+      end
+
+      case pattern
+      # when pattern contains /, do a pathname match
+      when /\/./
+        flags |= File::FNM_PATHNAME
+
+      # otherwise, match any file path
+      else
+        pattern = "**/#{pattern}"
+      end
+
+      File.fnmatch(pattern, name, flags)
+    end
+
+    def find_sockets(path)
+      all_files(path).select { |f| File.socket? f }
+    end
+
+    def determine_resources(path)
+      fingerprints, total_size = make_fingerprints(path)
+
+      return if total_size <= RESOURCE_CHECK_LIMIT
+
+      resources = @client.base.resource_match(fingerprints)
+
+      resources.each do |resource|
+        FileUtils.rm_f resource[:fn]
+        resource[:fn].sub!("#{path}/", "")
+      end
+
+      prune_empty_directories(path)
+
+      resources
+    end
+
+    # OK, HERES THE PLAN...
+    #
+    # 1. Get all the directories in the entire file tree.
+    # 2. Sort them by the length of their absolute path.
+    # 3. Go through the list, longest paths first, and remove
+    #    the directories that are empty.
+    #
+    # This ensures that directories containing empty directories
+    # are also pruned.
+    def prune_empty_directories(path)
+      all_files = all_files(path)
+
+      directories = all_files.select { |x| File.directory?(x) }
+      directories.sort! { |a, b| b.size <=> a.size }
+
+      directories.each do |directory|
+        entries = all_files(directory)
+        FileUtils.rmdir(directory) if entries.empty?
+      end
+    end
+
+    def make_fingerprints(path)
+      fingerprints = []
+      total_size = 0
+
+      all_files(path).each do |filename|
+        next if File.directory?(filename)
+
+        size = File.size(filename)
+
+        total_size += size
+
+        fingerprints << {
+          :size => size,
+          :sha1 => Digest::SHA1.file(filename).hexdigest,
+          :fn => filename
+        }
+      end
+
+      [fingerprints, total_size]
+    end
+
+    def all_files(path)
+      Dir.glob("#{path}/**/*", File::FNM_DOTMATCH).reject do |fn|
+        fn =~ /\.$/
+      end
+    end
+
+    def copy_tree(files, path, to)
+      files.each do |file|
+        dest = file.sub("#{path}/", "#{to}/")
+
+        if File.directory?(file)
+          FileUtils.mkdir_p(dest)
+        else
+          destdir = File.dirname(dest)
+          FileUtils.mkdir_p(destdir)
+          FileUtils.cp(file, dest)
+        end
+      end
+    end
+  end
+end