grifter 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 272206b2927fd805a4d0eebfaeb12aa9bb5544b8
-  data.tar.gz: 811ddf5f1872f2c8a82a0ccf7fb5082d7d5537ac
+  metadata.gz: b3c5acde96e3f1e8b052b55aa73c4c26e955d50e
+  data.tar.gz: 855578ed7e193ba868a043e3cc8c99b93451bbb4
 SHA512:
-  metadata.gz: e687565405a56497e50a449267895dc594006df2ae3679034003fc24132937fab025a76f3204e1707af89a8456c661b50a9766bec55d50b02343221262874759
-  data.tar.gz: 4aaf0eda705b509714ffbcb3b02d2d06df44dbf3645ebb412ccc6bb20dcef047bfac5693815bedfa4b26421689480cc432080e150931896031bf9a146418914f
+  metadata.gz: 49c8a5c1f5529e7f144921987a7032b26dff13e6b482c3a5095b458e940e7c6eb1085fd0ee120ff04de6a7479e93528edb760c635bc91cbcf2f036adbc9dba80
+  data.tar.gz: 1aad905c89ba51705b0aa203e0e5ad7fe3e93f324cea6c79bbdeb15604ee7e9e9dea8aa0dcba7dec1e6f11211f41d0eb66312fcc0b969db102db31cdb2e85583

data/Gemfile

@@ -2,7 +2,9 @@ source :rubygems
 
 #gemspec
 
+gem 'faraday'
 gem 'json'
+gem 'typhoeus'
 
 group :development do
   gem 'rspec'

data/Gemfile.lock

@@ -6,8 +6,12 @@ GEM
     builder (3.2.2)
     coderay (1.0.9)
     diff-lcs (1.2.4)
+    ethon (0.6.1)
+      ffi (>= 1.3.0)
+      mime-types (~> 1.18)
     faraday (0.8.8)
       multipart-post (~> 1.2.0)
+    ffi (1.9.3)
     git (1.2.5)
     github_api (0.10.1)
       addressable
@@ -32,6 +36,7 @@ GEM
     jwt (0.1.8)
       multi_json (>= 1.5)
     method_source (0.8.2)
+    mime-types (1.25.1)
     multi_json (1.7.8)
     multi_xml (0.5.4)
     multipart-post (1.2.0)
@@ -60,13 +65,17 @@ GEM
       diff-lcs (>= 1.1.3, < 2.0)
     rspec-mocks (2.14.2)
     slop (3.4.6)
+    typhoeus (0.6.6)
+      ethon (~> 0.6.1)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
   awesome_print
+  faraday
   jeweler
   json
   pry
   rspec
+  typhoeus

data/grifter.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "grifter"
-  s.version = "0.2.0"
+  s.version = "0.3.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Robert Schultheis"]
-  s.date = "2013-11-05"
+  s.date = "2013-12-04"
   s.description = "convention based approach to interfacing with an HTTP JSON API."
   s.email = "rob@knewton.com"
   s.executables = ["grift"]
@@ -28,6 +28,7 @@ Gem::Specification.new do |s|
     "lib/grifter/configuration.rb",
     "lib/grifter/helpers.rb",
     "lib/grifter/http_service.rb",
+    "lib/grifter/instrumentation.rb",
     "lib/grifter/json_helpers.rb",
     "lib/grifter/log.rb"
   ]
@@ -41,20 +42,26 @@ Gem::Specification.new do |s|
     s.specification_version = 4
 
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<faraday>, [">= 0"])
       s.add_runtime_dependency(%q<json>, [">= 0"])
+      s.add_runtime_dependency(%q<typhoeus>, [">= 0"])
       s.add_development_dependency(%q<rspec>, [">= 0"])
       s.add_development_dependency(%q<pry>, [">= 0"])
       s.add_development_dependency(%q<awesome_print>, [">= 0"])
       s.add_development_dependency(%q<jeweler>, [">= 0"])
     else
+      s.add_dependency(%q<faraday>, [">= 0"])
       s.add_dependency(%q<json>, [">= 0"])
+      s.add_dependency(%q<typhoeus>, [">= 0"])
       s.add_dependency(%q<rspec>, [">= 0"])
       s.add_dependency(%q<pry>, [">= 0"])
       s.add_dependency(%q<awesome_print>, [">= 0"])
       s.add_dependency(%q<jeweler>, [">= 0"])
     end
   else
+    s.add_dependency(%q<faraday>, [">= 0"])
     s.add_dependency(%q<json>, [">= 0"])
+    s.add_dependency(%q<typhoeus>, [">= 0"])
     s.add_dependency(%q<rspec>, [">= 0"])
     s.add_dependency(%q<pry>, [">= 0"])
     s.add_dependency(%q<awesome_print>, [">= 0"])

data/lib/grifter.rb

@@ -2,9 +2,11 @@ require_relative 'grifter/http_service'
 require_relative 'grifter/configuration'
 require_relative 'grifter/log'
 require_relative 'grifter/blankslate'
+require_relative 'grifter/instrumentation'
 
 class Grifter
   include Grifter::Configuration
+  include Grifter::Instrumentation
 
   DefaultConfigOptions = {
     #TODO: service_config: nil,
@@ -43,6 +45,8 @@ class Grifter
     if @config[:authenticate]
       self.grifter_authenticate_do
     end
+
+    start_instrumentation
   end
 
   attr_reader :services

data/lib/grifter/configuration.rb

@@ -23,6 +23,7 @@ class Grifter
         raise GrifterConfigurationError.new "url is not a proper aboslute URL: #{url}"
       end
       parsed = URI.parse url
+      #make the url faraday is configured with
       {
         :hostname => parsed.host,
         :port => parsed.port,
@@ -36,7 +37,7 @@ class Grifter
         config_file: ENV['GRIFTER_CONFIG_FILE'] ? ENV['GRIFTER_CONFIG_FILE'] : 'grifter.yml',
         environment: ENV['GRIFTER_ENVIRONMENT'],
       }.merge(options)
-      Log.debug "Loading config file '#{options[:config_file]}'"
+      Grifter::Log.debug "Loading config file '#{options[:config_file]}'"
       unless File.exist?(options[:config_file])
         raise GrifterConfigFileMissing.new "No such config file: '#{options[:config_file]}'"
       end
@@ -68,7 +69,6 @@ class Grifter
           base_uri: '',
           port: (service_config[:ssl] == true ? 443 : 80),
         }.merge(service_config))
-
       end
 
       #merge any environment overrides into the service block
@@ -95,6 +95,12 @@ class Grifter
         end
       end
 
+      #add in the faraday url as the final thing after figuring everything else out
+      config[:services].each_pair do |service_name, service_config|
+        #set the url we'll use to start faraday
+        service_config[:faraday_url] = "#{service_config[:ssl] ? 'https':'http'}://#{service_config[:hostname]}:#{service_config[:port].to_s}"
+      end
+
       #join the grift globs with the relative path to config file
       if config[:grift_globs] && options[:config_file]
         config_file_dir = File.dirname options[:config_file]

data/lib/grifter/http_service.rb

@@ -1,5 +1,6 @@
-require 'net/http'
-require 'openssl'
+require 'faraday'
+require 'typhoeus'
+require 'typhoeus/adapters/faraday' #https://github.com/typhoeus/typhoeus/issues/226#issuecomment-9919517
 
 require_relative 'json_helpers'
 require_relative 'log'
@@ -7,6 +8,7 @@ require_relative 'log'
 class Grifter
   class HTTPService
 
+    include Grifter::Log
     include Grifter::JsonHelpers
 
     def initialize config
@@ -14,30 +16,50 @@ class Grifter
       @config = config
       @name = config[:name]
       @base_uri = config[:base_uri]
-
-      Log.debug "Configuring service '#{@name}' with:\n\t#{@config.inspect}"
-
-      @http = Net::HTTP.new(@config[:hostname], @config[:port])
-      @http.use_ssl = @config[:ssl]
-      @http.verify_mode = OpenSSL::SSL::VERIFY_NONE if @config[:ignore_ssl_cert]
-      @http.read_timeout = @config[:timeout] if @config[:timeout]
+      @log_headers = config.fetch(:log_headers, true)
+      @log_bodies = config.fetch(:log_bodies, true)
+
+      logger.debug "Configuring service '#{@name}' with:\n\t#{@config.inspect}"
+
+      #@conn = Net::HTTP.new(@config[:hostname], @config[:port])
+      #@conn.use_ssl = @config[:ssl]
+      @conn = Faraday.new @config[:faraday_url] do |conn_builder|
+        #do our own logging
+        #conn_builder.response logger: logger
+        #conn_builder.adapter  Faraday.default_adapter  # make requests with Net::HTTP
+        conn_builder.adapter  :typhoeus
+        conn_builder.ssl[:verify] = false if @config[:ignore_ssl_cert]
+
+        #this nonsense dont work?!  https://github.com/lostisland/faraday_middleware/issues/76
+        #conn_builder.use :instrumentation
+      end
 
       @headers = {
         'accept' => 'application/json',
         'content-type' => 'application/json',
       }
       if @config[:default_headers]
-        Log.debug "Default headers configured: " + @config[:default_headers].inspect
+        logger.debug "Default headers configured: " + @config[:default_headers].inspect
         @config[:default_headers].each_pair do |k, v|
           @headers[k.to_s] = v.to_s
         end
       end
+      @default_timeout = @config.fetch(:timeout, 10)
+      logger.info "Initialized grifter service '#{@name}'"
+    end
+
+    def stubs &blk
+      stubs = Faraday::Adapter::Test::Stubs.new
+      @conn = Faraday.new @config[:faraday_url] do |conn_builder|
+        conn_builder.adapter :test, stubs
+      end
+      stubs
     end
 
     #allow stubbing http if we are testing
     attr_reader :http if defined?(RSpec)
 
-    attr_reader :headers, :name, :config
+    attr_reader :headers, :name, :config, :conn
 
     #this is useful for testing apis, and other times
     #you want to interrogate the http details of a response
@@ -49,34 +71,74 @@ class Grifter
     # options can include:
     # - :timeout, which specifies num secs the request should timeout in
     #   (this turns out to be kind of annoying to implement)
-    def do_request req, options={}
-      Log.debug RequestLogSeperator
-      Log.debug "#{req.class} #{req.path}"
-      Log.debug "HEADERS: #{req.to_hash}"
-      Log.debug "BODY:\n#{req.body}" if req.request_body_permitted?
-
-      if options.has_key? :timeout
-        cur_timeout = @http.read_timeout
-        Log.debug "Overriding timeout to: #{options[:timeout]}"
-        @http.read_timeout = options[:timeout]
+    def do_request method, path, obj=nil, options={}
+
+      #grifter clients pass in path possibly including query params.
+      #Faraday needs the query and path seperately.
+      parsed = URI.parse make_path(path)
+      #faraday needs the request params as a hash.
+      #this turns out to be non-trivial
+      query_hash = if parsed.query
+                     cgi_hash = CGI.parse(parsed.query)
+                     #make to account for one param having multiple values
+                     cgi_hash.inject({}) { |h,(k,v)| h[k] = v[1] ? v : v.first; h }
+                   else
+                     nil
+                   end
+
+      req_headers = make_headers(options)
+
+      body = if options[:form]
+        URI.encode_www_form obj
+      else
+        jsonify(obj)
       end
 
-      response = @http.request(req)
-
-      if cur_timeout
-        @http.read_timeout = cur_timeout
+      #log the request
+      logger.debug [
+        "Doing request: #{@name}: #{method.to_s.upcase} #{path}",
+        @log_headers ? ["Request Headers:",
+        req_headers.map{ |k, v| "#{k}: #{v.inspect}" }] : nil,
+        @log_bodies ? ["Request Body:", body] : nil,
+      ].flatten.compact.join("\n")
+
+      #doing it this way avoids problem with OPTIONS method: https://github.com/lostisland/faraday/issues/305
+      response = nil
+      metrics_obj = { method: method, service: @name, path: path }
+      ActiveSupport::Notifications.instrument('request.grifter', metrics_obj) do
+        response = @conn.run_request(method, nil, nil, nil) do |req|
+          req.path = parsed.path
+          req.params = metrics_obj[:params] = query_hash if query_hash
+
+          req.headers = req_headers
+          req.body = body
+          req.options[:timeout] = options.fetch(:timeout, @default_timeout)
+        end
+        metrics_obj[:response] = response
       end
 
-      Log.debug "RESPONSE CODE: #{response.code}"
-      Log.debug "RESPONSE HEADERS: #{response.to_hash}"
-      Log.debug "RESPONSE BODY:\n#{jsonify response.body}\n"
-
-      @last_request = req
+      logger.info "Request status: (#{response.status}) #{@name}: #{method.to_s.upcase} #{path}"
+      #@last_request = req
       @last_response = response
 
-      raise RequestException.new(req, response) unless response.kind_of? Net::HTTPSuccess
+      raise RequestException.new(nil, response) unless response.status >= 200 and response.status < 300
+
+      response_obj = objectify response.body
+      if response.headers['content-type'] =~ /json/
+        logger.debug [
+          "Response Details:",
+          @log_headers ? ["Response Headers:",
+                          response.headers.map { |k, v| "#{k}: #{v.inspect}" }] : nil,
+          @log_bodies ? [ "Response Body:", jsonify(response_obj)] : nil,
+          ''
+        ].flatten.compact.join("\n")
+      end
+      return response_obj
+    end
 
-      objectify response.body
+    def in_parallel &blk
+      @conn.headers = @headers
+      @conn.in_parallel &blk
     end
 
     #add base uri to request
@@ -90,13 +152,15 @@ class Grifter
     end
 
     def make_headers options
-      if options[:additional_headers]
+      headers = if options[:additional_headers]
         @headers.merge options[:additional_headers]
       elsif options[:headers]
         options[:headers]
       else
-        @headers
+        @headers.clone
       end
+      headers['content-type'] = 'application/x-www-form-urlencoded' if options[:form]
+      headers
     end
 
     def req_args path, options
@@ -104,47 +168,38 @@ class Grifter
     end
 
     def get path, options={}
-      req = Net::HTTP::Get.new(*req_args(path, options))
-      do_request req, options
+      do_request :get, path, nil, options
     end
 
     def head path, options={}
-      req = Net::HTTP::Head.new(*req_args(path, options))
-      do_request req, options
+      do_request :head, path, nil, options
     end
 
     def options path, options={}
-      req = Net::HTTP::Options.new(*req_args(path, options))
-      do_request req, options
+      do_request :options, path, nil, options
     end
 
     def delete path, options={}
-      req = Net::HTTP::Delete.new(*req_args(path, options))
-      do_request req, options
+      do_request :delete, path, nil, options
     end
 
     def post path, obj, options={}
-      req = Net::HTTP::Post.new(*req_args(path, options))
-      req.body = jsonify(obj)
-      do_request req, options
+      do_request :post, path, obj, options
     end
 
     def put path, obj, options={}
-      req = Net::HTTP::Put.new(*req_args(path, options))
-      req.body = jsonify(obj)
-      do_request req, options
+      do_request :put, path, obj, options
     end
 
     def patch path, obj, options={}
-      req = Net::HTTP::Patch.new(*req_args(path, options))
-      req.body = jsonify(obj)
-      do_request req, options
+      do_request :patch, path, obj, options
     end
 
     def post_form path, params, options={}
-      request_obj = Net::HTTP::Post.new(*req_args(path, options))
-      request_obj.set_form_data params
-      do_request request_obj, options
+      do_request :post, path, params, options.merge(form: true)
+      #request_obj = Net::HTTP::Post.new(*req_args(path, options))
+      #request_obj.set_form_data params
+      #do_request request_obj, options
     end
   end
 
@@ -155,12 +210,12 @@ class Grifter
 
     #this makes good info show up in rspec reports
     def to_s
-      "#{self.class}\nResponseCode: #{@response.code}\nResponseBody:\n#{@response.body}"
+      "#{self.class}\nResponseCode: #{self.code}\nResponseBody:\n#{self.body}"
     end
 
     #shortcut methods
     def code
-      @response.code
+      @response.status
     end
 
     def body

data/lib/grifter/instrumentation.rb

@@ -0,0 +1,30 @@
+require 'active_support/notifications'
+class Grifter
+  module Instrumentation
+
+    Sample = Struct.new(:service_name, :method, :path, :status, :duration_ms, :end_time)
+
+    def start_instrumentation
+      @all_requests ||= []
+      ActiveSupport::Notifications.subscribe('request.grifter') do |name, start_time, end_time, _, data|
+        #do nothing if exception happened, else we might interfere with exception handling
+        unless data[:exception]
+          duration_ms = ((end_time.to_f - start_time.to_f) * 1000).to_i
+          #$stderr.puts '[%s] %s %s (%.3f s)' % [url.host, http_method, url.request_uri, duration]
+          @all_requests << Sample.new(
+            data[:service],
+            data[:method].intern,
+            data[:path],
+            data[:response].status.to_s.intern,
+            duration_ms,
+            end_time
+          )
+        end
+      end
+    end
+
+    def metrics_all_requests
+      @all_requests
+    end
+  end
+end

data/lib/grifter/json_helpers.rb

@@ -21,14 +21,14 @@ class Grifter
 
     #attempts to parse json strings into native ruby objects
     def objectify json_string
+      return nil if json_string.nil? or json_string==''
       case json_string
       when Hash, Array
         return json_string
       else
         JSON.parse(json_string.to_s)
       end
-    rescue Exception => e
-      Log.debug "Unable to parse non-json object: #{e.to_s}"
+    rescue Exception
       json_string
     end
 

data/lib/grifter/log.rb

@@ -1,34 +1,44 @@
 require 'logger'
 
 class Grifter
-  class Log
+  module Log
+    extend self
+
     GrifterFormatter = proc do |severity, datetime, progname, msg|
       "#{severity[0]}: [#{datetime.strftime('%m/%d/%y %H:%M:%S')}][#{progname}] - #{msg}\n"
     end
 
     @@loggers = []
-    def self.add_logger handle
+    def loggers
+      @@loggers
+    end
+
+    def add_logger handle
       new_logger = Logger.new handle
       new_logger.progname = 'grifter'
       new_logger.formatter = GrifterFormatter
       @@loggers << new_logger
     end
 
-    self.add_logger(STDOUT)
+    add_logger(STDOUT)
 
-    def self.level= log_level
+    def level= log_level
       @@loggers.each { |logger| logger.level = log_level}
     end
 
-    def self.log level, msg
+    def log level, msg
       @@loggers.each {|logger| logger.send(level, msg)}
     end
 
     [:fatal, :error, :warn, :info,:debug].each do |log_method|
-      define_singleton_method log_method do |msg|
-        log(log_method, msg)
+      define_method log_method do |msg|
+        self.log(log_method, msg)
       end
     end
+
+    def logger
+      self
+    end
   end
 end
 

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: grifter
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Robert Schultheis
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-11-05 00:00:00.000000000 Z
+date: 2013-12-04 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: faraday
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: json
   requirement: !ruby/object:Gem::Requirement
@@ -24,6 +38,20 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: typhoeus
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -99,6 +127,7 @@ files:
 - lib/grifter/configuration.rb
 - lib/grifter/helpers.rb
 - lib/grifter/http_service.rb
+- lib/grifter/instrumentation.rb
 - lib/grifter/json_helpers.rb
 - lib/grifter/log.rb
 homepage: http://github.com/knewton/grifter