twingly-http 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: c96f42ded2e94d947bd2a15cc7de75669e3f2dc06a17ff7d648eb3c0cf862c5e
+  data.tar.gz: 5203e3833efe611708b1b185708735b2c6a027d20d8edd0e79c80dcca696d73a
+SHA512:
+  metadata.gz: 6869f39208327e4b834f999e96a1a0d7253768022aca54f259c1f0cf57bf60331e4aabc966919ad0a7bef08e3091ff7090c9e576986f7d4f5336894ac47b250b
+  data.tar.gz: 4cf992e7c60229b8fcbfd3ab62b0584e8531b229fd8a863d8ab1a47b248c5c2697eb225beecb6ec26377bce50009bf6ba295e1fee45622951be2809ffb5b4640

data/LICENSE

@@ -0,0 +1,20 @@
+The MIT License (MIT)
+
+Copyright (c) 2019 Twingly AB
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,23 @@
+# Twingly::HTTP
+
+Robust HTTP client
+
+
+## Release workflow
+
+* Bump the version in `lib/twingly/version.rb` in a commit, no need to push (the release task does that).
+
+* Ensure you are signed in to RubyGems.org as [twingly][twingly-rubygems] with `gem signin`.
+
+* Build and [publish](http://guides.rubygems.org/publishing/) the gem. This will create the proper tag in git, push the commit and tag and upload to RubyGems.
+
+        bundle exec rake release
+
+* Update the changelog with [GitHub Changelog Generator](https://github.com/skywinder/github-changelog-generator/) (`gem install github_changelog_generator` if you don't have it, set `CHANGELOG_GITHUB_TOKEN` to a personal access token to avoid rate limiting by GitHub). This command will update `CHANGELOG.md`. You need to commit and push manually.
+
+        github_changelog_generator
+
+[twingly-rubygems]: https://rubygems.org/profiles/twingly
+[ruby-prof]: http://ruby-prof.rubyforge.org/
+[memory_profiler]: https://github.com/SamSaffron/memory_profiler
+[examples]: examples/url.rb

data/lib/faraday/logfmt_logger.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require_relative "logfmt_logger/middleware"
+
+module Faraday
+  module LogfmtLogger
+  end
+end

data/lib/faraday/logfmt_logger/LICENSE.md

@@ -0,0 +1,20 @@
+Copyright (c) 2009-2015 Rick Olson, Zack Hobson
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/lib/faraday/logfmt_logger/middleware.rb

@@ -0,0 +1,130 @@
+# frozen_string_literal: true
+
+require "forwardable"
+
+module Faraday
+  module LogfmtLogger
+    class Middleware < Response::Middleware
+      extend Forwardable
+
+      DEFAULT_OPTIONS = { headers: true, bodies: false }.freeze
+
+      def initialize(app, logger = nil, options = {})
+        super(app)
+        @logger = logger || begin
+          require "logger"
+          ::Logger.new(STDOUT)
+        end
+
+        @filter = []
+        @options = DEFAULT_OPTIONS.merge(options)
+        yield self if block_given?
+      end
+
+      def_delegators :@logger, :debug, :info, :warn, :error, :fatal
+
+      def call(env)
+        info("request") do
+          log_entry = {
+            "source": "upstream-request",
+            "method": env.method.upcase,
+            "url": apply_filters(env.url.to_s),
+            "request_id": request_id,
+          }.merge(app_metadata)
+
+          Twingly::StringUtilities.logfmt(log_entry)
+        end
+        debug_log_request(env)
+        super
+      end
+
+      def on_complete(env)
+        info("response") do
+          log_entry = {
+            "source": "upstream-response",
+            "status": env.status,
+            "request_id": request_id,
+          }.merge(app_metadata)
+
+          Twingly::StringUtilities.logfmt(log_entry)
+        end
+        debug_log_response(env)
+      end
+
+      # Disable Rubocop here to keep the code look the same where it came from:
+      # https://github.com/lostisland/faraday/blob/v0.11.0/lib/faraday/response/logger.rb
+
+      # rubocop:disable all
+      def filter(filter_word, filter_replacement)
+        @filter.push([ filter_word, filter_replacement ])
+      end
+      # rubocop:enable all
+
+      private
+
+      def request_id
+        @options[:request_id]
+      end
+
+      def app_metadata
+        {
+          "release": Twingly::HTTP::Heroku.release_version,
+        }
+      end
+
+      # rubocop:disable all
+      def debug_log_request(env)
+        debug('request') { apply_filters( dump_headers env.request_headers ) } if log_headers?(:request)
+        debug('request') { apply_filters( dump_body(env[:body]) ) } if env[:body] && log_body?(:request)
+      end
+
+      def debug_log_response(env)
+        debug('response') { apply_filters( dump_headers env.response_headers ) } if log_headers?(:response)
+        debug('response') { apply_filters( dump_body env[:body] ) } if env[:body] && log_body?(:response)
+      end
+
+      def dump_headers(headers)
+        headers.map { |k, v| "#{k}: #{v.inspect}" }.join("\n")
+      end
+
+      def dump_body(body)
+        if body.respond_to?(:to_str)
+          body.to_str
+        else
+          pretty_inspect(body)
+        end
+      end
+
+      def pretty_inspect(body)
+        require 'pp' unless body.respond_to?(:pretty_inspect)
+        body.pretty_inspect
+      end
+
+      def log_headers?(type)
+        case @options[:headers]
+        when Hash then @options[:headers][type]
+        else @options[:headers]
+        end
+      end
+
+      def log_body?(type)
+        case @options[:bodies]
+        when Hash then @options[:bodies][type]
+        else @options[:bodies]
+        end
+      end
+
+      def apply_filters(output)
+        @filter.each do |pattern, replacement|
+          output = output.to_s.gsub(pattern, replacement)
+        end
+        output
+      end
+      # rubocop:enable all
+    end
+  end
+end
+
+Faraday::Response.register_middleware(
+  logfmt_logger: Faraday::LogfmtLogger::Middleware
+)

data/lib/faraday/url_size_limit.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require_relative "url_size_limit/middleware"
+
+module Faraday
+  module UrlSizeLimit
+  end
+end

data/lib/faraday/url_size_limit/middleware.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Faraday
+  module UrlSizeLimit
+    class LimitExceededError < StandardError; end
+
+    class Middleware < Faraday::Middleware
+      def initialize(app, max_size_bytes:)
+        super(app)
+
+        @max_size_bytes = max_size_bytes
+      end
+
+      def call(env)
+        url_bytesize = env.url.to_s.bytesize
+
+        if url_bytesize >= @max_size_bytes
+          raise LimitExceededError,
+                "Expected URL below #{@max_size_bytes} bytes, "\
+                "was #{url_bytesize} bytes"
+        end
+
+        @app.call(env)
+      end
+    end
+  end
+end
+
+Faraday::Request.register_middleware(
+  url_size_limit: Faraday::UrlSizeLimit::Middleware
+)

data/lib/twingly/heroku.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Twingly
+  module HTTP
+    class Heroku
+      def self.app_name
+        ENV.fetch("HEROKU_APP_NAME") { "unknown_heroku_app_name" }
+      end
+
+      def self.dyno_id
+        ENV.fetch("HEROKU_DYNO_ID") { "unknown_heroku_dyno_id" }
+      end
+
+      def self.slug_commit
+        ENV.fetch("HEROKU_SLUG_COMMIT") { "unknown_heroku_slug_commit" }
+      end
+
+      def self.release_version
+        ENV.fetch("HEROKU_RELEASE_VERSION") { "unknown_heroku_release_version" }
+      end
+
+      def self.review_app?
+        parent_name = ENV.fetch("HEROKU_PARENT_APP_NAME") {}
+
+        return false unless parent_name
+
+        app_name               = ENV.fetch("HEROKU_APP_NAME") { "" }
+        review_app_name_format = /\A#{parent_name}-pr-\d+\z/
+
+        review_app_name_format.match?(app_name)
+      end
+    end
+  end
+end

data/lib/twingly/http.rb

@@ -0,0 +1,200 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "faraday"
+
+require_relative "../faraday/logfmt_logger"
+require_relative "../faraday/url_size_limit"
+require_relative "heroku"
+require_relative "string_utilities"
+
+module Twingly
+  module HTTP
+    class ConnectionError < StandardError; end
+    class UrlSizeLimitExceededError < StandardError; end
+    class Client # rubocop:disable Metrics/ClassLength
+      DEFAULT_RETRYABLE_EXCEPTIONS = [
+        Faraday::ConnectionFailed,
+        Faraday::SSLError,
+        Zlib::BufError,
+        Zlib::DataError,
+      ].freeze
+      TIMEOUT_EXCEPTIONS = [
+        Faraday::TimeoutError,
+        Net::OpenTimeout,
+      ].freeze
+      DEFAULT_HTTP_TIMEOUT = 20
+      DEFAULT_HTTP_OPEN_TIMEOUT = 10
+      DEFAULT_NUMBER_OF_RETRIES = 0
+      DEFAULT_RETRY_INTERVAL = 1
+      DEFAULT_MAX_URL_SIZE_BYTES = Float::INFINITY
+
+      attr_writer :http_timeout
+      attr_writer :http_open_timeout
+      attr_writer :number_of_retries
+      attr_writer :retry_interval
+      attr_writer :on_retry_callback
+      attr_writer :max_url_size_bytes
+      attr_writer :request_id
+
+      attr_accessor :retryable_exceptions
+
+      def initialize(logger:, base_user_agent:)
+        @logger          = logger
+        @base_user_agent = base_user_agent
+        @request_id      = nil
+
+        @http_timeout      = DEFAULT_HTTP_TIMEOUT
+        @http_open_timeout = DEFAULT_HTTP_OPEN_TIMEOUT
+
+        @retryable_exceptions = DEFAULT_RETRYABLE_EXCEPTIONS
+        @number_of_retries    = DEFAULT_NUMBER_OF_RETRIES
+        @retry_interval       = DEFAULT_RETRY_INTERVAL
+        @on_retry_callback    = nil
+
+        @max_url_size_bytes = DEFAULT_MAX_URL_SIZE_BYTES
+      end
+
+      def get(url, params: {})
+        http_response_for(:get, url: url, params: params)
+      end
+
+      def post(url, body:, headers: {})
+        http_response_for(:post, url: url, body: body, headers: headers)
+      end
+
+      private
+
+      # rubocop:disable Metrics/MethodLength
+      def http_response_for(method, *args)
+        response = case method
+                   when :get
+                     http_get_response(*args)
+                   when :post
+                     http_post_response(*args)
+                   end
+
+        Response.new(headers: response.headers.to_h,
+                     status: response.status,
+                     body: response.body)
+      rescue *(@retryable_exceptions + TIMEOUT_EXCEPTIONS)
+        raise ConnectionError
+      rescue Faraday::UrlSizeLimit::LimitExceededError => error
+        raise UrlSizeLimitExceededError, error.message
+      end
+      # rubocop:enable all
+
+      def http_get_response(url:, params: {})
+        binary_url = url.dup.force_encoding(Encoding::BINARY)
+        http_client = create_http_client
+
+        headers = default_headers
+
+        http_client.get do |request|
+          request.url(binary_url)
+          request.params.merge!(params)
+          request.headers.merge!(headers)
+          request.options.timeout = @http_timeout
+          request.options.open_timeout = @http_open_timeout
+        end
+      end
+
+      def http_post_response(url:, body:, headers:)
+        binary_url = url.dup.force_encoding(Encoding::BINARY)
+        http_client = create_http_client
+
+        headers = default_headers.merge(headers)
+
+        http_client.post do |request|
+          request.url(binary_url)
+          request.headers.merge!(headers)
+          request.body = body
+          request.options.timeout = @http_timeout
+          request.options.open_timeout = @http_open_timeout
+        end
+      end
+
+      def create_http_client # rubocop:disable Metrics/MethodLength
+        Faraday.new do |faraday|
+          faraday.request :url_size_limit,
+                          max_size_bytes: @max_url_size_bytes
+          faraday.request :retry,
+                          max: @number_of_retries,
+                          interval: @retry_interval,
+                          exceptions: @retryable_exceptions,
+                          methods: [], # empty [] forces Faraday to run retry_if
+                          retry_if: retry_if
+          faraday.response :logfmt_logger, @logger.dup,
+                           headers: true,
+                           bodies: true,
+                           request_id: @request_id
+          faraday.adapter Faraday.default_adapter
+          faraday.headers[:user_agent] = user_agent
+        end
+      end
+
+      def retry_if
+        lambda do |env, exception|
+          unwrapped_exception = unwrap_exception(exception)
+
+          # we do not retry on timeouts due to our request time budget
+          if timeout_error?(unwrapped_exception)
+            false
+          else
+            @on_retry_callback&.call(env, unwrapped_exception)
+            true
+          end
+        end
+      end
+
+      def unwrap_exception(exception)
+        if exception.respond_to?(:wrapped_exception)
+          exception.wrapped_exception
+        else
+          exception
+        end
+      end
+
+      def timeout_error?(error)
+        TIMEOUT_EXCEPTIONS.include?(error.class)
+      end
+
+      def user_agent
+        format(
+          "%<base>s (Release/%<release>s; Commit/%<commit>s)",
+          base: @base_user_agent,
+          release: Heroku.release_version,
+          commit: Heroku.slug_commit
+        )
+      end
+
+      def app_metadata
+        {
+          "dyno_id": Heroku.dyno_id,
+          "release": Heroku.release_version,
+          "git_head": Heroku.slug_commit,
+        }
+      end
+
+      def default_headers
+        {
+          "X-Request-Id": @request_id,
+        }.delete_if { |_name, value| value.to_s.strip.empty? }
+      end
+    end
+
+    class Response
+      attr_reader :headers
+      attr_reader :status
+      attr_reader :body
+
+      def initialize(headers: nil,
+                     status: nil,
+                     body: nil)
+        @headers       = headers
+        @status        = status
+        @body          = body
+      end
+    end
+  end
+end

data/lib/twingly/string_utilities.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Twingly
+  module StringUtilities
+    module_function
+
+    def logfmt(hsh)
+      hsh.map { |key, value| "#{key}=#{value}" }.join(" ")
+    end
+
+    def strip_start_character(string, char:)
+      return unless string
+
+      if string[0] == char
+        string[1..-1]
+      else
+        string
+      end
+    end
+
+    def split_key_pair(key_pair)
+      key_pair.to_s.split(":")
+    end
+  end
+end

data/lib/twingly/version.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Twingly
+  module HTTP
+    VERSION = "0.1.0"
+  end
+end

metadata

@@ -0,0 +1,180 @@
+--- !ruby/object:Gem::Specification
+name: twingly-http
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Twingly AB
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2019-11-07 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: faraday
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.15'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.15'
+- !ruby/object:Gem::Dependency
+  name: climate_control
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.76'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.76'
+- !ruby/object:Gem::Dependency
+  name: rubocop-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.36'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.36'
+- !ruby/object:Gem::Dependency
+  name: toxiproxy
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: vcr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.7'
+description: Robust HTTP client tailored by Twingly
+email:
+- support@twingly.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE
+- README.md
+- lib/faraday/logfmt_logger.rb
+- lib/faraday/logfmt_logger/LICENSE.md
+- lib/faraday/logfmt_logger/middleware.rb
+- lib/faraday/url_size_limit.rb
+- lib/faraday/url_size_limit/middleware.rb
+- lib/twingly/heroku.rb
+- lib/twingly/http.rb
+- lib/twingly/string_utilities.rb
+- lib/twingly/version.rb
+homepage: http://github.com/twingly/twingly-http
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - "~>"
+    - !ruby/object:Gem::Version
+      version: '2.5'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.6
+signing_key: 
+specification_version: 4
+summary: Robust HTTP client
+test_files: []