faraday_json 0.1.0

data/lib/faraday_json/encoding.rb

@@ -0,0 +1,219 @@
+#
+# FaradayJSON
+# https://github.com/spriteCloud/faraday_json
+#
+# Copyright (c) 2015 spriteCloud B.V. and other FaradayJSON contributors.
+# All rights reserved.
+#
+
+module FaradayJSON
+
+# Character encoding helper functions
+module Encoding
+
+# Two versions of transcode, one for Ruby 1.8 and one for greater versions.
+
+if RUBY_VERSION.start_with?("1.8")
+
+  def transcode(data, input_charset, output_charset, opts = {})
+    # In Ruby 1.8, we pretty much have to believe the given charsets; there's
+    # not a lot of choice.
+
+    # If we don't have an input charset, we can't do better than US-ASCII.
+    if input_charset.nil? or input_charset.empty?
+      input_charset = opts.fetch('default_input_charset', 'us-ascii')
+    end
+
+    # The default output charset, on the other hand, should be UTF-8.
+    if output_charset.nil? or output_charset.empty?
+      output_charset = opts.fetch('default_output_charset', 'UTF-8//IGNORE')
+    end
+
+    # Transcode using iconv
+    require 'iconv'
+    return ::Iconv.conv(output_charset, input_charset, data)
+  end
+
+else # end ruby 1.8/start ruby > 1.8
+
+  def transcode(data, input_charset, output_charset, opts = {})
+    # Strings have an encode function in Ruby > 1.8
+    if not data.respond_to?(:encode)
+      return data
+    end
+
+    # If we don't have a charset, just use whatever is in the string
+    # currently. If we do have a charset, we'll have to run some extra
+    # checks.
+    if not (input_charset.nil? or input_charset.empty?)
+      # Check passed charset is *understood* by finding it. If this fails,
+      # an exception is raised, which it also should be.
+      canonical = ::Encoding.find(input_charset)
+
+      # Second, ensure the canonical charset and the actual string encoding
+      # are identical. If not, we'll have to do a little more than just
+      # transcode to UTF-8.
+      if canonical != data.encoding
+        if opts.fetch('force_input_charset', false)
+          data.force_encoding(canonical)
+        else
+          raise "Provided charset was #{canonical}, but data was #{data.encoding}"
+        end
+      end
+    end
+
+    # If there's no output charset, we should default to UTF-8.
+    if output_charset.nil? or output_charset.empty?
+      output_charset = opts.fetch('default_output_charset', 'UTF-8')
+    end
+
+    # Transcode!
+    return data.encode(output_charset)
+  end
+
+end # ruby > 1.8
+
+  # Convenient helper. Output is UTF-8. Input is either a string, or some data
+  # data. There's a Ruby 1.8 version mostly because it has to iteratively convert
+  # included strings.
+if RUBY_VERSION.start_with?("1.8")
+
+  def to_utf8(data, charset, opts = {})
+    if data.is_a? Hash
+      transcoded = {}
+      data.each do |key, value|
+        transcoded[to_utf8(key, charset, opts)] = to_utf8(value, charset, opts)
+      end
+      return transcoded
+    elsif data.is_a? Array
+      transcoded = []
+      data.each do |value|
+        transcoded << to_utf8(value, charest, opts)
+      end
+      return transcoded
+    elsif data.is_a? String
+      return transcode(data, charset, 'UTF-8//IGNORE', opts)
+    else
+      return data
+    end
+  end
+
+else # end ruby 1.8/start ruby > 1.8
+
+  def to_utf8(data, charset, opts = {})
+    return transcode(data, charset, 'UTF-8', opts)
+  end
+
+end # ruby > 1.8
+
+
+  # Helper function; strips a BOM for UTF-16 encodings
+  def strip_bom(data, charset, opts = {})
+    # Only need to do this on Strings
+    if not data.is_a? String
+      return data
+    end
+
+    # If the charset is given, it overrides string internal encoding.
+    enc = get_dominant_encoding(data, charset, opts)
+
+    # Make the encoding canonical (if we can find out about that).
+    canonical = get_canonical_encoding(enc)
+
+    # Determine what a BOM would look like.
+    bom = get_bom(canonical)
+
+    # We can't operate on data, we need a byte array.
+    arr = data.each_byte.to_a
+
+    # Match BOM
+    found = true
+    bom.each_index do |i|
+      if bom[i] != arr[i]
+        found = false
+        break
+      end
+    end
+
+    # So we may have found a BOM! Strip it.
+    if found
+      ret = arr[bom.length..-1].pack('c*')
+      if ret.respond_to? :force_encoding
+        ret.force_encoding(canonical)
+      end
+      return ret
+    end
+
+    # No BOM
+    return data
+  end
+
+  # Given a String with (potentially, this depends on Ruby version) an encoding,
+  # and a charset from a content-type header (which may be nil), determines the
+  # dominant encoding. (Charset, if given, overrides internal encoding,
+  # if present).
+  def get_dominant_encoding(str, charset, opts = {})
+    enc = nil
+    if str.respond_to? :encoding
+      enc = str.encoding
+    end
+
+    if charset.nil? or charset.empty?
+      if enc.nil?
+        default_encoding = opts.fetch('default_encoding', nil)
+        if default_encoding.nil?
+          raise "No charset provided, don't know what to do!" # FIXME
+        end
+        enc = default_encoding
+      end
+    else
+      enc = charset
+    end
+
+    return enc
+  end
+
+
+  # Returns a canonical version of an encoding.
+  def get_canonical_encoding(enc)
+    if defined? ::Encoding and ::Encoding.respond_to? :find
+      # Oh... Ruby 1.9.2 doesn't like passing an Encoding to find()...
+      if not enc.is_a? ::Encoding
+        enc = ::Encoding.find(enc)
+      end
+      return enc.to_s.downcase
+    end
+    return enc.downcase
+  end
+
+
+  # Given a (canonical) encoding, returns a BOM as an array of byte values. If
+  # the given encoding does not have a BOM, an empty array is returned.
+  def get_bom(enc)
+    bom = []
+    if enc.start_with?('utf16be') or enc.start_with?('utf-16be')
+      bom = [0xfe, 0xff]
+    elsif enc.start_with?('utf16le') or enc.start_with?('utf-16le')
+      bom = [0xff, 0xfe]
+    elsif enc.start_with?('utf8') or enc.start_with?('utf-8')
+      bom = [0xef, 0xbb, 0xbf]
+    elsif enc.start_with?('utf32be') or enc.start_with?('utf-32be')
+      bom = [0x00, 0x00, 0xfe, 0xff]
+    elsif enc.start_with?('utf32le') or enc.start_with?('utf-32le')
+      bom = [0xff, 0xfe, 0x00, 0x00]
+    end
+    return bom
+  end
+
+
+
+  # Helper function for testing
+  def bin_to_hex(data)
+    if data.respond_to? :each_byte
+      return data.each_byte.map { |b| b.to_s(16) }.join
+    end
+    return data
+  end
+
+end # module Encoding
+end # module FaradayJSON

data/lib/faraday_json/parse_json.rb

@@ -0,0 +1,165 @@
+#
+# FaradayJSON
+# https://github.com/spriteCloud/faraday_json
+#
+# Copyright (c) 2015 spriteCloud B.V. and other FaradayJSON contributors.
+# All rights reserved.
+#
+
+require 'faraday_json/encoding'
+
+module FaradayJSON
+  # Public: Parse response bodies as JSON.
+  class ParseJson < Faraday::Middleware
+    CONTENT_TYPE = 'Content-Type'.freeze
+
+    include ::FaradayJSON::Encoding
+
+    dependency do
+      require 'json' unless defined?(::JSON)
+    end
+
+    def initialize(app = nil, options = {})
+      super(app)
+      @options = options
+      @content_types = Array(options[:content_type])
+    end
+
+    def call(environment)
+      @app.call(environment).on_complete do |env|
+        if process_response_type?(response_type(env)) and parse_response?(env)
+          process_response(env)
+        end
+      end
+    end
+
+    def process_response(env)
+      env[:raw_body] = env[:body] if preserve_raw?(env)
+      body = env[:body]
+
+      # Body will be in an unknown encoding. Use charset field to coerce it to
+      # internal UTF-8.
+      charset = response_charset(env)
+
+      # We must ensure we're interpreting the body as the right charset. First,
+      # strip the BOM (if any).
+      body = strip_bom(body, charset, { 'default_encoding' => 'us-ascii' })
+
+      # Transcode to UTF-8
+      body = to_utf8(body, charset, { 'force_input_charset' => true })
+
+      # Now that's done, parse the JSON.
+      ret = nil
+      begin
+        ret = ::JSON.parse(body) unless body.strip.empty?
+      rescue StandardError, SyntaxError => err
+        raise err if err.is_a? SyntaxError and err.class.name != 'Psych::SyntaxError'
+        raise Faraday::Error::ParsingError, err
+      end
+      env[:body] = ret
+    end
+
+    def response_type(env)
+      type = env[:response_headers][CONTENT_TYPE].to_s
+      type = type.split(';', 2).first if type.index(';')
+      type
+    end
+
+    def response_charset(env)
+      header = env[:response_headers][CONTENT_TYPE].to_s
+      if header.index(';')
+        header.split(';').each do |part|
+          if part.index('charset=')
+            return part.split('charset=', 2).last
+          end
+        end
+      end
+      return nil
+    end
+
+    def process_response_type?(type)
+      @content_types.empty? or @content_types.any? { |pattern|
+        pattern.is_a?(Regexp) ? type =~ pattern : type == pattern
+      }
+    end
+
+    def parse_response?(env)
+      env[:body].respond_to? :to_str
+    end
+
+    def preserve_raw?(env)
+      env[:request].fetch(:preserve_raw, @options[:preserve_raw])
+    end
+
+
+
+    # DRAGONS
+    module OptionsExtension
+      attr_accessor :preserve_raw
+
+      def to_hash
+        super.update(:preserve_raw => preserve_raw)
+      end
+
+      def each
+        return to_enum(:each) unless block_given?
+        super
+        yield :preserve_raw, preserve_raw
+      end
+
+      def fetch(key, *args)
+        if :preserve_raw == key
+          value = __send__(key)
+          value.nil? ? args.fetch(0) : value
+        else
+          super
+        end
+      end
+    end
+
+    if defined?(Faraday::RequestOptions)
+      begin
+        Faraday::RequestOptions.from(:preserve_raw => true)
+      rescue NoMethodError
+        Faraday::RequestOptions.send(:include, OptionsExtension)
+      end
+    end
+  end # class ParseJson
+
+  # Public: Override the content-type of the response with "application/json"
+  # if the response body looks like it might be JSON, i.e. starts with an
+  # open bracket.
+  #
+  # This is to fix responses from certain API providers that insist on serving
+  # JSON with wrong MIME-types such as "text/javascript".
+  class ParseJsonMimeTypeFix < ParseJson
+    MIME_TYPE = 'application/json'.freeze
+
+    def process_response(env)
+      old_type = env[:response_headers][CONTENT_TYPE].to_s
+      new_type = MIME_TYPE.dup
+      new_type << ';' << old_type.split(';', 2).last if old_type.index(';')
+      env[:response_headers][CONTENT_TYPE] = new_type
+    end
+
+    BRACKETS = %w- [ { -
+    WHITESPACE = [ " ", "\n", "\r", "\t" ]
+
+    def parse_response?(env)
+      super and BRACKETS.include? first_char(env[:body])
+    end
+
+    def first_char(body)
+      idx = -1
+      begin
+        char = body[idx += 1]
+        char = char.chr if char
+      end while char and WHITESPACE.include? char
+      char
+    end
+  end # class ParseJson
+
+end
+
+# deprecated alias
+Faraday::Response::ParseJson = FaradayJSON::ParseJson

data/lib/faraday_json/version.rb

@@ -0,0 +1,10 @@
+#
+# FaradayJSON
+# https://github.com/spriteCloud/faraday_json
+#
+# Copyright (c) 2015 spriteCloud B.V. and other FaradayJSON contributors.
+# All rights reserved.
+#
+module FaradayJSON
+  VERSION = "0.1.0"
+end

data/spec/data/iso8859-15_file.json

@@ -0,0 +1 @@
+{"a":"Hell�, W�rld!"}

data/spec/data/utf8_file.json

@@ -0,0 +1 @@
+{"a":"Hellö, Wörld!"}

data/spec/encode_json_spec.rb

@@ -0,0 +1,376 @@
+# encoding: utf-8
+
+require 'helper'
+require 'faraday_json/encode_json'
+
+describe FaradayJSON::EncodeJson do
+  let(:middleware) { described_class.new(lambda{|env| env}) }
+
+  def process(body, content_type = nil)
+    env = {:body => body, :request_headers => Faraday::Utils::Headers.new}
+    env[:request_headers]['content-type'] = content_type if content_type
+    middleware.call(faraday_env(env))
+  end
+
+  def result_body() result[:body] end
+  def result_type() result[:request_headers]['content-type'] end
+  def result_length() result[:request_headers]['content-length'].to_i end
+
+  context "no body" do
+    let(:result) { process(nil) }
+
+    it "doesn't change body" do
+      expect(result_body).to be_nil
+    end
+
+    it "doesn't add content type" do
+      expect(result_type).to be_nil
+    end
+  end
+
+  context "empty body" do
+    let(:result) { process('') }
+
+    it "doesn't change body" do
+      expect(result_body).to be_empty
+    end
+
+    it "doesn't add content type" do
+      expect(result_type).to be_nil
+    end
+  end
+
+  context "string body" do
+    let(:result) { process('{"a":1}') }
+
+    it "doesn't change body" do
+      expect(result_body).to eq('{"a":1}')
+    end
+
+    it "adds content type" do
+      expect(result_type).to eq('application/json; charset=utf-8')
+    end
+  end
+
+  context "object body" do
+    let(:result) { process({:a => 1}) }
+
+    it "encodes body" do
+      expect(result_body).to eq('{"a":1}')
+    end
+
+    it "adds content type" do
+      expect(result_type).to eq('application/json; charset=utf-8')
+    end
+  end
+
+  context "empty object body" do
+    let(:result) { process({}) }
+
+    it "encodes body" do
+      expect(result_body).to eq('{}')
+    end
+  end
+
+  context "object body with json type" do
+    let(:result) { process({:a => 1}, 'application/json; charset=utf-8') }
+
+    it "encodes body" do
+      expect(result_body).to eq('{"a":1}')
+    end
+
+    it "doesn't change content type" do
+      expect(result_type).to eq('application/json; charset=utf-8')
+    end
+  end
+
+  context "object body with incompatible type" do
+    let(:result) { process({:a => 1}, 'application/xml; charset=utf-8') }
+
+    it "doesn't change body" do
+      expect(result_body).to eq({:a => 1})
+    end
+
+    it "doesn't change content type" do
+      expect(result_type).to eq('application/xml; charset=utf-8')
+    end
+  end
+
+  ### Unicode test cases
+  # Ruby 1.8 will almost certainly fail if there is no charset given in a header.
+  # In Ruby >1.8, we have some more methods for guessing well.
+
+  ### All Ruby versions should work with a charset given.
+  context "utf-8 in string body" do
+    let(:result) { process('{"a":"ä"}', 'application/json; charset=utf-8') }
+
+    it "doesn't change body" do
+      expect(result_body).to eq('{"a":"ä"}')
+    end
+
+    it "doesn't change content type" do
+      expect(result_type).to eq('application/json; charset=utf-8')
+    end
+
+    it "adds content length" do
+      expect(result_length).to eq(10)
+    end
+  end
+
+  context "utf-8 in object body" do
+    let(:result) { process({:a => "ä"}, 'application/json; charset=utf-8') }
+
+    it "encodes body" do
+      expect(result_body).to eq('{"a":"ä"}')
+    end
+
+    it "doesn't change content type" do
+      expect(result_type).to eq('application/json; charset=utf-8')
+    end
+
+    it "adds content length" do
+      expect(result_length).to eq(10)
+    end
+  end
+
+  context "non-unicode in string body" do
+    let(:result) {
+      process(test_encode('{"a":"ä"}', 'iso-8859-15'), 'application/json; charset=iso-8859-15')
+    }
+
+    it "changes body" do
+      expect(result_body).to eq('{"a":"ä"}')
+    end
+
+    it "changes content type" do
+      expect(result_type).to eq('application/json; charset=utf-8')
+    end
+
+    it "adds content length" do
+      expect(result_length).to eq(10)
+    end
+  end
+
+  context "non-unicode in object body" do
+    let(:result) {
+      process({:a => test_encode('ä', 'iso-8859-15')}, 'application/json; charset=iso-8859-15')
+    }
+
+    it "encodes body" do
+      expect(result_body).to eq('{"a":"ä"}')
+    end
+
+    it "changes content type" do
+      expect(result_type).to eq('application/json; charset=utf-8')
+    end
+
+    it "adds content length" do
+      expect(result_length).to eq(10)
+    end
+  end
+
+  context "non-utf-8 in string body" do
+    let(:result) {
+      process(test_encode('{"a":"ä"}', 'utf-16be'), 'application/json; charset=utf-16be')
+    }
+
+    it "changes body" do
+      expect(result_body).to eq('{"a":"ä"}')
+    end
+
+    it "changes content type" do
+      expect(result_type).to eq('application/json; charset=utf-8')
+    end
+
+    it "adds content length" do
+      expect(result_length).to eq(10)
+    end
+  end
+
+  context "non-utf-8 in object body" do
+    let(:result) {
+      process({:a => test_encode('ä', 'utf-16le')}, 'application/json; charset=utf-16le')
+    }
+
+    it "encodes body" do
+      expect(result_body).to eq('{"a":"ä"}')
+    end
+
+    it "changes content type" do
+      expect(result_type).to eq('application/json; charset=utf-8')
+    end
+
+    it "adds content length" do
+      expect(result_length).to eq(10)
+    end
+  end
+
+
+  ### Ruby versions > 1.8 should be able to guess missing charsets at times.
+  if not RUBY_VERSION.start_with?("1.8")
+    context "utf-8 in string body without content type" do
+      let(:result) { process('{"a":"ä"}') }
+
+      it "doesn't change body" do
+        expect(result_body).to eq('{"a":"ä"}')
+      end
+
+      it "adds content type" do
+        expect(result_type).to eq('application/json; charset=utf-8')
+      end
+
+      it "adds content length" do
+        expect(result_length).to eq(10)
+      end
+    end
+
+    context "utf-8 in object body without content type" do
+      let(:result) { process({:a => "ä"}) }
+
+      it "encodes body" do
+        expect(result_body).to eq('{"a":"ä"}')
+      end
+
+      it "adds content type" do
+        expect(result_type).to eq('application/json; charset=utf-8')
+      end
+
+      it "adds content length" do
+        expect(result_length).to eq(10)
+      end
+    end
+
+    context "non-unicode in string body without content type" do
+      let(:result) {
+        process(test_encode('{"a":"ä"}', 'iso-8859-15'))
+      }
+
+      it "doesn't change body" do
+        expect(result_body).to eq('{"a":"ä"}')
+      end
+
+      it "adds content type" do
+        expect(result_type).to eq('application/json; charset=utf-8')
+      end
+
+      it "adds content length" do
+        expect(result_length).to eq(10)
+      end
+    end
+
+    context "non-unicode in object body without content type" do
+      let(:result) {
+        process({:a => test_encode('ä', 'iso-8859-15')})
+      }
+
+      it "encodes body" do
+        expect(result_body).to eq('{"a":"ä"}')
+      end
+
+      it "adds content type" do
+        expect(result_type).to eq('application/json; charset=utf-8')
+      end
+
+      it "adds content length" do
+        expect(result_length).to eq(10)
+      end
+    end
+
+    context "non-utf-8 in string body without content type" do
+      let(:result) {
+        process(test_encode('{"a":"ä"}', 'utf-16be'))
+      }
+
+
+      it "doesn't change body" do
+        expect(result_body).to eq('{"a":"ä"}')
+      end
+
+      it "adds content type" do
+        expect(result_type).to eq('application/json; charset=utf-8')
+      end
+
+      it "adds content length" do
+        expect(result_length).to eq(10)
+      end
+    end
+
+    context "non-utf-8 in object body without content type" do
+      let(:result) {
+        process({:a => test_encode('ä', 'utf-16le')})
+      }
+
+      it "encodes body" do
+        expect(result_body).to eq('{"a":"ä"}')
+      end
+
+      it "adds content type" do
+        expect(result_type).to eq('application/json; charset=utf-8')
+      end
+
+      it "adds content length" do
+        expect(result_length).to eq(10)
+      end
+    end
+
+    ### Dealing with files in various encoding should ideally be easy
+    FILES = {
+      'spec/data/iso8859-15_file.json' => 'iso-8859-15',
+      'spec/data/utf16be_file.json' => 'utf-16be',
+      'spec/data/utf16le_file.json' => 'utf-16le',
+      'spec/data/utf8_file.json' => 'utf-8',
+    }
+
+
+    FILES.each do |fname, enc|
+      context "reading #{enc} encoded file '#{fname}'" do
+        # Read the string from file; read binary/with encoding. Ruby 1.8 will
+        # ignore this, but must still work.
+        data = File.new(fname, "rb:#{enc}").read
+
+        # Passing that data with a charset should do the right thing.
+        let(:result) {
+          process(data)
+        }
+
+        it "encodes body" do
+          expect(result_body).to eq("{\"a\":\"Hellö, Wörld!\"}\n")
+        end
+
+        it "adds content type" do
+          expect(result_type).to eq('application/json; charset=utf-8')
+        end
+
+        it "adds content length" do
+          expect(result_length).to eq(24)
+        end
+      end
+    end
+
+    FILES.each do |fname, enc|
+      context "reading #{enc} encoded file '#{fname}' as binary" do
+        # Read the string from file; read binary/with encoding. Ruby 1.8 will
+        # ignore this, but must still work.
+        data = File.new(fname, "rb").read
+
+        # Passing that data with a charset should do the right thing.
+        let(:result) {
+          process(data, "application/json; charset=#{enc}")
+        }
+
+        it "encodes body" do
+          expect(result_body).to eq("{\"a\":\"Hellö, Wörld!\"}\n")
+        end
+
+        it "adds content type" do
+          expect(result_type).to eq('application/json; charset=utf-8')
+        end
+
+        it "adds content length" do
+          expect(result_length).to eq(24)
+        end
+      end
+    end
+
+  end
+end