experian_consumer_view 1.0.0

data/Rakefile

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+task default: :spec

data/bin/console

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'experian_consumer_view'
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require 'irb'
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/experian_consumer_view.gemspec

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require_relative 'lib/experian_consumer_view/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'experian_consumer_view'
+  spec.version       = ExperianConsumerView::VERSION
+  spec.authors       = ['Andrew Sibley']
+  spec.email         = ['andrew.s@38degrees.org.uk']
+  spec.license       = 'MIT'
+  spec.homepage      = 'https://github.com/38degrees/experian_consumer_view'
+  spec.summary       = "Ruby wrapper for Experian's ConsumerView API."
+  spec.description   = "
+    Experian's ConsumerView API is a commercially licensed API which allows you
+    to obtain various demographic data on UK consumers at the postcode, household,
+    and individual level. This gem provides a simple Ruby wrapper to use the API.
+  "
+
+  spec.required_ruby_version = Gem::Requirement.new('>= 2.3.0')
+
+  # spec.metadata['allowed_push_host'] = "TODO: Set to 'http://mygemserver.com'"
+  # spec.metadata["homepage_uri"] = spec.homepage
+  # spec.metadata["source_code_uri"] = "TODO: Put your gem's public repo URL here."
+  # spec.metadata["changelog_uri"] = "TODO: Put your gem's CHANGELOG.md URL here."
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files = Dir.chdir(File.expand_path(__dir__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.bindir        = 'exe'
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'activesupport', '>= 5.2'
+  spec.add_dependency 'faraday', '~> 1.0'
+
+  spec.add_development_dependency 'bundler', '~> 2.1'
+  spec.add_development_dependency 'codecov', '>= 0.2'
+  spec.add_development_dependency 'pry', '>= 0.12'
+  spec.add_development_dependency 'rake', '>= 12.0'
+  spec.add_development_dependency 'rspec', '~> 3.0'
+  spec.add_development_dependency 'rubocop', '~> 0.91.0'
+  spec.add_development_dependency 'webmock', '~> 3.9'
+  spec.add_development_dependency 'yard', '~> 0.9.25'
+end

data/lib/experian_consumer_view.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+Dir[File.join(File.dirname(__FILE__), 'experian_consumer_view', '**', '*.rb')].sort.each { |file| require file }

data/lib/experian_consumer_view/api.rb

@@ -0,0 +1,158 @@
+# frozen_string_literal: true
+
+require_relative 'errors'
+
+require 'faraday'
+require 'json'
+
+module ExperianConsumerView
+  # Low-level class for accessing the Experian ConsumerView API. It is not recommended to use this class directly.
+  # The +ExperianConsumerView::Client+ class is designed to be directly used by applications.
+  #
+  # This class provides low-level access to make specific HTTP calls to the ConsumerView API, such as logging in to get
+  # an authorisation token, and performing lookups of an individual / household / postcode.
+  class Api
+    include ExperianConsumerView::Errors
+
+    PRODUCTION_URL = 'https://neartime.experian.co.uk'
+    STAGING_URL = 'https://stg.neartime.experian.co.uk'
+
+    LOGIN_PATH = '/overture/login'
+    SINGLE_LOOKUP_PATH = '/overture/lookup'
+    BATCH_LOOKUP_PATH = '/overture/batch'
+
+    # @param url [String] optional base URL for the API wrapper to connect to. Defaults to the Experian ConsumerView
+    #   production server.
+    def initialize(url: nil)
+      @httpclient = Faraday.new(
+        url: url || PRODUCTION_URL,
+        headers: { 'Content-Type' => 'application/json', 'Accept' => 'application/json' }
+      )
+    end
+
+    # Logs in to the Experian ConsumerView API, and gets an authorization token.
+    #
+    # @param user_id [String] the username / email used to authorize use of the ConsumerView API
+    # @param password [String] the password used to authorize use of the ConsumerView API
+    def get_auth_token(user_id:, password:)
+      query_params = { 'userid' => user_id, 'password' => password }
+
+      result = @httpclient.post(LOGIN_PATH, query_params.to_json)
+      check_http_result_status(result)
+
+      JSON.parse(result.body)['token']
+    end
+
+    # Looks up demographic data for a single individual / household / postcode.
+    #
+    # Note that the demographic / propensity keys returned will only be those which the given client & asset have access
+    # to. Refer to the Experian ConsumerView API Documentation for exact details of the keys & possible values.
+    #
+    # @param user_id [String] the username / email used to authorize use of the ConsumerView API
+    # @param token [String] the time-limited authorization token provided when logging into the API
+    # @param client_id [String] your 5-digit Experian client ID
+    # @param asset_id [String] your 6-character Experian asset ID
+    # @param search_keys [Hash] hash containing the keys required to look up an individual / household / postcode.
+    #   Refer to the Experian ConsumerView API Documentation for exact details on the required keys.
+    #
+    # @return [Hash] a hash containing a key/value pair for each demographic / propensity for the individual / household
+    #   / postcode which was successfully looked up. Returns an empty hash if the lookup does not find any matches.
+    def single_lookup(user_id:, token:, client_id:, asset_id:, search_keys:)
+      # TODO: Delete this if looking up a single item via the batch method isn't any slower - no point supporting both!
+
+      query_params = {
+        'ssoId' => user_id,
+        'token' => token,
+        'clientId' => client_id,
+        'assetId' => asset_id
+      }
+      query_params.merge!(search_keys)
+
+      result = @httpclient.post(SINGLE_LOOKUP_PATH, query_params.to_json)
+      check_http_result_status(result)
+
+      JSON.parse(result.body)
+    end
+
+    # Looks up demographic data for a batch of individuals / households / postcodes.
+    #
+    # Note that the demographic / propensity keys returned will only be those which the given client & asset have access
+    # to. Refer to the Experian ConsumerView API Documentation for exact details of the keys & possible values.
+    #
+    # @param user_id [String] the username / email used to authorize use of the ConsumerView API
+    # @param token [String] the time-limited authorization token provided when logging into the API
+    # @param client_id [String] your 5-digit Experian client ID
+    # @param asset_id [String] your 6-character Experian asset ID
+    # @param search_keys [Array<Hash>] an array of hashes, each hash containing the keys required to look up an
+    #   individual / household / postcode. Refer to the Experian ConsumerView API Documentation for exact details on the
+    #   required keys.
+    #
+    # @return [Array<Hash>] an array of hashes, each hash containing a key/value pair for each demographic / propensity
+    #   for the individual / household / postcode which was successfully looked up. Returns an empty hash for any items
+    #   in the batch where no matches were found. The order of the results array is the same as the order of the
+    #   supplied search array - ie. element 0 of the results array contains the hash of demographic data for the
+    #   individual / household / postcode supplied in position 0 of the batch of search keys.
+    def batch_lookup(user_id:, token:, client_id:, asset_id:, batched_search_keys:)
+      raise ApiBatchTooBigError if batched_search_keys.length > ExperianConsumerView::MAX_LOOKUP_BATCH_SIZE
+
+      query_params = {
+        'ssoId' => user_id,
+        'token' => token,
+        'clientId' => client_id,
+        'assetId' => asset_id,
+        'batch' => batched_search_keys
+      }
+
+      result = @httpclient.post(BATCH_LOOKUP_PATH, query_params.to_json)
+      check_http_result_status(result)
+
+      JSON.parse(result.body)
+    end
+
+    private
+
+    # Helper to check the result, and throw an appropriate error if something went wrong
+    def check_http_result_status(result)
+      return if result.status == 200
+
+      # An error occurred - attempt to extract the response string from the body if we can
+      response = get_response(result)
+
+      case result.status
+      when 401
+        raise ApiBadCredentialsError.new(result.status, response)
+      when 404
+        raise ApiEndpointNotFoundError.new(result.status, response)
+      when 417
+        raise ApiIncorrectJsonError.new(result.status, response)
+      when 500
+        raise ApiServerError.new(result.status, response)
+      when 503
+        raise ApiServerRefreshingError(result.status, response) if response == 'Internal refresh in progress'
+
+        raise ApiServerError.new(result.status, response)
+      when 515
+        raise ApiHttpVersionNotSupportedError.new(result.status, response)
+      else
+        raise ApiUnhandledHttpError.new(result.status, response)
+      end
+    end
+
+    def get_response(result)
+      # TODO: Temp debugging for convenience
+      # puts result.body.class.to_s
+      # puts result.body.to_s
+
+      # TODO: Is this complex handling necessary? Check if all types of error are consistent in the body they return...
+      if result.body&.is_a?(Hash)
+        result.body['response']
+      elsif result.body&.is_a?(String)
+        JSON.parse(result.body)['response']
+      else
+        ''
+      end
+    rescue JSON::ParserError
+      ''
+    end
+  end
+end

data/lib/experian_consumer_view/client.rb

@@ -0,0 +1,152 @@
+# frozen_string_literal: true
+
+require_relative 'errors'
+require_relative 'transformers/result_transformer'
+
+require 'active_support'
+require 'active_support/cache'
+
+module ExperianConsumerView
+  # Top-level wrapper for accessing the ExperianConsumerView API. Once an instance is created with the appropriate
+  # credentials, the +lookup+ method provides the ability to lookup individuals, households, or postcodes in the
+  # ConsumerView API and return all the data your account has access to.
+  #
+  # This class automatically handles logging in to the ConsumerView API, obtaining an authorisation token (which is
+  # valid for approximately 30 minutes), and then looking up the data. The authorisation token is cached so that it's
+  # not necessary to login again for every single lookup request.
+  #
+  # Note that by default the authorisation is cached in-memory using +ActiveSupport::Cache::MemoryStore+. This is
+  # suitable for single-server applications, but is unlikely to be suitable for distributed applications, or those
+  # hosted on cloud infrastructure. A distributed cache, such as +ActiveSupport::Cache::RedisCacheStore+ or
+  # +ActiveSupport::Cache::MemCacheStore+ is recommended for distributed or cloud-hosted applications.
+  #
+  # If an in-memory data-store were used in distributed or cloud-hosted applications, then the multiple servers will be
+  # unaware of each others tokens, and therefore each server would login to the ConsumerView API independently, even if
+  # another server already had a valid token. Logging in to the ConsumerView API multiple times with the same
+  # credentials will revoke prior tokens, meaning other servers will find their cached tokens are invalid the next time
+  # they try a lookup. This will likely lead to a situation where many lookup attempts fail the first time due to the
+  # server in question not having the most up-to-date token.
+  class Client
+    include ExperianConsumerView::Errors
+
+    CACHE_KEY = 'ExperianConsumerView::Client::CachedToken'
+
+    attr_writer :result_transformer
+
+    # @param user_id [String] the username / email used to authorize use of the ConsumerView API
+    # @param password [String] the password used to authorize use of the ConsumerView API
+    # @param client_id [String] your 5-digit Experian client ID
+    # @param asset_id [String] your 6-character Experian asset ID
+    # @param options [Hash] a hash of advanced options for configuring the client
+    #
+    # @option options [ActiveSupport::Cache] :token_cache optional cache to store login tokens. If no cache is provided,
+    #   a default in-memory cache is used, however such a cache is not suitable for distributed or cloud environments,
+    #   and will likely result in frequently invalidating the Experian ConsumerView authorization token.
+    # @option options [#transform] :result_transformer optional object whose +transform+ method accepts a hash
+    #   containing the results returned by the ConsumerView API for a single individual, household or postcode, and
+    #   transforms this hash into the desired output. By default, an instance of +ResultTransformer+ is used, which will
+    #   transform some common attributes returned by the ConsumerView API into hashes with richer details than returned
+    #   by the raw API.
+    # @option options [String] :api_base_url optional base URL to make ConsumerView API calls against. By default, uses
+    #   the Experian production ConsumerView server.
+    def initialize(user_id:, password:, client_id:, asset_id:, options: {})
+      @user_id = user_id
+      @password = password
+      @client_id = client_id
+      @asset_id = asset_id
+
+      @token_cache = options[:token_cache] || default_token_cache
+      @result_transformer = options[:result_transformer] || default_result_transformer
+      @api = ExperianConsumerView::Api.new(url: options[:api_base_url])
+    end
+
+    # Looks up 1 or more search items in the ConsumerView API.
+    #
+    # Note that the demographic / propensity keys returned will only be those which the client & asset have access to.
+    # Refer to the Experian ConsumerView API Documentation for exact details of the keys & possible values.
+    #
+    # @param search_items [Hash] a hash of identifiers to search keys for an individual / household / postcode as
+    #   required by the ConsumerView API. Eg.
+    #   <tt>{ "PersonA" => { "email" => "person.a@example.com" }, "Postcode1" => { "postcode" => "SW1A 1AA" } }</tt>.
+    #   Note that the top-level key is not passed to the ConsumerView API, it is just used for convenience when
+    #   returning results.
+    # @param auto_retries [Integer] optional number of times the lookup should be retried if a transient / potentially
+    #   recoverable error occurs. Defaults to 1.
+    #
+    # @returns [Hash] a hash of identifiers to the results returned by the ConsumerView API. Eg.
+    #   <tt>
+    #   {
+    #      "PersonA" => { "pc_mosaic_uk_7_group":"G", "Match":"P" } ,
+    #      "Postcode1" => { "pc_mosaic_uk_7_group":"G", "Match":"PC" }
+    #   }
+    #   </tt>
+    def lookup(search_items:, auto_retries: 1)
+      ordered_identifiers = search_items.keys
+      ordered_terms = search_items.values
+
+      token = auth_token
+      attempts = 0
+      begin
+        ordered_results = @api.batch_lookup(
+          user_id: @user_id,
+          token: token,
+          client_id: @client_id,
+          asset_id: @asset_id,
+          batched_search_keys: ordered_terms
+        )
+      rescue ApiBadCredentialsError, ApiServerRefreshingError => e
+        # Bad Credentials can sometimes be caused by race conditions - eg. one thread / server updating the cached
+        # token while another is querying with the old token. Retrying once should avoid the client throwing
+        # unnecessary errors to the calling code.
+        # Experian docs also recommend retrying when a server refresh is in progress, and if that fails, retrying again
+        # in approximately 10 minutes.
+        raise e unless attempts < auto_retries
+
+        token = auth_token(force_lookup: true)
+        attempts += 1
+        retry
+      end
+
+      results_hash(identifiers: ordered_identifiers, results: ordered_results)
+    end
+
+    private
+
+    def auth_token(force_lookup: false)
+      # ConsumerView auth tokens last for 30 minutes before expiring & becoming invalid.
+      # After 29 minutes, the cache entry will expire, and the first process to find the expired entry will refresh it,
+      # while allowing other processes to use the existing value for another 10s. This should alleviate race conditions,
+      # but will not eliminate them entirely. Note that in a distributed / cloud / multi-server environment, a shared
+      # cache MUST be used. An in-memory store would mean multiple instances logging to the ConsumerView API, and each
+      # login will change the active token, which other servers will not see, leading to frequent authorisation
+      # failures.
+      @token_cache.fetch(
+        CACHE_KEY, expires_in: 29.minutes, race_condition_ttl: 10.seconds, force: force_lookup
+      ) do
+        @api.get_auth_token(user_id: @user_id, password: @password)
+      end
+    end
+
+    def results_hash(identifiers:, results:)
+      raise ApiResultSizeMismatchError unless results.size == identifiers.size
+
+      # Construct a hash of { identifier => result_hash }
+      # Hash[identifiers.zip(results)]
+
+      results_hash = {}
+      results.each_with_index do |single_result, i|
+        results_hash[identifiers[i]] = @result_transformer.transform(single_result)
+      end
+
+      results_hash
+    end
+
+    def default_token_cache
+      ActiveSupport::Cache::MemoryStore.new
+    end
+
+    def default_result_transformer
+      ExperianConsumerView::Transformers::ResultTransformer.default
+    end
+  end
+end

data/lib/experian_consumer_view/constants.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module ExperianConsumerView
+  MAX_LOOKUP_BATCH_SIZE = 5_000
+end

data/lib/experian_consumer_view/errors.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module ExperianConsumerView
+  module Errors
+    # Base helper class for errors caused due to unexpected HTTP responses
+    class ApiHttpError < StandardError
+      attr_reader :code, :response
+
+      def initialize(code, response)
+        super()
+        @code = code
+        @response = response
+      end
+
+      def message
+        "HTTP code [#{@code}], response text [#{@response}]"
+      end
+    end
+
+    # Thrown for HTTP 401 codes. This means there is a problem with some of the supplied credentials. Race conditions
+    # may lead to out-of-date or invalid API tokens occasionally. These errors may be auto-retried.
+    class ApiBadCredentialsError < ApiHttpError; end
+
+    # Thrown for HTTP 404 codes. These imply the API endpoints may have changed, likely requiring a code change to
+    # this code library.
+    class ApiEndpointNotFoundError < ApiHttpError; end
+
+    # Thrown for HTTP 417 codes. These imply the API format may have changed, likely requiring a code change to this
+    # code library.
+    class ApiIncorrectJsonError < ApiHttpError; end
+
+    # Thrown for HTTP 500 codes, and HTTP 503 codes where the text response is "Server error". This means a serious
+    # server error has occurred. This can only be resolved on the server side by Experian - inform them if it persists.
+    class ApiServerError < ApiHttpError; end
+
+    # Thrown for HTTP 503 codes where the text response is "Internal refresh in progress". This means the server is
+    # temporarily down while data is being refreshed, but the request should complete successfully once the refresh
+    # operation is complete. These errors may be auto-retried.
+    class ApiServerRefreshingError < ApiHttpError; end
+
+    # Thrown for HTTP 505 codes. These imply an error which likely requires a code change to this code library.
+    class ApiHttpVersionNotSupportedError < ApiHttpError; end
+
+    # Thrown for unhandled HTTP codes.
+    class ApiUnhandledHttpError < ApiHttpError; end
+
+    # Thrown when the API is passed a batch which is too big to be given to the API
+    class ApiBatchTooBigError < StandardError; end
+
+    # Thrown when the API returns data successfully, but the size of the data returned does not match the size of the
+    # query data provided, meaning there is no way to know for sure which result relates to which query string. Such
+    # an error either implies a serious issue with the Experian API, or with this code library (or a change to the API
+    # contract).
+    class ApiResultSizeMismatchError < StandardError; end
+
+    # Thrown when the API returns a value for an attribute which is unrecognised and therefore can't be transformed
+    class AttributeValueUnrecognisedError < StandardError; end
+  end
+end