itrp-client 1.0.0

data/lib/itrp/client.rb

@@ -0,0 +1,314 @@
+require 'net/http'
+require 'json'
+require 'uri'
+require 'date'
+require 'time'
+require 'net/https'
+require 'open-uri'
+require 'itrp'
+
+require 'itrp/client/version'
+require 'itrp/client/response'
+require 'itrp/client/multipart'
+require 'itrp/client/attachments'
+
+# cherry-pick some core extensions from active support
+require 'active_support/core_ext/module/aliasing.rb'
+require 'active_support/core_ext/object/blank'
+require 'active_support/core_ext/object/try.rb'
+require 'active_support/core_ext/hash/indifferent_access'
+
+module Itrp
+  class Client
+    MAX_PAGE_SIZE = 100
+    DEFAULT_HEADER = {'Content-Type' => 'application/json'}
+
+    # Create a new ITRP Client
+    #
+    # Shared configuration for all ITRP Clients:
+    #   Itrp.configure do |config|
+    #     config.api_token = 'd41f5868feb65fc87fa2311a473a8766ea38bc40'
+    #     config.account = 'my-sandbox'
+    #     ...
+    #   end
+    #
+    # Override configuration per ITRP Client:
+    # itrp = Itrp::Client.new(account: 'trusted-sandbox')
+    #
+    # All options available:
+    #  - logger:      The Ruby Logger instance, default: Logger.new(STDOUT)
+    #  - host:        The ITRP API host, default: 'https://api.itrp.com'
+    #  - api_version: The ITRP API version, default: 'v1'
+    #  - api_token:   *required* The ITRP API token
+    #  - account:     Specify a different (trusted) account to work with
+    #                 @see http://developer.itrp.com/v1/#multiple-accounts
+    #  - source:      The Source used when creating new records
+    #                 @see http://developer.itrp.com/v1/general/source/
+    #
+    #  - max_retry_time: maximum nr of seconds to wait for server to respond (default = 5400 = 1.5 hours)
+    #                    the sleep time between retries starts at 2 seconds and doubles after each retry
+    #                    retry times: 2, 6, 18, 54, 162, 486, 1458, 4374, 13122, ... seconds
+    #                    one retry will always be performed unless you set the value to -1
+    #  - read_timeout:   HTTP GET read timeout in seconds (default = 25)
+    #  - block_at_rate_limit: Set to +true+ to block the request until the rate limit is lifted, default: +false+
+    #                         @see http://developer.itrp.com/v1/#rate-limiting
+    #
+    #  - proxy_host:     Define in case HTTP traffic needs to go through a proxy
+    #  - proxy_port:     Port of the proxy, defaults to 8080
+    #  - proxy_user:     Proxy user
+    #  - proxy_password: Proxy password
+    def initialize(options = {})
+      @options = Itrp.configuration.current.merge(options)
+      [:host, :api_version, :api_token].each do |required_option|
+        raise ::Itrp::Exception.new("Missing required configuration option #{required_option}") if option(required_option).blank?
+      end
+      @ssl, @domain, @port = ssl_domain_port_path(option(:host))
+      @logger = @options[:logger]
+    end
+
+    # Retrieve an option
+    def option(key)
+      @options[key]
+    end
+
+    # Yield all retrieved resources one-by-one for the given (paged) API query.
+    # Raises an ::Itrp::Exception with the response retrieved from ITRP is invalid
+    # Returns total nr of resources yielded (for logging)
+    def each(path, params = {}, &block)
+      # retrieve the resources using the max page size (least nr of API calls)
+      next_path = expand_path(path, {per_page: MAX_PAGE_SIZE, page: 1}.merge(params))
+      size = 0
+      while next_path
+        # retrieve the records (with retry and optionally wait for rate-limit)
+        response = get(next_path)
+        # raise exception in case the response is invalid
+        raise ::Itrp::Exception.new(response.message) unless response.valid?
+        # yield the resources
+        response.json.each{ |resource| yield resource }
+        size += response.json.size
+        # go to the next page
+        next_path = response.pagination_relative_link(:next)
+      end
+      size
+    end
+
+    # send HTTPS GET request and return instance of Itrp::Response
+    def get(path, params = {}, header = {})
+      _send(Net::HTTP::Get.new(expand_path(path, params), expand_header(header)))
+    end
+
+    # send HTTPS PUT request and return instance of Itrp::Response
+    def put(path, data = {}, header = {})
+      _send(json_request(Net::HTTP::Put, path, data, header))
+    end
+    alias_method :patch, :put
+
+    # send HTTPS POST request and return instance of Itrp::Response
+    def post(path, data = {}, header = {})
+      _send(json_request(Net::HTTP::Post, path, data, header))
+    end
+
+    # upload a CSV file to import
+    # @param csv: The CSV File or the location of the CSV file
+    # @param type: The type, e.g. person, organization, people_contact_details
+    # @raise Itrp::UploadFailed in case the file could was not accepted by ITRP and +block_until_completed+ is +true+
+    # @raise Itrp::Exception in case the import progress could not be monitored
+    def import(csv, type, block_until_completed = false)
+      csv = File.open(csv, 'r') unless csv.respond_to?(:path) && csv.respond_to?(:read)
+      data, headers = Itrp::Multipart::Post.prepare_query(type: type, file: csv)
+      request = Net::HTTP::Post.new(expand_path('/import'), expand_header(headers))
+      request.body = data
+      response = _send(request)
+      @logger.info { "Import file '#{csv.path}' successfully uploaded with token '#{response[:token]}'." } if response.valid?
+
+      if block_until_completed
+        raise ::Itrp::UploadFailed.new("Failed to queue #{type} import. #{response.message}") unless response.valid?
+        token = response[:token]
+        while true
+          response = get("/import/#{token}")
+          raise ::Itrp::Exception.new("Unable to monitor progress for #{type} import. #{response.message}") unless response.valid?
+          # wait 30 seconds while the response is OK and import is still busy
+          break unless ['queued', 'processing'].include?(response[:state])
+          @logger.debug { "Import of '#{csv.path}' is #{response[:state]}. Checking again in 30 seconds." }
+          sleep(30)
+        end
+      end
+
+      response
+    end
+
+    # Export CSV files
+    # @param types: The types to export, e.g. person, organization, people_contact_details
+    # @param from: Retrieve all files since a given data and time
+    # @param block_until_completed: Set to true to monitor the export progress
+    # @raise Itrp::Exception in case the export progress could not be monitored
+    def export(types, from = nil, block_until_completed = false)
+      data = {type: [types].flatten.join(',')}
+      data[:from] = from unless from.blank?
+      response = post('/export', data)
+      @logger.info { "Export for '#{data[:type]}' successfully queued with token '#{response[:token]}'." } if response.valid?
+
+      if block_until_completed
+        raise ::Itrp::UploadFailed.new("Failed to queue '#{data[:type]}' export. #{response.message}") unless response.valid?
+        token = response[:token]
+        while true
+          response = get("/export/#{token}")
+          raise ::Itrp::Exception.new("Unable to monitor progress for '#{data[:type]}' export. #{response.message}") unless response.valid?
+          # wait 30 seconds while the response is OK and export is still busy
+          break unless ['queued', 'processing'].include?(response[:state])
+          @logger.debug { "Export of '#{data[:type]}' is #{response[:state]}. Checking again in 30 seconds." }
+          sleep(30)
+        end
+      end
+
+      response
+    end
+
+    def logger
+      @logger
+    end
+
+    private
+
+    # create a request (place data in body if the request becomes too large)
+    def json_request(request_class, path, data = {}, header = {})
+      Itrp::Attachments.new(self).upload_attachments!(path, data)
+      request = request_class.new(expand_path(path), expand_header(header))
+      body = {}
+      data.each{ |k,v| body[k.to_s] = typecast(v, false) }
+      request.body = body.to_json
+      request
+    end
+
+    URI_ESCAPE_PATTERN = Regexp.new("[^#{URI::PATTERN::UNRESERVED}]")
+    def uri_escape(value)
+      URI.escape(value, URI_ESCAPE_PATTERN).gsub('.', '%2E')
+    end
+
+    # Expand the given header with the default header
+    def expand_header(header = {})
+      header = DEFAULT_HEADER.merge(header)
+      header['X-ITRP-Account'] = option(:account) if option(:account)
+      header['AUTHORIZATION'] = 'Basic ' + ["#{option(:api_token)}:"].pack("m*")
+      if option(:source)
+        header['X-ITRP-Source'] = option(:source)
+        header['HTTP_USER_AGENT'] = option(:source)
+      end
+      header
+    end
+
+    # Expand the given path with the parameters
+    # Examples:
+    #   person_id: 5
+    #   :"updated_at=>" => yesterday
+    #   fields: ["id", "created_at", "sourceID"]
+    def expand_path(path, params = {})
+      path = path.dup
+      path = "/#{path}" unless path =~ /^\// # make sure path starts with /
+      path = "/#{option(:api_version)}#{path}" unless path =~ /^\/v[\d.]+\// # preprend api version
+      params.each do |key, value|
+        path << (path['?'] ? '&' : '?')
+        path << expand_param(key, value)
+      end
+      path
+    end
+
+    # Expand one parameter, e.g. (:"created_at=>", DateTime.now) to "created_at=%3E22011-12-16T12:24:41+01:00"
+    def expand_param(key, value)
+      param = uri_escape(key.to_s).gsub('%3D', '=') # handle :"updated_at=>" or :"person_id!=" parameters
+      param << '=' unless key['=']
+      param << typecast(value)
+      param
+    end
+
+    # Parameter value typecasting
+    def typecast(value, escape = true)
+      case value.class.name.to_sym
+        when :NilClass    then ''
+        when :String      then escape ? URI.escape(value) : value
+        when :TrueClass   then 'true'
+        when :FalseClass  then 'false'
+        when :DateTime    then value.new_offset(0).iso8601
+        when :Date        then value.strftime("%Y-%m-%d")
+        when :Time        then value.strftime("%H:%M")
+        when :Array       then value.map{ |v| typecast(v, escape) }.join(',')
+        else value.to_s
+      end
+    end
+
+    # Send a request to ITRP and wrap the HTTP Response in an Itrp::Response
+    # Guaranteed to return a Response, thought it may be +empty?+
+    def _send(request, domain = @domain, port = @port, ssl = @ssl)
+      @logger.debug { "Sending #{request.method} request to #{domain}:#{port}#{request.path}" }
+      _response = begin
+        http_with_proxy = option(:proxy_host).blank? ? Net::HTTP : Net::HTTP::Proxy(option(:proxy_host), option(:proxy_port), option(:proxy_user), option(:proxy_password))
+        http = http_with_proxy.new(domain, port)
+        http.read_timeout = option(:read_timeout)
+        http.use_ssl = ssl
+        http.start{ |_http| _http.request(request) }
+      rescue ::Exception => e
+        Struct.new(:body, :message, :code, :header).new(nil, "No Response from Server - #{e.message} for '#{domain}:#{port}#{request.path}'", 500, {})
+      end
+      response = Itrp::Response.new(request, _response)
+      if response.valid?
+        @logger.debug { "Response:\n#{JSON.pretty_generate(response.json)}" }
+      elsif response.raw.body =~ /^\s*<\?xml/i
+        @logger.debug { "XML response:\n#{response.raw.body}" }
+      elsif '303' == response.raw.code.to_s
+        @logger.debug { "Redirect: #{response.raw.header['Location']}" }
+      else
+        @logger.error { "Request failed: #{response.message}" }
+      end
+      response
+    end
+
+    # Wraps the _send method with retries when the server does not responsd, see +initialize+ option +:rate_limit_block+
+    def _send_with_rate_limit_block(request, domain = @domain, port = @port, ssl = @ssl)
+      return _send_without_rate_limit_block(request, domain, port, ssl) unless option(:block_at_rate_limit)
+      now = Time.now
+      begin
+        _response = _send_without_rate_limit_block(request, domain, port, ssl)
+        @logger.warn { "Request throttled, trying again in 5 minutes: #{_response.message}" } and sleep(300) if _response.throttled?
+      end while _response.throttled? && (Time.now - now) < 3660 # max 1 hour and 1 minute
+      _response
+    end
+    alias_method_chain :_send, :rate_limit_block
+
+    # Wraps the _send method with retries when the server does not responsd, see +initialize+ option +:retries+
+    def _send_with_retries(request, domain = @domain, port = @port, ssl = @ssl)
+      retries = 0
+      sleep_time = 2
+      total_retry_time = 0
+      begin
+        _response = _send_without_retries(request, domain, port, ssl)
+        @logger.warn { "Request failed, retry ##{retries += 1} in #{sleep_time} seconds: #{_response.message}" } and sleep(sleep_time) if _response.empty? && option(:max_retry_time) > 0
+        total_retry_time += sleep_time
+        sleep_time *= 2
+      end while _response.empty? && total_retry_time < option(:max_retry_time)
+      _response
+    end
+    alias_method_chain :_send, :retries
+
+    # parse the given URI to [domain, port, ssl, path]
+    def ssl_domain_port_path(uri)
+      uri = URI.parse(uri)
+      ssl = uri.scheme == 'https'
+      [ssl, uri.host, uri.port, uri.path]
+    end
+
+  end
+end
+
+# HTTPS with certificate bundle
+module Net
+  class HTTP
+    alias_method :original_use_ssl=, :use_ssl=
+
+    def use_ssl=(flag)
+      self.ca_file = File.expand_path("../ca-bundle.crt", __FILE__) if flag
+      self.verify_mode = OpenSSL::SSL::VERIFY_PEER
+      self.original_use_ssl = flag
+    end
+  end
+end
+

data/lib/itrp/client/attachments.rb

@@ -0,0 +1,99 @@
+module Itrp
+  class Attachments
+
+    AWS_PROVIDER = 'aws'
+    FILENAME_TEMPLATE = '${filename}'
+
+    def initialize(client)
+      @client = client
+    end
+
+    # upload the attachments in :attachments to ITRP and return the data with the uploaded attachment info
+    def upload_attachments!(path, data)
+      raise_exceptions = !!data.delete(:attachments_exception)
+      attachments = [data.delete(:attachments)].flatten.compact
+      return if attachments.empty?
+
+      # retrieve the upload configuration for this record from ITRP
+      storage = @client.get(path =~ /\d+$/ ? path : "#{path}/new", {attachment_upload_token: true}, @client.send(:expand_header))[:storage_upload]
+      report_error("Attachments not allowed for #{path}", raise_exceptions) and return unless storage
+
+      # upload each attachment and store the {key, filesize} has in the note_attachments parameter
+      data[:note_attachments] = attachments.map {|attachment| upload_attachment(storage, attachment, raise_exceptions) }.compact.to_json
+    end
+
+    private
+
+    def report_error(message, raise_exceptions)
+      if raise_exceptions
+        raise Itrp::UploadFailed.new(message)
+      else
+        @client.logger.error{ message }
+      end
+    end
+
+    # upload a single attachment and return the data for the note_attachments
+    # returns nil and provides an error in case the attachment upload failed
+    def upload_attachment(storage, attachment, raise_exceptions)
+      begin
+        # attachment is already a file or we need to open the file from disk
+        unless attachment.respond_to?(:path) && attachment.respond_to?(:read)
+          raise "file does not exist: #{attachment}" unless File.exists?(attachment)
+          attachment = File.open(attachment, 'r')
+        end
+
+        # there are two different upload methods: AWS S3 and ITRP local storage
+        key_template = "#{storage[:upload_path]}#{FILENAME_TEMPLATE}"
+        key = key_template.gsub(FILENAME_TEMPLATE, File.basename(attachment.path))
+        upload_method = storage[:provider] == AWS_PROVIDER ? :aws_upload : :itrp_upload
+        send(upload_method, storage, key_template, key, attachment)
+
+        # return the values for the note_attachments param
+        {key: key, filesize: File.size(attachment.path)}
+      rescue ::Exception => e
+        report_error("Attachment upload failed: #{e.message}", raise_exceptions)
+        nil
+      end
+    end
+
+    def aws_upload(aws, key_template, key, attachment)
+      # upload the file to AWS
+      response = send_file(aws[:upload_uri], {
+        key: key_template,
+        AWSAccessKeyId: aws[:access_key],
+        acl: 'private',
+        signature: aws[:signature],
+        success_action_redirect: aws[:success_url],
+        policy: aws[:policy],
+        file: attachment # file must be last (will that work in Ruby 1.9.3)?
+      })
+      # this is a bit of a hack, but Amazon S3 returns only XML :(
+      xml = response.raw.body || ''
+      error = xml[/<Error>.*<Message>(.*)<\/Message>.*<\/Error>/, 1]
+      raise "AWS upload to #{aws[:upload_uri]} for #{key} failed: #{error}" if error
+
+      # inform ITRP of the successful upload
+      response = @client.get(aws[:success_url].split('/').last, {key: key}, @client.send(:expand_header))
+      raise "ITRP confirmation #{aws[:success_url].split('/').last} for #{key} failed: #{response.message}" unless response.valid?
+    end
+
+    # upload the file directly to ITRP
+    def itrp_upload(itrp, key_template, key, attachment)
+      response = send_file(itrp[:upload_uri], {
+        file: attachment,
+        key: key_template
+      })
+      raise "ITRP upload to #{itrp[:upload_uri]} for #{key} failed: #{response.message}" unless response.valid?
+    end
+
+    def send_file(uri, params)
+      params = {:'Content-Type' => MIME::Types.type_for(params[:key])[0] || MIME::Types["application/octet-stream"][0]}.merge(params)
+      data, header = Itrp::Multipart::Post.prepare_query(params)
+      ssl, domain, port, path = @client.send(:ssl_domain_port_path, uri)
+      request = Net::HTTP::Post.new(path, header)
+      request.body = data
+      @client.send(:_send, request, domain, port, ssl)
+    end
+
+  end
+end

data/lib/itrp/client/multipart.rb

@@ -0,0 +1,75 @@
+require 'cgi'
+require 'mime/types'
+
+# Takes a hash of string and file parameters and returns a string of text
+# formatted to be sent as a multipart form post.
+#
+# Author:: Cody Brimhall <mailto:cbrimhall@ucdavis.edu>
+# Created:: 22 Feb 2008
+module Itrp
+  module Multipart
+    VERSION = "1.0.0" unless const_defined?(:VERSION)
+
+    # Formats a given hash as a multipart form post
+    # If a hash value responds to :string or :read messages, then it is
+    # interpreted as a file and processed accordingly; otherwise, it is assumed
+    # to be a string
+    class Post
+      # We have to pretend like we're a web browser...
+      USERAGENT = "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-us) AppleWebKit/523.10.6 (KHTML, like Gecko) Version/3.0.4 Safari/523.10.6" unless const_defined?(:USERAGENT)
+      BOUNDARY = "0123456789ABLEWASIEREISAWELBA9876543210" unless const_defined?(:BOUNDARY)
+      CONTENT_TYPE = "multipart/form-data; boundary=#{ BOUNDARY }" unless const_defined?(:CONTENT_TYPE)
+      HEADER = { "Content-Type" => CONTENT_TYPE, "User-Agent" => USERAGENT } unless const_defined?(:HEADER)
+
+      def self.prepare_query(params)
+        fp = []
+
+        params.each do |k, v|
+          if v.respond_to?(:path) && v.respond_to?(:read)
+            fp.push(FileParam.new(k, v.path, v.read))
+          else
+            fp.push(StringParam.new(k, v))
+          end
+        end
+
+        # Assemble the request body using the special multipart format
+        query = fp.map{ |p| "--#{BOUNDARY}\r\n#{p.to_multipart}" }.join  + "--#{BOUNDARY}--"
+        return query, HEADER
+      end
+    end
+
+    private
+
+    # Formats a basic string key/value pair for inclusion with a multipart post
+    class StringParam
+      attr_accessor :k, :v
+
+      def initialize(k, v)
+        @k = k
+        @v = v
+      end
+
+      def to_multipart
+        return %(Content-Disposition: form-data; name="#{CGI::escape(k.to_s)}"\r\n\r\n#{v}\r\n)
+      end
+    end
+
+    # Formats the contents of a file or string for inclusion with a multipart
+    # form post
+    class FileParam
+      attr_accessor :k, :filename, :content
+
+      def initialize(k, filename, content)
+        @k = k
+        @filename = filename
+        @content = content
+      end
+
+      def to_multipart
+        # If we can tell the possible mime-type from the filename, use the first in the list; otherwise, use "application/octet-stream"
+        mime_type = MIME::Types.type_for(filename)[0] || MIME::Types["application/octet-stream"][0]
+        return %(Content-Disposition: form-data; name="#{CGI::escape(k.to_s)}"; filename="#{filename}"\r\nContent-Type: #{ mime_type.simplified }\r\n\r\n#{ content }\r\n)
+      end
+    end
+  end
+end