rets 0.4.0 → 0.5.0

data/CHANGELOG.md

@@ -1,3 +1,12 @@
+### 0.5.0 / 2013-09-05
+
+* feature: Allow client.count to get integer count
+* feature: Allow for downcased capability names
+* fix: Handle the rets element being empty
+* feature: Instrument rets client with stats reporting
+* feature: Add a locking client
+* feature: Support Basic Authentication
+
 ### 0.4.0 / 2012-08-29
 
 * fix: update authentication header to uri matches path

data/Manifest.txt

@@ -4,7 +4,6 @@ README.md
 Rakefile
 bin/rets
 lib/rets.rb
-lib/rets/authentication.rb
 lib/rets/client.rb
 lib/rets/metadata.rb
 lib/rets/metadata/containers.rb

data/Rakefile

@@ -1,5 +1,6 @@
 require 'rubygems'
 require 'hoe'
+require 'rake/testtask'
 
 Hoe.plugin :git, :doofus
 Hoe.plugin :travis
@@ -8,7 +9,7 @@ Hoe.plugin :gemspec
 Hoe.spec 'rets' do
   developer 'Estately, Inc. Open Source', 'opensource@estately.com'
 
-  extra_deps << [ "net-http-persistent", "~> 1.7" ]
+  extra_deps << [ "httpclient", "~> 2.3.0" ]
   extra_deps << [ "nokogiri",            "~> 1.5.2" ]
 
   extra_dev_deps << [ "mocha", "~> 0.11.0" ]
@@ -19,3 +20,8 @@ Hoe.spec 'rets' do
   self.history_file = 'CHANGELOG.md'
   self.readme_file  = 'README.md'
 end
+
+
+Rake::TestTask.new do |t|
+  t.pattern = "test/test_*.rb"
+end

data/bin/rets

@@ -4,12 +4,14 @@ require "optparse"
 require "pp"
 
 require "rubygems"
+
+$:.unshift File.join(File.dirname(__FILE__), '..', 'lib')
 require "rets"
 
 class RetsCli
   def self.parse(args)
 
-    actions = %w(metadata search object)
+    actions = %w(metadata search count object)
     options = {:count => 5}
 
     opts = OptionParser.new do |opts|
@@ -107,8 +109,6 @@ query   = ARGV[1]
 
 client = Rets::Client.new(options.merge(:login_url => url))
 
-COUNT = Struct.new(:exclude, :include, :only).new(0,1,2)
-
 if options[:capabilities]
   pp client.capabilities
 end
@@ -161,9 +161,17 @@ case options[:action]
                   :search_type => options[:resource],
                   :class       => options[:class],
                   :query       => query,
-                  :count       => COUNT.exclude,
+                  :count       => Rets::Client::COUNT.exclude,
                   :limit       => options[:limit])
 
+  when "count" then
+    pp client.find(:all,
+              :search_type => options[:resource],
+              :class       => options[:class],
+              :query       => query,
+              :count       => Rets::Client::COUNT.only,
+              :limit       => options[:limit])
+
   when "object" then
 
     def write_objects(parts)
@@ -191,4 +199,4 @@ case options[:action]
 
 end
 
-
+client.logout

data/lib/rets.rb

@@ -1,24 +1,45 @@
-require 'net/http'
 require 'uri'
-require 'cgi'
 require 'digest/md5'
-
-require 'rubygems'
-require 'net/http/persistent'
 require 'nokogiri'
 
 module Rets
-  VERSION = '0.4.0'
+  VERSION = '0.5.0'
 
-  AuthorizationFailure = Class.new(ArgumentError)
-  InvalidRequest       = Class.new(ArgumentError)
   MalformedResponse    = Class.new(ArgumentError)
   UnknownResponse      = Class.new(ArgumentError)
+  NoLogout             = Class.new(ArgumentError)
+
+  class AuthorizationFailure < ArgumentError
+    attr_reader :status, :body
+    def initialize(status, body)
+      @status = status
+      @body = body
+      super("HTTP status: #{status} (#{body})")
+    end
+  end
+
+  class InvalidRequest < ArgumentError
+    attr_reader :error_code, :reply_text
+    def initialize(error_code, reply_text)
+      @error_code = error_code
+      @reply_text = reply_text
+      super("Got error code #{error_code} (#{reply_text})")
+    end
+  end
+
+  class UnknownCapability < ArgumentError
+    attr_reader :capability_name
+    def initialize(capability_name)
+      @capability_name = capability_name
+      super("unknown capabilitiy #{capability_name}")
+    end
+  end
 end
 
-require 'rets/authentication'
+require 'rets/client'
 require 'rets/metadata'
 require 'rets/parser/compact'
 require 'rets/parser/multipart'
-
-require 'rets/client'
+require 'rets/measuring_http_client'
+require 'rets/locking_http_client'
+require 'rets/client_progress_reporter'

data/lib/rets/client.rb

@@ -1,12 +1,16 @@
+require 'httpclient'
+require 'logger'
+require_relative 'http_client'
+
 module Rets
-  Session = Struct.new(:auth_digest, :capabilities, :cookies)
+  class HttpError < StandardError ; end
 
   class Client
-    DEFAULT_OPTIONS = { :persistent => true }
+    DEFAULT_OPTIONS = {}
 
-    include Authentication
+    COUNT = Struct.new(:exclude, :include, :only).new(0,1,2)
 
-    attr_accessor :login_uri, :options, :logger, :auth_digest
+    attr_accessor :login_url, :options, :logger
     attr_writer   :capabilities, :metadata
 
     def initialize(options)
@@ -15,41 +19,47 @@ module Rets
     end
 
     def clean_setup
+      self.options     = DEFAULT_OPTIONS.merge(@options)
+      self.login_url   = self.options[:login_url]
 
-      @auth_digest       = nil
       @cached_metadata   = nil
       @capabilities      = nil
-      @connection        = nil
-      @cookies           = nil
       @metadata          = nil
       @tries             = nil
       self.capabilities  = nil
 
-      uri              = URI.parse(@options[:login_url])
-
-      uri.user         = @options.key?(:username) ? CGI.escape(@options[:username]) : nil
-      uri.password     = @options.key?(:password) ? CGI.escape(@options[:password]) : nil
-
-      self.options     = DEFAULT_OPTIONS.merge(@options)
-      self.login_uri   = uri
-
       self.logger      = @options[:logger] || FakeLogger.new
-
-      self.session     = @options.delete(:session)  if @options[:session]
-      @cached_metadata = @options[:metadata] || nil
+      @client_progress = ClientProgressReporter.new(self.logger, options[:stats_collector], options[:stats_prefix])
+      @cached_metadata = @options[:metadata]
+      @http = HTTPClient.new
+      @http.set_cookie_store(options[:cookie_store]) if options[:cookie_store]
+
+      @http_client = Rets::HttpClient.new(@http, @options, @logger, @login_url)
+      if options[:http_timing_stats_collector]
+        @http_client = Rets::MeasuringHttpClient.new(@http_client, options.fetch(:http_timing_stats_collector), options.fetch(:http_timing_stats_prefix))
+      end
+      if options[:lock_around_http_requests]
+        @http_client = Rets::LockingHttpClient.new(@http_client, options.fetch(:locker), options.fetch(:lock_name), options.fetch(:lock_options))
+      end
     end
 
-
     # Attempts to login by making an empty request to the URL
     # provided in initialize. Returns the capabilities that the
     # RETS server provides, per http://retsdoc.onconfluence.com/display/rets172/4.10+Capability+URL+List.
     def login
-      response = request(login_uri.path)
-      self.capabilities = extract_capabilities(Nokogiri.parse(response.body))
+      res = http_get(login_url)
+      self.capabilities = extract_capabilities(Nokogiri.parse(res.body))
       raise UnknownResponse, "Cannot read rets server capabilities." unless @capabilities
       @capabilities
     end
 
+    def logout
+      unless capabilities["Logout"]
+        raise NoLogout.new('No logout method found for rets client')
+      end
+      http_get(capability_url("Logout"))
+    end
+
     # Finds records.
     #
     # [quantity]  Return the first record, or an array of records.
@@ -81,51 +91,41 @@ module Rets
 
     def find_with_retries(opts = {})
       retries = 0
+      resolve = opts.delete(:resolve)
       begin
-        find_every(opts)
+        find_every(opts, resolve)
       rescue AuthorizationFailure, InvalidRequest => e
         if retries < 3
           retries += 1
-          self.logger.warn "Failed with message: #{e.message}"
-          self.logger.info "Retry #{retries}/3"
+          @client_progress.find_with_retries_failed_a_retry(e, retries)
           clean_setup
           retry
         else
+          @client_progress.find_with_retries_exceeded_retry_count(e)
           raise e
         end
       end
     end
 
-    def find_every(opts = {})
-      search_uri = capability_url("Search")
-
-      resolve = opts.delete(:resolve)
-
-      extras = fixup_keys(opts)
-
-      defaults = {"QueryType" => "DMQL2", "Format" => "COMPACT"}
-
-      query = defaults.merge(extras)
-
-      body = build_key_values(query)
-
-      extra_headers = {
-        "Content-Type"   => "application/x-www-form-urlencoded",
-        "Content-Length" => body.size.to_s
-      }
-
-      results = request_with_compact_response(search_uri.path, body, extra_headers)
+    def find_every(opts, resolve)
+      params = {"QueryType" => "DMQL2", "Format" => "COMPACT"}.merge(fixup_keys(opts))
+      res = http_post(capability_url("Search"), params)
 
-      if resolve
-        rets_class = find_rets_class(opts[:search_type], opts[:class])
-        decorate_results(results, rets_class)
+      if opts[:count] == COUNT.only
+        Parser::Compact.get_count(res.body)
       else
-        results
+        results = Parser::Compact.parse_document(res.body)
+        if resolve
+          rets_class = find_rets_class(opts[:search_type], opts[:class])
+          decorate_results(results, rets_class)
+        else
+          results
+        end
       end
     end
 
     def find_rets_class(resource_name, rets_class_name)
-      metadata.build_tree[resource_name].find_rets_class(rets_class_name)
+      metadata.tree[resource_name].find_rets_class(rets_class_name)
     end
 
     def decorate_results(results, rets_class)
@@ -141,12 +141,11 @@ module Rets
           result[key] = table.resolve(value.to_s)
         else
           #can't resolve just leave the value be
-          logger.warn "Can't resolve find metadata for #{key.inspect}"
+          @client_progress.could_not_resolve_find_metadata(key)
         end
       end
     end
 
-
     # Returns an array of all objects associated with the given resource.
     def all_objects(opts = {})
       objects("*", opts)
@@ -164,7 +163,7 @@ module Rets
     end
 
     def create_parts_from_response(response)
-      content_type = response["content-type"]
+      content_type = response.header["content-type"][0]
 
       if content_type.nil?
         raise MalformedResponse, "Unable to read content-type from response: #{response.inspect}"
@@ -175,13 +174,13 @@ module Rets
 
         parts = Parser::Multipart.parse(response.body, boundary)
 
-        logger.debug "Found #{parts.size} parts"
+        logger.debug "Rets::Client: Found #{parts.size} parts"
 
         return parts
       else
         # fake a multipart for interface compatibility
         headers = {}
-        response.each { |k,v| headers[k] = v }
+        response.headers.each { |k,v| headers[k] = v[0] }
 
         part = Parser::Multipart::Part.new(headers, response.body)
 
@@ -194,30 +193,25 @@ module Rets
     # resource     RETS resource as defined in the resource metadata.
     # object_type  an object type defined in the object metadata.
     # resource_id  the KeyField value of the given resource instance.
-    # object_id    can be "*", or a comma delimited string of one or more integers.
+    # object_id    can be "*" or a colon delimited string of integers or an array of integers.
     def object(object_id, opts = {})
-      response = fetch_object(object_id, opts)
-
+      response = fetch_object(Array(object_id).join(':'), opts)
       response.body
     end
 
     def fetch_object(object_id, opts = {})
-      object_uri = capability_url("GetObject")
-
-      body = build_key_values(
-        "Resource" => opts[:resource],
-        "Type"     => opts[:object_type],
-        "ID"       => "#{opts[:resource_id]}:#{object_id}",
-        "Location" => opts[:location] || 0
-      )
+      params = {
+        "Resource" => opts.fetch(:resource),
+        "Type"     => opts.fetch(:object_type),
+        "ID"       => "#{opts.fetch(:resource_id)}:#{object_id}",
+        "Location" => opts.fetch(:location, 0)
+      }
 
       extra_headers = {
-        "Accept"         => "image/jpeg, image/png;q=0.5, image/gif;q=0.1",
-        "Content-Type"   => "application/x-www-form-urlencoded",
-        "Content-Length" => body.size.to_s
+        "Accept" => "image/jpeg, image/png;q=0.5, image/gif;q=0.1",
       }
 
-      request(object_uri.path, body, extra_headers)
+      http_post(capability_url("GetObject"), params, extra_headers)
     end
 
     # Changes keys to be camel cased, per the RETS standard for queries.
@@ -238,166 +232,24 @@ module Rets
 
       if @cached_metadata && (@options[:skip_metadata_uptodate_check] ||
           @cached_metadata.current?(capabilities["MetadataTimestamp"], capabilities["MetadataVersion"]))
-        logger.info "Use cached metadata"
+        @client_progress.use_cached_metadata
         self.metadata = @cached_metadata
       else
-        logger.info @cached_metadata ? "Cached metadata out of date" : "Cached metadata unavailable"
+        @client_progress.bad_cached_metadata(@cached_metadata)
         metadata_fetcher = lambda { |type| retrieve_metadata_type(type) }
         self.metadata = Metadata::Root.new(&metadata_fetcher)
       end
     end
 
     def retrieve_metadata_type(type)
-      metadata_uri = capability_url("GetMetadata")
-
-      body = build_key_values(
-        "Format" => "COMPACT",
-        "Type"   => "METADATA-#{type}",
-        "ID"     => "0"
-      )
-
-      extra_headers = {
-        "Content-Type"   => "application/x-www-form-urlencoded",
-        "Content-Length" => body.size.to_s
-      }
-
-      response = request(metadata_uri.path, body, extra_headers)
-
-      response.body
-    end
-
-    def raw_request(path, body = nil, extra_headers = {}, &reader)
-      headers = build_headers(path).merge(extra_headers)
-
-      post = Net::HTTP::Post.new(path, headers)
-      post.body = body.to_s
-
-      logger.debug <<EOF
->>>> Request
-POST #{path}
-#{format_headers(headers)}
-
-#{binary?(body.to_s) ? '<<< BINARY BODY >>>' : body.to_s}
-EOF
-
-      connection_args = [Net::HTTP::Persistent === connection ? login_uri : nil, post].compact
-
-      response = connection.request(*connection_args) do |res|
-        res.read_body(&reader)
-      end
-
-      logger.debug <<EOF
-<<<< Response
-#{response.code} #{response.message}
-#{format_headers(response.to_hash)}
-
-#{binary?(response.body.to_s) ? '<<< BINARY BODY >>>' : response.body.to_s}
-EOF
-
-      handle_cookies(response)
-      return response
-    end
-
-    def digest_auth_request(path, body = nil, extra_headers = {}, &reader)
-      response = raw_request(path, body, extra_headers, &reader)
-      if Net::HTTPUnauthorized === response
-        @auth_digest = extract_digest_header(response)
-        if @auth_digest
-          response = raw_request(path, body, extra_headers, &reader)
-          if Net::HTTPUnauthorized === response
-            raise AuthorizationFailure, "Authorization failed, check credentials?"
-          end
-        end
-      end
-      response
-    end
-
-    def authorization(path)
-      return nil unless @auth_digest
-      uri2 = URI.parse(login_uri.to_s)
-      uri2.user     = login_uri.user
-      uri2.password = login_uri.password
-      uri2.path     = path
-      build_auth(@auth_digest, uri2, tries)
-    end
-
-
-    def request(*args, &block)
-      handle_response(digest_auth_request(*args, &block))
-    end
-
-    def request_with_compact_response(path, body, headers)
-      response = request(path, body, headers)
-
-      Parser::Compact.parse_document response.body
-    end
-
-    def extract_digest_header(response)
-      authenticate_headers = response.get_fields("www-authenticate")
-      if authenticate_headers
-        authenticate_headers.detect {|h| h =~ /Digest/}
-      else
-        nil
-      end
-    end
-
-    def handle_response(response)
-      if Net::HTTPSuccess === response
-        ErrorChecker.check(response)
-      elsif Net::HTTPUnauthorized === response
-        raise AuthorizationFailure, "Authorization failed, check credentials?"
-      else
-        raise UnknownResponse, "Unable to handle response #{response.class}"
-      end
-      response
-    end
-
-    def handle_cookies(response)
-      if cookies?(response)
-        self.cookies = response.get_fields('set-cookie')
-        logger.info "Cookies set to #{cookies.inspect}"
-      end
-    end
-
-    def cookies?(response)
-      response['set-cookie']
-    end
-
-    def cookies=(cookies)
-      @cookies ||= {}
-
-      Array(cookies).each do |cookie|
-        cookie.match(/(\S+)=([^;]+);?/)
-
-        @cookies[$1] = $2
-      end
-
-      nil
-    end
-
-    def cookies
-      return if @cookies.nil? or @cookies.empty?
-
-      @cookies.map{ |k,v| "#{k}=#{v}" }.join("; ")
-    end
-
-    def cookie(name)
-      return if @cookies.nil? or @cookies.empty?
-
-      @cookies[name]
-    end
-
-    def session=(session)
-      self.auth_digest  = session.auth_digest
-      self.capabilities = session.capabilities
-      self.cookies      = session.cookies
-    end
-
-    def session
-      Session.new(auth_digest, capabilities, cookies)
+      res = http_post(capability_url("GetMetadata"),
+                      { "Format" => "COMPACT",
+                        "Type"   => "METADATA-#{type}",
+                        "ID"     => "0"
+                      })
+      res.body
     end
 
-
     # The capabilies as provided by the RETS server during login.
     #
     # Currently, only the path in the endpoint URLs is used[1]. Host,
@@ -410,15 +262,21 @@ EOF
     end
 
     def capability_url(name)
-      url = capabilities[name]
+      val = capabilities[name] || capabilities[name.downcase]
+
+      raise UnknownCapability.new(name) unless val
 
       begin
-        capability_uri = URI.parse(url)
+        if val.downcase.match(/^https?:\/\//)
+          uri = URI.parse(val)
+        else
+          uri = URI.parse(login_url)
+          uri.path = val
+        end
       rescue URI::InvalidURIError
-        raise MalformedResponse, "Unable to parse capability URL: #{url.inspect}"
+        raise MalformedResponse, "Unable to parse capability URL: #{name} => #{val.inspect}"
       end
-
-      capability_uri
+      uri.to_s
     end
 
     def extract_capabilities(document)
@@ -435,59 +293,16 @@ EOF
       hash
     end
 
-
-
-    def connection
-      @connection ||= options[:persistent] ?
-        persistent_connection :
-        Net::HTTP.new(login_uri.host, login_uri.port)
-    end
-
-    def persistent_connection
-      conn = Net::HTTP::Persistent.new
-
-      def conn.idempotent?(*)
-	true
-      end
-
-      conn
-    end
-
-
-    def user_agent
-      options[:agent] || "Client/1.0"
+    def save_cookie_store(force=nil)
+      @http_client.save_cookie_store(force)
     end
 
-    def rets_version
-      options[:version] || "RETS/1.7.2"
+    def http_get(url, params=nil, extra_headers={})
+      @http_client.http_get(url, params, extra_headers)
     end
 
-    def build_headers(path)
-      headers = {
-        "User-Agent"   => user_agent,
-        "Host"         => "#{login_uri.host}:#{login_uri.port}",
-        "RETS-Version" => rets_version
-      }
-
-      auth = authorization(path)
-      headers.merge!("Authorization" => auth) if auth
-      headers.merge!("Cookie" => cookies)     if cookies
-
-      if options[:ua_password]
-        headers.merge!(
-          "RETS-UA-Authorization" => build_user_agent_auth(
-            user_agent, options[:ua_password], '', cookie('RETS-Session-ID'), rets_version))
-      end
-
-      headers
-    end
-
-    def build_key_values(data)
-      data.map{|k,v| "#{CGI.escape(k.to_s)}=#{CGI.escape(v.to_s)}" }.join("&")
-    end
-
-    def binary?(data)
-      data.slice(0, 1024).chars.any? {|b| b >= "\x0" && b < " " && b != '-' && b != '~' && b != "\t" && b != "\r" && b != "\n"}
+    def http_post(url, params, extra_headers = {})
+      @http_client.http_post(url, params, extra_headers)
     end
 
     def tries
@@ -496,50 +311,46 @@ EOF
       (@tries += 1) - 1
     end
 
-    class FakeLogger
-      def fatal(*); end
-      def error(*); end
-      def warn(*);  end
-      def info(*);  end
-      def debug(*); end
-    end
-
-    def format_headers(headers)
-      out = []
-
-      headers.each do |name, value|
-        if Array === value
-          value.each do |v|
-            out << "#{name}: #{v}"
-          end
-        else
-          out << "#{name}: #{value}"
-        end
+    class FakeLogger < Logger
+      def initialize
+        super("/dev/null")
       end
-
-      out.join("\n")
     end
 
     class ErrorChecker
       def self.check(response)
-        begin
-          if !response.body.empty?
+        # some RETS servers return success code in XML body but failure code 4xx in http status
+        # If xml body is present we ignore http status
+
+        if !response.body.empty?
+          begin
             xml = Nokogiri::XML.parse(response.body, nil, nil, Nokogiri::XML::ParseOptions::STRICT)
 
             rets_element = xml.xpath("/RETS")
+            if rets_element.empty?
+              return
+            end
             reply_text = (rets_element.attr("ReplyText") || rets_element.attr("replyText")).value
             reply_code = (rets_element.attr("ReplyCode") || rets_element.attr("replyCode")).value.to_i
 
             if reply_code.nonzero?
-              raise InvalidRequest, "Got error code #{reply_code} (#{reply_text})."
+              raise InvalidRequest.new(reply_code, reply_text)
+            else
+              return
             end
+          rescue Nokogiri::XML::SyntaxError
+            #Not xml
           end
+        end
 
-        rescue Nokogiri::XML::SyntaxError
-          #Not xml
+        if response.respond_to?(:ok?) && ! response.ok?
+          if response.status_code == 401
+            raise AuthorizationFailure.new(response.status_code, response.body)
+          else
+            raise HttpError, "HTTP status: #{response.status_code}, body: #{response.body}"
+          end
         end
       end
     end
-
   end
 end