rets 0.9.0 → 0.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6a9bf5e37008d7651782e347a520f24cb291931f
-  data.tar.gz: 22742e9f4a45db910f03358e45024ee25d8cb6e9
+  metadata.gz: cedf0e419db8f2c813a4c2c97fd210909b9b58af
+  data.tar.gz: b8eb9ad36844f79503c64f020830ef477586fa20
 SHA512:
-  metadata.gz: a742d37803a7c1354bb4f8d2c087b9a20b745c984286df5c243022cbb053612cb281350f589ca259494e955b7404bfd37db18ff2812441b2a560ad33485976f0
-  data.tar.gz: 250667d8536d504b41fc4192d8bb124237cd48bd663d061baccca0728b6408abd2d8776cd3ccf70ba6d7458a9a56e61f83a853510be725b264940be0e0c777b9
+  metadata.gz: e116e7a292cb5e8118cfff090aae8147a431299fd79b81572bf0afb6d093c0152f1ad35c671e2199898ddaf5b8ea12061a011184776179a239b44c8a8baf9b74
+  data.tar.gz: d8f6da1337852f4ba871eabb531e31765e5a1a1167f57a9aad5b02ce01c6a242d260d7f6e63492e8fd64c89f9cb4da12a1ec344c20b39f519195cf075cdce666

data/CHANGELOG.md

@@ -1,3 +1,22 @@
+### 0.10.0 / 2016-02-29
+
+* fix: ensure cookie store exists #133
+* feature: make cached capabilities case insensitive #136
+* feature: add specific classes for each rets error #137
+* feature: whitelist RETS search options #142
+* feature: simplify metadata caching #134
+* feature: use a SAX parser #98
+* fix: save capabilities to avoid double logins #148
+* feature: login on authorization error #155
+* add basic support for DataDictionary feeds #156
+* fix: count always returns a number #161
+* feature: make lookup tables case insensitive #163
+* feature: update to httpclient 2.7 #165
+* fix: getObject now works with non-multipart responses #166
+* fix: getObject works with multiple ids #167
+* feature: store rets object metadata #168
+* feature: add a code of conduct #171
+
 ### 0.9.0 / 2015-06-11
 
 * feature: update to httpclient 2.6

data/Manifest.txt

@@ -13,22 +13,46 @@ lib/rets/http_client.rb
 lib/rets/locking_http_client.rb
 lib/rets/measuring_http_client.rb
 lib/rets/metadata.rb
+lib/rets/metadata/caching.rb
 lib/rets/metadata/containers.rb
+lib/rets/metadata/file_cache.rb
+lib/rets/metadata/json_serializer.rb
+lib/rets/metadata/lookup_table.rb
 lib/rets/metadata/lookup_type.rb
+lib/rets/metadata/marshal_serializer.rb
+lib/rets/metadata/multi_lookup_table.rb
+lib/rets/metadata/null_cache.rb
 lib/rets/metadata/resource.rb
 lib/rets/metadata/rets_class.rb
+lib/rets/metadata/rets_object.rb
 lib/rets/metadata/root.rb
 lib/rets/metadata/table.rb
+lib/rets/metadata/table_factory.rb
+lib/rets/metadata/yaml_serializer.rb
 lib/rets/parser/compact.rb
 lib/rets/parser/error_checker.rb
 lib/rets/parser/multipart.rb
 test/fixtures.rb
 test/helper.rb
+test/test_caching.rb
 test/test_client.rb
 test/test_error_checker.rb
+test/test_file_cache.rb
 test/test_http_client.rb
+test/test_json_serializer.rb
 test/test_locking_http_client.rb
+test/test_marshal_serializer.rb
 test/test_metadata.rb
+test/test_metadata_class.rb
+test/test_metadata_lookup_table.rb
+test/test_metadata_lookup_type.rb
+test/test_metadata_multi_lookup_table.rb
+test/test_metadata_object.rb
+test/test_metadata_resource.rb
+test/test_metadata_root.rb
+test/test_metadata_table.rb
+test/test_metadata_table_factory.rb
 test/test_parser_compact.rb
 test/test_parser_multipart.rb
+test/test_yaml_serializer.rb
 test/vcr_cassettes/unauthorized_response.yml

data/README.md

@@ -28,7 +28,79 @@ gem 'rets'
 ## EXAMPLE USAGE:
 
 We need work in this area! There are currently a few guideline examples in the `example` folder on connecting, fetching a property's data, and fetching a property's photos.
- 
+
+## Metadata caching
+
+Metadata, which is loaded when a client is first started, can be slow
+to fetch.  To avoid the cost of fetching metadata every time the
+client is started, metadata can be cached.
+
+To cache metadata, pass the :metadata_cache option to the client when
+you start it.  The library comes with a predefined metadata cache that
+persists the metadata to a file.  It is created with the path to which
+the cached metadata should be written:
+
+    metadata_cache = Rets::Metadata::FileCache.new("/tmp/metadata")
+
+When you create the RETS client, pass it the metadata cache:
+
+    client = Rets::Client.new(
+      ...
+      metadata_cache: metadata_cache
+    )
+
+If you want to persist to something other than a file, create your own
+metadata cache object and pass it in.  It should have the same interface
+as the built-in Metadata::FileCache class:
+
+    class MyMetadataCache
+
+      # Save the metadata.  Should yield an IO-like object to a block;
+      # that block will serialize the metadata to that object.
+      def save(&block)
+      end
+
+      # Load the metadata.  Should yield an IO-like object to a block;
+      # that block will deserialize the metadata from that object and
+      # return the metadata.  Returns the metadata, or nil if it could
+      # not be loaded.
+      def load(&block)
+      end
+      
+    end
+
+By default, the metadata is serialized using Marshal.  You may select
+JSON or YAML instead, or define your own serialization mechanism, using the
+:metadata_serializer option when you create the Rets::Client:
+
+    client = Rets::Client.new(
+      ...
+      metadata_serializer: Rets::Metadata::JsonSerializer.new
+    )
+
+The built-in serializers are:
+
+* Rets::Metadata::JsonSerializer
+* Rets::Metadata::MarshalSerializer
+* Rets::Metadata::YamlSerializer
+
+To define your own serializer, create an object with this interface:
+
+    class MySerializer
+
+      # Serialize to a file.  The library reserves the right to change
+      # the type or contents of o, so don't depend on it being
+      # anything in particular.
+      def save(file, o)
+      end
+
+      # Deserialize from a file.  If the metadata cannot be
+      # deserialized, return nil.
+      def load(file)
+      end
+      
+    end
+
 ## LICENSE:
 
 (The MIT License)

data/Rakefile

@@ -9,7 +9,7 @@ Hoe.plugin :gemspec
 Hoe.spec 'rets' do
   developer 'Estately, Inc. Open Source', 'opensource@estately.com'
 
-  extra_deps << [ "httpclient", "~> 2.6.0" ]
+  extra_deps << [ "httpclient", "~> 2.7.0" ]
   extra_deps << [ "http-cookie", "~> 1.0.0" ]
   extra_deps << [ "nokogiri",   "~> 1.5" ]
 

data/lib/rets.rb

@@ -3,8 +3,9 @@ require 'digest/md5'
 require 'nokogiri'
 
 module Rets
-  VERSION = '0.9.0'
+  VERSION = '0.10.0'
 
+  HttpError            = Class.new(StandardError)
   MalformedResponse    = Class.new(ArgumentError)
   UnknownResponse      = Class.new(ArgumentError)
   NoLogout             = Class.new(ArgumentError)
@@ -55,6 +56,206 @@ module Rets
       super("unknown capability #{capability_name}, available capabilities #{capabilities}")
     end
   end
+
+  class SystemError < InvalidRequest
+    ERROR_CODE = 10000
+  end
+
+  class ZeroBalance < InvalidRequest
+    ERROR_CODE = 20003
+  end
+
+  class BrokerCodeRequired < InvalidRequest
+    ERROR_CODE = 20012
+  end
+
+  class AdditionalLoginNotPermitted < InvalidRequest
+    ERROR_CODE = 20022
+  end
+
+  class ServerLoginError < InvalidRequest
+    ERROR_CODE = 20036
+  end
+
+  class ClientAuthenticationFailed < InvalidRequest
+    ERROR_CODE = 20037
+  end
+
+  class UserAgentAuthenticationRequired < InvalidRequest
+    ERROR_CODE = 20041
+  end
+
+  class ServerTemporarilyDisabled < InvalidRequest
+    ERROR_CODE = 20050
+  end
+
+  class InsecurePassword < InvalidRequest
+    ERROR_CODE = 20140
+  end
+
+  class SameAsPreviousPassword < InvalidRequest
+    ERROR_CODE = 20141
+  end
+
+  class InvalidUserName < InvalidRequest
+    ERROR_CODE = 20142
+  end
+
+  class UnknownQueryField < InvalidRequest
+    ERROR_CODE = 20200
+  end
+
+  class InvalidSelect < InvalidRequest
+    ERROR_CODE = 20202
+  end
+
+  class MiscellaneousSearchError < InvalidRequest
+    ERROR_CODE = 20203
+  end
+
+  class InvalidQuerySyntax < InvalidRequest
+    ERROR_CODE = 20206
+  end
+
+  class UnauthorizedQuery < InvalidRequest
+    ERROR_CODE = 20207
+  end
+
+  class MaximumRecordsExceeded < InvalidRequest
+    ERROR_CODE = 20208
+  end
+
+  class RequestTimeout < InvalidRequest
+    ERROR_CODE = 20209
+  end
+
+  class TooManyOutstandingQueries < InvalidRequest
+    ERROR_CODE = 20210
+  end
+
+  class QueryTooComplex < InvalidRequest
+    ERROR_CODE = 20211
+  end
+
+  class InvalidKeyRequest < InvalidRequest
+    ERROR_CODE = 20212
+  end
+
+  class InvalidKey < InvalidRequest
+    ERROR_CODE = 20213
+  end
+
+  class InvalidParameter < InvalidRequest
+    ERROR_CODE = 20301
+  end
+
+  class UnableToSaveRecord < InvalidRequest
+    ERROR_CODE = 20301
+  end
+
+  class MiscellaneousUpdateError < InvalidRequest
+    ERROR_CODE = 20301
+  end
+
+  class WarningResponseNotGivenForAllWarnings < InvalidRequest
+    ERROR_CODE = 20311
+  end
+
+  class WarningResponseGivenForWarningNotRequired < InvalidRequest
+    ERROR_CODE = 20312
+  end
+
+  class InvalidResource < InvalidRequest
+    ERROR_CODE = 20400
+  end
+
+  class InvalidType < InvalidRequest
+    ERROR_CODE = 20401
+  end
+
+  class InvalidIdentifier < InvalidRequest
+    ERROR_CODE = 20402
+  end
+
+  class UnsupportedMimeType < InvalidRequest
+    ERROR_CODE = 20406
+  end
+
+  class UnauthorizedRetrieval < InvalidRequest
+    ERROR_CODE = 20407
+  end
+
+  class ResourceUnavailable < InvalidRequest
+    ERROR_CODE = 20408
+  end
+
+  class ObjectUnavailable < InvalidRequest
+    ERROR_CODE = 20409
+  end
+
+  class RequestTooLarge < InvalidRequest
+    ERROR_CODE = 20410
+  end
+
+  class ExecutionTimeout < InvalidRequest
+    ERROR_CODE = 20411
+  end
+
+  class TooManyOutstandingRequests < InvalidRequest
+    ERROR_CODE = 20412
+  end
+
+  class InvalidResourceRequested < InvalidRequest
+    ERROR_CODE = 20500
+  end
+
+  class InvalidMetadataType < InvalidRequest
+    ERROR_CODE = 20501
+  end
+
+  class InvalidIdentifierRequested < InvalidRequest
+    ERROR_CODE = 20502
+  end
+
+  class NoMetadataFound < InvalidRequest
+    ERROR_CODE = 20503
+  end
+
+  class UnsupportedMetadataMimeType < InvalidRequest
+    ERROR_CODE = 20506
+  end
+
+  class UnauthorizedMetadataRetrieval < InvalidRequest
+    ERROR_CODE = 20507
+  end
+
+  class MetadataResourceUnavailable < InvalidRequest
+    ERROR_CODE = 20508
+  end
+
+  class MetadataUnavailable < InvalidRequest
+    ERROR_CODE = 20509
+  end
+
+  class MetadataRequestTooLarge < InvalidRequest
+    ERROR_CODE = 20510
+  end
+
+  class MetadataRequestTimeout < InvalidRequest
+    ERROR_CODE = 20511
+  end
+
+  class TooManyOutstandingMetadataRequests < InvalidRequest
+    ERROR_CODE = 20512
+  end
+
+  class RequestedDTDVersionUnavailable < InvalidRequest
+    ERROR_CODE = 20514
+  end
+
+  class NotLoggedIn < InvalidRequest
+    ERROR_CODE = 20701
+  end
 end
 
 require 'rets/http_client'

data/lib/rets/client.rb

@@ -1,17 +1,11 @@
-require 'http-cookie'
-require 'httpclient'
 require 'logger'
 
 module Rets
-  class HttpError < StandardError ; end
-
   class Client
-    DEFAULT_OPTIONS = {}
-
     COUNT = Struct.new(:exclude, :include, :only).new(0,1,2)
+    CASE_INSENSITIVE_PROC = Proc.new { |h,k| h.key?(k.downcase) ? h[k.downcase] : nil }
 
-    attr_accessor :login_url, :options, :logger
-    attr_writer   :capabilities, :metadata
+    attr_accessor :cached_metadata, :client_progress, :logger, :login_url, :options
 
     def initialize(options)
       @options = options
@@ -19,41 +13,15 @@ module Rets
     end
 
     def clean_setup
-      self.options     = DEFAULT_OPTIONS.merge(@options)
-      self.login_url   = self.options[:login_url]
-
-      @cached_metadata   = nil
-      @capabilities      = nil
-      @metadata          = nil
-      @tries             = nil
-      self.capabilities  = nil
-
-      self.logger      = @options[:logger] || FakeLogger.new
-      @client_progress = ClientProgressReporter.new(self.logger, options[:stats_collector], options[:stats_prefix])
-      @cached_metadata = @options[:metadata]
-      if @options[:http_proxy]
-        @http = HTTPClient.new(options.fetch(:http_proxy))
-
-        if @options[:proxy_username]
-          @http.set_proxy_auth(options.fetch(:proxy_username), options.fetch(:proxy_password))
-        end
-      else
-        @http = HTTPClient.new
-      end
-
-      if @options[:receive_timeout]
-        @http.receive_timeout = @options[:receive_timeout]
-      end
-
-      @http.set_cookie_store(options[:cookie_store]) if options[:cookie_store]
-
-      @http_client = Rets::HttpClient.new(@http, @options, @logger, @login_url)
-      if options[:http_timing_stats_collector]
-        @http_client = Rets::MeasuringHttpClient.new(@http_client, options.fetch(:http_timing_stats_collector), options.fetch(:http_timing_stats_prefix))
-      end
-      if options[:lock_around_http_requests]
-        @http_client = Rets::LockingHttpClient.new(@http_client, options.fetch(:locker), options.fetch(:lock_name), options.fetch(:lock_options))
-      end
+      @metadata            = nil
+      @tries               = nil
+      @login_url           = options[:login_url]
+      @cached_metadata     = options[:metadata]
+      @cached_capabilities = options[:capabilities]
+      @logger              = options[:logger] || FakeLogger.new
+      @client_progress     = ClientProgressReporter.new(logger, options[:stats_collector], options[:stats_prefix])
+      @http_client         = Rets::HttpClient.from_options(options, logger)
+      @caching             = Metadata::Caching.make(options)
     end
 
     # Attempts to login by making an empty request to the URL provided in
@@ -63,9 +31,11 @@ module Rets
       res = http_get(login_url)
       Parser::ErrorChecker.check(res)
 
-      self.capabilities = extract_capabilities(Nokogiri.parse(res.body))
-      raise UnknownResponse, "Cannot read rets server capabilities." unless @capabilities
-      @capabilities
+      new_capabilities = extract_capabilities(Nokogiri.parse(res.body))
+      unless new_capabilities
+        raise UnknownResponse, "Cannot read rets server capabilities."
+      end
+      @capabilities = new_capabilities
     end
 
     def logout
@@ -110,39 +80,58 @@ module Rets
 
     def find_with_retries(opts = {})
       retries = 0
-      resolve = opts.delete(:resolve)
-      find_with_given_retry(retries, resolve, opts)
+      find_with_given_retry(retries, opts)
     end
 
-    def find_with_given_retry(retries, resolve, opts)
+    def find_with_given_retry(retries, opts)
       begin
-        find_every(opts, resolve)
+        find_every(opts)
       rescue NoRecordsFound => e
         if opts.fetch(:no_records_not_an_error, false)
-          @client_progress.no_records_found
+          client_progress.no_records_found
           opts[:count] == COUNT.only ? 0 : []
         else
-          handle_find_failure(retries, resolve, opts, e)
+          handle_find_failure(retries, opts, e)
         end
-      rescue AuthorizationFailure, InvalidRequest => e
-        handle_find_failure(retries, resolve, opts, e)
+      rescue InvalidRequest, HttpError => e
+        handle_find_failure(retries, opts, e)
+      rescue AuthorizationFailure => e
+        login
+        handle_find_failure(retries, opts, e)
       end
     end
 
-    def handle_find_failure(retries, resolve, opts, e)
+    def handle_find_failure(retries, opts, e)
       if retries < opts.fetch(:max_retries, 3)
         retries += 1
-        @client_progress.find_with_retries_failed_a_retry(e, retries)
+        client_progress.find_with_retries_failed_a_retry(e, retries)
         clean_setup
-        find_with_given_retry(retries, resolve, opts)
+        find_with_given_retry(retries, opts)
       else
-        @client_progress.find_with_retries_exceeded_retry_count(e)
+        client_progress.find_with_retries_exceeded_retry_count(e)
         raise e
       end
     end
 
-    def find_every(opts, resolve)
-      params = {"QueryType" => "DMQL2", "Format" => "COMPACT"}.merge(fixup_keys(opts))
+    def find_every(opts)
+      raise ArgumentError.new("missing option :search_type (provide the name of a RETS resource)") unless opts[:search_type]
+      raise ArgumentError.new("missing option :class (provide the name of a RETS class)") unless opts[:class]
+
+      params = {
+        "SearchType"          => opts.fetch(:search_type),
+        "Class"               => opts.fetch(:class),
+
+        "Count"               => opts[:count],
+        "Format"              => opts.fetch(:format, "COMPACT"),
+        "Limit"               => opts[:limit],
+        "Offset"              => opts[:offset],
+        "Select"              => opts[:select],
+        "RestrictedIndicator" => opts[:RestrictedIndicator],
+        "StandardNames"       => opts[:standard_name],
+        "Payload"             => opts[:payload],
+        "Query"               => opts[:query],
+        "QueryType"           => opts.fetch(:query_type, "DMQL2"),
+      }.reject { |k,v| v.nil? }
       res = http_post(capability_url("Search"), params)
 
       if opts[:count] == COUNT.only
@@ -151,7 +140,7 @@ module Rets
         results = Parser::Compact.parse_document(
           res.body.encode("UTF-8", res.body.encoding, :invalid => :replace, :undef => :replace)
         )
-        if resolve
+        if opts[:resolve]
           rets_class = find_rets_class(opts[:search_type], opts[:class])
           decorate_results(results, rets_class)
         else
@@ -177,7 +166,7 @@ module Rets
           result[key] = table.resolve(value.to_s)
         else
           #can't resolve just leave the value be
-          @client_progress.could_not_resolve_find_metadata(key)
+          client_progress.could_not_resolve_find_metadata(key)
         end
       end
     end
@@ -191,7 +180,7 @@ module Rets
     def objects(object_ids, opts = {})
       response = case object_ids
         when String then fetch_object(object_ids, opts)
-        when Array  then fetch_object(object_ids.join(","), opts)
+        when Array  then fetch_object(object_ids.join(":"), opts)
         else raise ArgumentError, "Expected instance of String or Array, but got #{object_ids.inspect}."
       end
 
@@ -214,10 +203,11 @@ module Rets
 
         return parts
       else
+        logger.debug "Rets::Client: Found 1 part (the whole body)"
+
         # fake a multipart for interface compatibility
         headers = {}
-        response.headers.each { |k,v| headers[k] = v[0] }
-
+        response.headers.each { |k,v| headers[k.downcase] = v }
         part = Parser::Multipart::Part.new(headers, response.body)
 
         return [part]
@@ -250,35 +240,24 @@ module Rets
       http_post(capability_url("GetObject"), params, extra_headers)
     end
 
-    # Changes keys to be camel cased, per the RETS standard for queries.
-    def fixup_keys(hash)
-      fixed_hash = {}
-
-      hash.each do |key, value|
-        camel_cased_key = key.to_s.capitalize.gsub(/_(\w)/) { $1.upcase }
-
-        fixed_hash[camel_cased_key] = value
-      end
-
-      fixed_hash
-    end
-
-    def metadata
+    def metadata(types=nil)
       return @metadata if @metadata
-
-      if @cached_metadata && (@options[:skip_metadata_uptodate_check] ||
-          @cached_metadata.current?(capabilities["MetadataTimestamp"], capabilities["MetadataVersion"]))
-        @client_progress.use_cached_metadata
-        self.metadata = @cached_metadata
+      @cached_metadata ||= @caching.load(@logger)
+      if cached_metadata && (options[:skip_metadata_uptodate_check] ||
+          cached_metadata.current?(capabilities["MetadataTimestamp"], capabilities["MetadataVersion"]))
+        client_progress.use_cached_metadata
+        @metadata = cached_metadata
       else
-        @client_progress.bad_cached_metadata(@cached_metadata)
-        self.metadata = Metadata::Root.new(logger, retrieve_metadata)
+        client_progress.bad_cached_metadata(cached_metadata)
+        @metadata = Metadata::Root.new(logger, retrieve_metadata(types))
+        @caching.save(metadata)
       end
+      @metadata
     end
 
-    def retrieve_metadata
+    def retrieve_metadata(types=nil)
       raw_metadata = {}
-      Metadata::METADATA_TYPES.each {|type|
+      (types || Metadata::METADATA_TYPES).each {|type|
         raw_metadata[type] = retrieve_metadata_type(type)
       }
       raw_metadata
@@ -301,7 +280,13 @@ module Rets
     #
     # [1] In fact, sometimes only a path is returned from the server.
     def capabilities
-      @capabilities || login
+      if @capabilities
+        @capabilities
+      elsif @cached_capabilities
+        @capabilities = add_case_insensitive_default_proc(@cached_capabilities)
+      else
+        login
+      end
     end
 
     def capability_url(name)
@@ -325,15 +310,19 @@ module Rets
     def extract_capabilities(document)
       raw_key_values = document.xpath("/RETS/RETS-RESPONSE").text.strip
 
-      hash = Hash.new{|h,k| h.key?(k.downcase) ? h[k.downcase] : nil }
-
       # ... :(
       # Feel free to make this better. It has a test.
-      raw_key_values.split(/\n/).
+      hash = raw_key_values.split(/\n/).
         map  { |r| r.split(/\=/, 2) }.
-        each { |k,v| hash[k.strip.downcase] = v.strip }
+        each_with_object({}) { |(k,v), h| h[k.strip.downcase] = v.strip }
+
+      add_case_insensitive_default_proc(hash)
+    end
 
-      hash
+    def add_case_insensitive_default_proc(hash)
+      new_hash = hash.dup
+      new_hash.default_proc = CASE_INSENSITIVE_PROC
+      new_hash
     end
 
     def save_cookie_store