rets 0.10.0 → 0.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: cedf0e419db8f2c813a4c2c97fd210909b9b58af
-  data.tar.gz: b8eb9ad36844f79503c64f020830ef477586fa20
+  metadata.gz: 966f5afd3517f512f4a54b235b7e52aa5437c7b7
+  data.tar.gz: 5b29080c3a5c2a6c48c2b4e8c5d32cd51b90add9
 SHA512:
-  metadata.gz: e116e7a292cb5e8118cfff090aae8147a431299fd79b81572bf0afb6d093c0152f1ad35c671e2199898ddaf5b8ea12061a011184776179a239b44c8a8baf9b74
-  data.tar.gz: d8f6da1337852f4ba871eabb531e31765e5a1a1167f57a9aad5b02ce01c6a242d260d7f6e63492e8fd64c89f9cb4da12a1ec344c20b39f519195cf075cdce666
+  metadata.gz: bb19f04bc071b36c147cc697b4b0c616be5c47a2e0919f9289d6fac7f4d112f356a0fbe25ca5a0fb56dbac62c8c3aa0d11438411ccafe43c2d5f16972f3f4990
+  data.tar.gz: 9d64e1063bbd640985bc0820c58e403b1391d62838bd92f353719caa7805de98e1188e196a96af5f5e92c55691ebeb8c1ee25eb54e26640c6e6aaf01c89be40b

data/CHANGELOG.md

@@ -1,3 +1,14 @@
+### 0.11.0 / NOT RELEASED YET
+
+* fix: fix retry logging
+* feature: allow retries to be configured for all query types in client settings
+* feature: allow configrable wait time between retries
+* feature: detect errors as error messages in a response body delivered with HTTP 200
+
+### 0.10.1 / 2016-05-04
+
+* fix: handle invalid codepoints in character references
+
 ### 0.10.0 / 2016-02-29
 
 * fix: ensure cookie store exists #133

data/README.md

@@ -7,6 +7,8 @@
 [![Build Status](https://secure.travis-ci.org/estately/rets.png?branch=master)](http://travis-ci.org/estately/rets)
 A pure-ruby library for fetching data from [RETS] servers.
 
+If you're looking for a slick CLI interface check out [retscli](https://github.com/summera/retscli), which is an awesome tool for exploring metadata or learning about RETS.
+
 [RETS]: http://www.rets.org
 
 ## REQUIREMENTS:

data/Rakefile

@@ -13,7 +13,7 @@ Hoe.spec 'rets' do
   extra_deps << [ "http-cookie", "~> 1.0.0" ]
   extra_deps << [ "nokogiri",   "~> 1.5" ]
 
-  extra_dev_deps << [ "mocha", "~> 0.11" ]
+  extra_dev_deps << [ "mocha", "~> 1.1.0" ]
   extra_dev_deps << [ "vcr", "~> 2.2" ]
   extra_dev_deps << [ "webmock", "~> 1.8" ]
 

data/lib/rets.rb

@@ -3,7 +3,7 @@ require 'digest/md5'
 require 'nokogiri'
 
 module Rets
-  VERSION = '0.10.0'
+  VERSION = '0.11.0'
 
   HttpError            = Class.new(StandardError)
   MalformedResponse    = Class.new(ArgumentError)

data/lib/rets/client.rb

@@ -13,6 +13,9 @@ module Rets
     end
 
     def clean_setup
+      if options.fetch(:login_after_error, true)
+        @capabilities = nil
+      end
       @metadata            = nil
       @tries               = nil
       @login_url           = options[:login_url]
@@ -102,9 +105,11 @@ module Rets
     end
 
     def handle_find_failure(retries, opts, e)
-      if retries < opts.fetch(:max_retries, 3)
+      max_retries = fetch_max_retries(opts)
+      if retries < max_retries
         retries += 1
-        client_progress.find_with_retries_failed_a_retry(e, retries)
+        wait_before_next_request
+        client_progress.find_with_retries_failed_a_retry(e, retries, max_retries)
         clean_setup
         find_with_given_retry(retries, opts)
       else
@@ -113,6 +118,18 @@ module Rets
       end
     end
 
+    def fetch_max_retries(hash)
+      hash[:max_retries] || options.fetch(:max_retries, 3)
+    end
+
+    def wait_before_next_request
+      sleep_time = Float(options.fetch(:recoverable_error_wait_secs, 0))
+      if sleep_time > 0
+        logger.info "Waiting #{sleep_time} seconds before next attempt"
+        sleep sleep_time
+      end
+    end
+
     def find_every(opts)
       raise ArgumentError.new("missing option :search_type (provide the name of a RETS resource)") unless opts[:search_type]
       raise ArgumentError.new("missing option :class (provide the name of a RETS class)") unless opts[:class]
@@ -120,7 +137,6 @@ module Rets
       params = {
         "SearchType"          => opts.fetch(:search_type),
         "Class"               => opts.fetch(:class),
-
         "Count"               => opts[:count],
         "Format"              => opts.fetch(:format, "COMPACT"),
         "Limit"               => opts[:limit],
@@ -132,13 +148,13 @@ module Rets
         "Query"               => opts[:query],
         "QueryType"           => opts.fetch(:query_type, "DMQL2"),
       }.reject { |k,v| v.nil? }
-      res = http_post(capability_url("Search"), params)
+      res = clean_response(http_post(capability_url("Search"), params))
 
       if opts[:count] == COUNT.only
         Parser::Compact.get_count(res.body)
       else
         results = Parser::Compact.parse_document(
-          res.body.encode("UTF-8", res.body.encoding, :invalid => :replace, :undef => :replace)
+          res.body
         )
         if opts[:resolve]
           rets_class = find_rets_class(opts[:search_type], opts[:class])
@@ -269,7 +285,7 @@ module Rets
                         "Type"   => "METADATA-#{type}",
                         "ID"     => "0"
                       })
-      res.body
+      clean_response(res).body
     end
 
     # The capabilies as provided by the RETS server during login.
@@ -330,7 +346,12 @@ module Rets
     end
 
     def http_get(url, params=nil, extra_headers={})
-      @http_client.http_get(url, params, extra_headers)
+      clean_response(@http_client.http_get(url, params, extra_headers))
+    end
+
+    def clean_response(res)
+      res.body.encode!("UTF-8", res.body.encoding, :invalid => :replace, :undef => :replace)
+      res
     end
 
     def http_post(url, params, extra_headers = {})

data/lib/rets/client_progress_reporter.rb

@@ -18,10 +18,10 @@ module Rets
       @stats_prefix = stats_prefix
     end
 
-    def find_with_retries_failed_a_retry(exception, retries)
+    def find_with_retries_failed_a_retry(exception, retries, max_retries)
       @stats.count("#{@stats_prefix}find_with_retries_failed_retry")
       @logger.warn("Rets::Client: Failed with message: #{exception.message}")
-      @logger.info("Rets::Client: Retry #{retries}/3")
+      @logger.info("Rets::Client: Retry #{retries}/#{max_retries}")
     end
 
     def find_with_retries_exceeded_retry_count(exception)

data/lib/rets/metadata/lookup_table.rb

@@ -22,7 +22,7 @@ module Rets
       #
       # [out] The file to print to.  Defaults to $stdout.
       def print_tree(out = $stdout)
-        out.puts "    LookupTable: #{name}"
+        out.puts "### LookupTable: #{name}"
         out.puts "      Resource: #{resource_id}"
         out.puts "      Required: #{table_fragment['Required']}"
         out.puts "      Searchable: #{ table_fragment["Searchable"] }"
@@ -30,7 +30,7 @@ module Rets
         out.puts "      ShortName: #{ table_fragment["ShortName"] }"
         out.puts "      LongName: #{ long_name }"
         out.puts "      StandardName: #{ table_fragment["StandardName"] }"
-        out.puts "      Types:"
+        out.puts "####  Types:"
 
         lookup_types.each do |lookup_type|
           lookup_type.print_tree(out)

data/lib/rets/metadata/lookup_type.rb

@@ -4,8 +4,8 @@ module Rets
       attr_reader :long_value, :value
 
       def initialize(lookup_type_fragment)
-        @value       = lookup_type_fragment["Value"]
-        @long_value  = lookup_type_fragment["LongValue"]
+        @value       = lookup_type_fragment["Value"].strip
+        @long_value  = lookup_type_fragment["LongValue"].strip
       end
 
       # Print the tree to a file

data/lib/rets/metadata/multi_lookup_table.rb

@@ -22,7 +22,7 @@ module Rets
       #
       # [out] The file to print to.  Defaults to $stdout.
       def print_tree(out = $stdout)
-        out.puts "    MultiLookupTable: #{name}"
+        out.puts "### MultiLookupTable: #{name}"
         out.puts "      Resource: #{resource_id}"
         out.puts "      Required: #{table_fragment['Required']}"
         out.puts "      Searchable: #{ table_fragment["Searchable"] }"

data/lib/rets/metadata/resource.rb

@@ -29,7 +29,12 @@ module Rets
       end
 
       def self.find_rets_objects(metadata, resource_id)
-        metadata[:object].select { |object| object.resource == resource_id }.map(&:objects).flatten
+        objects = metadata[:object]
+        if objects
+          objects.select { |object| object.resource == resource_id }.map(&:objects).flatten
+        else
+          []
+        end
       end
 
       def self.build_lookup_tree(resource_id, metadata)
@@ -80,7 +85,7 @@ module Rets
       #
       # [out] The file to print to.  Defaults to $stdout.
       def print_tree(out = $stdout)
-        out.puts "Resource: #{id} (Key Field: #{key_field})"
+        out.puts "# Resource: #{id} (Key Field: #{key_field})"
         rets_classes.each do |rets_class|
           rets_class.print_tree(out)
         end

data/lib/rets/metadata/rets_class.rb

@@ -41,7 +41,7 @@ module Rets
       #
       # [out] The file to print to.  Defaults to $stdout.
       def print_tree(out = $stdout)
-        out.puts "  Class: #{name}"
+        out.puts "## Class: #{name}"
         out.puts "    Visible Name: #{visible_name}"
         out.puts "    Description : #{description}"
         tables.each do |table|

data/lib/rets/metadata/rets_object.rb

@@ -1,24 +1,28 @@
 module Rets
   module Metadata
     class RetsObject
-      attr_reader :name, :mime_type, :description
+      attr_reader :name, :mime_type, :description, :type
 
-      def initialize(name, mime_type, description)
+      def initialize(type, name, mime_type, description)
         @name = name
         @mime_type = mime_type
         @description = description
+        @type = type
       end
 
       def self.build(rets_object_fragment)
-        name = rets_object_fragment["VisibleName"]
-        mime_type = rets_object_fragment["MIMEType"]
-        description = rets_object_fragment["Description"]
-        new(name, mime_type, description)
+        rets_object_fragment = downcase_hash_keys(rets_object_fragment)
+        name = rets_object_fragment["visiblename"]
+        mime_type = rets_object_fragment["mimetype"]
+        description = rets_object_fragment["description"]
+        type = rets_object_fragment['objecttype']
+        new(type, name, mime_type, description)
       end
 
       def print_tree(out = $stdout)
-        out.puts "  Object: #{name}"
-        out.puts "    MimeType: #{mime_type}"
+        out.puts "  Object: #{type}"
+        out.puts "    Visible Name: #{name}"
+        out.puts "    Mime Type: #{mime_type}"
         out.puts "    Description: #{description}"
       end
 
@@ -27,6 +31,11 @@ module Rets
           mime_type == other.mime_type &&
           description == other.description
       end
+
+      private
+      def self.downcase_hash_keys(hash)
+        Hash[hash.map { |k, v| [k.downcase, v] }]
+      end
     end
   end
 end

data/lib/rets/metadata/table.rb

@@ -15,7 +15,7 @@ module Rets
       #
       # [out] The file to print to.  Defaults to $stdout.
       def print_tree(out = $stdout)
-        out.puts "    Table: #{name}"
+        out.puts "### Table: #{name}"
         out.puts "      Resource: #{resource_id}"
         out.puts "      ShortName: #{ table_fragment["ShortName"] }"
         out.puts "      LongName: #{ long_name }"

data/lib/rets/parser/compact.rb

@@ -1,5 +1,6 @@
 # coding: utf-8
 require 'cgi'
+
 module Rets
   module Parser
     class Compact
@@ -76,7 +77,10 @@ module Rets
         end
 
         column_names = columns.split(delimiter)
-        data_values = data.split(delimiter, INCLUDE_NULL_FIELDS).map { |x| CGI.unescapeHTML(x) }
+        data_values = data.split(delimiter, INCLUDE_NULL_FIELDS).map do |x|
+          safely_decode_character_references!(x)
+          CGI.unescape_html(x)
+        end
 
         zipped_key_values = column_names.zip(data_values).map { |k, v| [k.freeze, v.to_s] }
 
@@ -84,6 +88,21 @@ module Rets
         hash.reject { |key, value| key.empty? && value.to_s.empty? }
       end
 
+      def self.safely_decode_character_references!(string)
+        string.gsub!(/&#(x)?([\h]+);/) do
+          if $2
+            base = $1 == "x" ? 16 : 10
+            int = Integer($2, base)
+            begin
+              int.chr(Encoding::UTF_8)
+            rescue RangeError
+              ""
+            end
+          end
+        end
+        string
+      end
+
       def self.get_count(xml)
         doc = Nokogiri.parse(xml.to_s)
         if node = doc.at("//COUNT")

data/test/fixtures.rb

@@ -266,6 +266,14 @@ SAMPLE_COMPACT_WITH_SPECIAL_CHARS_2 = <<EOF
 </RETS>
 EOF
 
+SAMPLE_COMPACT_WITH_DOUBLY_ENCODED_BAD_CHARACTER_REFERENCES = <<EOF
+<RETS ReplyCode=\"0\" ReplyText=\"Operation Success.\">
+  <DELIMITER value=\"09\" />
+  <COLUMNS>	PublicRemarksNew	WindowCoverings	YearBuilt	Zoning	ZoningCompatibleYN	</COLUMNS>
+  <DATA>	foo &amp;#56324; bar		1999	00		</DATA>
+</RETS>
+EOF
+
 SAMPLE_PROPERTY_WITH_LOTS_OF_COLUMNS = <<EOF
 <RETS ReplyCode=\"0\" ReplyText=\"Operation Success.\">
   <DELIMITER value=\"09\" />
@@ -309,7 +317,8 @@ Resource: Properties (Key Field: matrix_unique_key)
       Types:
         Quarterly -> Q
         Annually -> A
-  Object: Photo
-    MimeType: photo/jpg
+  Object: HiRes
+    Visible Name: Photo
+    Mime Type: photo/jpg
     Description: photo description
 EOF

data/test/test_client.rb

@@ -97,7 +97,7 @@ class TestClient < MiniTest::Test
     logger.debug "foo"
   end
 
-  def test_find_first_calls_find_every_with_limit_one
+  def test_find_every_raises_on_missing_required_arguments
     assert_raises ArgumentError do
       @client.find_every({})
     end
@@ -129,38 +129,27 @@ class TestClient < MiniTest::Test
 
   def test_response_text_encoding_from_ascii
     @client.stubs(:capability_url).with("Search").returns("search_url")
-
     response = mock
-    response.stubs(:body).returns(("An ascii string").encode("binary", "UTF-8"))
-    @client.stubs(:http_post).with("search_url", anything).returns(response)
+    response.stubs(:body).returns("An ascii string".encode("binary", "UTF-8"))
 
-    Rets::Parser::Compact.expects(:parse_document).with("An ascii string")
-
-    @client.find_every(:search_type => "Foo", :class => "Bar")
+    assert_equal @client.clean_response(response).body, "An ascii string"
   end
 
   def test_response_text_encoding_from_utf_8
     @client.stubs(:capability_url).with("Search").returns("search_url")
-
     response = mock
     response.stubs(:body).returns("Some string with non-ascii characters \u0119")
-    @client.stubs(:http_post).with("search_url", anything).returns(response)
 
-    Rets::Parser::Compact.expects(:parse_document).with("Some string with non-ascii characters \u0119")
+    assert_equal @client.clean_response(response).body, "Some string with non-ascii characters \u0119"
 
-    @client.find_every(:search_type => "Foo", :class => "Bar")
   end
 
   def test_response_text_encoding_from_utf_16
     @client.stubs(:capability_url).with("Search").returns("search_url")
-
     response = mock
     response.stubs(:body).returns("Some string with non-utf-8 characters \xC2")
-    @client.stubs(:http_post).with("search_url", anything).returns(response)
-
-    Rets::Parser::Compact.expects(:parse_document).with("Some string with non-utf-8 characters \uFFFD")
 
-    @client.find_every(:search_type => "Foo", :class => "Bar")
+    assert_equal @client.clean_response(response).body, "Some string with non-utf-8 characters \uFFFD"
   end
 
   def test_find_retries_when_receiving_no_records_found
@@ -187,6 +176,13 @@ class TestClient < MiniTest::Test
     @client.find(:all, :foo => :bar)
   end
 
+  def test_find_waits_configured_time_before_next_request
+    @client.options[:recoverable_error_wait_secs] = 3.14
+    @client.expects(:sleep).with(3.14).times(3)
+    @client.stubs(:find_every).raises(Rets::MiscellaneousSearchError.new(0, 'Foo'))
+    @client.find(:all, :foo => :bar) rescue nil
+  end
+
   def test_find_eventually_reraises_errors
     @client.stubs(:find_every).raises(Rets::AuthorizationFailure.new(401, 'Not Authorized'))
     @client.stubs(:login)

data/test/test_metadata.rb

@@ -58,7 +58,7 @@ class TestMetadata < MiniTest::Test
     ]
 
     rets_objects = [
-      Rets::Metadata::RetsObject.new("Photo", "photo/jpg", "photo description")
+      Rets::Metadata::RetsObject.new("HiRes", "Photo", "photo/jpg", "photo description")
     ]
 
     resource = Rets::Metadata::Resource.new(resource_id, "matrix_unique_key", rets_classes, rets_objects)

data/test/test_metadata_lookup_type.rb

@@ -9,4 +9,13 @@ class TestMetadataLookupType < MiniTest::Test
     assert_equal('a', lookup_type.value)
     assert_equal('c', lookup_type.long_value)
   end
+
+  def test_lookup_type_ignores_trailing_whitespace
+    fragment = { "Value" => 'a     ', "LongValue" => 'c     ' }
+
+    lookup_type = Rets::Metadata::LookupType.new(fragment)
+
+    assert_equal('a', lookup_type.value)
+    assert_equal('c', lookup_type.long_value)
+  end
 end

data/test/test_metadata_object.rb

@@ -5,16 +5,29 @@ class TestMetadataObject < MiniTest::Test
     name = "Name"
     mime_type = "mimetype"
     description = "description"
+    object_type = "type"
 
     object_fragment = {
-        "VisibleName" => name,
-        "MIMEType"    => mime_type,
-        "Description" => description,
+      "ObjectType" => object_type,
+      "VisibleName" => name,
+      "MIMEType"    => mime_type,
+      "Description" => description,
     }
 
     assert_equal(
       Rets::Metadata::RetsObject.build(object_fragment),
-      Rets::Metadata::RetsObject.new(name, mime_type, description)
+      Rets::Metadata::RetsObject.new(object_type, name, mime_type, description)
+    )
+  end
+
+  def test_rets_object_building_not_case_dependent
+    object_fragment = {
+      "MiMeTyPe" => "image/jpeg"
+    }
+
+    assert_equal(
+      Rets::Metadata::RetsObject.build(object_fragment).mime_type,
+      "image/jpeg"
     )
   end
 end

data/test/test_metadata_resource.rb

@@ -53,12 +53,20 @@ class TestMetadataResource < MiniTest::Test
     assert_equal([rets_object], objects)
   end
 
+  def test_resource_build_objects_when_objects_were_not_loaded
+    resource_id = "id"
+    metadata    = {} # doesn't contain metadata for :object key
+
+    objects = Rets::Metadata::Resource.build_objects(resource_id, metadata)
+    assert_equal [], objects
+  end
+
   def test_resource_build
     fragment = { "ResourceID" => "test" }
 
     lookup_types = stub(:lookup_types)
     classes = stub(:classes)
-    objects = stub(:classes)
+    objects = stub(:objects)
     metadata = stub(:metadata)
 
     Rets::Metadata::Resource.stubs(:build_lookup_tree => lookup_types)

data/test/test_parser_compact.rb

@@ -86,9 +86,30 @@ class TestParserCompact < MiniTest::Test
     assert_equal "text with <tag>", rows.last["PublicRemarksNew"]
   end
 
+  def test_parse_doubly_encoded_bad_character_references
+    rows = Rets::Parser::Compact.parse_document(SAMPLE_COMPACT_WITH_DOUBLY_ENCODED_BAD_CHARACTER_REFERENCES)
+    assert_equal "foo  bar", rows.last["PublicRemarksNew"]
+  end
+
   def test_parse_property_with_lots_of_columns
     row = Rets::Parser::Compact.parse_document(SAMPLE_PROPERTY_WITH_LOTS_OF_COLUMNS).first
     assert_equal 800, row.keys.size
     assert_equal 800.times.map { |x| "K%03d" % x }, row.keys
   end
+
+  def test_safely_decode_character_references!
+    assert_decoded "a", "&#97;"
+    assert_decoded "a", "&#097;"
+    assert_decoded "a", "&#x61;"
+    assert_decoded "a", "&#x061;"
+    assert_decoded "", "&#xDC04;"
+    assert_decoded "", "&#56324;"
+    assert_decoded "", "&#x2000FF;"
+  end
+
+
+  def assert_decoded(a, b)
+    recoded = Rets::Parser::Compact.safely_decode_character_references!(b)
+    assert_equal a, recoded
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rets
 version: !ruby/object:Gem::Version
-  version: 0.10.0
+  version: 0.11.0
 platform: ruby
 authors:
 - Estately, Inc. Open Source
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-02-29 00:00:00.000000000 Z
+date: 2017-01-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: httpclient
@@ -72,14 +72,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.11'
+        version: 1.1.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.11'
+        version: 1.1.0
 - !ruby/object:Gem::Dependency
   name: vcr
   requirement: !ruby/object:Gem::Requirement
@@ -114,18 +114,20 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.13'
+        version: '3.15'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.13'
+        version: '3.15'
 description: |-
   [![Build Status](https://secure.travis-ci.org/estately/rets.png?branch=master)](http://travis-ci.org/estately/rets)
   A pure-ruby library for fetching data from [RETS] servers.
 
+  If you're looking for a slick CLI interface check out [retscli](https://github.com/summera/retscli), which is an awesome tool for exploring metadata or learning about RETS.
+
   [RETS]: http://www.rets.org
 email:
 - opensource@estately.com
@@ -137,7 +139,6 @@ extra_rdoc_files:
 - Manifest.txt
 - README.md
 files:
-- ".gemtest"
 - CHANGELOG.md
 - Manifest.txt
 - README.md
@@ -218,7 +219,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5.1
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: "[![Build Status](https://secure.travis-ci.org/estately/rets.png?branch=master)](http://travis-ci.org/estately/rets)