rets-hack 0.11

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: aa7eec964ae1e040ba2d291f5283432661b1a08c
+  data.tar.gz: 597bbab53e0a16701d8b070cf193f7d4bc015053
+SHA512:
+  metadata.gz: 4946f647c2cc1a67f644018c50918cfb6fd992ce13eda38de79db7ea5f842f7e7aa10dadb8244b7f4ef908cac4c4ebe564cad1a2191bc1819972a70e66d7278f
+  data.tar.gz: 994a48a22bef9a8f03a1195a685339e9d6d2a29a985172962e312d072b1487b8f37a51819afe22765f1d5fbd2627afa65f6c0a0dbe1917ce6a61302b77b17466

data/CHANGELOG.md

@@ -0,0 +1,142 @@
+### 0.11.0 / NOT RELEASED YET
+
+* fix: fix retry logging
+* feature: allow retries to be configured for all query types in client settings
+* feature: allow configrable wait time between retries
+* feature: detect errors as error messages in a response body delivered with HTTP 200
+
+### 0.10.1 / 2016-05-04
+
+* fix: handle invalid codepoints in character references
+
+### 0.10.0 / 2016-02-29
+
+* fix: ensure cookie store exists #133
+* feature: make cached capabilities case insensitive #136
+* feature: add specific classes for each rets error #137
+* feature: whitelist RETS search options #142
+* feature: simplify metadata caching #134
+* feature: use a SAX parser #98
+* fix: save capabilities to avoid double logins #148
+* feature: login on authorization error #155
+* add basic support for DataDictionary feeds #156
+* fix: count always returns a number #161
+* feature: make lookup tables case insensitive #163
+* feature: update to httpclient 2.7 #165
+* fix: getObject now works with non-multipart responses #166
+* fix: getObject works with multiple ids #167
+* feature: store rets object metadata #168
+* feature: add a code of conduct #171
+
+### 0.9.0 / 2015-06-11
+
+* feature: update to httpclient 2.6
+
+### 0.8.1 / 2015-06-09
+
+* fix: actually make the httpclient version more specific this time
+
+### 0.8.0 / 2015-06-09
+
+* feature: reduce memory usage on parsing metadata
+* fix: correctly raise authorization error when given XHTML instead of XML
+* fix: unescape HTML encoded responses
+* fix: make httpclient version requirement more specific
+* feature: add ability to print metadata to a file
+* fix: remove Gemfile.lock from repository
+
+### 0.7.0 / 2015-01-16
+
+* feature: optionally treat No Records Found as not an error
+* fix: update httpclient version, patches SSL vulnerabilities
+* feature: work around bogus http status codes that don't agree with XML body
+
+### 0.6.0 / 2014-11-26
+
+* fix: fix spelling error that created misleading exceptions
+* feature: track stats for http requests sent
+* feature: raise an exception if the login action doesn't return an http 200 status
+* feature: add better class description and more fields to print tree
+* feature: support http proxies
+* feature: customizable http timeouts
+* feature: add logging http headers when in debug mode
+* feature: strip invalid utf8 from responses before parsing
+* fix: don't raise an exception on a 401 after logout
+* fix: treat no matching records status without a count node as a zero count
+* feature: add an option for loading custom ca_certs
+* feature: remove invalid resource types from metadata
+* feature: special case http 412
+* feature: add max_retries option
+
+### 0.5.1 / 2013-10-30
+
+* fix: 0.5.0 was broken, fix gem Manifest to fix gem
+
+### 0.5.0 / 2013-09-05
+
+* feature: Allow client.count to get integer count
+* feature: Allow for downcased capability names
+* fix: Handle the rets element being empty
+* feature: Instrument rets client with stats reporting
+* feature: Add a locking client
+* feature: Support Basic Authentication
+
+### 0.4.0 / 2012-08-29
+
+* fix: update authentication header to uri matches path
+
+### 0.3.0 / 2012-07-31
+
+* correctly handle digest authentication
+
+### 0.3.0.rc.0 / 2012-07-26
+
+* feature: significantly better handling of authorization failures
+
+### 0.2.1 / 2012-04-20
+
+* fix: better handling of malformed RETS responses
+
+### 0.2.0 / 2012-04-20
+
+* feature: Ruby 1.9 compatibility!
+
+### 0.1.7 / 2012-04-05
+
+* feature: key_field lookup for resources
+
+### 0.1.6 / 2012-04-03
+
+* fix: user_agent authentication
+
+### 0.1.5 / 2012-03-17
+
+* fix: retries raise error after too many failures
+* fix: raise error for failed multipart object request
+* fix: retries start with a clean slate, fixing authorization errors during retry
+
+### 0.1.4 / 2012-03-12
+
+* fix: an MLS uses lower case in RETS tag
+
+### 0.1.3 / 2012-03-05
+
+* fixes to support location=1 in getobject query
+
+### 0.1.2 / 2012-02-17
+
+* bugfix - check ReplyCode in login, retry on errors
+
+### 0.1.1 / 2012-01-11
+
+* bugfix - prevent infinite loop in login
+
+# rets Changelog
+
+### 0.1.0 / 2011-06-23
+
+* First public release!
+
+### 0.0.1 / 2011-03-24
+
+* Project Created

data/Manifest.txt

@@ -0,0 +1,58 @@
+CHANGELOG.md
+Manifest.txt
+README.md
+Rakefile
+bin/rets
+example/connect.rb
+example/get-photos.rb
+example/get-property.rb
+lib/rets.rb
+lib/rets/client.rb
+lib/rets/client_progress_reporter.rb
+lib/rets/http_client.rb
+lib/rets/locking_http_client.rb
+lib/rets/measuring_http_client.rb
+lib/rets/metadata.rb
+lib/rets/metadata/caching.rb
+lib/rets/metadata/containers.rb
+lib/rets/metadata/file_cache.rb
+lib/rets/metadata/json_serializer.rb
+lib/rets/metadata/lookup_table.rb
+lib/rets/metadata/lookup_type.rb
+lib/rets/metadata/marshal_serializer.rb
+lib/rets/metadata/multi_lookup_table.rb
+lib/rets/metadata/null_cache.rb
+lib/rets/metadata/resource.rb
+lib/rets/metadata/rets_class.rb
+lib/rets/metadata/rets_object.rb
+lib/rets/metadata/root.rb
+lib/rets/metadata/table.rb
+lib/rets/metadata/table_factory.rb
+lib/rets/metadata/yaml_serializer.rb
+lib/rets/parser/compact.rb
+lib/rets/parser/error_checker.rb
+lib/rets/parser/multipart.rb
+test/fixtures.rb
+test/helper.rb
+test/test_caching.rb
+test/test_client.rb
+test/test_error_checker.rb
+test/test_file_cache.rb
+test/test_http_client.rb
+test/test_json_serializer.rb
+test/test_locking_http_client.rb
+test/test_marshal_serializer.rb
+test/test_metadata.rb
+test/test_metadata_class.rb
+test/test_metadata_lookup_table.rb
+test/test_metadata_lookup_type.rb
+test/test_metadata_multi_lookup_table.rb
+test/test_metadata_object.rb
+test/test_metadata_resource.rb
+test/test_metadata_root.rb
+test/test_metadata_table.rb
+test/test_metadata_table_factory.rb
+test/test_parser_compact.rb
+test/test_parser_multipart.rb
+test/test_yaml_serializer.rb
+test/vcr_cassettes/unauthorized_response.yml

data/README.md

@@ -0,0 +1,129 @@
+# rets
+
+* http://github.com/estately/rets
+
+## DESCRIPTION:
+
+[![Build Status](https://secure.travis-ci.org/estately/rets.png?branch=master)](http://travis-ci.org/estately/rets)
+A pure-ruby library for fetching data from [RETS] servers.
+
+If you're looking for a slick CLI interface check out [retscli](https://github.com/summera/retscli), which is an awesome tool for exploring metadata or learning about RETS.
+
+[RETS]: http://www.rets.org
+
+## REQUIREMENTS:
+
+* [httpclient]
+* [nokogiri]
+
+[httpclient]: https://github.com/nahi/httpclient
+[nokogiri]: http://nokogiri.org
+
+## INSTALLATION:
+```
+gem install rets
+
+# or add it to your Gemfile if using Bundler then run bundle install
+gem 'rets'
+```
+
+## EXAMPLE USAGE:
+
+We need work in this area! There are currently a few guideline examples in the `example` folder on connecting, fetching a property's data, and fetching a property's photos.
+
+## Metadata caching
+
+Metadata, which is loaded when a client is first started, can be slow
+to fetch.  To avoid the cost of fetching metadata every time the
+client is started, metadata can be cached.
+
+To cache metadata, pass the :metadata_cache option to the client when
+you start it.  The library comes with a predefined metadata cache that
+persists the metadata to a file.  It is created with the path to which
+the cached metadata should be written:
+
+    metadata_cache = Rets::Metadata::FileCache.new("/tmp/metadata")
+
+When you create the RETS client, pass it the metadata cache:
+
+    client = Rets::Client.new(
+      ...
+      metadata_cache: metadata_cache
+    )
+
+If you want to persist to something other than a file, create your own
+metadata cache object and pass it in.  It should have the same interface
+as the built-in Metadata::FileCache class:
+
+    class MyMetadataCache
+
+      # Save the metadata.  Should yield an IO-like object to a block;
+      # that block will serialize the metadata to that object.
+      def save(&block)
+      end
+
+      # Load the metadata.  Should yield an IO-like object to a block;
+      # that block will deserialize the metadata from that object and
+      # return the metadata.  Returns the metadata, or nil if it could
+      # not be loaded.
+      def load(&block)
+      end
+      
+    end
+
+By default, the metadata is serialized using Marshal.  You may select
+JSON or YAML instead, or define your own serialization mechanism, using the
+:metadata_serializer option when you create the Rets::Client:
+
+    client = Rets::Client.new(
+      ...
+      metadata_serializer: Rets::Metadata::JsonSerializer.new
+    )
+
+The built-in serializers are:
+
+* Rets::Metadata::JsonSerializer
+* Rets::Metadata::MarshalSerializer
+* Rets::Metadata::YamlSerializer
+
+To define your own serializer, create an object with this interface:
+
+    class MySerializer
+
+      # Serialize to a file.  The library reserves the right to change
+      # the type or contents of o, so don't depend on it being
+      # anything in particular.
+      def save(file, o)
+      end
+
+      # Deserialize from a file.  If the metadata cannot be
+      # deserialized, return nil.
+      def load(file)
+      end
+      
+    end
+
+## LICENSE:
+
+(The MIT License)
+
+Copyright (c) 2011 Estately, Inc. <opensource@estately.com>
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be included
+in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,28 @@
+require 'rubygems'
+require 'hoe'
+require 'rake/testtask'
+
+Hoe.plugin :git, :doofus
+Hoe.plugin :travis
+Hoe.plugin :gemspec
+
+Hoe.spec 'rets' do
+  developer 'Estately, Inc. Open Source', 'opensource@estately.com'
+
+  extra_deps << [ "httpclient", "~> 2.7.0" ]
+  extra_deps << [ "http-cookie", "~> 1.0.0" ]
+  extra_deps << [ "nokogiri",   "~> 1.5" ]
+
+  extra_dev_deps << [ "mocha", "~> 1.1.0" ]
+  extra_dev_deps << [ "vcr", "~> 2.2" ]
+  extra_dev_deps << [ "webmock", "~> 1.8" ]
+
+  ### Use markdown for changelog and readme
+  self.history_file = 'CHANGELOG.md'
+  self.readme_file  = 'README.md'
+end
+
+
+Rake::TestTask.new do |t|
+  t.pattern = "test/test_*.rb"
+end

data/bin/rets

@@ -0,0 +1,202 @@
+#!/usr/bin/env ruby
+
+require "optparse"
+require "pp"
+
+require "rubygems"
+
+$:.unshift File.join(File.dirname(__FILE__), '..', 'lib')
+require "rets"
+
+class RetsCli
+  def self.parse(args)
+
+    actions = %w(metadata search count object)
+    options = {:count => 5}
+
+    opts = OptionParser.new do |opts|
+      opts.banner = "Usage: #{File.basename($0)} URL [options] [query]"
+
+      opts.separator ""
+      opts.separator "Authentication options:"
+
+      opts.on("-U", "--username USERNAME", "The username to authenticate with.") do |username|
+        options[:username] = username
+      end
+
+      opts.on("-P", "--password [PASSWORD]", "The password to authenticate with.","Prompts if no argument is provided.") do |password|
+        options[:password] = password #or prompt # TODO
+      end
+
+      opts.on("-A", "--agent AGENT", "User-Agent header to provide.") do |agent|
+        options[:agent] = agent
+      end
+
+      opts.on("-B", "--agent-password [PASSWORD]", "User-Agent password to provide.") do |ua_password|
+        options[:ua_password] = ua_password
+      end
+
+      opts.separator ""
+      opts.separator "Actions:"
+
+      opts.on("-p", "--capabilities", "Print capabilities of the RETS server.") do |capabilities|
+        options[:capabilities] = capabilities
+      end
+
+      opts.on("-a", "--action ACTION", actions, "Action to perform (#{actions.join(",")}).") do |action|
+        options[:action] = action
+      end
+
+      opts.on("-m", "--metadata [FORMAT]", %w(tree long short), "Print metadata.", "Format is short, long or tree.", "Defaults to short.") do |format|
+        options[:action] = "metadata"
+        options[:metadata_format] = format || "short"
+      end
+
+      opts.separator ""
+      opts.separator "Search action options:"
+
+      opts.on("-r", "--resource NAME", "Name of resource to search for.") do |name|
+        options[:resource] = name
+      end
+
+      opts.on("-c", "--class NAME", "Name of class to search for.") do |name|
+        options[:class] = name
+      end
+
+      opts.on("-n", "--number LIMIT", Integer, "Return LIMIT results. Defaults to 5.") do |limit|
+        options[:limit] = limit
+      end
+
+      opts.separator ""
+      opts.separator "Misc options:"
+
+      opts.on_tail("-v", "--verbose", "Be verbose.") do |verbose|
+        logger = Class.new do
+          def method_missing(method, *a, &b)
+            puts a
+          end
+        end
+
+        options[:logger] = logger.new
+      end
+
+      opts.on_tail("-h", "--help", "Show this message") do
+        puts opts
+        exit
+      end
+
+      opts.on_tail("--version", "Show version") do
+        puts Rets::VERSION
+        exit
+      end
+
+    end
+
+    begin
+      opts.parse!(args.empty? ? ["-h"] : args)
+    rescue OptionParser::InvalidArgument => e
+      abort e.message
+    end
+
+    options
+  end
+
+end
+
+options = RetsCli.parse(ARGV)
+url     = ARGV[0] or abort "Need login URL"
+query   = ARGV[1]
+
+client = Rets::Client.new(options.merge(:login_url => url))
+
+if options[:capabilities]
+  pp client.capabilities
+end
+
+case options[:action]
+  when "metadata" then
+    metadata = client.metadata
+
+    if options[:metadata_format] != "tree"
+      preferred_fields = %w(ClassName SystemName ResourceID StandardName VisibleName MetadataEntryID KeyField)
+
+
+      # All types except system
+      types = Rets::METADATA_TYPES.map { |t| t.downcase.to_sym } - [:system]
+
+      types.each do |type|
+        # if RowContainer ...
+        rows = metadata[type]
+
+        puts type.to_s.capitalize
+        puts "="*40
+
+        print_key_value = lambda do |k,v|
+          key   = "#{k}:".ljust(35)
+          value = "#{v}".ljust(35)
+
+          puts [key, value].join
+        end
+
+        rows.each do |row|
+          top, rest = row.partition { |k,v| preferred_fields.include?(k) }
+
+          top.each(&print_key_value)
+
+          rest.sort_by{|k,v|k}.each(&print_key_value) if options[:metadata_format] == "long"
+
+          puts
+        end
+
+        puts
+      end
+
+    # Tree format
+    else
+      metadata.print_tree
+    end
+
+  when "search" then
+    pp client.find(:all,
+                  :search_type => options[:resource],
+                  :class       => options[:class],
+                  :query       => query,
+                  :count       => Rets::Client::COUNT.exclude,
+                  :limit       => options[:limit])
+
+  when "count" then
+    pp client.find(:all,
+              :search_type => options[:resource],
+              :class       => options[:class],
+              :query       => query,
+              :count       => Rets::Client::COUNT.only,
+              :limit       => options[:limit])
+
+  when "object" then
+
+    def write_objects(parts)
+      parts.each do |part|
+        cid = part.headers["content-id"].to_i
+        oid = part.headers["object-id"].to_i
+
+        File.open("tmp/#{cid}-#{oid}", "wb") do |f|
+          puts f.path
+
+          f.write part.body
+        end
+      end
+    end
+
+    parts = client.all_objects(
+      :resource    => "Property",
+      :resource_id => 90020062739, # id from KeyField for a given property
+      :object_type => "Photo"
+    )
+
+    parts.each { |pt| p pt.headers }
+
+    write_objects(parts)
+
+end
+
+client.logout

data/example/connect.rb

@@ -0,0 +1,19 @@
+require 'rets'
+
+# Pass the :login_url, :username, :password and :version of RETS
+client = Rets::Client.new({
+  login_url: 'login_url',
+  username: 'xxx',
+  password: 'xxx',
+  version: 'RETS/1.5'
+})
+
+begin
+  client.login
+rescue => e
+  puts 'Error: ' + e.message
+  exit!
+end
+
+puts 'We connected! Lets log out...'
+client.logout

data/example/get-photos.rb

@@ -0,0 +1,20 @@
+require 'rets'
+
+
+# Get all photos (*) for MLS ID 'mls_id'
+# Pass :object_id (ie '0', '1,2', wildcard '*')
+# The pass :resource (Property, Agent, MetaData, ...), :object_type (Photo, PhotoLarge), :rescource_id (ID of agent, MLS, ...)
+photos = client.objects '*', {
+  resource: 'Property',
+  object_type: 'Photo',
+  resource_id: 'mls_id'
+}
+
+photos.each_with_index do |data, index|
+  File.open("property-#{index.to_s}.jpg", 'w') do |file|
+    file.write data.body
+  end
+end
+
+puts photos.length.to_s + ' photos saved.'
+client.logout

data/example/get-property.rb

@@ -0,0 +1,16 @@
+require 'rets'
+
+
+# Get one property
+# Pass :first or :all
+# Then :search_type (Property, Agent, ...), :class (Condo, Commerical, ...), :query and :limit
+property = client.find :first, {
+  search_type: 'Property',
+  class: 'CLASS_NAME',
+  query: 'RETS_QUERY',
+  limit: 1
+}
+
+puts 'received property: '
+puts property.inspect
+client.logout