caboose-rets 0.0.57 → 0.0.58

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    ODAxYTczY2U0NzIxNWY3NTgwNzc1MDQ5NTQ5NGMxMWNmZmMwNjIxNQ==
+    YzA2ZjFiZWNjODQ0YTQ1NWI3ZmMwZTY4MGUzOWEwZGE5YTU0NzQ0Nw==
   data.tar.gz: !binary |-
-    ZjQzOGVhYWJmMGVjOGU3MzQwNWFjYTQyNTUxYzdmNGUwMWIyYzhmYQ==
-!binary "U0hBNTEy":
+    YzNjZmJhMWM2NjI4YjI5MjZjYTI3MGFkNGZiNzliYTQwYjk3Yjg4NQ==
+SHA512:
   metadata.gz: !binary |-
-    OGM0ZjVhNThjOTc0ZjI2YThmZjE0MDk5YzI0NmY2NjI0ZGRjOGMyNzVkM2M5
-    MDgyNWM5OTJiNWY1ZWQxYWE1ZDVlZTE0OTcxMmYzZmE1NTYxODdhOWE2NTNi
-    ZDRiNTZiNjJjMTcwNDBjOTY1YzE3MDcwNzA2Nzk5NjhmMDFlZDg=
+    ZDM2YTE0OGM1ZjdkNjMzMTk1N2Q2NzAwYWQwZWQ2Nzg4NmI0MGRjNjJmZDZh
+    MjNlNjBmOTY5NGI5Yjg2NWM3YjI5ZDBhMTBmMTRmNWZhYTgwODJjZGFlN2U4
+    NjJkNmFhYjJmZGNmMDRlOWMyYjYyOWU5YjRjZjYzYjQ3MjVhYWQ=
   data.tar.gz: !binary |-
-    OWE1YTU0OWNmMzExM2Q2N2U2YWNkNzA5ZjFlNTdhYzk5MmQ3NTQ0MmEwZDQ1
-    MjgzZTA5N2JkODhmOTE3NmVkMTNhM2JlMzQ2MzY3MWYzNTNjZjFmMGVhZTky
-    Y2ZjMmRhYjkzZGJlYTU3MzBhODVkZTVkZTI0NGE2NmZkNjg3ZmQ=
+    YWNiZWNkZTExYTA3ZTI2Y2FkZDBiNTUyOWM0ZjUzMDhjYmI0MjY4Y2ZjOGQy
+    MmVhMjIxNmI5OGJhMzg0OThlMTg0YjVlZGFlNGZhNWZmOWVmOGEwNTBkZWMx
+    YmYyZmFjYmY4NzM2MzI2NDM4YzQ5MGY4YTIzMjA2YzI1MWY3OTA=

data/app/models/caboose_rets/rets_importer.rb

@@ -1,4 +1,13 @@
-require 'ruby-rets'
+#require 'ruby-rets'
+require "rets/version"
+require "rets/exceptions"
+require "rets/client"
+require "rets/http"
+require "rets/stream_http"
+require "rets/base/core"
+require "rets/base/sax_search"
+require "rets/base/sax_metadata"
+
 require 'httparty'
 require 'json'
 
@@ -118,22 +127,22 @@ class CabooseRets::RetsImporter # < ActiveRecord::Base
   # Main updater
   #=============================================================================
 
-  def self.update_after(date_modified)
-    self.update_residential_properties_modified_after(date_modified)  
-    self.update_commercial_properties_modified_after(date_modified)  
-    self.update_land_properties_modified_after(date_modified)
-    self.update_multi_family_properties_modified_after(date_modified)
-    self.update_offices_modified_after(date_modified)
-    self.update_agents_modified_after(date_modified)
-    self.update_open_houses_modified_after(date_modified)
+  def self.update_after(date_modified, save_images = true)
+    self.update_residential_properties_modified_after(date_modified, save_images)  
+    self.update_commercial_properties_modified_after(date_modified, save_images)  
+    self.update_land_properties_modified_after(date_modified, save_images)
+    self.update_multi_family_properties_modified_after(date_modified, save_images)
+    self.update_offices_modified_after(date_modified, save_images)
+    self.update_agents_modified_after(date_modified, save_images)
+    self.update_open_houses_modified_after(date_modified, save_images)
   end                       
-  def self.update_residential_properties_modified_after(date_modified)  self.client.search({ :search_type => 'Property' , :class => 'RES', :select => ['MLS_ACCT']   , :query => "(DATE_MODIFIED=#{date_modified.strftime("%FT%T")}+)"   , :standard_names_only => true, :timeout => -1 }) { |data| self.delay(:priority => 10).import_residential_property  data['MLS_ACCT'  ] } end          
-  def self.update_commercial_properties_modified_after(date_modified)   self.client.search({ :search_type => 'Property' , :class => 'COM', :select => ['MLS_ACCT']   , :query => "(DATE_MODIFIED=#{date_modified.strftime("%FT%T")}+)"   , :standard_names_only => true, :timeout => -1 }) { |data| self.delay(:priority => 10).import_commercial_property   data['MLS_ACCT'  ] } end          
-  def self.update_land_properties_modified_after(date_modified)         self.client.search({ :search_type => 'Property' , :class => 'LND', :select => ['MLS_ACCT']   , :query => "(DATE_MODIFIED=#{date_modified.strftime("%FT%T")}+)"   , :standard_names_only => true, :timeout => -1 }) { |data| self.delay(:priority => 10).import_land_property         data['MLS_ACCT'  ] } end          
-  def self.update_multi_family_properties_modified_after(date_modified) self.client.search({ :search_type => 'Property' , :class => 'MUL', :select => ['MLS_ACCT']   , :query => "(DATE_MODIFIED=#{date_modified.strftime("%FT%T")}+)"   , :standard_names_only => true, :timeout => -1 }) { |data| self.delay(:priority => 10).import_multi_family_property data['MLS_ACCT'  ] } end
-  def self.update_offices_modified_after(date_modified)                 self.client.search({ :search_type => 'Office'   , :class => 'OFF', :select => ['LO_LO_CODE'] , :query => "(LO_DATE_MODIFIED=#{date_modified.strftime("%FT%T")}+)", :standard_names_only => true, :timeout => -1 }) { |data| self.delay(:priority => 10).import_office                data['LO_LO_CODE'] } end      
-  def self.update_agents_modified_after(date_modified)                  self.client.search({ :search_type => 'Agent'    , :class => 'AGT', :select => ['LA_LA_CODE'] , :query => "(LA_DATE_MODIFIED=#{date_modified.strftime("%FT%T")}+)", :standard_names_only => true, :timeout => -1 }) { |data| self.delay(:priority => 10).import_agent                 data['LA_LA_CODE'] } end     
-  def self.update_open_houses_modified_after(date_modified)             self.client.search({ :search_type => 'OpenHouse', :class => 'OPH', :select => ['ID']         , :query => "(DATE_MODIFIED=#{date_modified.strftime("%FT%T")}+)"   , :standard_names_only => true, :timeout => -1 }) { |data| self.delay(:priority => 10).import_open_house            data['ID'        ] } end
+  def self.update_residential_properties_modified_after(date_modified, save_images = true)  self.client.search({ :search_type => 'Property' , :class => 'RES', :select => ['MLS_ACCT']   , :query => "(DATE_MODIFIED=#{date_modified.strftime("%FT%T")}+)"   , :standard_names_only => true, :timeout => -1 }) { |data| self.delay(:priority => 10).import_residential_property(  data['MLS_ACCT'  ], save_images) } end          
+  def self.update_commercial_properties_modified_after(date_modified, save_images = true)   self.client.search({ :search_type => 'Property' , :class => 'COM', :select => ['MLS_ACCT']   , :query => "(DATE_MODIFIED=#{date_modified.strftime("%FT%T")}+)"   , :standard_names_only => true, :timeout => -1 }) { |data| self.delay(:priority => 10).import_commercial_property(   data['MLS_ACCT'  ], save_images) } end          
+  def self.update_land_properties_modified_after(date_modified, save_images = true)         self.client.search({ :search_type => 'Property' , :class => 'LND', :select => ['MLS_ACCT']   , :query => "(DATE_MODIFIED=#{date_modified.strftime("%FT%T")}+)"   , :standard_names_only => true, :timeout => -1 }) { |data| self.delay(:priority => 10).import_land_property(         data['MLS_ACCT'  ], save_images) } end          
+  def self.update_multi_family_properties_modified_after(date_modified, save_images = true) self.client.search({ :search_type => 'Property' , :class => 'MUL', :select => ['MLS_ACCT']   , :query => "(DATE_MODIFIED=#{date_modified.strftime("%FT%T")}+)"   , :standard_names_only => true, :timeout => -1 }) { |data| self.delay(:priority => 10).import_multi_family_property( data['MLS_ACCT'  ], save_images) } end
+  def self.update_offices_modified_after(date_modified, save_images = true)                 self.client.search({ :search_type => 'Office'   , :class => 'OFF', :select => ['LO_LO_CODE'] , :query => "(LO_DATE_MODIFIED=#{date_modified.strftime("%FT%T")}+)", :standard_names_only => true, :timeout => -1 }) { |data| self.delay(:priority => 10).import_office(                data['LO_LO_CODE'], save_images) } end      
+  def self.update_agents_modified_after(date_modified, save_images = true)                  self.client.search({ :search_type => 'Agent'    , :class => 'AGT', :select => ['LA_LA_CODE'] , :query => "(LA_DATE_MODIFIED=#{date_modified.strftime("%FT%T")}+)", :standard_names_only => true, :timeout => -1 }) { |data| self.delay(:priority => 10).import_agent(                 data['LA_LA_CODE'], save_images) } end     
+  def self.update_open_houses_modified_after(date_modified, save_images = true)             self.client.search({ :search_type => 'OpenHouse', :class => 'OPH', :select => ['ID']         , :query => "(DATE_MODIFIED=#{date_modified.strftime("%FT%T")}+)"   , :standard_names_only => true, :timeout => -1 }) { |data| self.delay(:priority => 10).import_open_house(            data['ID'        ], save_images) } end
     
   #=============================================================================
   # Single model import methods (called from a worker dyno)
@@ -158,47 +167,47 @@ class CabooseRets::RetsImporter # < ActiveRecord::Base
     self.download_property_images(p)
   end
   
-  def self.import_residential_property(mls_acct)    
+  def self.import_residential_property(mls_acct, save_images = true)    
     self.import("(MLS_ACCT=*#{mls_acct}*)", 'Property', 'RES')
     p = CabooseRets::ResidentialProperty.where(:id => mls_acct.to_i).first    
-    self.download_property_images(p)    
+    self.download_property_images(p, save_images)    
     self.update_coords(p)        
   end
   
-  def self.import_commercial_property(mls_acct)    
+  def self.import_commercial_property(mls_acct, save_images = true)    
     self.import("(MLS_ACCT=*#{mls_acct}*)", 'Property', 'COM')
     p = CabooseRets::CommercialProperty.where(:id => mls_acct.to_i).first
-    self.download_property_images(p)
+    self.download_property_images(p, save_images)
     self.update_coords(p)
   end
   
-  def self.import_land_property(mls_acct)    
+  def self.import_land_property(mls_acct, save_images = true)    
     self.import("(MLS_ACCT=*#{mls_acct}*)", 'Property', 'LND')
     p = CabooseRets::LandProperty.where(:id => mls_acct.to_i).first    
-    self.download_property_images(p)
+    self.download_property_images(p, save_images)
     self.update_coords(p)
   end
   
-  def self.import_multi_family_property(mls_acct)    
+  def self.import_multi_family_property(mls_acct, save_images = true)    
     self.import("(MLS_ACCT=*#{mls_acct}*)", 'Property', 'MUL')
     p = CabooseRets::MultiFamilyProperty.where(:id => mls_acct.to_i).first
-    self.download_property_images(p)
+    self.download_property_images(p, save_images)
     self.update_coords(p)
   end
   
-  def self.import_office(lo_code)
+  def self.import_office(lo_code, save_images = true)
     self.import("(LO_LO_CODE=*#{lo_code}*)", 'Office', 'OFF')
     office = CabooseRets::Office.where(:lo_code => lo_code.to_s).first
-    self.download_office_image(office)
+    self.download_office_image(office) if save_images == true
   end
   
-  def self.import_agent(la_code)
+  def self.import_agent(la_code, save_images = true)
     self.import("(LA_LA_CODE=*#{la_code}*)", 'Agent', 'AGT')
     a = CabooseRets::Agent.where(:la_code => la_code.to_s).first
-    self.download_agent_image(a)
+    self.download_agent_image(a) if save_images == true
   end
   
-  def self.import_open_house(id)
+  def self.import_open_house(id, save_images = true)
     self.import("(ID=*#{id}*)", 'OpenHouse', 'OPH')        
   end
 
@@ -206,8 +215,9 @@ class CabooseRets::RetsImporter # < ActiveRecord::Base
   # Images
   #=============================================================================
     
-  def self.download_property_images(p)
+  def self.download_property_images(p, save_images = true)    
     self.refresh_property_media(p)
+    return if save_images == false
     
     self.log("-- Downloading images and resizing for #{p.mls_acct}")
     media = []
@@ -265,7 +275,7 @@ class CabooseRets::RetsImporter # < ActiveRecord::Base
     end    
   end
   
-  def self.download_office_image(office)            
+  def self.download_office_image(office)
     self.log "Saving image for #{office.lo_name}..."
     begin
       self.client.get_object(:resource => :Office, :type => :Photo, :location => true, :id => office.lo_code) do |headers, content|
@@ -319,6 +329,45 @@ class CabooseRets::RetsImporter # < ActiveRecord::Base
     end
   end
   
+  #=============================================================================
+  # Purging
+  #=============================================================================
+  
+  def self.purge    
+    self.purge_residential
+    self.purge_commercial
+    self.purge_land
+    self.purge_multi_family
+    self.purge_offices
+    self.purge_agents
+    self.purge_open_houses
+    self.purge_media
+  end
+      
+  def self.purge_residential()  self.purge_helper('Property'  , 'RES', 'MLS_ACCT'   , 'DATE_MODIFIED'    , "delete from rets_residential  where mls_acct not in (?)") end    
+  def self.purge_commercial()   self.purge_helper('Property'  , 'COM', 'MLS_ACCT'   , 'DATE_MODIFIED'    , "delete from rets_commercial   where mls_acct not in (?)") end        
+  def self.purge_land()         self.purge_helper('Property'  , 'LND', 'MLS_ACCT'   , 'DATE_MODIFIED'    , "delete from rets_land         where mls_acct not in (?)") end    
+  def self.purge_multi_family() self.purge_helper('Property'  , 'MUL', 'MLS_ACCT'   , 'DATE_MODIFIED'    , "delete from rets_multi_family where mls_acct not in (?)") end        
+  def self.purge_offices()      self.purge_helper('Office'    , 'OFF', 'LO_LO_CODE' , 'LO_DATE_MODIFIED' , "delete from rets_offices      where lo_code  not in (?)") end    
+  def self.purge_agents()       self.purge_helper('Agent'     , 'AGT', 'LA_LA_CODE' , 'LA_DATE_MODIFIED' , "delete from rets_agents       where la_code  not in (?)") end  
+  def self.purge_open_houses()  self.purge_helper('OpenHouse' , 'OPH', 'ID'         , 'DATE_MODIFIED'    , "delete from rets_open_houses  where id       not in (?)") end
+  def self.purge_media()        self.purge_helper('Media'     , 'GFX', 'MEDIA_ID'   , 'DATE_MODIFIED'    , "delete from rets_media        where media_id not in (?)") end
+    
+  def self.purge_helper(search_type, class_type, key_field, date_modified_field, delete_query)
+    
+    # Get the total number of records
+    params = { :search_type => search_type, :class => class_type, :query => "(#{date_modified_field}=2013-08-06T00:00:01+)", :standard_names_only => true, :timeout => -1 }        
+    self.client.search(params.merge({ :count_mode => :only }))
+    count = self.client.rets_data[:code] == "20201" ? 0 : self.client.rets_data[:count]            
+    batch_count = (count.to_f/5000.0).ceil
+
+    ids = []
+    (0...batch_count).each do |i|            
+      self.client.search(params.merge({ :select => [key_field], :limit => 5000, :offset => 5000*i })){ |data| ids << data[key_field] }
+    end    
+    ActiveRecord::Base.connection.execute(ActiveRecord::Base.send(:sanitize_sql_array, [delete_query, ids]))            
+  end
+
   #=============================================================================
   # Logging
   #=============================================================================
@@ -336,8 +385,14 @@ class CabooseRets::RetsImporter # < ActiveRecord::Base
     return if self.task_is_locked
     task_started = self.lock_task
     
-    begin      
-      self.update_after(self.last_updated - 30.seconds)		  
+    begin
+      overlap = 30.seconds    
+      if (DateTime.now - self.last_purged) > 24.hours      
+        self.purge
+        self.save_last_purged(task_started)
+        overlap = 1.month
+      end              
+      self.update_after(self.last_updated - overlap)		  
 		  self.save_last_updated(task_started)
 		  self.unlock_task
 		rescue
@@ -358,12 +413,26 @@ class CabooseRets::RetsImporter # < ActiveRecord::Base
     return DateTime.parse(s.value)
   end
   
+  def self.last_purged
+    if !Caboose::Setting.exists?(:name => 'rets_last_purged')
+      Caboose::Setting.create(:name => 'rets_last_purged', :value => '2013-08-06T00:00:01')
+    end
+    s = Caboose::Setting.where(:name => 'rets_last_purged').first
+    return DateTime.parse(s.value)
+  end
+  
   def self.save_last_updated(d)
     s = Caboose::Setting.where(:name => 'rets_last_updated').first
     s.value = d.strftime('%FT%T')
     s.save
   end
   
+  def self.save_last_purged(d)
+    s = Caboose::Setting.where(:name => 'rets_last_purged').first
+    s.value = d.strftime('%FT%T')
+    s.save
+  end
+  
   def self.task_is_locked
     return Caboose::Setting.exists?(:name => 'rets_update_running')
   end

data/app/models/caboose_rets/schema.rb

@@ -4,7 +4,8 @@ class CabooseRets::Schema < Caboose::Utilities::Schema
   def self.removed_columns
     {
       CabooseRets::CommercialProperty => [ :virtual_tour ],
-      CabooseRets::ResidentialProperty => [ :virtual_tour ]
+      CabooseRets::ResidentialProperty => [ :virtual_tour ],
+      CabooseRets::ResidentialProperty => [ :acreage_temp ]
     }
   end
   
@@ -619,7 +620,9 @@ class CabooseRets::Schema < Caboose::Utilities::Schema
         [ :elem_school                    , :text ], 
         [ :price_sqft                     , :text ], 
         [ :rm_recrm                       , :text ], 
-        [ :acreage                        , :text ], 
+        #[ :acreage                        , :text ],
+        [ :acreage                        , :float, { :default => 0.0 }],
+        #[ :acreage_temp                   , :float, { :default => 0.0 }],
         [ :expire_date                    , :text ], 
         [ :prop_type                      , :text ], 
         [ :rm_recrm_desc                  , :text ], 

data/lib/caboose_rets/version.rb

@@ -1,3 +1,3 @@
 module CabooseRets
-  VERSION = '0.0.57'
+  VERSION = '0.0.58'
 end

data/lib/rets/base/core.rb

@@ -0,0 +1,290 @@
+# For more information on what the possible values of fields that are passed to the RETS server can be, see {http://www.rets.org/documentation}.
+module RETS
+  module Base
+    class Core
+      GET_OBJECT_DATA = ["object-id", "description", "content-id", "content-description", "location", "content-type", "preferred"]
+
+      # Can be called after any {RETS::Base::Core} call that hits the RETS Server.
+      # @return [String] How big the request was
+      attr_reader :request_size
+
+      # Can be called after any {RETS::Base::Core} call that hits the RETS Server.
+      # @return [String] SHA1 hash of the request
+      attr_reader :request_hash
+
+      # Can be called after any {#get_object} or {#search} call that hits the RETS Server.
+      # @return [Float] How long the request took
+      attr_reader :request_time
+
+      # Can be called after any {RETS::Base::Core} call that hits the RETS Server.
+      # @return [Hash]
+      #   Gives access to the miscellaneous RETS data, such as reply text, code, delimiter, count and so on depending on the API call made.
+      #   * *text* (String) - Reply text from the server
+      #   * *code* (String) - Reply code from the server
+      attr_reader :rets_data
+
+      def initialize(http, urls)
+        @http = http
+        @urls = urls
+      end
+
+      ##
+      # Attempts to logout of the RETS server.
+      #
+      # @raise [RETS::CapabilityNotFound]
+      # @raise [RETS::APIError]
+      # @raise [RETS::HTTPError]
+      def logout
+        unless @urls[:logout]
+          raise RETS::CapabilityNotFound.new("No Logout capability found for given user.")
+        end
+
+        @http.request(:url => @urls[:logout])
+
+        nil
+      end
+
+      ##
+      # Whether the RETS server has the requested capability.
+      #
+      # @param [Symbol] type Lowercase of the capability, "getmetadata", "getobject" and so on
+      # @return [Boolean]
+      def has_capability?(type)
+        @urls.has_key?(type)
+      end
+
+      ##
+      # Requests metadata from the RETS server.
+      #
+      # @param [Hash] args
+      # @option args [String] :type Metadata to request, the same value if you were manually making the request, "METADATA-SYSTEM", "METADATA-CLASS" and so on
+      # @option args [String] :id Filter the data returned by ID, "*" would return all available data
+      # @option args [Integer, Optional] :read_timeout How many seconds to wait before giving up
+      #
+      # @yield For every group of metadata downloaded
+      # @yieldparam [String] :type Type of data that was parsed with "METADATA-" stripped out, for "METADATA-SYSTEM" this will be "SYSTEM"
+      # @yieldparam [Hash] :attrs Attributes of the data, generally *Version*, *Date* and *Resource* but can vary depending on what metadata you requested
+      # @yieldparam [Array] :metadata Array of hashes with metadata info
+      #
+      # @raise [RETS::CapabilityNotFound]
+      # @raise [RETS::APIError]
+      # @raise [RETS::HTTPError]
+      # @see #rets_data
+      # @see #request_size
+      # @see #request_hash
+      def get_metadata(args, &block)
+        raise ArgumentError, "No block passed" unless block_given?
+
+        unless @urls[:getmetadata]
+          raise RETS::CapabilityNotFound.new("No GetMetadata capability found for given user.")
+        end
+
+        @request_size, @request_hash, @request_time, @rets_data = nil, nil, nil, nil
+        @http.request(:url => @urls[:getmetadata], :read_timeout => args[:read_timeout], :params => {:Format => :COMPACT, :Type => args[:type], :ID => args[:id]}) do |response|
+          stream = RETS::StreamHTTP.new(response)
+          sax = RETS::Base::SAXMetadata.new(block)
+
+          Nokogiri::XML::SAX::Parser.new(sax).parse_io(stream)
+
+          @request_size, @request_hash = stream.size, stream.hash
+          @rets_data = sax.rets_data
+        end
+
+        nil
+      end
+
+      ##
+      # Requests an object from the RETS server.
+      #
+      # @param [Hash] args
+      # @option args [String] :resource Resource to load, typically *Property*
+      # @option args [String] :type Type of object you want, usually *Photo*
+      # @option args [String] :id What objects to return
+      # @option args [Array, Optional] :accept Array of MIME types to accept, by default this is *image/png*, *image/gif* and *image/jpeg*
+      # @option args [Boolean, Optional] :location Return the location of the object rather than the contents of it
+      # @option args [Integer, Optional] :read_timeout How many seconds to wait before timing out
+      #
+      # @yield For every object downloaded
+      # @yieldparam [Hash] :headers Object headers
+      #     * *object-id* (String) - Objects ID
+      #     * *content-id* (String) - Content ID
+      #     * *content-type* (String) - MIME type of the content
+      #     * *description* (String, Optional) - A description of the object
+      #     * *location* (String, Optional) - Where the file is located, only returned is *location* is true
+      # @yieldparam [String, Optional] :content Content for the object, not called when *location* is set
+      #
+      # @raise [RETS::CapabilityNotFound]
+      # @raise [RETS::APIError]
+      # @raise [RETS::HTTPError]
+      # @see #rets_data
+      # @see #request_size
+      # @see #request_hash
+      def get_object(args, &block)
+        raise ArgumentError, "No block passed" unless block_given?
+
+        unless @urls[:getobject]
+          raise RETS::CapabilityNotFound.new("No GetObject capability found for given user.")
+        end
+
+        req = {:url => @urls[:getobject], :read_timeout => args[:read_timeout], :headers => {}}
+        req[:params] = {:Resource => args[:resource], :Type => args[:type], :Location => (args[:location] ? 1 : 0), :ID => args[:id]}
+        if args[:accept].is_a?(Array)
+          req[:headers]["Accept"] = args[:accept].join(",")
+        else
+          req[:headers]["Accept"] = "image/png,image/gif,image/jpeg"
+        end
+
+        # Will get swapped to a streaming call rather than a download-and-parse later, easy to do as it's called with a block now
+        start = Time.now.utc.to_f
+
+        @request_size, @request_hash, @request_time, @rets_data = nil, nil, nil, nil
+        @http.request(req) do |response|
+          body = response.read_body
+
+          @request_time = Time.now.utc.to_f - start
+          @request_size, @request_hash = body.length, Digest::SHA1.hexdigest(body)
+
+          # Make sure we aren't erroring
+          if body =~ /(<RETS (.+)\>)/
+            # RETSIQ (and probably others) return a <RETS> tag on a location request without any error inside
+            # since parsing errors out of full image data calls is a tricky pain. We're going to keep the
+            # loose error checking, but will confirm that it has an actual error code
+            code, text = @http.get_rets_response(Nokogiri::XML($1).xpath("//RETS").first)
+            unless code == "0"
+              @rets_data = {:code => code, :text => text}
+
+              if code == "20403"
+                return
+              else
+                raise RETS::APIError.new("#{code}: #{text}", code, text)
+              end
+            end
+          end
+
+          # Using a wildcard somewhere
+          if response.content_type == "multipart/parallel"
+            boundary = response.type_params["boundary"]
+            boundary.gsub!(/^"|"$/, "")
+
+            parts = body.split("--#{boundary}\r\n")
+            parts.last.gsub!("\r\n--#{boundary}--", "")
+            parts.each do |part|
+              part.strip!
+              next if part == ""
+
+              headers, content = part.split("\r\n\r\n", 2)
+
+              parsed_headers = {}
+              headers.split("\r\n").each do |line|
+                name, value = line.split(":", 2)
+                next unless value and value != ""
+
+                parsed_headers[name.downcase] = value.strip
+              end
+
+              # Check off the first children because some Rap Rets seems to use RETS-Status
+              # and it will include it with an object while returning actual data.
+              # It only does this for multipart requests, single image pulls will use <RETS> like it should.
+              if parsed_headers["content-type"] == "text/xml"
+                code, text = @http.get_rets_response(Nokogiri::XML(content).children.first)
+                next if code == "20403"
+              end
+
+              if block.arity == 1
+                yield parsed_headers
+              else
+                yield parsed_headers, content
+              end
+
+            end
+
+          # Either text (error) or an image of some sorts, which is irrelevant for this
+          else
+            headers = {}
+            GET_OBJECT_DATA.each do |field|
+              next unless response.header[field] and response.header[field] != ""
+              headers[field] = response.header[field].strip
+            end
+
+            if block.arity == 1
+              yield headers
+            else
+              yield headers, body
+            end
+          end
+        end
+
+        nil
+      end
+
+      ##
+      # Searches the RETS server for data.
+      #
+      # @param [Hash] args
+      # @option args [String] :search_type What to search on, typically *Property*, *Office* or *Agent*
+      # @option args [String] :class What class of data to return, varies between RETS implementations and can be anything from *1* to *ResidentialProperty*
+      # @option args [String] :query How to filter data, should be unescaped as CGI::escape will be called on the string
+      # @option args [Symbol, Optional] :count_mode Either *:only* to return just the total records found or *:both* to get count and records returned
+      # @option args [Integer, Optional] :limit Limit total records returned
+      # @option args [Integer, Optional] :offset Offset to start returning records from
+      # @option args [Array, Optional] :select Restrict the fields the RETS server returns
+      # @option args [Boolean, Optional] :standard_names Whether to use standard names for all fields
+      # @option args [String, Optional] :restricted String to show in place of a field value for any restricted fields the user cannot see
+      # @option args [Integer, Optional] :read_timeout How long to wait for data from the socket before giving up
+      # @option args [Boolean, Optional] :disable_stream Disables the streaming setup for data and instead loads it all and then parses
+      #
+      # @yield Called for every <DATA></DATA> group from the RETS server
+      # @yieldparam [Hash] :data One record of data from the RETS server
+      #
+      # @raise [RETS::CapabilityNotFound]
+      # @raise [RETS::APIError]
+      # @raise [RETS::HTTPError]
+      # @see #rets_data
+      # @see #request_size
+      # @see #request_hash
+      def search(args, &block)
+        if !block_given? and args[:count_mode] != :only
+          raise ArgumentError, "No block found"
+        end
+
+        unless @urls[:search]
+          raise RETS::CapabilityNotFound.new("Cannot find URL for Search call")
+        end
+
+        req = {:url => @urls[:search], :read_timeout => args[:read_timeout]}
+        req[:params] = {:Format => "COMPACT-DECODED", :SearchType => args[:search_type], :QueryType => "DMQL2", :Query => args[:query], :Class => args[:class], :Limit => args[:limit], :Offset => args[:offset], :RestrictedIndicator => args[:restricted]}
+        req[:params][:Select] = args[:select].join(",") if args[:select].is_a?(Array)
+        req[:params][:StandardNames] = 1 if args[:standard_names]
+
+        if args[:count_mode] == :only
+          req[:params][:Count] = 2
+        elsif args[:count_mode] == :both
+          req[:params][:Count] = 1
+        end
+
+        @request_size, @request_hash, @request_time, @rets_data = nil, nil, nil, {}
+
+        start = Time.now.utc.to_f
+        @http.request(req) do |response|
+          if args[:disable_stream]
+            stream = StringIO.new(response.body)
+            @request_time = Time.now.utc.to_f - start
+          else
+            stream = RETS::StreamHTTP.new(response)
+          end
+
+          sax = RETS::Base::SAXSearch.new(@rets_data, block)
+          Nokogiri::XML::SAX::Parser.new(sax).parse_io(stream)
+
+          if args[:disable_stream]
+            @request_size, @request_hash = response.body.length, Digest::SHA1.hexdigest(response.body)
+          else
+            @request_size, @request_hash = stream.size, stream.hash
+          end
+        end
+
+        nil
+      end
+    end
+  end
+end