caboose-rets 0.0.69 → 0.0.70

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    YjRhYzQ1ZWY3YjMzZmFhNzYwZjFhNTUyNzk2NjU5OWRlZjNhN2FiNw==
+    MWExNTE4ZTRhZDM1MDUxY2YwNzllZTA4ZjgxMTJhZjNmNjQ2ZDdhNg==
   data.tar.gz: !binary |-
-    ZmNmMzg1NDMzZTQxZGViZWE0OTZmNzBjYzFkY2ZhZjQ5MGRiYzA1Mw==
+    ZmM4YmY1MWM3MWUxOGExMjY2M2NhYmVlYzA1MWRhOWVmMWE1ZTY2OA==
 SHA512:
   metadata.gz: !binary |-
-    NzE2YzRjODQ2MjU2ZGE1YzA2MTRmZjI2OGMxMWVmNzg2YWQzNGYwMmE0NGVm
-    ZDc0MzJmN2E5YjZjMTkwOTgwZTZmNzk5NWExYmVmODRkMjg5NjUwZDIxNzhj
-    NWQ3NDkwZDJmYTIyN2EzYTIwZWI3MWZjY2YwMzY0NGI3NDI4ZTI=
+    MjcwNzQyZjkwMjg5NTYwN2M1ODkxMDhkNzQxNWQ5ZGVlYTQ5ZDZkYjkxY2Qy
+    YzAxZDAxMTFmNjI4Y2U0ODYzNmQyM2YwMzIyOGUzNjkzNDUwMWRlNjFkMTdl
+    OGVhN2QxMjZmZDRhNWVlNzI2MDJhNjRjYzRiZDEyNTg4ZDU2Mjk=
   data.tar.gz: !binary |-
-    NzFiMDQ1ZGI5NTA0NWI1NTI3NmFmYjgwNTBiMDZhY2I2NjI3YjllZGM4NDlh
-    MzI0NmU4YmYzODIxYWMxOWExMGU0YmM2ODFkMmQ4MGQxOWQ2NzIzYmU0MzY0
-    NzEwZDIyNmVlZDEzNTE5OWEzYzY3MTNiZjVjMmUxNGZkNGU1OTk=
+    OGYzODJjYTBmNzQwMTI4MWUyZDBkNDMwMGU3ZmY1YmU5MjdjZGE5ZDMyMzZi
+    YzI0NDgwNmVjZTQ5NDE5ODY4NzZmYTEzODIwM2I1YmQ4N2JlMWMwMzgwMmM4
+    ZTY0OTg4OTcxNWI5MWY2MjJhYTllZTE0ZmUzYTI1Y2U3Y2NhMmM=

data/app/controllers/caboose_rets/media_controller.rb

@@ -102,9 +102,9 @@ module CabooseRets
           :content_type => m.image_content_type,
           :file_size    => m.image_file_size,
           :update_at    => m.image_updated_at,
-          :tiny_url     => m.image.url(:tiny),
-          :thumb_url    => m.image.url(:thumb),
-          :large_url    => m.image.url(:large)            
+          :tiny_url     => m.image_url(:tiny),
+          :thumb_url    => m.image_url(:thumb),
+          :large_url    => m.image_url(:large)            
         },
         :file => {
           :file_name    => m.file_file_name,

data/app/models/caboose_rets/agent.rb

@@ -76,5 +76,13 @@ class CabooseRets::Agent < ActiveRecord::Base
 	  self.photo_date_modified  = data['PHOTO_DATE_MODIFIED']
 	  #self.photo_url            = ""
 	end
+	
+	def image_url(style)
+    if CabooseRets::use_hosted_images == true
+      return "#{CabooseRets::agents_base_url}/#{self.image_file_name}"
+    end
+    return "" if self.image.nil?
+    return self.image.url(style)           
+  end
 
 end

data/app/models/caboose_rets/media.rb

@@ -18,7 +18,7 @@ class CabooseRets::Media < ActiveRecord::Base
     self.date_modified  = data['DATE_MODIFIED']
     self.file_name 		  = data['FILE_NAME']
     self.media_id 		  = data['MEDIA_ID']
-    self.media_order 	  = data['MEDIA_ORDER']
+    self.media_order 	  = data['MEDIA_ORDER'].to_i
     self.media_remarks  = data['MEDIA_REMARKS']
     self.media_type 		= data['MEDIA_TYPE']
     self.mls_acct 		  = data['MLS_ACCT']
@@ -122,5 +122,13 @@ class CabooseRets::Media < ActiveRecord::Base
     
     return true
   end
-    
+  
+  def image_url(style)
+    if CabooseRets::use_hosted_images == true
+      return "#{CabooseRets::media_base_url}/#{self.file_name}"
+    end
+    return "" if self.image.nil?
+    return self.image.url(style)           
+  end
+        
 end

data/app/models/caboose_rets/rets_importer.rb

@@ -78,16 +78,22 @@ class CabooseRets::RetsImporter # < ActiveRecord::Base
       :timeout => -1
     }
     obj = nil
-    self.client.search(params) do |data|
-      obj = self.get_instance_with_id(class_type, data)
-      if obj.nil?
-        self.log("Error: object is nil")
-        self.log(data.inspect)
-        next
+    begin        
+      self.client.search(params) do |data|
+        obj = self.get_instance_with_id(class_type, data)
+        if obj.nil?
+          self.log("Error: object is nil")
+          self.log(data.inspect)
+          next
+        end
+        obj.parse(data)
+        obj.save
       end
-      obj.parse(data)
-      obj.save
+    rescue RETS::HTTPError => err
+      self.log "Import error for #{class_type}: #{query}"
+      self.log err.message
     end
+    
   end
 
   def self.get_instance_with_id(class_type, data)
@@ -204,7 +210,7 @@ class CabooseRets::RetsImporter # < ActiveRecord::Base
     self.update_coords(p)
   end
 
-  def self.import_office(lo_code, save_images = true)
+  def self.import_office(lo_code, save_images = true)    
     self.import('OFF', "(LO_LO_CODE=*#{lo_code}*)")
     office = CabooseRets::Office.where(:lo_code => lo_code.to_s).first
     self.download_office_image(office) if save_images == true
@@ -221,97 +227,43 @@ class CabooseRets::RetsImporter # < ActiveRecord::Base
   end
 
   def self.import_media(id, save_images = true)
-    self.import('GFX', "((MEDIA_ID=#{id}+),(MEDIA_ID=#{id}-))")    
-    #self.client.get_object(:resource => :Property, :type => :Photo, :location => true, :id => p.id) do |headers, content|
-    #
-    #  # Find the associated media record for the image
-    #  filename = File.basename(headers['location'])
-    #  m = CabooseRets::Media.where(:mls_acct => p.mls_acct, :file_name => filename).first
-    #
-    #  if m.nil?
-    #    self.log("Can't find media record for #{p.mls_acct} #{filename}.")
-    #  else
-    #    m.image = URI.parse(headers['location'])
-    #    media << m
-    #    #m.save
-    #  end
-    #end    
+    self.import('GFX', "((MEDIA_ID=#{id}+),(MEDIA_ID=#{id}-))")        
   end
 
   #=============================================================================
   # Images
   #=============================================================================
-
+    
   def self.download_property_images(p, save_images = true)
-    self.refresh_property_media(p)
     return if save_images == false
-
-    self.log("-- Downloading images and resizing for #{p.mls_acct}")
-    media = []
-    self.client.get_object(:resource => :Property, :type => :Photo, :location => true, :id => p.id) do |headers, content|
-
-      # Find the associated media record for the image
-      filename = File.basename(headers['location'])
-      m = CabooseRets::Media.where(:mls_acct => p.mls_acct, :file_name => filename).first
-
-      if m.nil?
-        self.log("Can't find media record for #{p.mls_acct} #{filename}.")
-      else
-        m.image = URI.parse(headers['location'])
-        media << m
-        #m.save
-      end
-    end
-
-    self.log("-- Uploading images to S3 for #{p.mls_acct}")
-    media.each do |m|
-      m.save
-    end
-  end
-
-  def self.refresh_property_media(p)
-    self.log("-- Deleting images and metadata for #{p.mls_acct}...")
-    #CabooseRets::Media.where(:mls_acct => p.mls_acct, :media_type => 'Photo').destroy_all
-    CabooseRets::Media.where(:mls_acct => p.mls_acct).destroy_all
-
-    self.log("-- Downloading image metadata for #{p.mls_acct}...")
+    
+    self.log("- Downloading GFX records for #{p.mls_acct}...")
     params = {
       :search_type => 'Media',
-      :class => 'GFX',
-      #:query => "(MLS_ACCT=*#{p.id}*),(MEDIA_TYPE=|I)",
+      :class => 'GFX',      
       :query => "(MLS_ACCT=*#{p.id}*)",
       :timeout => -1
     }
+    ids = []
     self.client.search(params) do |data|
-      m = CabooseRets::Media.new
-      m.parse(data)
-      #m.id = m.media_id
+      ids << data['MEDIA_ID']      
+      m = CabooseRets::Media.where(:media_id => data['MEDIA_ID']).first
+      m = CabooseRets::Media.new if m.nil?             
+      m.parse(data)      
       m.save
     end
-  end
-
-  def self.download_agent_image(agent)
-    self.log "Saving image for #{agent.first_name} #{agent.last_name}..."
-    begin
-      self.client.get_object(:resource => :Agent, :type => :Photo, :location => true, :id => agent.la_code) do |headers, content|
-        agent.image = URI.parse(headers['location'])
-        agent.save
-      end
-    rescue RETS::APIError => err
-      self.log "No image for #{agent.first_name} #{agent.last_name}."
-    end
-  end
-
-  def self.download_office_image(office)
-    self.log "Saving image for #{office.lo_name}..."
-    begin
-      self.client.get_object(:resource => :Office, :type => :Photo, :location => true, :id => office.lo_code) do |headers, content|
-        office.image = URI.parse(headers['location'])
-        office.save
-      end
-    rescue RETS::APIError => err
-      self.log "No image for #{office.lo_name}."
+    
+    # Delete any records in the local database that shouldn't be there
+    puts "- Deleting GFX records for MLS ##{p.mls_acct} in the local database that are not in the remote database..."
+    query = "select media_id from rets_media where mls_acct = '#{p.mls_acct}'"
+    rows = ActiveRecord::Base.connection.select_all(ActiveRecord::Base.send(:sanitize_sql_array, query))
+    local_ids = rows.collect{ |row| row['media_id'] }
+    ids_to_remove = local_ids - ids
+    if ids_to_remove && ids_to_remove.count > 0
+      query = ["delete from rets_media where media_id not in (?)", ids_to_remove]
+      ActiveRecord::Base.connection.execute(ActiveRecord::Base.send(:sanitize_sql_array, query))
     end
+        
   end
 
   #=============================================================================
@@ -343,6 +295,7 @@ class CabooseRets::RetsImporter # < ActiveRecord::Base
   end
 
   def self.coords_from_address(address)
+    return false
     begin
       uri = "https://maps.googleapis.com/maps/api/geocode/json?address=#{address}&sensor=false"
       uri.gsub!(" ", "+")
@@ -383,6 +336,78 @@ class CabooseRets::RetsImporter # < ActiveRecord::Base
   def self.purge_helper(class_type, date_modified)
     m = self.meta(class_type)
 
+    puts "Purging #{class_type}..."
+    
+    # Get the total number of records
+    puts "- Getting total number of records for #{class_type}..."
+    params = {
+      :search_type => m.search_type,
+      :class => class_type,
+      :query => "(#{m.date_modified_field}=#{date_modified}T00:00:01+)",
+      :standard_names_only => true,
+      :timeout => -1
+    }
+    self.client.search(params.merge({ :count_mode => :only }))
+    count = self.client.rets_data[:code] == "20201" ? 0 : self.client.rets_data[:count]
+    batch_count = (count.to_f/5000.0).ceil
+    
+    ids = []
+    k = m.remote_key_field
+    (0...batch_count).each do |i|
+      puts "- Getting ids for #{class_type} (batch #{i+1} of #{batch_count})..."
+      self.client.search(params.merge({ :select => [k], :limit => 5000, :offset => 5000*i })) do |data|
+        ids << case class_type
+          when 'RES' then data[k] 
+          when 'COM' then data[k] 
+          when 'LND' then data[k] 
+          when 'MUL' then data[k] 
+          when 'OFF' then data[k]  
+          when 'AGT' then data[k]
+          when 'OPH' then data[k].to_i
+          when 'GFX' then data[k]
+        end                
+      end
+    end
+
+    # Delete any records in the local database that shouldn't be there
+    puts "- Finding #{class_type} records in the local database that are not in the remote database..."    
+    t = m.local_table
+    k = m.local_key_field        
+    query = "select distinct #{k} from #{t}"
+    rows = ActiveRecord::Base.connection.select_all(ActiveRecord::Base.send(:sanitize_sql_array, query))
+    local_ids = rows.collect{ |row| row[k] }
+    ids_to_remove = local_ids - ids    
+    puts "- Found #{ids_to_remove.count} #{class_type} records in the local database that are not in the remote database."
+    puts "- Deleting #{class_type} records in the local database that shouldn't be there..."
+    query = ["delete from #{t} where #{k} not in (?)", ids_to_remove]
+    ActiveRecord::Base.connection.execute(ActiveRecord::Base.send(:sanitize_sql_array, query))
+
+    # Find any ids in the remote database that should be in the local database
+    puts "- Finding #{class_type} records in the remote database that should be in the local database..."
+    query = "select distinct #{k} from #{t}"
+    rows = ActiveRecord::Base.connection.select_all(ActiveRecord::Base.send(:sanitize_sql_array, query))
+    local_ids = rows.collect{ |row| row[k] }
+    ids_to_add = ids - local_ids    
+    puts "- Found #{ids_to_add.count} #{class_type} records in the remote database that we need to add to the local database."
+    ids_to_add.each do |id|
+      puts "- Importing #{id}..."
+      case class_type
+        when 'RES' then self.delay.import_residential_property(id, false)
+        when 'COM' then self.delay.import_commercial_property(id, false)
+        when 'LND' then self.delay.import_land_property(id, false)
+        when 'MUL' then self.delay.import_multi_family_property(id, false)
+        when 'OFF' then self.delay.import_office(id, false)
+        when 'AGT' then self.delay.import_agent(id, false)
+        when 'OPH' then self.delay.import_open_house(id, false)
+        when 'GFX' then self.delay.import_media(id, false)
+      end
+    end
+
+  end
+  
+  def self.get_media_urls
+    m = self.meta(class_type)
+
     # Get the total number of records
     params = {
       :search_type => m.search_type,
@@ -444,8 +469,8 @@ class CabooseRets::RetsImporter # < ActiveRecord::Base
   #=============================================================================
 
   def self.log(msg)
-    #puts "[rets_importer] #{msg}"
-    Rails.logger.info("[rets_importer] #{msg}")
+    puts "[rets_importer] #{msg}"
+    #Rails.logger.info("[rets_importer] #{msg}")
   end
 
   #=============================================================================
@@ -458,6 +483,9 @@ class CabooseRets::RetsImporter # < ActiveRecord::Base
 
     begin
       overlap = 30.seconds
+      puts DateTime.now
+      puts self.last_purged
+      puts (DateTime.now - self.last_purged)
       if (DateTime.now - self.last_purged).to_i > 1
         self.purge
         self.save_last_purged(task_started)

data/app/models/caboose_rets/rets_importer_bak.rb

@@ -0,0 +1,621 @@
+##require 'ruby-rets'
+#require "rets/version"
+#require "rets/exceptions"
+#require "rets/client"
+#require "rets/http"
+#require "rets/stream_http"
+#require "rets/base/core"
+#require "rets/base/sax_search"
+#require "rets/base/sax_metadata"
+#
+#require 'httparty'
+#require 'json'
+#
+## http://rets.solidearth.com/ClientHome.aspx
+#
+#class CabooseRets::RetsImporter # < ActiveRecord::Base
+#
+#  @@rets_client = nil
+#  @@config = nil
+#
+#  def self.config
+#    return @@config
+#  end
+#
+#  def self.get_config
+#    @@config = {
+#      'url'                 => nil, # URL to the RETS login
+#      'username'            => nil,
+#      'password'            => nil,
+#      'temp_path'           => nil,
+#      'log_file'            => nil,
+#      'media_base_url'      => nil
+#    }
+#    config = YAML::load(File.open("#{Rails.root}/config/rets_importer.yml"))
+#    config = config[Rails.env]
+#    config.each { |key,val| @@config[key] = val }
+#  end
+#
+#  def self.client
+#    self.get_config if @@config.nil? || @@config['url'].nil?
+#
+#    if @@rets_client.nil?
+#      @@rets_client = RETS::Client.login(
+#        :url      => @@config['url'],
+#        :username => @@config['username'],
+#        :password => @@config['password']
+#      )
+#    end
+#    return @@rets_client
+#  end
+#
+#  def self.meta(class_type)
+#    case class_type
+#      when 'RES' then Caboose::StdClass.new({ :search_type => 'Property'  , :remote_key_field => 'MLS_ACCT'   , :local_key_field => 'mls_acct' , :local_table => 'rets_residential'  , :date_modified_field => 'DATE_MODIFIED'    })
+#      when 'COM' then Caboose::StdClass.new({ :search_type => 'Property'  , :remote_key_field => 'MLS_ACCT'   , :local_key_field => 'mls_acct' , :local_table => 'rets_commercial'   , :date_modified_field => 'DATE_MODIFIED'    })
+#      when 'LND' then Caboose::StdClass.new({ :search_type => 'Property'  , :remote_key_field => 'MLS_ACCT'   , :local_key_field => 'mls_acct' , :local_table => 'rets_land'         , :date_modified_field => 'DATE_MODIFIED'    })
+#      when 'MUL' then Caboose::StdClass.new({ :search_type => 'Property'  , :remote_key_field => 'MLS_ACCT'   , :local_key_field => 'mls_acct' , :local_table => 'rets_multi_family' , :date_modified_field => 'DATE_MODIFIED'    })
+#      when 'OFF' then Caboose::StdClass.new({ :search_type => 'Office'    , :remote_key_field => 'LO_LO_CODE' , :local_key_field => 'lo_code'  , :local_table => 'rets_offices'      , :date_modified_field => 'LO_DATE_MODIFIED' })
+#      when 'AGT' then Caboose::StdClass.new({ :search_type => 'Agent'     , :remote_key_field => 'LA_LA_CODE' , :local_key_field => 'la_code'  , :local_table => 'rets_agents'       , :date_modified_field => 'LA_DATE_MODIFIED' })
+#      when 'OPH' then Caboose::StdClass.new({ :search_type => 'OpenHouse' , :remote_key_field => 'ID'         , :local_key_field => 'id'       , :local_table => 'rets_open_houses'  , :date_modified_field => 'DATE_MODIFIED'    })
+#      when 'GFX' then Caboose::StdClass.new({ :search_type => 'Media'     , :remote_key_field => 'MEDIA_ID'   , :local_key_field => 'media_id' , :local_table => 'rets_media'        , :date_modified_field => 'DATE_MODIFIED'    })
+#    end
+#  end
+#
+#  #=============================================================================
+#  # Import method
+#  #=============================================================================
+#
+#  def self.import(class_type, query)    
+#    m = self.meta(class_type)
+#    self.log("Importing #{m.search_type}:#{class_type} with query #{query}...")
+#    self.get_config if @@config.nil? || @@config['url'].nil?
+#    params = {
+#      :search_type => m.search_type,
+#      :class => class_type,
+#      :query => query,
+#      :limit => -1,
+#      :timeout => -1
+#    }
+#    obj = nil
+#    begin        
+#      self.client.search(params) do |data|
+#        obj = self.get_instance_with_id(class_type, data)
+#        if obj.nil?
+#          self.log("Error: object is nil")
+#          self.log(data.inspect)
+#          next
+#        end
+#        obj.parse(data)
+#        obj.save
+#      end
+#    rescue RETS::HTTPError => err
+#      self.log "Import error for #{class_type}: #{query}"
+#      self.log err.message
+#    end
+#    
+#  end
+#
+#  def self.get_instance_with_id(class_type, data)
+#    obj = nil
+#    m = case class_type
+#      when 'OPH' then CabooseRets::OpenHouse
+#      when 'GFX' then CabooseRets::Media
+#      when 'COM' then CabooseRets::CommercialProperty
+#      when 'LND' then CabooseRets::LandProperty
+#      when 'MUL' then CabooseRets::MultiFamilyProperty
+#      when 'RES' then CabooseRets::ResidentialProperty
+#      when 'AGT' then CabooseRets::Agent
+#      when 'OFF' then CabooseRets::Office
+#    end
+#    obj = case class_type
+#      when 'OPH' then m.where(:id       => data['ID'].to_i       ).exists? ? m.where(:id       => data['ID'].to_i       ).first : m.new(:id       => data['ID'].to_i       )
+#      when 'GFX' then m.where(:media_id => data['MEDIA_ID']      ).exists? ? m.where(:media_id => data['MEDIA_ID']      ).first : m.new(:media_id => data['MEDIA_ID']      )
+#      when 'COM' then m.where(:id       => data['MLS_ACCT'].to_i ).exists? ? m.where(:id       => data['MLS_ACCT'].to_i ).first : m.new(:id       => data['MLS_ACCT'].to_i )
+#      when 'LND' then m.where(:id       => data['MLS_ACCT'].to_i ).exists? ? m.where(:id       => data['MLS_ACCT'].to_i ).first : m.new(:id       => data['MLS_ACCT'].to_i )
+#      when 'MUL' then m.where(:id       => data['MLS_ACCT'].to_i ).exists? ? m.where(:id       => data['MLS_ACCT'].to_i ).first : m.new(:id       => data['MLS_ACCT'].to_i )
+#      when 'RES' then m.where(:id       => data['MLS_ACCT'].to_i ).exists? ? m.where(:id       => data['MLS_ACCT'].to_i ).first : m.new(:id       => data['MLS_ACCT'].to_i )
+#      when 'AGT' then m.where(:la_code  => data['LA_LA_CODE']    ).exists? ? m.where(:la_code  => data['LA_LA_CODE']    ).first : m.new(:la_code  => data['LA_LA_CODE']    )
+#      when 'OFF' then m.where(:lo_code  => data['LO_LO_CODE']    ).exists? ? m.where(:lo_code  => data['LO_LO_CODE']    ).first : m.new(:lo_code  => data['LO_LO_CODE']    )
+#    end
+#    return obj
+#  end
+#
+#  #=============================================================================
+#  # Main updater
+#  #=============================================================================
+#
+#  def self.update_after(date_modified, save_images = true)
+#    self.update_helper('RES', date_modified, save_images)
+#    self.update_helper('COM', date_modified, save_images)
+#    self.update_helper('LND', date_modified, save_images)
+#    self.update_helper('MUL', date_modified, save_images)
+#    self.update_helper('OFF', date_modified, save_images)
+#    self.update_helper('AGT', date_modified, save_images)
+#    self.update_helper('OPH', date_modified, save_images)
+#  end
+#
+#  def self.update_helper(class_type, date_modified, save_images = true)
+#    m = self.meta(class_type)
+#    k = m.remote_key_field
+#    params = {
+#      :search_type => m.search_type,
+#      :class => class_type,
+#      :select => [m.remote_key_field],
+#      :query => "(#{m.date_modified_field}=#{date_modified.strftime("%FT%T")}+)",
+#      :standard_names_only => true,
+#      :timeout => -1
+#    }
+#    self.client.search(params) do |data|
+#      case class_type
+#        when 'RES' then self.delay(:priority => 10).import_residential_property(  data[k], save_images)
+#        when 'COM' then self.delay(:priority => 10).import_commercial_property(   data[k], save_images)
+#        when 'LND' then self.delay(:priority => 10).import_land_property(         data[k], save_images)
+#        when 'MUL' then self.delay(:priority => 10).import_multi_family_property( data[k], save_images)
+#        when 'OFF' then self.delay(:priority => 10).import_office(                data[k], save_images)
+#        when 'AGT' then self.delay(:priority => 10).import_agent(                 data[k], save_images)
+#        when 'OPH' then self.delay(:priority => 10).import_open_house(            data[k], save_images)
+#      end
+#    end
+#  end
+#
+#  #=============================================================================
+#  # Single model import methods (called from a worker dyno)
+#  #=============================================================================
+#
+#  def self.import_property(mls_acct, save_images = true)
+#    self.import('RES', "(MLS_ACCT=*#{mls_acct}*)")
+#    p = CabooseRets::ResidentialProperty.where(:id => mls_acct.to_i).first
+#    if p.nil?
+#      self.import('COM', "(MLS_ACCT=*#{mls_acct}*)")
+#      p = CabooseRets::CommercialProperty.where(:id => mls_acct.to_i).first
+#      if p.nil?
+#        self.import('LND', "(MLS_ACCT=*#{mls_acct}*)")
+#        p = CabooseRets::LandProperty.where(:id => mls_acct.to_i).first
+#        if p.nil?
+#          self.import('MUL', "(MLS_ACCT=*#{mls_acct}*)")
+#          p = CabooseRets::MultiFamilyProperty.where(:id => mls_acct.to_i).first
+#          return if p.nil?
+#        end
+#      end
+#    end
+#    self.download_property_images(p, save_images)
+#  end
+#
+#  def self.import_residential_property(mls_acct, save_images = true)
+#    self.import('RES', "(MLS_ACCT=*#{mls_acct}*)")
+#    p = CabooseRets::ResidentialProperty.where(:id => mls_acct.to_i).first
+#    self.download_property_images(p, save_images)
+#    self.update_coords(p)
+#  end
+#
+#  def self.import_commercial_property(mls_acct, save_images = true)
+#    self.import('COM', "(MLS_ACCT=*#{mls_acct}*)")
+#    p = CabooseRets::CommercialProperty.where(:id => mls_acct.to_i).first
+#    self.download_property_images(p, save_images)
+#    self.update_coords(p)
+#  end
+#
+#  def self.import_land_property(mls_acct, save_images = true)
+#    self.import('LND', "(MLS_ACCT=*#{mls_acct}*)")
+#    p = CabooseRets::LandProperty.where(:id => mls_acct.to_i).first
+#    self.download_property_images(p, save_images)
+#    self.update_coords(p)
+#  end
+#
+#  def self.import_multi_family_property(mls_acct, save_images = true)
+#    self.import('MUL', "(MLS_ACCT=*#{mls_acct}*)")
+#    p = CabooseRets::MultiFamilyProperty.where(:id => mls_acct.to_i).first
+#    self.download_property_images(p, save_images)
+#    self.update_coords(p)
+#  end
+#
+#  def self.import_office(lo_code, save_images = true)    
+#    self.import('OFF', "(LO_LO_CODE=*#{lo_code}*)")
+#    office = CabooseRets::Office.where(:lo_code => lo_code.to_s).first
+#    self.download_office_image(office) if save_images == true
+#  end
+#
+#  def self.import_agent(la_code, save_images = true)
+#    self.import('AGT', "(LA_LA_CODE=*#{la_code}*)")
+#    a = CabooseRets::Agent.where(:la_code => la_code.to_s).first
+#    self.download_agent_image(a) if save_images == true
+#  end
+#
+#  def self.import_open_house(id, save_images = true)
+#    self.import('OPH', "((ID=#{id}+),(ID=#{id}-))")
+#  end
+#
+#  def self.import_media(id, save_images = true)
+#    self.import('GFX', "((MEDIA_ID=#{id}+),(MEDIA_ID=#{id}-))")
+#    #m = CabooseRets::Media.where(:media_id => id.to_s).first
+#    #if m.nil?
+#    #  self.log("Can't find media record for #{id}.")
+#    #  return
+#    #end        
+#    #self.client.get_object(:resource => :Property, :type => :Photo, :location => true, :id => m.mls_acct) do |headers, content|      
+#    #  if headers['object-id'].to_i == m.media_order
+#    #    m.url = headers['location']
+#    #    #m.image = URI.parse(headers['location'])
+#    #  end
+#    #end
+#    #m.save    
+#  end
+#
+#  #=============================================================================
+#  # Images
+#  #=============================================================================
+#    
+#  def self.download_property_images(p, save_images = true)
+#    return if save_images == false
+#    self.refresh_property_media(p)    
+#
+#    #self.log("-- Downloading images and resizing for #{p.mls_acct}")
+#    #media = []
+#    #self.client.get_object(:resource => :Property, :type => :Photo, :location => true, :id => p.id) do |headers, content|
+#    #
+#    #  # Find the associated media record for the image
+#    #  #filename = File.basename(headers['location'])
+#    #  #m = CabooseRets::Media.where(:mls_acct => p.mls_acct, :file_name => filename).first
+#    #  m = CabooseRets::Media.where(:mls_acct => p.mls_acct, :media_order => headers['object-id'].to_i).first
+#    #
+#    #  if m.nil?
+#    #    self.log("Can't find media record for #{p.mls_acct} #{filename}.")
+#    #  else
+#    #    m.url = headers['location']
+#    #    #m.image = URI.parse(headers['location'])
+#    #    media << m
+#    #    #m.save
+#    #  end
+#    #end
+#    #
+#    #self.log("-- Uploading images to S3 for #{p.mls_acct}")
+#    #media.each do |m|
+#    #  m.save
+#    #end
+#  end
+#
+#  def self.refresh_property_media(p)
+#    #self.log("-- Deleting images and metadata for #{p.mls_acct}...")
+#    #CabooseRets::Media.where(:mls_acct => p.mls_acct, :media_type => 'Photo').destroy_all
+#    #CabooseRets::Media.where(:mls_acct => p.mls_acct).destroy_all
+#
+#    self.log("-- Downloading GFX records for #{p.mls_acct}...")
+#    params = {
+#      :search_type => 'Media',
+#      :class => 'GFX',      
+#      :query => "(MLS_ACCT=*#{p.id}*)",
+#      :timeout => -1
+#    }
+#    ids = []
+#    self.client.search(params) do |data|
+#      ids << data['MEDIA_ID']      
+#      m = CabooseRets::Media.new
+#      m.parse(data)      
+#      m.save
+#    end
+#    
+#    # Delete any records in the local database that shouldn't be there
+#    puts "- Finding GFX records for MLS ##{p.mls_acct} in the local database that are not in the remote database..."
+#    query = "select media_id from rets_media where mls_acct = '#{p.mls_acct}'"
+#    rows = ActiveRecord::Base.connection.select_all(ActiveRecord::Base.send(:sanitize_sql_array, query))
+#    local_ids = rows.collect{ |row| row[k] }
+#    ids_to_remove = local_ids - ids
+#    if ids_to_remove && ids_to_remove > 0
+#      query = ["delete from rets_media where media_id not in (?)", ids_to_remove]
+#      ActiveRecord::Base.connection.execute(ActiveRecord::Base.send(:sanitize_sql_array, query))
+#    end
+#        
+#  end
+#
+#  #def self.download_agent_image(agent)
+#  #  self.log "Saving image for #{agent.first_name} #{agent.last_name}..."
+#  #  begin
+#  #    self.client.get_object(:resource => :Agent, :type => :Photo, :location => true, :id => agent.la_code) do |headers, content|
+#  #      puts headers.inspect
+#  #      agent.image = URI.parse(headers['location'])
+#  #      agent.save
+#  #    end
+#  #  rescue RETS::APIError => err
+#  #    self.log "No image for #{agent.first_name} #{agent.last_name}."
+#  #  end
+#  #end
+#
+#  #def self.download_office_image(office)
+#  #  self.log "Saving image for #{office.lo_name}..."
+#  #  begin
+#  #    self.client.get_object(:resource => :Office, :type => :Photo, :location => true, :id => office.lo_code) do |headers, content|
+#  #      office.image = URI.parse(headers['location'])
+#  #      office.save
+#  #    end
+#  #  rescue RETS::APIError => err
+#  #    self.log "No image for #{office.lo_name}."
+#  #  end
+#  #end
+#
+#  #=============================================================================
+#  # GPS
+#  #=============================================================================
+#
+#  def self.update_coords(p = nil)
+#    if p.nil?
+#      models = [CabooseRets::CommercialProperty, CabooseRets::LandProperty, CabooseRets::MultiFamilyProperty, CabooseRets::ResidentialProperty]
+#      names = ["commercial", "land", "multi-family", "residential"]
+#      i = 0
+#      models.each do |model|
+#        self.log "Updating coords #{names[i]} properties..."
+#        model.where(:latitude => nil).reorder(:mls_acct).each do |p|
+#          self.update_coords(p)
+#        end
+#        i = i + 1
+#      end
+#      return
+#    end
+#
+#    self.log "Getting coords for mls_acct #{p.mls_acct}..."
+#    coords = self.coords_from_address(CGI::escape "#{p.street_num} #{p.street_name}, #{p.city}, #{p.state} #{p.zip}")
+#    return if coords.nil? || coords == false
+#
+#    p.latitude = coords['lat']
+#    p.longitude = coords['lng']
+#    p.save
+#  end
+#
+#  def self.coords_from_address(address)
+#    return false
+#    begin
+#      uri = "https://maps.googleapis.com/maps/api/geocode/json?address=#{address}&sensor=false"
+#      uri.gsub!(" ", "+")
+#      resp = HTTParty.get(uri)
+#      json = JSON.parse(resp.body)
+#      return json['results'][0]['geometry']['location']
+#    rescue
+#      self.log "Error: #{uri}"
+#      sleep(2)
+#      return false
+#    end
+#  end
+#
+#  #=============================================================================
+#  # Purging
+#  #=============================================================================
+#
+#  def self.purge
+#    self.purge_residential
+#    self.purge_commercial
+#    self.purge_land
+#    self.purge_multi_family
+#    self.purge_offices
+#    self.purge_agents
+#    self.purge_open_houses
+#    self.purge_media
+#  end
+#
+#  def self.purge_residential()  self.purge_helper('RES', '2012-01-01') end
+#  def self.purge_commercial()   self.purge_helper('COM', '2012-01-01') end
+#  def self.purge_land()         self.purge_helper('LND', '2012-01-01') end
+#  def self.purge_multi_family() self.purge_helper('MUL', '2012-01-01') end
+#  def self.purge_offices()      self.purge_helper('OFF', '2012-01-01') end
+#  def self.purge_agents()       self.purge_helper('AGT', '2012-01-01') end
+#  def self.purge_open_houses()  self.purge_helper('OPH', '2012-01-01') end
+#  def self.purge_media()        self.purge_helper('GFX', '2012-01-01') end
+#
+#  def self.purge_helper(class_type, date_modified)
+#    m = self.meta(class_type)
+#
+#    puts "Purging #{class_type}..."
+#    
+#    # Get the total number of records
+#    puts "- Getting total number of records for #{class_type}..."
+#    params = {
+#      :search_type => m.search_type,
+#      :class => class_type,
+#      :query => "(#{m.date_modified_field}=#{date_modified}T00:00:01+)",
+#      :standard_names_only => true,
+#      :timeout => -1
+#    }
+#    self.client.search(params.merge({ :count_mode => :only }))
+#    count = self.client.rets_data[:code] == "20201" ? 0 : self.client.rets_data[:count]
+#    batch_count = (count.to_f/5000.0).ceil
+#    
+#    ids = []
+#    k = m.remote_key_field
+#    (0...batch_count).each do |i|
+#      puts "- Getting ids for #{class_type} (batch #{i+1} of #{batch_count})..."
+#      self.client.search(params.merge({ :select => [k], :limit => 5000, :offset => 5000*i })) do |data|
+#        ids << case class_type
+#          when 'RES' then data[k] 
+#          when 'COM' then data[k] 
+#          when 'LND' then data[k] 
+#          when 'MUL' then data[k] 
+#          when 'OFF' then data[k]  
+#          when 'AGT' then data[k]
+#          when 'OPH' then data[k].to_i
+#          when 'GFX' then data[k]
+#        end                
+#      end
+#    end
+#
+#    # Delete any records in the local database that shouldn't be there
+#    puts "- Finding #{class_type} records in the local database that are not in the remote database..."    
+#    t = m.local_table
+#    k = m.local_key_field        
+#    query = "select distinct #{k} from #{t}"
+#    rows = ActiveRecord::Base.connection.select_all(ActiveRecord::Base.send(:sanitize_sql_array, query))
+#    local_ids = rows.collect{ |row| row[k] }
+#    ids_to_remove = local_ids - ids    
+#    puts "- Found #{ids_to_remove.count} #{class_type} records in the local database that are not in the remote database."
+#    puts "- Deleting #{class_type} records in the local database that shouldn't be there..."
+#    query = ["delete from #{t} where #{k} not in (?)", ids_to_remove]
+#    ActiveRecord::Base.connection.execute(ActiveRecord::Base.send(:sanitize_sql_array, query))
+#
+#    # Find any ids in the remote database that should be in the local database
+#    puts "- Finding #{class_type} records in the remote database that should be in the local database..."
+#    query = "select distinct #{k} from #{t}"
+#    rows = ActiveRecord::Base.connection.select_all(ActiveRecord::Base.send(:sanitize_sql_array, query))
+#    local_ids = rows.collect{ |row| row[k] }
+#    ids_to_add = ids - local_ids    
+#    puts "- Found #{ids_to_add.count} #{class_type} records in the remote database that we need to add to the local database."
+#    ids_to_add.each do |id|
+#      puts "- Importing #{id}..."
+#      case class_type
+#        when 'RES' then self.delay.import_residential_property(id, false)
+#        when 'COM' then self.delay.import_commercial_property(id, false)
+#        when 'LND' then self.delay.import_land_property(id, false)
+#        when 'MUL' then self.delay.import_multi_family_property(id, false)
+#        when 'OFF' then self.delay.import_office(id, false)
+#        when 'AGT' then self.delay.import_agent(id, false)
+#        when 'OPH' then self.delay.import_open_house(id, false)
+#        when 'GFX' then self.delay.import_media(id, false)
+#      end
+#    end
+#
+#  end
+#  
+#  def self.get_media_urls
+#    m = self.meta(class_type)
+#
+#    # Get the total number of records
+#    params = {
+#      :search_type => m.search_type,
+#      :class => class_type,
+#      :query => "(#{m.date_modified_field}=#{date_modified}T00:00:01+)",
+#      :standard_names_only => true,
+#      :timeout => -1
+#    }
+#    self.client.search(params.merge({ :count_mode => :only }))
+#    count = self.client.rets_data[:code] == "20201" ? 0 : self.client.rets_data[:count]
+#    batch_count = (count.to_f/5000.0).ceil
+#
+#    ids = []
+#    k = m.remote_key_field
+#    (0...batch_count).each do |i|
+#      self.client.search(params.merge({ :select => [k], :limit => 5000, :offset => 5000*i })) do |data|
+#        ids << case class_type
+#          when 'RES' then data[k] 
+#          when 'COM' then data[k] 
+#          when 'LND' then data[k] 
+#          when 'MUL' then data[k] 
+#          when 'OFF' then data[k]  
+#          when 'AGT' then data[k]
+#          when 'OPH' then data[k].to_i
+#          when 'GFX' then data[k]
+#        end                
+#      end
+#    end
+#
+#    # Delete any records in the local database that shouldn't be there
+#    t = m.local_table
+#    k = m.local_key_field
+#    query = ["delete from #{t} where #{k} not in (?)", ids]
+#    ActiveRecord::Base.connection.execute(ActiveRecord::Base.send(:sanitize_sql_array, query))
+#
+#    # Find any ids in the remote database that should be in the local database
+#    query = "select distinct #{k} from #{t}"
+#    rows = ActiveRecord::Base.connection.select_all(ActiveRecord::Base.send(:sanitize_sql_array, query))
+#    local_ids = rows.collect{ |row| row[k] }
+#    ids_to_add = ids - local_ids
+#    ids_to_add.each do |id|
+#      puts "Importing #{id}..."
+#      case class_type
+#        when 'RES' then self.delay.import_residential_property(id, false)
+#        when 'COM' then self.delay.import_commercial_property(id, false)
+#        when 'LND' then self.delay.import_land_property(id, false)
+#        when 'MUL' then self.delay.import_multi_family_property(id, false)
+#        when 'OFF' then self.delay.import_office(id, false)
+#        when 'AGT' then self.delay.import_agent(id, false)
+#        when 'OPH' then self.delay.import_open_house(id, false)
+#        when 'GFX' then self.delay.import_media(id)
+#      end
+#    end
+#
+#  end
+#
+#  #=============================================================================
+#  # Logging
+#  #=============================================================================
+#
+#  def self.log(msg)
+#    puts "[rets_importer] #{msg}"
+#    #Rails.logger.info("[rets_importer] #{msg}")
+#  end
+#
+#  #=============================================================================
+#  # Locking update task
+#  #=============================================================================
+#
+#  def self.update_rets
+#    return if self.task_is_locked
+#    task_started = self.lock_task
+#
+#    begin
+#      overlap = 30.seconds
+#      puts DateTime.now
+#      puts self.last_purged
+#      puts (DateTime.now - self.last_purged)
+#      if (DateTime.now - self.last_purged).to_i > 1
+#        self.purge
+#        self.save_last_purged(task_started)
+#        #overlap = 1.month
+#      end
+#      self.update_after(self.last_updated - overlap)
+#		  self.save_last_updated(task_started)
+#		  self.unlock_task
+#		rescue
+#		  raise
+#		ensure
+#		  self.unlock_task_if_last_updated(task_started)
+#		end
+#
+#		# Start the same update process in five minutes
+#		self.delay(:run_at => 1.minutes.from_now).update_rets
+#	end
+#
+#  def self.last_updated
+#    if !Caboose::Setting.exists?(:name => 'rets_last_updated')
+#      Caboose::Setting.create(:name => 'rets_last_updated', :value => '2013-08-06T00:00:01')
+#    end
+#    s = Caboose::Setting.where(:name => 'rets_last_updated').first
+#    return DateTime.parse(s.value)
+#  end
+#
+#  def self.last_purged
+#    if !Caboose::Setting.exists?(:name => 'rets_last_purged')
+#      Caboose::Setting.create(:name => 'rets_last_purged', :value => '2013-08-06T00:00:01')
+#    end
+#    s = Caboose::Setting.where(:name => 'rets_last_purged').first
+#    return DateTime.parse(s.value)
+#  end
+#
+#  def self.save_last_updated(d)
+#    s = Caboose::Setting.where(:name => 'rets_last_updated').first
+#    s.value = d.strftime('%FT%T')
+#    s.save
+#  end
+#
+#  def self.save_last_purged(d)
+#    s = Caboose::Setting.where(:name => 'rets_last_purged').first
+#    s.value = d.strftime('%FT%T')
+#    s.save
+#  end
+#
+#  def self.task_is_locked
+#    return Caboose::Setting.exists?(:name => 'rets_update_running')
+#  end
+#
+#  def self.lock_task
+#    d = DateTime.now.utc - 5.hours
+#    Caboose::Setting.create(:name => 'rets_update_running', :value => d.strftime('%F %T'))
+#    return d
+#  end
+#
+#  def self.unlock_task
+#    Caboose::Setting.where(:name => 'rets_update_running').first.destroy
+#  end
+#
+#  def self.unlock_task_if_last_updated(d)
+#    setting = Caboose::Setting.where(:name => 'rets_update_running').first
+#    self.unlock_task if setting && d.strftime('%F %T') == setting.value
+#  end
+#
+#end

data/app/models/caboose_rets/schema.rb

@@ -9,6 +9,12 @@ class CabooseRets::Schema < Caboose::Utilities::Schema
     }
   end
   
+  def self.indexes
+    {
+      CabooseRets::Media => [:media_id]
+    }    
+  end
+  
   # The schema of the database
   # { Model => [[name, data_type, options]] }
   def self.schema      
@@ -377,10 +383,10 @@ class CabooseRets::Schema < Caboose::Utilities::Schema
         [ :file_name          , :string ], 
         [ :media_id           , :string ], 
         [ :media_order        , :integer, { :default => 0 }], 
-        [ :media_remarks      , :text ], 
+        [ :media_remarks      , :text   ], 
         [ :media_type         , :string ], 
         [ :mls_acct           , :string ], 
-        [ :url                , :text ]
+        [ :url                , :text   ]        
       ],
       CabooseRets::MultiFamilyProperty => [
         [ :acreage                   , :text ], 

data/app/views/caboose_rets/residential/index.html.erb

@@ -9,7 +9,7 @@
     <% @properties.each do |p| %>
       <li>
         <% if p.images && p.images.count > 0 %>
-          <div class='image'><a href="/residential/<%= p.mls_acct %>/details"><img src="<%= p.images[0].url(:thumb) %>" /></a></div>
+          <div class='image'><a href="/residential/<%= p.mls_acct %>/details"><img src="<%= p.images[0].image_url(:thumb) %>" /></a></div>
         <% end %>
         <div class="address"><% if !p.unit_num.nil? && p.unit_num.strip.length > 0 %>Unit <%= p.unit_num %>, <% end %><%= p.street_num %> <%= p.street_name.titleize %></div>                  
         <% if p.agent %>

data/lib/caboose-rets.rb

@@ -4,5 +4,17 @@ module CabooseRets
 
   mattr_accessor :default_property_sort
   @@default_property_sort = 'current_price DESC, mls_acct'
-
+  
+  mattr_accessor :use_hosted_images
+  @@use_hosted_images = true
+  
+  mattr_accessor :media_base_url
+  @@media_base_url = ''
+  
+  mattr_accessor :agents_base_url
+  @@agents_base_url = ''
+  
+  mattr_accessor :offices_base_url
+  @@offices_base_url = ''
+  
 end

data/lib/caboose_rets/version.rb

@@ -1,3 +1,3 @@
 module CabooseRets
-  VERSION = '0.0.69'
+  VERSION = '0.0.70'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: caboose-rets
 version: !ruby/object:Gem::Version
-  version: 0.0.69
+  version: 0.0.70
 platform: ruby
 authors:
 - William Barry
@@ -58,6 +58,7 @@ files:
 - app/models/caboose_rets/open_house.rb
 - app/models/caboose_rets/residential_property.rb
 - app/models/caboose_rets/rets_importer.rb
+- app/models/caboose_rets/rets_importer_bak.rb
 - app/models/caboose_rets/rets_plugin.rb
 - app/models/caboose_rets/saved_property.rb
 - app/models/caboose_rets/saved_search.rb