caboose-rets 0.1.194 → 0.1.196

data/app/models/caboose_rets/rets_importer_old.rb

@@ -0,0 +1,622 @@
+#require 'ruby-rets'
+require "rets/version"
+require "rets/exceptions"
+require "rets/client"
+require "rets/http"
+require "rets/stream_http"
+require "rets/base/core"
+require "rets/base/sax_search"
+require "rets/base/sax_metadata"
+
+require 'httparty'
+require 'json'
+
+# http://rets.solidearth.com/ClientHome.aspx
+
+class CabooseRets::RetsImporter # < ActiveRecord::Base
+
+  @@rets_client = nil   
+  @@config = nil
+
+  def self.config
+    return @@config
+  end
+
+  def self.get_config
+    @@config = {
+      'url'                 => nil, # URL to the RETS login
+      'username'            => nil,
+      'password'            => nil,
+      'temp_path'           => nil,
+      'log_file'            => nil,
+      'media_base_url'      => nil
+    }
+    config = YAML::load(File.open("#{Rails.root}/config/rets_importer.yml"))    
+    config = config[Rails.env]
+    config.each { |key,val| @@config[key] = val }
+  end
+
+  def self.client
+    self.get_config if @@config.nil? || @@config['url'].nil?
+    if @@rets_client.nil?
+      @@rets_client = RETS::Client.login(
+        :url      => @@config['url'],
+        :username => @@config['username'],
+        :password => @@config['password']
+      )
+    end
+    return @@rets_client
+  end
+
+  def self.meta(class_type)
+    case class_type
+      when 'Office'    then Caboose::StdClass.new({ :search_type => 'Office'    , :remote_key_field => 'OfficeMlsId' , :local_key_field => 'lo_mls_id'    , :local_table => 'rets_offices'      , :date_modified_field => 'ModificationTimestamp'})
+      when 'Member'    then Caboose::StdClass.new({ :search_type => 'Member'    , :remote_key_field => 'MemberMlsId' , :local_key_field => 'mls_id'    , :local_table => 'rets_agents'       , :date_modified_field => 'ModificationTimestamp'})
+      when 'OpenHouse' then Caboose::StdClass.new({ :search_type => 'OpenHouse' , :remote_key_field => 'OpenHouseKey' , :local_key_field => 'matrix_unique_id'    , :local_table => 'rets_open_houses'  , :date_modified_field => 'ModificationTimestamp'})
+      when 'Property'  then Caboose::StdClass.new({ :search_type => 'Property'  , :remote_key_field => 'ListingId' , :local_key_field => 'mls_number'    , :local_table => 'rets_properties'   , :date_modified_field => 'ModificationTimestamp'})
+      when 'Media'     then Caboose::StdClass.new({ :search_type => 'Media'     , :remote_key_field => 'MediaObjectID'         , :local_key_field => 'media_id'            , :local_table => 'rets_media'        , :date_modified_field => 'ModificationTimestamp'   })
+      end
+  end
+
+  #=============================================================================
+  # Import method
+  #=============================================================================
+
+  def self.import(class_type, query)
+    m = self.meta(class_type)
+    self.log3(class_type,nil,"Importing #{m.search_type}:#{class_type} with query #{query}...") 
+    self.get_config if @@config.nil? || @@config['url'].nil?
+    params = {
+      :search_type => m.search_type,
+      :class => class_type,
+      :query => query,
+      :timeout => -1
+    }
+    obj = nil
+
+    begin
+      self.client.search(params) do |data|
+        obj = self.get_instance_with_id(class_type, data)
+        if obj.nil?
+          self.log3(class_type,nil,"Error: object is nil")
+          self.log3(class_type,nil,data.inspect)
+          next
+        end
+        obj.parse(data)
+        obj.save
+      end
+    rescue RETS::HTTPError => err
+      self.log3(class_type,nil,"Import error for #{class_type}: #{query}")
+      self.log3(class_type,nil,err.message)
+    end
+  end
+
+  def self.get_instance_with_id(class_type, data)
+    obj = nil
+    self.log "Getting instance of #{class_type}..."
+    m = case class_type
+      when 'Property'  then CabooseRets::Property
+      when 'OpenHouse' then CabooseRets::OpenHouse
+      when 'Member'    then CabooseRets::Agent
+      when 'Office'    then CabooseRets::Office
+      when 'Media'     then CabooseRets::Media
+    end  
+    obj  = case class_type
+      when 'Property'  then m.where(:mls_number => data['ListingId']).exists? ? m.where(:mls_number => data['ListingId']).first : m.create(:mls_number => data['ListingId'])
+      when 'OpenHouse' then m.where(:matrix_unique_id => data['OpenHouseKey']).exists? ? m.where(:matrix_unique_id => data['OpenHouseKey']).first : m.create(:matrix_unique_id => data['OpenHouseKey'])
+      when 'Member'    then m.where(:mls_id => data['MemberMlsId']).exists? ? m.where(:mls_id => data['MemberMlsId']).first : m.create(:mls_id => data['MemberMlsId'])
+      when 'Office'    then m.where(:lo_mls_id => data['OfficeMlsId']).exists? ? m.where(:lo_mls_id => data['OfficeMlsId']).first : m.create(:lo_mls_id => data['OfficeMlsId'])
+      when 'Media'     then m.where(:media_id => data['MediaObjectID']      ).exists? ? m.where(:media_id => data['MediaObjectID']      ).first : m.create(:media_id => data['MediaObjectID']      )
+    end
+    self.log "Found matching object ID #{obj.id}"
+    return obj
+  end
+
+  #=============================================================================
+  # Main updater
+  #=============================================================================
+
+  def self.update_after(date_modified, save_images = true)
+    si = save_images ? 'saving images' : 'not saving images'
+    self.log3(nil,nil,"Updating everything after #{date_modified} and #{si}")
+    self.delay(:priority => 10, :queue => 'rets').update_helper('Property' , date_modified, save_images)
+    self.delay(:priority => 10, :queue => 'rets').update_helper('Office'   , date_modified, false)
+    self.delay(:priority => 10, :queue => 'rets').update_helper('Member'   , date_modified, false)
+    self.delay(:priority => 10, :queue => 'rets').update_helper('OpenHouse', date_modified, false)
+  end
+
+  def self.update_helper(class_type, date_modified, save_images = true)
+    si = save_images ? 'saving images' : 'not saving images'
+    self.log3(class_type,nil,"Updating #{class_type} modified after #{date_modified} and #{si}")
+    m = self.meta(class_type)
+    k = m.remote_key_field
+    d = date_modified.in_time_zone(CabooseRets::timezone).strftime("%FT%T")
+
+    statusquery = ""
+    case class_type
+      when 'Property'  then statusquery = "OriginatingSystemName=WESTAL"
+      when 'Office'    then statusquery = "OfficeStatus=Active"
+      when 'Member'    then statusquery = "MemberStatus=Active"
+      when 'OpenHouse' then statusquery = "OpenHouseKeyNumeric=0+"
+    end
+
+    quer = "(#{m.date_modified_field}=#{d}+)AND(OriginatingSystemName=WESTAL)AND(#{statusquery})"
+    params = {
+      :search_type => m.search_type,
+      :class => class_type,
+      :select => [m.remote_key_field],
+      :querytype => 'DMQL2',
+      :query => quer,
+      :limit => 1000,
+      :standard_names_only => true,
+      :timeout => -1
+    }    
+    self.log3(class_type,nil,"Searching with params: " + params.to_s)
+    self.client.search(params) do |data|
+      self.log3(class_type,nil,"Resulting data: " + data.to_s)
+      case class_type
+        when 'Property'  then self.delay(:priority => 10, :queue => 'rets').import_properties(data[k], save_images)
+        when 'Office'    then self.delay(:priority => 10, :queue => 'rets').import_office(    data[k], false)
+        when 'Member'    then self.delay(:priority => 10, :queue => 'rets').import_agent(     data[k], false)
+        when 'OpenHouse' then self.delay(:priority => 10, :queue => 'rets').import_open_house(data[k], false)
+      end
+    end
+
+    # Check for changed images
+    if class_type == 'Property' && Rails.env.production?
+      self.log3("Property",nil,"Checking for modified images on Properties...")
+      d1 = (self.last_updated - 1.hours).in_time_zone(CabooseRets::timezone).strftime("%FT%T")
+      params = {
+        :search_type => m.search_type,
+        :class => class_type,
+        :select => [m.remote_key_field],
+        :querytype => 'DMQL2',
+        :limit => 1000,
+        :query => "(PhotosChangeTimestamp=#{d1}+)AND(OriginatingSystemName=WESTAL)AND(MlsStatus=Active)",
+        :standard_names_only => true,
+        :timeout => -1
+      }
+      self.log3(class_type,nil,"Searching with params: " + params.to_s)
+      self.client.search(params) do |data|
+        self.log3(class_type,nil,"Resulting data: " + data.to_s)
+        self.delay(:priority => 10, :queue => 'rets').import_properties(data[k], true)
+      end
+    end
+
+  end
+
+  #=============================================================================
+  # Single model import methods (called from a worker dyno)
+  #=============================================================================
+
+  def self.import_properties(mls_id, save_images = true)
+    si = save_images ? 'saving images' : 'not saving images'
+    self.log3('Property',mls_id,"Importing Property #{mls_id} and #{si}...")
+    save_images = true if !CabooseRets::Property.where(:mls_number => mls_id.to_s).exists?
+    self.import('Property', "(ListingId=#{mls_id})")
+    p = CabooseRets::Property.where(:mls_number => mls_id.to_s).first
+    if p != nil && p.status == 'Active'
+      self.download_property_images(p) if save_images == true && Rails.env.production?
+      if p.latitude.blank? || p.latitude == '0.0' || p.longitude.blank? || p.longitude == '0.0'
+        self.update_coords(p) 
+      end
+    else
+      self.log3(nil,nil,"No Active Property associated with #{mls_id}, not downloading images")
+    end
+  end
+
+  def self.import_office(mls_id, save_images = true)
+    self.log3('Office',mls_id,"Importing Office #{mls_id}...")
+    self.import('Office', "(OfficeMlsId=#{mls_id})")
+    office = CabooseRets::Office.where(:matrix_unique_id => mls_id.to_s).first
+  end
+
+  def self.import_agent(mls_id, save_images = true)
+    return if mls_id == "T/ISC-SA-MATRIXMONITOR"
+    a = CabooseRets::Agent.where(:mls_id => mls_id.to_s).first
+    if a.nil?
+      self.log3('Agent',mls_id,"Importing new Agent #{mls_id}...")
+      self.import('Member', "(MemberMlsId=#{mls_id})")
+      a = CabooseRets::Agent.where(:mls_id => mls_id.to_s).first
+      if a
+        a.last_updated = DateTime.now
+        a.save
+      end
+    else
+      lu = a.last_updated.blank? ? 0 : a.last_updated.to_time.to_i
+      now = DateTime.now.to_time.to_i
+      diff = now - lu
+      is_old = diff > 86400 # 24 hours
+      if is_old
+        self.log3('Agent',mls_id,"Updating existing Agent #{mls_id}...")
+        self.import('Member', "(MemberMlsId=#{mls_id})")
+        a.last_updated = DateTime.now
+        a.save
+      else
+        self.log3('Agent',mls_id,"Skipping importing Agent #{mls_id} because last_updated is today...")
+      end
+    end  
+  end 
+
+  def self.import_open_house(oh_id, save_images = true)
+    self.log3('OpenHouse',oh_id,"Importing Open House #{oh_id}...")
+    self.import('OpenHouse', "(OpenHouseKey=#{oh_id})")
+  end
+
+  def self.import_media(id, save_images = true)
+    self.log3('Media',id,"Importing Media #{id}...")
+    self.import('Media', "((MediaObjectID=#{id}+),(MediaObjectID=#{id}-))")
+  end
+
+  #=============================================================================
+  # Images go here
+  #=============================================================================
+
+  def self.download_property_images(p)
+    return if Rails.env.development?
+    self.log3('Property',p.mls_number,"Downloading images for #{p.mls_number}...")
+    ids_to_keep = []
+    begin
+      self.client.get_object(:resource => 'Property', :type => 'Photo', :location => false, :id => "#{p.matrix_unique_id}:*") do |headers, content|
+        next if headers.blank?
+        ind = headers['orderhint'] ? headers['orderhint'].to_i : 1 
+        self.log3('Media',p.mls_number,headers.to_s)
+        self.log3('Media',p.mls_number,"Downloading photo with content-id #{headers['content-id']}, index #{ind}")
+        is_new = false
+        m = CabooseRets::Media.where(:media_mui => headers['content-id'], :media_order => ind).first
+        is_new = true if m.nil?
+        m = CabooseRets::Media.new if m.nil?
+        tmp_path = "#{Rails.root}/tmp/rets_media_#{headers['content-id']}_#{ind}.jpeg"
+        File.open(tmp_path, "wb") do |f|
+          f.write(content)
+        end
+        m.media_mui     = headers['content-id']
+        m.media_order   = ind
+        m.media_type    = 'Photo'
+        cm = nil
+        old_cm_id = is_new ? nil : m.media_id
+        begin
+          cm               = Caboose::Media.new
+          cm.image         = File.open(tmp_path)
+          cm.name          = "rets_media_#{headers['content-id']}_#{ind}"
+          cm.original_name = "rets_media_#{headers['content-id']}_#{ind}.jpeg"
+          cm.processed     = true
+          cm.save
+          if cm && !cm.id.blank?
+            m.media_id = cm.id
+            m.save
+            ids_to_keep << m.id
+            if is_new
+              self.log3("Media",p.mls_number,"Created new RetsMedia object #{m.id}, media_id = #{m.media_id}")
+            else
+              old_media = Caboose::Media.where(:id => old_cm_id).first
+              if old_media
+                self.log3("Media",p.mls_number,"Deleting old CabooseMedia #{old_media.id}")
+                old_media.destroy
+              end
+              self.log3("Media",p.mls_number,"RetsMedia object already existed #{m.id}, updated media_id = #{m.media_id}")
+            end
+            self.log3("Media",p.mls_number,"Image rets_media_#{headers['content-id']}_#{ind} saved")
+          else
+            self.log3("Media",p.mls_number,"CabooseMedia was not created for some reason, not saving RetsMedia")
+          end
+        rescue
+          self.log3("Media",p.mls_number,"Error processing image #{ind} from RETS")
+        end
+        `rm #{tmp_path}`
+      end
+    rescue
+      self.log3("Media",p.mls_number,"Error downloading images for property with MLS # #{p.mls_number}")
+    end
+
+    # If we downloaded new images, look for old images to delete. 
+    if ids_to_keep.count > 0
+      self.log3("Media",p.mls_number,"Keeping new RetsMedia ids: #{ids_to_keep}")
+      self.log3("Media",p.mls_number,"Looking for old RetsMedia to delete")
+      CabooseRets::Media.where(:media_mui => p.matrix_unique_id).where("id not in (?)",ids_to_keep).each do |med|
+        self.log3("Media",p.mls_number,"Deleting old RetsMedia #{med.id} and CabooseMedia #{med.media_id}...")
+        m = Caboose::Media.where(:id => med.media_id).where("name ILIKE ?","rets_media%").first
+        m.destroy if m
+        med.destroy
+      end
+    end
+
+  end
+
+  def self.download_missing_images
+    self.log3("Property",nil,"Downloading all missing images...")
+    CabooseRets::Property.where("photo_count = ? OR photo_count is null", '').where(:status => "Active").all.each do |p|
+      self.delay(:priority => 10, :queue => 'rets').import_properties(p.mls_number, true)
+    end
+  end
+
+  def self.download_agent_image(agent)
+
+  end
+
+  def self.download_office_image(office)
+
+  end
+
+  #=============================================================================
+  # GPS
+  #=============================================================================
+
+  def self.update_coords(p = nil)
+    if p.nil?
+      model = CabooseRets::Property      
+      i = 0      
+        self.log3('Property',nil,"Updating coords properties...")
+        model.where(:latitude => nil).reorder(:mls_number).each do |p1|
+          self.delay(:priority => 10, :queue => 'rets').update_coords(p1)
+        end
+      return
+    end
+
+    self.log3('Property',p.mls_number,"Getting coords for MLS # #{p.mls_number}...")
+    coords = self.coords_from_address(CGI::escape "#{p.street_number} #{p.street_name}, #{p.city}, #{p.state_or_province} #{p.postal_code}")
+    if coords.nil? || coords == false
+      self.log3('Property',nil,"Can't set coords for MLS # #{p.mls_number}...")
+      return
+    end
+
+    p.latitude = coords['lat'].to_f
+    p.longitude = coords['lng'].to_f
+    p.save
+  end
+
+  def self.coords_from_address(address)
+    begin
+      uri = "https://maps.googleapis.com/maps/api/geocode/json?key=AIzaSyB9Wwx7sdWaUnFyLcdQ61NOV7DE2NZkDUE&address=#{address}"
+      uri.gsub!(" ", "+")
+      resp = HTTParty.get(uri)
+      json = JSON.parse(resp.body)
+      return json['results'][0]['geometry']['location']
+    rescue
+      self.log "Error: #{uri}"
+      sleep(2)
+      return false
+    end
+  end
+
+  #=============================================================================
+  # Purging
+  #=============================================================================
+
+  def self.purge
+    self.log3(nil,nil,'purging')
+    self.purge_properties
+    self.purge_offices
+    self.purge_agents
+    self.purge_open_houses
+  end  
+  
+  def self.purge_properties()   self.delay(:priority => 10, :queue => 'rets').purge_helper('Property', '2012-01-01') end
+  def self.purge_offices()      self.delay(:priority => 10, :queue => 'rets').purge_helper('Office', '2012-01-01') end
+  def self.purge_agents()       self.delay(:priority => 10, :queue => 'rets').purge_helper('Member', '2012-01-01') end
+  def self.purge_open_houses()  self.delay(:priority => 10, :queue => 'rets').purge_helper('OpenHouse', '2012-01-01') end
+  
+
+  # Adds/removes records in the database
+  def self.purge_helper(class_type, date_modified)    
+    m = self.meta(class_type)    
+    self.log(m.search_type)
+
+    self.log3(class_type,nil,"Purging #{class_type}...")
+
+    # Get the total number of records
+    self.log3(class_type,nil,"Getting total number of records for #{class_type}...")
+
+    statusquery = ""
+
+    case class_type
+      when 'Property'  then statusquery = "MlsStatus=Active"
+      when 'Office'    then statusquery = "OfficeStatus=Active"
+      when 'Member'    then statusquery = "MemberStatus=Active"
+      when 'OpenHouse' then statusquery = "OpenHouseKeyNumeric=0+"
+    end
+
+    params = {
+      :search_type => m.search_type,
+      :class => class_type,
+      :query => "(#{m.date_modified_field}=#{date_modified}T00:00:01+)AND(#{statusquery})AND(OriginatingSystemName=WESTAL)",
+      :standard_names_only => true,
+      :limit => 1000,
+      :timeout => -1
+    }
+    count = 0
+    self.client.search(params.merge({ :count => 1})) do |data| end        
+    count = self.client.rets_data[:code] == "20201" ? 0 : self.client.rets_data[:count]    
+    batch_count = (count.to_f/1000.0).ceil
+
+    ids = []
+    k = m.remote_key_field
+    (0...batch_count).each do |i|
+      self.log3(class_type,nil,"Getting ids for #{class_type} (batch #{i+1} of #{batch_count})...")
+      self.client.search(params.merge({ :select => [k], :limit => 1000, :offset => 1000*i })) do |data|
+        ids << data[k]
+      end
+    end
+
+    # Only do stuff if we got a real response from the server 
+    if ids.count > 0
+
+      self.log3(class_type,nil,"Remote IDs found: #{ids.to_s}")
+
+      # Delete any records in the local database that shouldn't be there
+      self.log3(class_type,nil,"Finding #{class_type} records in the local database that are not in the remote database...")
+      t = m.local_table
+      k = m.local_key_field
+      query = "select distinct #{k} from #{t}"
+      rows = ActiveRecord::Base.connection.select_all(ActiveRecord::Base.send(:sanitize_sql_array, query))
+      local_ids = rows.collect{ |row| row[k] }
+      self.log3(class_type,nil,"Local IDs found: #{local_ids.to_s}")
+      ids_to_remove = local_ids - ids
+      self.log3(class_type,nil,"Found #{ids_to_remove.count} #{class_type} records in the local database that are not in the remote database.")
+      
+      # Delete all RetsMedia and CabooseMedia for the deleted property listings (except keep the first image) 
+      if class_type == 'Property' && ids_to_remove && ids_to_remove.count > 0
+        self.log3(class_type,nil,"Deleting Media objects that shouldn't be there...")
+        muis = CabooseRets::Property.where("#{k} in (?)", ids_to_remove).pluck(:matrix_unique_id)
+        if muis && muis.count > 0 && Rails.env.production?
+          CabooseRets::Media.where("media_mui in (?)", muis).where("media_order != ?", 1).each do |med|
+            self.log3("Media",med.id,"Deleting old RetsMedia #{med.id} and CabooseMedia #{med.media_id}...")
+            m = Caboose::Media.where(:id => med.media_id).where("name ILIKE ?","rets_media%").first
+            m.destroy if m
+            med.destroy
+          end
+        end
+      end
+
+      if class_type != 'Property' # keep all properties in the DB
+        self.log3(class_type,nil,"Deleting #{class_type} records in the local database that shouldn't be there...")
+        query = ["delete from #{t} where #{k} in (?)", ids_to_remove]
+        ActiveRecord::Base.connection.execute(ActiveRecord::Base.send(:sanitize_sql_array, query))
+      else # mark deleted properties as Deleted status
+        self.log3(class_type,nil,"Setting deleted properties as Deleted status...")
+        query = ["update #{t} set status = ? where #{k} in (?)", "Deleted", ids_to_remove]
+        ActiveRecord::Base.connection.execute(ActiveRecord::Base.send(:sanitize_sql_array, query))
+      end
+
+      # Find any ids in the remote database that should be in the local database
+      self.log3(class_type,nil,"Finding #{class_type} records in the remote database that should be in the local database...")
+      query = "select distinct #{k} from #{t}"
+      rows = ActiveRecord::Base.connection.select_all(ActiveRecord::Base.send(:sanitize_sql_array, query))
+      local_ids = rows.collect{ |row| row[k] }
+      self.log3(class_type,nil,"Local IDs found: #{local_ids.to_s}")
+      ids_to_add = ids - local_ids
+      ids_to_add = ids_to_add.sort.reverse
+      self.log3(class_type,nil,"Found #{ids_to_add.count} #{class_type} records in the remote database that we need to add to the local database.")
+      ids_to_add.each do |id|
+        self.log3(class_type,nil,"Importing #{id}...")
+        case class_type
+          when "Property"   then self.delay(:priority => 10, :queue => 'rets').import_properties(id, true)
+          when "Office"    then self.delay(:priority => 10, :queue => 'rets').import_office(id, false)
+          when "Member"     then self.delay(:priority => 10, :queue => 'rets').import_agent(id, false)
+          when "OpenHouse" then self.delay(:priority => 10, :queue => 'rets').import_open_house(id, false)
+        end
+      end
+    end
+  end
+
+  #=============================================================================
+  # Logging
+  #=============================================================================
+
+  def self.log(msg)
+    puts "[rets_importer] #{msg}"
+  end
+
+  def self.log2(msg)
+    puts "======================================================================"
+    puts "[rets_importer] #{msg}"
+    puts "======================================================================"
+  end
+
+  def self.log3(class_name, object_id, text)
+    self.log2(text)
+    log = CabooseRets::Log.new
+    log.class_name = class_name
+    log.object_id = object_id
+    log.text = text
+    log.timestamp = DateTime.now
+    log.save
+  end
+
+  #=============================================================================
+  # Locking update task
+  #=============================================================================
+
+  def self.update_rets
+    self.log3(nil,nil,"Updating rets...")
+    if self.task_is_locked
+      self.log2("Task is locked, aborting.")
+      return
+    end
+    self.log2("Locking task...")
+    task_started = self.lock_task
+    begin
+      overlap = 2.hours
+      if (DateTime.now - self.last_purged).to_f >= 0.5
+        self.purge
+        self.save_last_purged(task_started)
+        overlap = 1.week
+      end
+      self.update_after((self.last_updated - overlap), false)
+      self.download_missing_images if Rails.env.production?
+      self.log3(nil,nil,"Saving the timestamp for when we updated to #{task_started.to_s}...")
+		  self.save_last_updated(task_started)
+		  self.log2("Unlocking the task...")
+		  self.unlock_task
+		rescue Exception => err
+		  puts err
+		  raise
+		ensure
+		  self.log2("Unlocking task if last updated...")
+		  self.unlock_task_if_last_updated(task_started)
+    end
+		# Start the same update process in 20 minutes
+		self.log3(nil,nil,"Adding the update rets task for 20 minutes from now...")
+		q = "handler like '%update_rets%'"
+		count = Delayed::Job.where(q).count
+		if count == 0 || (count == 1 && Delayed::Job.where(q).first.locked_at)
+		  self.delay(:run_at => 20.minutes.from_now, :priority => 10, :queue => 'rets').update_rets
+		end
+
+    # Delete RETS logs over 7 days old
+    dt = DateTime.now - 5.days
+    sql = "delete from rets_logs where timestamp < '#{dt}';"
+    ActiveRecord::Base.connection.select_all(sql)
+
+    # Update search options
+    CabooseRets::SearchOption.delay(:queue => "rets", :priority => 15).update_search_options
+  end
+
+  def self.last_updated
+    if !Caboose::Setting.exists?(:name => 'rets_last_updated')
+      Caboose::Setting.create(:name => 'rets_last_updated', :value => '2013-08-06T00:00:01')
+    end
+    s = Caboose::Setting.where(:name => 'rets_last_updated').first
+    return DateTime.parse(s.value)
+  end
+
+  def self.last_purged
+    if !Caboose::Setting.exists?(:name => 'rets_last_purged')
+      Caboose::Setting.create(:name => 'rets_last_purged', :value => '2013-08-06T00:00:01')
+    end
+    s = Caboose::Setting.where(:name => 'rets_last_purged').first
+    return DateTime.parse(s.value)
+  end
+
+  def self.save_last_updated(d)
+    s = Caboose::Setting.where(:name => 'rets_last_updated').first
+    s.value = d.in_time_zone(CabooseRets::timezone).strftime("%FT%T%:z")
+    s.save
+  end
+
+  def self.save_last_purged(d)
+    s = Caboose::Setting.where(:name => 'rets_last_purged').first
+    s.value = d.in_time_zone(CabooseRets::timezone).strftime("%FT%T%:z")
+    s.save
+  end
+
+  def self.task_is_locked
+    return Caboose::Setting.exists?(:name => 'rets_update_running')
+  end
+
+  def self.lock_task
+    d = DateTime.now.utc.in_time_zone(CabooseRets::timezone)
+    Caboose::Setting.create(:name => 'rets_update_running', :value => d.strftime("%FT%T%:z"))
+    return d
+  end
+
+  def self.unlock_task
+    Caboose::Setting.where(:name => 'rets_update_running').first.destroy
+  end
+
+  def self.unlock_task_if_last_updated(d)
+    setting = Caboose::Setting.where(:name => 'rets_update_running').first
+    self.unlock_task if setting && d.in_time_zone.strftime("%FT%T%:z") == setting.value
+  end
+
+end

data/lib/caboose_rets/version.rb

@@ -1,3 +1,3 @@
 module CabooseRets
-  VERSION = '0.1.194'
+  VERSION = '0.1.196'
 end

data/lib/tasks/caboose_rets.rake

@@ -211,10 +211,14 @@ namespace :caboose_rets do
 
   desc "Import rets data"
   task :import => :environment do
-    CabooseRets::RetsImporter.import('Member'    , "(MemberStatus=Active)")
-    CabooseRets::RetsImporter.import('Property'  , "(MlsStatus=Active)")
-    CabooseRets::RetsImporter.import('Office'    , "(OfficeStatus=Active)")    
-    CabooseRets::RetsImporter.import('OpenHouse' , "(OpenHouseKeyNumeric=0+)")
+    CabooseRets::RetsImporter.import('Member'    , "MemberStatus eq 'Active'")
+    CabooseRets::RetsImporter.import('Property'  , "MlsStatus eq 'Active'")
+    CabooseRets::RetsImporter.import('Office'    , "OfficeStatus eq 'Active'")
+    CabooseRets::RetsImporter.import('OpenHouse' , "OpenHouseKeyNumeric gt 0")
+  end
+
+  task :test_import => :environment do
+    CabooseRets::RetsImporter.import_one_property('146546')
   end
   
   desc "Single Import Test"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: caboose-rets
 version: !ruby/object:Gem::Version
-  version: 0.1.194
+  version: 0.1.196
 platform: ruby
 authors:
 - William Barry
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-03-30 00:00:00.000000000 Z
+date: 2023-01-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: caboose-cms
@@ -78,6 +78,7 @@ files:
 - app/models/caboose_rets/property.rb
 - app/models/caboose_rets/rets_config.rb
 - app/models/caboose_rets/rets_importer.rb
+- app/models/caboose_rets/rets_importer_old.rb
 - app/models/caboose_rets/rets_plugin.rb
 - app/models/caboose_rets/saved_property.rb
 - app/models/caboose_rets/saved_search.rb