caboose-rets 0.0.2

data/app/models/caboose_rets/rets_importer.rb

@@ -0,0 +1,303 @@
+require 'ruby-rets'
+require 'httparty'
+require 'json'
+
+# Check for examples:
+# http://rets.solidearth.com/ClientHome.aspx
+# https://www.flexmls.com/developers/rets/tutorials/dmql-tutorial/
+# https://www.flexmls.com/developers/rets/tutorials/how-to-efficiently-replicate-rets-data/
+
+class CabooseRets::RetsImporter # < ActiveRecord::Base
+   
+  @@rets_client = nil
+  @@config = nil
+  @@object_types = {
+    'OpenHouse' => ['OPH'],
+    'Media'     => ['GFX'],
+    'Property'  => ['COM', 'LND', 'MUL', 'RES'],
+    'Agent'     => ['AGT'],
+    'Office'    => ['OFF']
+  }
+  @@key_fields = {
+    'OpenHouse' => 'ID',
+    'Media'     => 'MEDIA_ID',
+    'Property'  => 'MLS_ACCT',
+    'Agent'     => 'LA_LA_CODE',
+    'Office'    => 'LO_LO_CODE'
+  }
+  @@models = {
+    'OPH' => 'CabooseRets::OpenHouse',
+    'GFX' => 'CabooseRets::Media',
+    'COM' => 'CabooseRets::CommercialProperty',
+    'LND' => 'CabooseRets::LandProperty',
+    'MUL' => 'CabooseRets::MultiFamilyProperty',
+    'RES' => 'CabooseRets::ResidentialProperty',
+    'AGT' => 'CabooseRets::Agent',
+    'OFF' => 'CabooseRets::Office'
+  }
+  @@date_modified_fields = {
+    'OPH' => 'DATE_MODIFIED',
+    'GFX' => 'DATE_MODIFIED',
+    'COM' => 'DATE_MODIFIED',
+    'LND' => 'DATE_MODIFIED',
+    'MUL' => 'DATE_MODIFIED',
+    'RES' => 'DATE_MODIFIED',
+    'AGT' => 'LA_DATE_MODIFIED',
+    'OFF' => 'LO_DATE_MODIFIED'
+  }
+
+  def self.config
+    return @@config
+  end
+    
+  def self.get_config
+    @@config = {
+      'url'                 => nil, # URL to the RETS login
+      'username'            => nil,
+      'password'            => nil,
+      'limit'               => nil, # How many records to limit per request
+      'days_per_batch'      => nil, # When performing a large property import, how many days to search on per batch 
+      'temp_path'           => nil,
+      'log_file'            => nil,
+      'media_base_url'      => nil
+    }
+    config = YAML::load(File.open("#{Rails.root}/config/rets_importer.yml"))
+    config = config[Rails.env]
+    config.each { |key,val| @@config[key] = val }
+  end
+             
+  def self.client
+    self.get_config if @@config.nil? || @@config['url'].nil?
+        
+    if (@@rets_client.nil?)     
+      @@rets_client = RETS::Client.login(
+        :url      => @@config['url'],
+        :username => @@config['username'],
+        :password => @@config['password']
+      )
+    end
+    return @@rets_client
+  end
+  
+  #=============================================================================
+  # Main updater
+  #=============================================================================
+  
+  def self.update_after(date_modified)
+    self.get_config if @@config.nil? || @@config['url'].nil?
+        
+    # Import the data
+    self.import_modified_after(date_modified, 'Agent'     , 'AGT')
+    self.import_modified_after(date_modified, 'Office'    , 'OFF')
+    self.import_modified_after(date_modified, 'OpenHouse' , 'OPH')        
+    self.import_modified_after(date_modified, 'Property'  , 'COM')
+    self.import_modified_after(date_modified, 'Property'  , 'LND')
+    self.import_modified_after(date_modified, 'Property'  , 'MUL')
+    self.import_modified_after(date_modified, 'Property'  , 'RES')
+    #self.update_coords(obj)
+    
+    # Import the images
+    self.download_agent_images_modified_after(date_modified)
+    self.download_property_images_modified_after(date_modified)
+  end
+  
+  #=============================================================================
+  # Data
+  #=============================================================================
+  
+  def self.import_modified_after(date_modified, search_type = nil, class_type = nil)
+    self.get_config if @@config.nil? || @@config['url'].nil?
+                
+    d = date_modified
+    date_modified_field = @@date_modified_fields[class_type]
+    
+    while d.strftime('%FT%T') <= DateTime.now.strftime('%FT%T') do      
+      break if d.nil?
+    
+      d2 = d.strftime('%FT%T')
+      d2 << "-"
+      d2 << (d+@@config['days_per_batch']).strftime('%FT%T')
+            
+      query = "(#{date_modified_field}=#{d2})"            
+      self.import(query, search_type, class_type)
+      
+      d = d + @@config['days_per_batch']
+    end
+  end
+  
+  def self.import(query, search_type, class_type)
+    # See how many records we have
+    self.client.search(
+      :search_type => search_type,
+      :class => class_type,
+      :query => query,
+      :count_mode => :only,
+      :timeout => -1,
+    )
+    # Return if no records found
+    if (self.client.rets_data[:code] == "20201")
+      self.log "No #{search_type}:#{class_type} records found for query: #{query}"
+      return
+    else
+      count = self.client.rets_data[:count]            
+      self.log "Importing #{count} #{search_type}:#{class_type} record" + (count == 1 ? "" : "s") + "..."
+    end
+
+    count = self.client.rets_data[:count]    
+    batch_count = (count.to_f/@@config['limit'].to_f).ceil
+    
+    (0...batch_count).each do |i|  
+      params = {
+        :search_type => search_type,
+        :class => class_type,
+        :query => query,
+        :limit => @@config['limit'],
+        :offset => @@config['limit'] * i,
+        :timeout => -1
+      }
+      obj = nil
+      self.client.search(params) do |data|
+        m = @@models[class_type].constantize
+        key_field = @@key_fields[search_type]
+        id = data[key_field].to_i
+        obj = m.exists?(id) ? m.find(id) : m.new
+        obj.parse(data)
+        obj.id = id
+        obj.save        
+      end
+    end
+  end
+  
+  #=============================================================================
+  # Agent Images
+  #=============================================================================
+  
+  def self.download_agent_images_modified_after(date_modified, agent = nil)
+    if agent.nil?
+      CabooseRets::Agent.where('photo_date_modified > ?', date_modified.strftime('%FT%T')).reorder('last_name, first_name').all.each do |a|
+        self.download_agent_images_modified_after(date_modified, a)
+      end
+      return
+    end
+    
+    a = agent    
+    self.log "Saving image for #{a.first_name} #{a.last_name}..."
+    begin
+      self.client.get_object(:resource => :Agent, :type => :Photo, :location => true, :id => a.la_code) do |headers, content|
+        a.image = URI.parse(headers['location'])
+        a.save
+      end
+    rescue RETS::APIError => err
+      self.log "No image for #{a.first_name} #{a.last_name}."
+    end    
+  end
+  
+  #=============================================================================
+  # Property Images
+  #=============================================================================
+  
+  def self.download_property_images_modified_after(date_modified)
+    [CommercialProperty, LandProperty, MultiFamilyProperty, ResidentialProperty].each do |model|
+      model.where("photo_date_modified > ?", date_modified.strftime('%FT%T')).each do |p|        
+        self.download_property_images(p)                        
+      end
+    end    
+  end
+    
+  def self.download_property_images(p)
+    self.refresh_property_media(p)
+    self.client.get_object(:resource => :Property, :type => :Photo, :location => true, :id => p.id) do |headers, content|
+      
+      # Find the associated media record for the image
+      filename = File.basename(headers['location'])
+      m = CabooseRets::Media.where(:mls_acct => p.mls_acct, :file_name => filename).first
+      
+      if m.nil?
+        self.log("Can't find media record for #{p.mls_acct} #{filename}.")
+      else                                 
+        m.image = URI.parse(headers['location'])
+        m.save
+      end
+      
+    end
+  end
+  
+  def self.refresh_property_media(p)
+    CabooseRets::Media.where(:mls_acct => p.mls_acct, :media_type => 'Photo').destroy_all
+    
+    params = {
+      :search_type => 'Media',
+      :class => 'GFX',
+      :query => "(MLS_ACCT=*#{p.id}*),(MEDIA_TYPE=|I)",
+      :timeout => -1
+    }    
+    self.client.search(params) do |data|      
+      m = CabooseRets::Media.new
+      m.parse(data)
+      m.id = m.media_id
+      m.save
+    end
+  end
+
+  #=============================================================================
+  # GPS
+  #=============================================================================
+  
+  def self.update_coords(p, type = 'RES')
+    if type == 'RES'
+      self.update_property_coords(ResidentialProperty, 'residential', p)
+    elsif type == 'COM'
+      self.update_property_coords(CommercialProperty, 'commercial', p)
+    elsif type == 'LND'
+      self.update_property_coords(LandProperty, 'land', p)
+    end
+  end
+  
+  def self.update_property_coords(property_class, symbolic_name, property = nil)
+    if (property.nil?)
+      if (property_class.where(:latitude => nil).count == 0)
+        self.log "All #{symbolic_name} properties have GPS coordinates."
+        return
+      end
+      property_class.where(:latitude => nil).reorder(:mls_acct).each do |p|
+        self.update_property_coords(property_class, symbolic_name, p)
+      end                         
+      self.update_property_coords(property_class, symbolic_name, nil)
+      return
+    end
+    p = property
+      
+    coords = self.coords_from_address(CGI::escape "#{p.street_num} #{p.street_name}, #{p.city}, #{p.state} #{p.zip}")
+    return if coords.nil? || coords == false
+    
+    p.latitude = coords['lat']
+    p.longitude = coords['lng']
+    p.save
+    self.log "Saved #{symbolic_name} property #{p.mls_acct} coords"
+  end
+  
+  def self.coords_from_address(address)   
+    begin
+      uri = "https://maps.googleapis.com/maps/api/geocode/json?address=#{address}&sensor=false"
+      uri.gsub!(" ", "+")
+      
+      resp = HTTParty.get(uri)
+      json = JSON.parse(resp.body)
+      return json['results'][0]['geometry']['location']          
+    rescue
+      self.log "Error: #{uri}"
+      sleep(2)
+      return false      
+    end
+  end
+  
+  #=============================================================================
+  # Logging
+  #=============================================================================
+  
+  def self.log(msg)
+    puts "[rets_importer] #{msg}"
+    Rails.logger.info("[rets_importer] #{msg}")
+  end
+  
+end

data/app/models/caboose_rets/rets_plugin.rb

@@ -0,0 +1,26 @@
+class AdvantagePlugin < Caboose::CaboosePlugin
+
+  def self.admin_nav(nav, user = nil, page = nil)
+    return nav if user.nil? || !user.is_allowed('properties', 'view')
+    
+    item = {
+      'id' => 'rets',
+      'text' => 'RETS', 
+      'children' => [],
+      'modal' => true
+    }
+    
+    item['children'] << { 'id' => 'agents'      , 'href' => '/admin/agents'           , 'text' => 'Agents'                , 'modal' => false }
+    item['children'] << { 'id' => 'agents'      , 'href' => '/admin/offices'          , 'text' => 'Offices'               , 'modal' => false }
+    item['children'] << { 'id' => 'open-houses' , 'href' => '/admin/open-houses'      , 'text' => 'Open Houses'           , 'modal' => false }
+    item['children'] << { 'id' => 'residential' , 'href' => '/admin/residential'      , 'text' => 'Residential Property'  , 'modal' => false }
+    item['children'] << { 'id' => 'commercial'  , 'href' => '/admin/commercial'       , 'text' => 'Commercial Property'   , 'modal' => false }
+    item['children'] << { 'id' => 'commercial'  , 'href' => '/admin/multi-family'     , 'text' => 'Multi-Family Property' , 'modal' => false }
+    item['children'] << { 'id' => 'land'        , 'href' => '/admin/land'             , 'text' => 'Land Property'         , 'modal' => false }    
+    
+    nav << item
+    
+    return nav
+  end
+  
+end

data/app/views/caboose_rets/agents/_admin_top_nav.html.erb

@@ -0,0 +1,7 @@
+<div class='top_right_controls'>
+<input type='button' value='General'      onclick="window.location='/admin/agents/<%= @agent.id %>/edit';" />
+<input type='button' value='Bio'          onclick="window.location='/admin/agents/<%= @agent.id %>/edit-bio';" />
+<input type='button' value='Contact Info' onclick="window.location='/admin/agents/<%= @agent.id %>/edit-contact-info';" />
+<input type='button' value='MLS Info'     onclick="window.location='/admin/agents/<%= @agent.id %>/edit-mls-info';" />
+</div>
+<p>Agent: <%= "#{@agent.first_name} #{@agent.last_name}, #{@agent.la_code}" %></p>

data/app/views/caboose_rets/agents/admin_edit.html.erb

@@ -0,0 +1,28 @@
+<h1>Edit Agent</h1>
+<%= render :partial => 'agents/admin_top_nav' %>
+
+<div id='agent_<%= @agent.id %>_hide'         ></div>
+<div id='agent_<%= @agent.id %>_assistant_to' ></div>
+
+<p style='clear: both;'><input type='button' value='< Back' onclick="window.location='/admin/agents';" /></p>
+
+<% content_for :caboose_js do %>
+<%= javascript_include_tag "caboose/model/all" %>
+<script type='text/javascript'>
+$(document).ready(function() {
+  m = new ModelBinder({
+    name: 'Agent',
+    id: <%= @agent.id %>,
+    update_url: '/admin/agents/<%= @agent.id %>',
+    authenticity_token: '<%= form_authenticity_token %>',
+    attributes: [
+      { name: 'hide'            , nice_name: 'Hide from website'  , type: 'checkbox'  , value: <%= raw @agent.hide                        %>, width: 400 },
+      { name: 'assistant_to'    , nice_name: 'Assistant To'       , type: 'select'    , value: <%= raw Caboose.json(@agent.assistant_to)  %>, width: 400, text: <%= raw Caboose.json(@boss ? "#{@boss.first_name} #{@boss.last_name}" : 'empty') %>, options_url: '/admin/agents/assistant-to-options', empty_text: 'Not an assistant' }
+    ]
+  });
+});
+$(window).load(function() {
+  var modal = new CabooseModal(800);
+});
+</script>
+<% end %>

data/app/views/caboose_rets/agents/admin_edit_bio.html.erb

@@ -0,0 +1,31 @@
+<h1>Edit Agent</h1>
+<%= render :partial => 'agents/admin_top_nav' %>
+
+<div id='agent_<%= @agent.id %>_bio'></div>
+
+<p style='clear: both;'><input type='button' value='< Back' onclick="window.location='/admin/agents';" /></p>
+
+<% content_for :caboose_js do %>
+<%= javascript_include_tag "caboose/model/all" %>
+<script type='text/javascript'>
+$(document).ready(function() {
+  m = new ModelBinder({
+    name: 'Agent',
+    id: <%= @agent.id %>,
+    update_url: '/admin/agents/<%= @agent.id %>',
+    authenticity_token: '<%= form_authenticity_token %>',
+    attributes: [{ 
+      name: 'bio',
+      nice_name: 'Bio',
+      type: 'textarea',
+      value: <%= raw Caboose.json(@agent.bio) %>,
+      width: 600, 
+      height: 250 
+    }]
+  });
+});
+$(window).load(function() {
+  var modal = new CabooseModal(800);
+});
+</script>
+<% end %>

data/app/views/caboose_rets/agents/admin_edit_contact_info.html.erb

@@ -0,0 +1,31 @@
+<h1>Edit Agent</h1>
+<%= render :partial => 'agents/admin_top_nav' %>
+
+<div id='agent_<%= @agent.id %>_contact_info' ></div>
+
+<p style='clear: both;'><input type='button' value='< Back' onclick="window.location='/admin/agents';" /></p>
+
+<% content_for :caboose_js do %>
+<%= javascript_include_tag "caboose/model/all" %>
+<script type='text/javascript'>
+$(document).ready(function() {
+  m = new ModelBinder({
+    name: 'Agent',
+    id: <%= @agent.id %>,
+    update_url: '/admin/agents/<%= @agent.id %>',
+    authenticity_token: '<%= form_authenticity_token %>',
+    attributes: [{ 
+      name: 'contact_info',
+      nice_name: 'Contact Info',
+      type: 'textarea',
+      value: <%= raw Caboose.json(@agent.contact_info) %>,
+      width: 750, 
+      height: 250 
+    }]
+  });
+});
+$(window).load(function() {
+  var modal = new CabooseModal(800);
+});
+</script>
+<% end %>

data/app/views/caboose_rets/agents/admin_edit_mls_info.html.erb

@@ -0,0 +1,71 @@
+<h1>Edit Agent</h1>
+<%= render :partial => 'agents/admin_top_nav' %>
+
+<p>
+<input type='button' value='< Back' onclick="window.location='/admin/agents';" />
+<input type='button' value='Refresh from MLS' onclick="refresh_agent_from_mls(<%= @agent.id %>);" />
+</p>
+<div id='message'><%= raw flash[:message] ? flash[:message] : "" %></div>
+
+<table class='data'>
+<tr><td>beeper_phone        </td><td><%= @agent.beeper_phone        %></td></tr> 
+<tr><td>car_phone           </td><td><%= @agent.car_phone           %></td></tr> 
+<tr><td>date_created        </td><td><%= @agent.date_created        %></td></tr> 
+<tr><td>date_modified       </td><td><%= @agent.date_modified       %></td></tr> 
+<tr><td>defaultemail        </td><td><%= @agent.defaultemail        %></td></tr> 
+<tr><td>fax_phone           </td><td><%= @agent.fax_phone           %></td></tr> 
+<tr><td>first_name          </td><td><%= @agent.first_name          %></td></tr> 
+<tr><td>home_phone          </td><td><%= @agent.home_phone          %></td></tr> 
+<tr><td>la_code             </td><td><%= @agent.la_code             %></td></tr> 
+<tr><td>last_name           </td><td><%= @agent.last_name           %></td></tr> 
+<tr><td>lo_code             </td><td><%= @agent.lo_code             %></td></tr> 
+<tr><td>mail_addr1          </td><td><%= @agent.mail_addr1          %></td></tr> 
+<tr><td>mail_addr2          </td><td><%= @agent.mail_addr2          %></td></tr> 
+<tr><td>mail_city           </td><td><%= @agent.mail_city           %></td></tr> 
+<tr><td>mail_state          </td><td><%= @agent.mail_state          %></td></tr> 
+<tr><td>mail_zip            </td><td><%= @agent.mail_zip            %></td></tr> 
+<tr><td>member_email        </td><td><%= @agent.member_email        %></td></tr> 
+<tr><td>member_page         </td><td><%= @agent.member_page         %></td></tr> 
+<tr><td>member_status       </td><td><%= @agent.member_status       %></td></tr> 
+<tr><td>nrds_id             </td><td><%= @agent.nrds_id             %></td></tr> 
+<tr><td>office_phone        </td><td><%= @agent.office_phone        %></td></tr> 
+<tr><td>other_phone         </td><td><%= @agent.other_phone         %></td></tr> 
+<tr><td>phone_change_date   </td><td><%= @agent.phone_change_date   %></td></tr> 
+<tr><td>phone_direct_office </td><td><%= @agent.phone_direct_office %></td></tr> 
+<tr><td>phone_home_fax      </td><td><%= @agent.phone_home_fax      %></td></tr> 
+<tr><td>phone_pager         </td><td><%= @agent.phone_pager         %></td></tr> 
+<tr><td>phone_second_home   </td><td><%= @agent.phone_second_home   %></td></tr> 
+<tr><td>phone_toll_free     </td><td><%= @agent.phone_toll_free     %></td></tr> 
+<tr><td>phone_voice_mail    </td><td><%= @agent.phone_voice_mail    %></td></tr> 
+<tr><td>phone_voice_pager   </td><td><%= @agent.phone_voice_pager   %></td></tr> 
+<tr><td>photo_count         </td><td><%= @agent.photo_count         %></td></tr> 
+<tr><td>photo_date_modified </td><td><%= @agent.photo_date_modified %></td></tr>  
+</table>
+
+<p><input type='button' value='< Back' onclick="window.location='/admin/agents';" /></p>
+
+<% content_for :caboose_js do %>
+<%= javascript_include_tag "caboose/model/all" %>
+<script type='text/javascript'>
+
+function refresh_agent_from_mls(agent_id)
+{
+  //modal.autosize("<p class='loading'>Refreshing agent info from MLS...</p>");
+  $.ajax({
+    url: '/admin/agents/' + agent_id + '/refresh',
+    success: function(resp) {
+      if (resp.error)
+        modal.autosize("<p class='note error'>" + resp.error + "</p>");
+      if (resp.reload)
+        location.reload();    
+    }
+  });
+}
+
+var modal = false;
+$(window).load(function() {
+  modal = new CabooseModal(800);
+});
+
+</script>
+<% end %>