caboose-rets 0.0.62 → 0.0.63

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (4) hide show
  1. checksums.yaml +8 -8
  2. data/app/models/caboose_rets/rets_importer.rb +217 -171
  3. data/lib/caboose_rets/version.rb +1 -1
  4. metadata +1 -1
checksums.yaml CHANGED
@@ -1,15 +1,15 @@
1
1
  ---
2
2
  !binary "U0hBMQ==":
3
3
  metadata.gz: !binary |-
4
- NWU0NDk2NmI3ZjJhNjA0M2RlYzdjMTZmNmMyZjkzN2Q2OTVlNzdlMA==
4
+ MDA2YWIxNjM3YWYyYmYwYjNjNGNkZDQ1YThmMGI3MDU4ZDFjYzYzZQ==
5
5
  data.tar.gz: !binary |-
6
- MDhjOWMwODBiNDg5OTUwOWUyNTg1NGY5ZmQ5NWVkNGQzMTk0ZmQxZQ==
6
+ NmMyNTFkYWE1NjBlYWZkNzNlNDk2ZDg0MGQyOGYxYmEwYmVlYTIzYg==
7
7
  SHA512:
8
8
  metadata.gz: !binary |-
9
- MmNjYzBiZWIxYzM3ZjlhMzBjNDAyYmZkOTA2NzA1MThhMDkxZWJkMjNlMjdi
10
- ZWYzNjM4ZjU4MjJmZDRlMTNmZjljODFmYWY4NjAxOTE4M2Y3OGVhMWQ4Nzg5
11
- MGRiY2Y0OTMyYjA0ZjI4OTdiY2U5NGQ3Yzk2NTJhMGVkODQ4NGY=
9
+ ODhkYWYyOWJhZmQyNzczNzQ1N2UzZTAwZTVkZGJmNDY4MjlmZGM3ODY0ZjI3
10
+ ZGMwNTkxODk5NWNiZWM5MWE2Y2VhYTc1OTBhYmUyN2Q1Y2IzYjVlYjNmYmI5
11
+ MWY5YTY0YmJlNWFjOTMwMjc3ZWY0MjIyYWE3MWViMGQxYTRjOGQ=
12
12
  data.tar.gz: !binary |-
13
- OTkxYzMyMmUxODZhNWYxZDk4NGM3ZDM0M2E2MDM5MWQ3NGRiNmE5NmY3NjJj
14
- ODQxMjRhMzllNmJiMmQzOWI2OWI0ZTk4NTMwYTU2ZTc0YWUxMzI4OTVkZjgz
15
- NDJhMjA0ZmQxY2NkZDRmYjQ1YzllNjJkOWMyMzY0YWRkNmM5MjM=
13
+ NmE1YTVkMWI2OTU2Nzg2MDhiMTM0ZDgyYTIwOTNmM2RlZDM0MmI1MTZmMDc4
14
+ ZTc1YWVmNDhiYWZkNmViNmI0MzQ1YzA5NmY0NWVjNzVkZjkyNmJkMTYxNWEy
15
+ NWQ0OTVmZGUxY2NlNGI4MzBkZWIwNGE2MDYzZGM2MjI0MmJlN2E=
data/app/models/caboose_rets/rets_importer.rb CHANGED
@@ -14,14 +14,14 @@ require 'json'
14
14
  # http://rets.solidearth.com/ClientHome.aspx
15
15
 
16
16
  class CabooseRets::RetsImporter # < ActiveRecord::Base
17
-
17
+
18
18
  @@rets_client = nil
19
19
  @@config = nil
20
-
20
+
21
21
  def self.config
22
22
  return @@config
23
23
  end
24
-
24
+
25
25
  def self.get_config
26
26
  @@config = {
27
27
  'url' => nil, # URL to the RETS login
@@ -35,11 +35,11 @@ class CabooseRets::RetsImporter # < ActiveRecord::Base
35
35
  config = config[Rails.env]
36
36
  config.each { |key,val| @@config[key] = val }
37
37
  end
38
-
38
+
39
39
  def self.client
40
40
  self.get_config if @@config.nil? || @@config['url'].nil?
41
-
42
- if (@@rets_client.nil?)
41
+
42
+ if @@rets_client.nil?
43
43
  @@rets_client = RETS::Client.login(
44
44
  :url => @@config['url'],
45
45
  :username => @@config['username'],
@@ -48,35 +48,49 @@ class CabooseRets::RetsImporter # < ActiveRecord::Base
48
48
  end
49
49
  return @@rets_client
50
50
  end
51
-
51
+
52
+ def self.meta(class_type)
53
+ case class_type
54
+ when 'RES' then Caboose::StdClass.new({ :search_type => 'Property' , :remote_key_field => 'MLS_ACCT' , :local_key_field => 'mls_acct' , :local_table => 'rets_residential' , :date_modified_field => 'DATE_MODIFIED' })
55
+ when 'COM' then Caboose::StdClass.new({ :search_type => 'Property' , :remote_key_field => 'MLS_ACCT' , :local_key_field => 'mls_acct' , :local_table => 'rets_commercial' , :date_modified_field => 'DATE_MODIFIED' })
56
+ when 'LND' then Caboose::StdClass.new({ :search_type => 'Property' , :remote_key_field => 'MLS_ACCT' , :local_key_field => 'mls_acct' , :local_table => 'rets_land' , :date_modified_field => 'DATE_MODIFIED' })
57
+ when 'MUL' then Caboose::StdClass.new({ :search_type => 'Property' , :remote_key_field => 'MLS_ACCT' , :local_key_field => 'mls_acct' , :local_table => 'rets_multi_family' , :date_modified_field => 'DATE_MODIFIED' })
58
+ when 'OFF' then Caboose::StdClass.new({ :search_type => 'Office' , :remote_key_field => 'LO_LO_CODE' , :local_key_field => 'lo_code' , :local_table => 'rets_offices' , :date_modified_field => 'LO_DATE_MODIFIED' })
59
+ when 'AGT' then Caboose::StdClass.new({ :search_type => 'Agent' , :remote_key_field => 'LA_LA_CODE' , :local_key_field => 'la_code' , :local_table => 'rets_agents' , :date_modified_field => 'LA_DATE_MODIFIED' })
60
+ when 'OPH' then Caboose::StdClass.new({ :search_type => 'OpenHouse' , :remote_key_field => 'ID' , :local_key_field => 'id' , :local_table => 'rets_open_houses' , :date_modified_field => 'DATE_MODIFIED' })
61
+ when 'GFX' then Caboose::StdClass.new({ :search_type => 'Media' , :remote_key_field => 'MEDIA_ID' , :local_key_field => 'media_id' , :local_table => 'rets_media' , :date_modified_field => 'DATE_MODIFIED' })
62
+ end
63
+ end
64
+
52
65
  #=============================================================================
53
66
  # Import method
54
67
  #=============================================================================
55
-
56
- def self.import(query, search_type, class_type)
57
- self.log("Importing #{search_type}:#{class_type} with query #{query}...")
68
+
69
+ def self.import(class_type, query)
70
+ m = self.meta(class_type)
71
+ self.log("Importing #{m.search_type}:#{class_type} with query #{query}...")
58
72
  self.get_config if @@config.nil? || @@config['url'].nil?
59
73
  params = {
60
- :search_type => search_type,
74
+ :search_type => m.search_type,
61
75
  :class => class_type,
62
76
  :query => query,
63
- :limit => -1,
77
+ :limit => -1,
64
78
  :timeout => -1
65
79
  }
66
80
  obj = nil
67
81
  self.client.search(params) do |data|
68
82
  obj = self.get_instance_with_id(class_type, data)
69
83
  if obj.nil?
70
- self.log("Error: object is nil")
84
+ self.log("Error: object is nil")
71
85
  self.log(data.inspect)
72
86
  next
73
87
  end
74
- obj.parse(data)
75
- obj.save
76
- end
88
+ obj.parse(data)
89
+ obj.save
90
+ end
77
91
  end
78
-
79
- def self.get_instance_with_id(class_type, data)
92
+
93
+ def self.get_instance_with_id(class_type, data)
80
94
  obj = nil
81
95
  m = case class_type
82
96
  when 'OPH' then CabooseRets::OpenHouse
@@ -87,183 +101,182 @@ class CabooseRets::RetsImporter # < ActiveRecord::Base
87
101
  when 'RES' then CabooseRets::ResidentialProperty
88
102
  when 'AGT' then CabooseRets::Agent
89
103
  when 'OFF' then CabooseRets::Office
90
- end
104
+ end
91
105
  obj = case class_type
92
106
  when 'OPH' then m.where(:id => data['ID'].to_i ).exists? ? m.where(:id => data['ID'].to_i ).first : m.new(:id => data['ID'].to_i )
93
107
  when 'GFX' then m.where(:media_id => data['MEDIA_ID'] ).exists? ? m.where(:media_id => data['MEDIA_ID'] ).first : m.new(:media_id => data['MEDIA_ID'] )
94
108
  when 'COM' then m.where(:id => data['MLS_ACCT'].to_i ).exists? ? m.where(:id => data['MLS_ACCT'].to_i ).first : m.new(:id => data['MLS_ACCT'].to_i )
95
- when 'LND' then m.where(:id => data['MLS_ACCT'].to_i ).exists? ? m.where(:id => data['MLS_ACCT'].to_i ).first : m.new(:id => data['MLS_ACCT'].to_i )
96
- when 'MUL' then m.where(:id => data['MLS_ACCT'].to_i ).exists? ? m.where(:id => data['MLS_ACCT'].to_i ).first : m.new(:id => data['MLS_ACCT'].to_i )
97
- when 'RES' then m.where(:id => data['MLS_ACCT'].to_i ).exists? ? m.where(:id => data['MLS_ACCT'].to_i ).first : m.new(:id => data['MLS_ACCT'].to_i )
109
+ when 'LND' then m.where(:id => data['MLS_ACCT'].to_i ).exists? ? m.where(:id => data['MLS_ACCT'].to_i ).first : m.new(:id => data['MLS_ACCT'].to_i )
110
+ when 'MUL' then m.where(:id => data['MLS_ACCT'].to_i ).exists? ? m.where(:id => data['MLS_ACCT'].to_i ).first : m.new(:id => data['MLS_ACCT'].to_i )
111
+ when 'RES' then m.where(:id => data['MLS_ACCT'].to_i ).exists? ? m.where(:id => data['MLS_ACCT'].to_i ).first : m.new(:id => data['MLS_ACCT'].to_i )
98
112
  when 'AGT' then m.where(:la_code => data['LA_LA_CODE'] ).exists? ? m.where(:la_code => data['LA_LA_CODE'] ).first : m.new(:la_code => data['LA_LA_CODE'] )
99
113
  when 'OFF' then m.where(:lo_code => data['LO_LO_CODE'] ).exists? ? m.where(:lo_code => data['LO_LO_CODE'] ).first : m.new(:lo_code => data['LO_LO_CODE'] )
100
114
  end
101
- return obj
102
- end
103
-
104
- #def self.delete_old_properties
105
- # self.log("Deleting old residential properties...")
106
- # self.get_config if @@config.nil? || @@config['url'].nil?
107
- #
108
- # d = DateTime.new(2000, 1, 1)
109
- # now = DateTime.now
110
- # while d < now
111
- # d2 = d + 3.months
112
- # puts "- Getting ids for #{d.strftime("%Y-%m-%d")} - #{d2.strftime("%Y-%m-%d")}..."
113
- # q = "((DATE_CREATED=#{d.strftime("%Y-%m-%d")}+),(DATE_CREATED=#{d2.strftime("%Y-%m-%d")}-))",
114
- # params = { :search_type => 'Property', :class => 'RES', :query => q, :limit => -1, :timeout => -1, :select => 'MLS_ACCT' }
115
- # ids = []
116
- # self.client.search(params) do |data|
117
- # ids << data['MLS_ACCT'].to_i
118
- # end
119
- # d = d + 3.months
120
- # end
121
- #
122
- # puts "Deleting properties..."
123
- # #CabooseRets::ResidentialProperty.where("id not in (?)", ids).destroy_all
124
- #end
125
-
115
+ return obj
116
+ end
117
+
126
118
  #=============================================================================
127
119
  # Main updater
128
120
  #=============================================================================
129
121
 
130
122
  def self.update_after(date_modified, save_images = true)
131
- self.update_residential_properties_modified_after(date_modified, save_images)
132
- self.update_commercial_properties_modified_after(date_modified, save_images)
133
- self.update_land_properties_modified_after(date_modified, save_images)
134
- self.update_multi_family_properties_modified_after(date_modified, save_images)
135
- self.update_offices_modified_after(date_modified, save_images)
136
- self.update_agents_modified_after(date_modified, save_images)
137
- self.update_open_houses_modified_after(date_modified, save_images)
138
- end
139
- def self.update_residential_properties_modified_after(date_modified, save_images = true) self.client.search({ :search_type => 'Property' , :class => 'RES', :select => ['MLS_ACCT'] , :query => "(DATE_MODIFIED=#{date_modified.strftime("%FT%T")}+)" , :standard_names_only => true, :timeout => -1 }) { |data| self.delay(:priority => 10).import_residential_property( data['MLS_ACCT' ], save_images) } end
140
- def self.update_commercial_properties_modified_after(date_modified, save_images = true) self.client.search({ :search_type => 'Property' , :class => 'COM', :select => ['MLS_ACCT'] , :query => "(DATE_MODIFIED=#{date_modified.strftime("%FT%T")}+)" , :standard_names_only => true, :timeout => -1 }) { |data| self.delay(:priority => 10).import_commercial_property( data['MLS_ACCT' ], save_images) } end
141
- def self.update_land_properties_modified_after(date_modified, save_images = true) self.client.search({ :search_type => 'Property' , :class => 'LND', :select => ['MLS_ACCT'] , :query => "(DATE_MODIFIED=#{date_modified.strftime("%FT%T")}+)" , :standard_names_only => true, :timeout => -1 }) { |data| self.delay(:priority => 10).import_land_property( data['MLS_ACCT' ], save_images) } end
142
- def self.update_multi_family_properties_modified_after(date_modified, save_images = true) self.client.search({ :search_type => 'Property' , :class => 'MUL', :select => ['MLS_ACCT'] , :query => "(DATE_MODIFIED=#{date_modified.strftime("%FT%T")}+)" , :standard_names_only => true, :timeout => -1 }) { |data| self.delay(:priority => 10).import_multi_family_property( data['MLS_ACCT' ], save_images) } end
143
- def self.update_offices_modified_after(date_modified, save_images = true) self.client.search({ :search_type => 'Office' , :class => 'OFF', :select => ['LO_LO_CODE'] , :query => "(LO_DATE_MODIFIED=#{date_modified.strftime("%FT%T")}+)", :standard_names_only => true, :timeout => -1 }) { |data| self.delay(:priority => 10).import_office( data['LO_LO_CODE'], save_images) } end
144
- def self.update_agents_modified_after(date_modified, save_images = true) self.client.search({ :search_type => 'Agent' , :class => 'AGT', :select => ['LA_LA_CODE'] , :query => "(LA_DATE_MODIFIED=#{date_modified.strftime("%FT%T")}+)", :standard_names_only => true, :timeout => -1 }) { |data| self.delay(:priority => 10).import_agent( data['LA_LA_CODE'], save_images) } end
145
- def self.update_open_houses_modified_after(date_modified, save_images = true) self.client.search({ :search_type => 'OpenHouse', :class => 'OPH', :select => ['ID'] , :query => "(DATE_MODIFIED=#{date_modified.strftime("%FT%T")}+)" , :standard_names_only => true, :timeout => -1 }) { |data| self.delay(:priority => 10).import_open_house( data['ID' ], save_images) } end
146
-
123
+ self.update_helper('RES', date_modified, save_images)
124
+ self.update_helper('COM', date_modified, save_images)
125
+ self.update_helper('LND', date_modified, save_images)
126
+ self.update_helper('MUL', date_modified, save_images)
127
+ self.update_helper('OFF', date_modified, save_images)
128
+ self.update_helper('AGT', date_modified, save_images)
129
+ self.update_helper('OPH', date_modified, save_images)
130
+ end
131
+
132
+ def self.update_helper(class_type, date_modified, save_images = true)
133
+ m = self.meta(class_type)
134
+ k = m.remote_key_field
135
+ params = {
136
+ :search_type => m.search_type,
137
+ :class => class_type,
138
+ :select => [m.remote_key_field],
139
+ :query => "(#{m.date_modified_field}=#{date_modified.strftime("%FT%T")}+)",
140
+ :standard_names_only => true,
141
+ :timeout => -1
142
+ }
143
+ self.client.search(params) do |data|
144
+ case class_type
145
+ when 'RES' then self.delay(:priority => 10).import_residential_property( data[k], save_images)
146
+ when 'COM' then self.delay(:priority => 10).import_commercial_property( data[k], save_images)
147
+ when 'LND' then self.delay(:priority => 10).import_land_property( data[k], save_images)
148
+ when 'MUL' then self.delay(:priority => 10).import_multi_family_property( data[k], save_images)
149
+ when 'OFF' then self.delay(:priority => 10).import_office( data[k], save_images)
150
+ when 'AGT' then self.delay(:priority => 10).import_agent( data[k], save_images)
151
+ when 'OPH' then self.delay(:priority => 10).import_open_house( data[k], save_images)
152
+ end
153
+ end
154
+ end
155
+
147
156
  #=============================================================================
148
157
  # Single model import methods (called from a worker dyno)
149
158
  #=============================================================================
150
-
151
- def self.import_property(mls_acct)
152
- self.import("(MLS_ACCT=*#{mls_acct}*)", 'Property', 'RES')
159
+
160
+ def self.import_property(mls_acct, save_images = true)
161
+ self.import('RES', "(MLS_ACCT=*#{mls_acct}*)")
153
162
  p = CabooseRets::ResidentialProperty.where(:id => mls_acct.to_i).first
154
163
  if p.nil?
155
- self.import("(MLS_ACCT=*#{mls_acct}*)", 'Property', 'COM')
156
- p = CabooseRets::CommercialProperty.where(:id => mls_acct.to_i).first
164
+ self.import('COM', "(MLS_ACCT=*#{mls_acct}*)")
165
+ p = CabooseRets::CommercialProperty.where(:id => mls_acct.to_i).first
157
166
  if p.nil?
158
- self.import("(MLS_ACCT=*#{mls_acct}*)", 'Property', 'LND')
167
+ self.import('LND', "(MLS_ACCT=*#{mls_acct}*)")
159
168
  p = CabooseRets::LandProperty.where(:id => mls_acct.to_i).first
160
169
  if p.nil?
161
- self.import("(MLS_ACCT=*#{mls_acct}*)", 'Property', 'MUL')
170
+ self.import('MUL', "(MLS_ACCT=*#{mls_acct}*)")
162
171
  p = CabooseRets::MultiFamilyProperty.where(:id => mls_acct.to_i).first
163
172
  return if p.nil?
164
173
  end
165
174
  end
166
175
  end
167
- self.download_property_images(p)
168
- end
169
-
170
- def self.import_residential_property(mls_acct, save_images = true)
171
- self.import("(MLS_ACCT=*#{mls_acct}*)", 'Property', 'RES')
172
- p = CabooseRets::ResidentialProperty.where(:id => mls_acct.to_i).first
173
- self.download_property_images(p, save_images)
174
- self.update_coords(p)
175
- end
176
-
177
- def self.import_commercial_property(mls_acct, save_images = true)
178
- self.import("(MLS_ACCT=*#{mls_acct}*)", 'Property', 'COM')
176
+ self.download_property_images(p, save_images)
177
+ end
178
+
179
+ def self.import_residential_property(mls_acct, save_images = true)
180
+ self.import('RES', "(MLS_ACCT=*#{mls_acct}*)")
181
+ p = CabooseRets::ResidentialProperty.where(:id => mls_acct.to_i).first
182
+ self.download_property_images(p, save_images)
183
+ self.update_coords(p)
184
+ end
185
+
186
+ def self.import_commercial_property(mls_acct, save_images = true)
187
+ self.import('COM', "(MLS_ACCT=*#{mls_acct}*)")
179
188
  p = CabooseRets::CommercialProperty.where(:id => mls_acct.to_i).first
180
189
  self.download_property_images(p, save_images)
181
190
  self.update_coords(p)
182
191
  end
183
-
184
- def self.import_land_property(mls_acct, save_images = true)
185
- self.import("(MLS_ACCT=*#{mls_acct}*)", 'Property', 'LND')
186
- p = CabooseRets::LandProperty.where(:id => mls_acct.to_i).first
192
+
193
+ def self.import_land_property(mls_acct, save_images = true)
194
+ self.import('LND', "(MLS_ACCT=*#{mls_acct}*)")
195
+ p = CabooseRets::LandProperty.where(:id => mls_acct.to_i).first
187
196
  self.download_property_images(p, save_images)
188
197
  self.update_coords(p)
189
198
  end
190
-
191
- def self.import_multi_family_property(mls_acct, save_images = true)
192
- self.import("(MLS_ACCT=*#{mls_acct}*)", 'Property', 'MUL')
199
+
200
+ def self.import_multi_family_property(mls_acct, save_images = true)
201
+ self.import('MUL', "(MLS_ACCT=*#{mls_acct}*)")
193
202
  p = CabooseRets::MultiFamilyProperty.where(:id => mls_acct.to_i).first
194
203
  self.download_property_images(p, save_images)
195
204
  self.update_coords(p)
196
205
  end
197
-
206
+
198
207
  def self.import_office(lo_code, save_images = true)
199
- self.import("(LO_LO_CODE=*#{lo_code}*)", 'Office', 'OFF')
208
+ self.import('OFF', "(LO_LO_CODE=*#{lo_code}*)")
200
209
  office = CabooseRets::Office.where(:lo_code => lo_code.to_s).first
201
210
  self.download_office_image(office) if save_images == true
202
211
  end
203
-
212
+
204
213
  def self.import_agent(la_code, save_images = true)
205
- self.import("(LA_LA_CODE=*#{la_code}*)", 'Agent', 'AGT')
214
+ self.import('AGT', "(LA_LA_CODE=*#{la_code}*)")
206
215
  a = CabooseRets::Agent.where(:la_code => la_code.to_s).first
207
216
  self.download_agent_image(a) if save_images == true
208
217
  end
209
-
218
+
210
219
  def self.import_open_house(id, save_images = true)
211
- self.import("(ID=*#{id}*)", 'OpenHouse', 'OPH')
220
+ self.import('OPH', "(ID=*#{id}*)")
221
+ end
222
+
223
+ def self.import_media(id, save_images = true)
224
+ self.import('GFX', "(MEDIA_ID=*#{id}*)")
212
225
  end
213
226
 
214
227
  #=============================================================================
215
228
  # Images
216
229
  #=============================================================================
217
-
218
- def self.download_property_images(p, save_images = true)
230
+
231
+ def self.download_property_images(p, save_images = true)
219
232
  self.refresh_property_media(p)
220
233
  return if save_images == false
221
-
234
+
222
235
  self.log("-- Downloading images and resizing for #{p.mls_acct}")
223
236
  media = []
224
237
  self.client.get_object(:resource => :Property, :type => :Photo, :location => true, :id => p.id) do |headers, content|
225
-
238
+
226
239
  # Find the associated media record for the image
227
240
  filename = File.basename(headers['location'])
228
241
  m = CabooseRets::Media.where(:mls_acct => p.mls_acct, :file_name => filename).first
229
-
242
+
230
243
  if m.nil?
231
244
  self.log("Can't find media record for #{p.mls_acct} #{filename}.")
232
- else
245
+ else
233
246
  m.image = URI.parse(headers['location'])
234
247
  media << m
235
248
  #m.save
236
- end
249
+ end
237
250
  end
238
-
251
+
239
252
  self.log("-- Uploading images to S3 for #{p.mls_acct}")
240
- media.each do |m|
253
+ media.each do |m|
241
254
  m.save
242
- end
255
+ end
243
256
  end
244
-
257
+
245
258
  def self.refresh_property_media(p)
246
- self.log("-- Deleting images and metadata for #{p.mls_acct}...")
259
+ self.log("-- Deleting images and metadata for #{p.mls_acct}...")
247
260
  #CabooseRets::Media.where(:mls_acct => p.mls_acct, :media_type => 'Photo').destroy_all
248
261
  CabooseRets::Media.where(:mls_acct => p.mls_acct).destroy_all
249
-
250
- self.log("-- Downloading image metadata for #{p.mls_acct}...")
262
+
263
+ self.log("-- Downloading image metadata for #{p.mls_acct}...")
251
264
  params = {
252
265
  :search_type => 'Media',
253
266
  :class => 'GFX',
254
267
  #:query => "(MLS_ACCT=*#{p.id}*),(MEDIA_TYPE=|I)",
255
268
  :query => "(MLS_ACCT=*#{p.id}*)",
256
269
  :timeout => -1
257
- }
258
- self.client.search(params) do |data|
270
+ }
271
+ self.client.search(params) do |data|
259
272
  m = CabooseRets::Media.new
260
273
  m.parse(data)
261
274
  #m.id = m.media_id
262
275
  m.save
263
276
  end
264
277
  end
265
-
266
- def self.download_agent_image(agent)
278
+
279
+ def self.download_agent_image(agent)
267
280
  self.log "Saving image for #{agent.first_name} #{agent.last_name}..."
268
281
  begin
269
282
  self.client.get_object(:resource => :Agent, :type => :Photo, :location => true, :id => agent.la_code) do |headers, content|
@@ -272,9 +285,9 @@ class CabooseRets::RetsImporter # < ActiveRecord::Base
272
285
  end
273
286
  rescue RETS::APIError => err
274
287
  self.log "No image for #{agent.first_name} #{agent.last_name}."
275
- end
288
+ end
276
289
  end
277
-
290
+
278
291
  def self.download_office_image(office)
279
292
  self.log "Saving image for #{office.lo_name}..."
280
293
  begin
@@ -282,58 +295,58 @@ class CabooseRets::RetsImporter # < ActiveRecord::Base
282
295
  office.image = URI.parse(headers['location'])
283
296
  office.save
284
297
  end
285
- rescue RETS::APIError => err
298
+ rescue RETS::APIError => err
286
299
  self.log "No image for #{office.lo_name}."
287
- end
300
+ end
288
301
  end
289
302
 
290
303
  #=============================================================================
291
304
  # GPS
292
305
  #=============================================================================
293
-
294
- def self.update_coords(p = nil)
306
+
307
+ def self.update_coords(p = nil)
295
308
  if p.nil?
296
309
  models = [CabooseRets::CommercialProperty, CabooseRets::LandProperty, CabooseRets::MultiFamilyProperty, CabooseRets::ResidentialProperty]
297
310
  names = ["commercial", "land", "multi-family", "residential"]
298
311
  i = 0
299
- models.each do |model|
312
+ models.each do |model|
300
313
  self.log "Updating coords #{names[i]} properties..."
301
314
  model.where(:latitude => nil).reorder(:mls_acct).each do |p|
302
- self.update_coords(p)
315
+ self.update_coords(p)
303
316
  end
304
317
  i = i + 1
305
318
  end
306
319
  return
307
320
  end
308
-
321
+
309
322
  self.log "Getting coords for mls_acct #{p.mls_acct}..."
310
323
  coords = self.coords_from_address(CGI::escape "#{p.street_num} #{p.street_name}, #{p.city}, #{p.state} #{p.zip}")
311
324
  return if coords.nil? || coords == false
312
-
325
+
313
326
  p.latitude = coords['lat']
314
327
  p.longitude = coords['lng']
315
- p.save
328
+ p.save
316
329
  end
317
-
318
- def self.coords_from_address(address)
330
+
331
+ def self.coords_from_address(address)
319
332
  begin
320
333
  uri = "https://maps.googleapis.com/maps/api/geocode/json?address=#{address}&sensor=false"
321
- uri.gsub!(" ", "+")
334
+ uri.gsub!(" ", "+")
322
335
  resp = HTTParty.get(uri)
323
336
  json = JSON.parse(resp.body)
324
- return json['results'][0]['geometry']['location']
337
+ return json['results'][0]['geometry']['location']
325
338
  rescue
326
339
  self.log "Error: #{uri}"
327
340
  sleep(2)
328
- return false
341
+ return false
329
342
  end
330
343
  end
331
-
344
+
332
345
  #=============================================================================
333
346
  # Purging
334
347
  #=============================================================================
335
-
336
- def self.purge
348
+
349
+ def self.purge
337
350
  self.purge_residential
338
351
  self.purge_commercial
339
352
  self.purge_land
@@ -343,56 +356,89 @@ class CabooseRets::RetsImporter # < ActiveRecord::Base
343
356
  self.purge_open_houses
344
357
  self.purge_media
345
358
  end
346
-
347
- def self.purge_residential() self.purge_helper('Property' , 'RES', 'MLS_ACCT' , 'DATE_MODIFIED' , '2012-01-01', "delete from rets_residential where mls_acct not in (?)") end
348
- def self.purge_commercial() self.purge_helper('Property' , 'COM', 'MLS_ACCT' , 'DATE_MODIFIED' , '2012-01-01', "delete from rets_commercial where mls_acct not in (?)") end
349
- def self.purge_land() self.purge_helper('Property' , 'LND', 'MLS_ACCT' , 'DATE_MODIFIED' , '2012-01-01', "delete from rets_land where mls_acct not in (?)") end
350
- def self.purge_multi_family() self.purge_helper('Property' , 'MUL', 'MLS_ACCT' , 'DATE_MODIFIED' , '2012-01-01', "delete from rets_multi_family where mls_acct not in (?)") end
351
- def self.purge_offices() self.purge_helper('Office' , 'OFF', 'LO_LO_CODE' , 'LO_DATE_MODIFIED' , '2012-01-01', "delete from rets_offices where lo_code not in (?)") end
352
- def self.purge_agents() self.purge_helper('Agent' , 'AGT', 'LA_LA_CODE' , 'LA_DATE_MODIFIED' , '2012-01-01', "delete from rets_agents where la_code not in (?)") end
353
- def self.purge_open_houses() self.purge_helper('OpenHouse' , 'OPH', 'ID' , 'DATE_MODIFIED' , '2012-01-01', "delete from rets_open_houses where id not in (?)") end
354
- def self.purge_media() self.purge_helper('Media' , 'GFX', 'MEDIA_ID' , 'DATE_MODIFIED' , '2012-01-01', "delete from rets_media where media_id not in (?)") end
355
-
356
- def self.purge_helper(search_type, class_type, key_field, date_modified_field, date_modified, delete_query)
357
-
359
+
360
+ def self.purge_residential() self.purge_helper('RES', '2012-01-01') end
361
+ def self.purge_commercial() self.purge_helper('COM', '2012-01-01') end
362
+ def self.purge_land() self.purge_helper('LND', '2012-01-01') end
363
+ def self.purge_multi_family() self.purge_helper('MUL', '2012-01-01') end
364
+ def self.purge_offices() self.purge_helper('OFF', '2012-01-01') end
365
+ def self.purge_agents() self.purge_helper('AGT', '2012-01-01') end
366
+ def self.purge_open_houses() self.purge_helper('OPH', '2012-01-01') end
367
+ def self.purge_media() self.purge_helper('GFX', '2012-01-01') end
368
+
369
+ def self.purge_helper(class_type, date_modified)
370
+ m = self.meta(class_type)
371
+
358
372
  # Get the total number of records
359
- params = { :search_type => search_type, :class => class_type, :query => "(#{date_modified_field}=#{date_modified}T00:00:01+)", :standard_names_only => true, :timeout => -1 }
373
+ params = {
374
+ :search_type => m.search_type,
375
+ :class => class_type,
376
+ :query => "(#{m.date_modified_field}=#{date_modified}T00:00:01+)",
377
+ :standard_names_only => true,
378
+ :timeout => -1
379
+ }
360
380
  self.client.search(params.merge({ :count_mode => :only }))
361
- count = self.client.rets_data[:code] == "20201" ? 0 : self.client.rets_data[:count]
381
+ count = self.client.rets_data[:code] == "20201" ? 0 : self.client.rets_data[:count]
362
382
  batch_count = (count.to_f/5000.0).ceil
363
383
 
364
384
  ids = []
365
- (0...batch_count).each do |i|
366
- self.client.search(params.merge({ :select => [key_field], :limit => 5000, :offset => 5000*i })){ |data| ids << data[key_field] }
367
- end
368
- ActiveRecord::Base.connection.execute(ActiveRecord::Base.send(:sanitize_sql_array, [delete_query, ids]))
385
+ k = m.remote_key_field
386
+ (0...batch_count).each do |i|
387
+ self.client.search(params.merge({ :select => [k], :limit => 5000, :offset => 5000*i })){ |data| ids << data[k] }
388
+ end
389
+
390
+ # Delete any records in the local database that shouldn't be there
391
+ t = m.local_table
392
+ k = m.local_key_field
393
+ query = ["delete from #{t} where #{k} not in (?)", ids]
394
+ ActiveRecord::Base.connection.execute(ActiveRecord::Base.send(:sanitize_sql_array, query))
395
+
396
+ # Find any ids in the remote database that should be in the local database
397
+ query = "select distinct #{k} from #{t}"
398
+ rows = ActiveRecord::Base.connection.select_all(ActiveRecord::Base.send(:sanitize_sql_array, query))
399
+ local_ids = rows.collect{ |row| row[k] }
400
+ ids_to_add = ids - local_ids
401
+ ids_to_add.each do |id|
402
+ puts "Importing #{id}..."
403
+ case class_type
404
+ when 'RES' then self.delay.import_residential_property(id)
405
+ when 'COM' then self.delay.import_commercial_property(id)
406
+ when 'LND' then self.delay.import_land_property(id)
407
+ when 'MUL' then self.delay.import_multi_family_property(id)
408
+ when 'OFF' then self.delay.import_office(id)
409
+ when 'AGT' then self.delay.import_agent(id)
410
+ when 'OPH' then self.delay.import_open_house(id)
411
+ when 'GFX' then self.delay.import_media(id)
412
+ end
413
+ end
414
+
369
415
  end
370
416
 
371
417
  #=============================================================================
372
418
  # Logging
373
419
  #=============================================================================
374
-
420
+
375
421
  def self.log(msg)
376
422
  #puts "[rets_importer] #{msg}"
377
423
  Rails.logger.info("[rets_importer] #{msg}")
378
- end
379
-
424
+ end
425
+
380
426
  #=============================================================================
381
427
  # Locking update task
382
428
  #=============================================================================
383
-
384
- def self.update_rets
429
+
430
+ def self.update_rets
385
431
  return if self.task_is_locked
386
432
  task_started = self.lock_task
387
-
433
+
388
434
  begin
389
- overlap = 30.seconds
390
- if (DateTime.now - self.last_purged).to_i > 1
435
+ overlap = 30.seconds
436
+ if (DateTime.now - self.last_purged).to_i > 1
391
437
  self.purge
392
438
  self.save_last_purged(task_started)
393
439
  overlap = 1.month
394
- end
395
- self.update_after(self.last_updated - overlap)
440
+ end
441
+ self.update_after(self.last_updated - overlap)
396
442
  self.save_last_updated(task_started)
397
443
  self.unlock_task
398
444
  rescue
@@ -400,11 +446,11 @@ class CabooseRets::RetsImporter # < ActiveRecord::Base
400
446
  ensure
401
447
  self.unlock_task_if_last_updated(task_started)
402
448
  end
403
-
449
+
404
450
  # Start the same update process in five minutes
405
- self.delay(:run_at => 1.minutes.from_now).update_rets
451
+ self.delay(:run_at => 1.minutes.from_now).update_rets
406
452
  end
407
-
453
+
408
454
  def self.last_updated
409
455
  if !Caboose::Setting.exists?(:name => 'rets_last_updated')
410
456
  Caboose::Setting.create(:name => 'rets_last_updated', :value => '2013-08-06T00:00:01')
@@ -412,7 +458,7 @@ class CabooseRets::RetsImporter # < ActiveRecord::Base
412
458
  s = Caboose::Setting.where(:name => 'rets_last_updated').first
413
459
  return DateTime.parse(s.value)
414
460
  end
415
-
461
+
416
462
  def self.last_purged
417
463
  if !Caboose::Setting.exists?(:name => 'rets_last_purged')
418
464
  Caboose::Setting.create(:name => 'rets_last_purged', :value => '2013-08-06T00:00:01')
@@ -420,36 +466,36 @@ class CabooseRets::RetsImporter # < ActiveRecord::Base
420
466
  s = Caboose::Setting.where(:name => 'rets_last_purged').first
421
467
  return DateTime.parse(s.value)
422
468
  end
423
-
469
+
424
470
  def self.save_last_updated(d)
425
471
  s = Caboose::Setting.where(:name => 'rets_last_updated').first
426
472
  s.value = d.strftime('%FT%T')
427
473
  s.save
428
474
  end
429
-
475
+
430
476
  def self.save_last_purged(d)
431
477
  s = Caboose::Setting.where(:name => 'rets_last_purged').first
432
478
  s.value = d.strftime('%FT%T')
433
479
  s.save
434
480
  end
435
-
481
+
436
482
  def self.task_is_locked
437
483
  return Caboose::Setting.exists?(:name => 'rets_update_running')
438
484
  end
439
-
485
+
440
486
  def self.lock_task
441
487
  d = DateTime.now.utc - 5.hours
442
488
  Caboose::Setting.create(:name => 'rets_update_running', :value => d.strftime('%F %T'))
443
489
  return d
444
490
  end
445
-
491
+
446
492
  def self.unlock_task
447
493
  Caboose::Setting.where(:name => 'rets_update_running').first.destroy
448
494
  end
449
-
495
+
450
496
  def self.unlock_task_if_last_updated(d)
451
497
  setting = Caboose::Setting.where(:name => 'rets_update_running').first
452
498
  self.unlock_task if setting && d.strftime('%F %T') == setting.value
453
499
  end
454
-
500
+
455
501
  end
data/lib/caboose_rets/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  module CabooseRets
2
- VERSION = '0.0.62'
2
+ VERSION = '0.0.63'
3
3
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: caboose-rets
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.62
4
+ version: 0.0.63
5
5
  platform: ruby
6
6
  authors:
7
7
  - William Barry