caboose-rets 0.1.151 → 0.1.152
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/app/models/caboose_rets/agent.rb +1 -1
- data/app/models/caboose_rets/agent_meta.rb +10 -10
- data/app/models/caboose_rets/log.rb +1 -1
- data/app/models/caboose_rets/media.rb +1 -1
- data/app/models/caboose_rets/office.rb +1 -1
- data/app/models/caboose_rets/office_meta.rb +1 -1
- data/app/models/caboose_rets/open_house.rb +1 -1
- data/app/models/caboose_rets/property.rb +1 -1
- data/app/models/caboose_rets/rets_config.rb +10 -10
- data/app/models/caboose_rets/saved_property.rb +1 -1
- data/app/models/caboose_rets/saved_search.rb +1 -1
- data/app/models/caboose_rets/search_option.rb +1 -1
- data/lib/caboose_rets/version.rb +1 -1
- metadata +2 -4
- data/app/models/caboose_rets/rets_importer_bak.rb +0 -621
- data/app/models/caboose_rets/rets_importer_old.rb +0 -625
@@ -1,625 +0,0 @@
|
|
1
|
-
#require 'ruby-rets'
|
2
|
-
require "rets/version"
|
3
|
-
require "rets/exceptions"
|
4
|
-
require "rets/client"
|
5
|
-
require "rets/http"
|
6
|
-
require "rets/stream_http"
|
7
|
-
require "rets/base/core"
|
8
|
-
require "rets/base/sax_search"
|
9
|
-
require "rets/base/sax_metadata"
|
10
|
-
|
11
|
-
require 'httparty'
|
12
|
-
require 'json'
|
13
|
-
|
14
|
-
# http://rets.solidearth.com/ClientHome.aspx
|
15
|
-
|
16
|
-
class CabooseRets::RetsImporter # < ActiveRecord::Base
|
17
|
-
|
18
|
-
@@rets_client = nil
|
19
|
-
@@config = nil
|
20
|
-
|
21
|
-
def self.config
|
22
|
-
return @@config
|
23
|
-
end
|
24
|
-
|
25
|
-
def self.get_config
|
26
|
-
@@config = {
|
27
|
-
'url' => nil, # URL to the RETS login
|
28
|
-
'username' => nil,
|
29
|
-
'password' => nil,
|
30
|
-
'temp_path' => nil,
|
31
|
-
'log_file' => nil,
|
32
|
-
'media_base_url' => nil
|
33
|
-
}
|
34
|
-
config = YAML::load(File.open("#{Rails.root}/config/rets_importer.yml"))
|
35
|
-
config = config[Rails.env]
|
36
|
-
config.each { |key,val| @@config[key] = val }
|
37
|
-
end
|
38
|
-
|
39
|
-
def self.client
|
40
|
-
self.get_config if @@config.nil? || @@config['url'].nil?
|
41
|
-
|
42
|
-
if @@rets_client.nil?
|
43
|
-
@@rets_client = RETS::Client.login(
|
44
|
-
:url => 'http://rets.wamls.mlsmatrix.com/rets/Login.ashx', #@@config['url'],
|
45
|
-
:username => 'RETS_6', #@@config['username'],
|
46
|
-
:password => 'ellis' #@@config['password']
|
47
|
-
)
|
48
|
-
end
|
49
|
-
return @@rets_client
|
50
|
-
end
|
51
|
-
|
52
|
-
def self.meta(class_type)
|
53
|
-
Caboose::StdClass.new({ :search_type => 'Property' , :remote_key_field => 'MLS' , :local_key_field => 'mls' , :local_table => 'rets_properties' , :date_modified_field => 'DATE_MODIFIED'})
|
54
|
-
# case class_type
|
55
|
-
# when 'RES' then Caboose::StdClass.new({ :search_type => 'Property' , :remote_key_field => 'MLS' , :local_key_field => 'mls' , :local_table => 'rets_properties' , :date_modified_field => 'DATE_MODIFIED' })
|
56
|
-
# when 'OFF' then Caboose::StdClass.new({ :search_type => 'Office' , :remote_key_field => 'LO_LO_CODE' , :local_key_field => 'lo_code' , :local_table => 'rets_offices' , :date_modified_field => 'LO_DATE_MODIFIED' })
|
57
|
-
# when 'AGT' then Caboose::StdClass.new({ :search_type => 'Agent' , :remote_key_field => 'LA_LA_CODE' , :local_key_field => 'la_code' , :local_table => 'rets_agents' , :date_modified_field => 'LA_DATE_MODIFIED' })
|
58
|
-
# when 'OPH' then Caboose::StdClass.new({ :search_type => 'OpenHouse' , :remote_key_field => 'ID' , :local_key_field => 'id' , :local_table => 'rets_open_houses' , :date_modified_field => 'DATE_MODIFIED' })
|
59
|
-
# when 'GFX' then Caboose::StdClass.new({ :search_type => 'Media' , :remote_key_field => 'MEDIA_ID' , :local_key_field => 'media_id' , :local_table => 'rets_media' , :date_modified_field => 'DATE_MODIFIED' })
|
60
|
-
# end
|
61
|
-
end
|
62
|
-
|
63
|
-
#=============================================================================
|
64
|
-
# Import method
|
65
|
-
#=============================================================================
|
66
|
-
|
67
|
-
def self.import(class_type, query)
|
68
|
-
m = self.meta(class_type)
|
69
|
-
#self.log("Importing #{m.search_type}:#{class_type} with query #{query}...")
|
70
|
-
self.get_config if @@config.nil? || @@config['url'].nil?
|
71
|
-
params = {
|
72
|
-
:search_type => m.search_type,
|
73
|
-
:class => class_type,
|
74
|
-
:query => query,
|
75
|
-
:limit => -1,
|
76
|
-
:timeout => -1
|
77
|
-
}
|
78
|
-
obj = nil
|
79
|
-
begin
|
80
|
-
self.client.search(params) do |data|
|
81
|
-
obj = self.get_instance_with_id(class_type, data)
|
82
|
-
if obj.nil?
|
83
|
-
self.log("Error: object is nil")
|
84
|
-
self.log(data.inspect)
|
85
|
-
next
|
86
|
-
end
|
87
|
-
obj.parse(data)
|
88
|
-
obj.save
|
89
|
-
end
|
90
|
-
rescue RETS::HTTPError => err
|
91
|
-
self.log "Import error for #{class_type}: #{query}"
|
92
|
-
self.log err.message
|
93
|
-
end
|
94
|
-
|
95
|
-
end
|
96
|
-
|
97
|
-
def self.get_instance_with_id(class_type, data)
|
98
|
-
obj = nil
|
99
|
-
m = CabooseRets::Property
|
100
|
-
# m = case class_type
|
101
|
-
# when 'OPH' then CabooseRets::OpenHouse
|
102
|
-
# when 'GFX' then CabooseRets::Media
|
103
|
-
# when 'COM' then CabooseRets::CommercialProperty
|
104
|
-
# when 'LND' then CabooseRets::LandProperty
|
105
|
-
# when 'MUL' then CabooseRets::MultiFamilyProperty
|
106
|
-
# when 'RES' then CabooseRets::ResidentialProperty
|
107
|
-
# when 'AGT' then CabooseRets::Agent
|
108
|
-
# when 'OFF' then CabooseRets::Office
|
109
|
-
# end
|
110
|
-
obj = m.where(:id => data['ID'].to_i).exists? ? m.where(:id => data['ID'].to_i).first : m.new(:id=> data['ID'].to_i)
|
111
|
-
# obj = case class_type
|
112
|
-
# when 'OPH' then m.where(:id => data['ID'].to_i ).exists? ? m.where(:id => data['ID'].to_i ).first : m.new(:id => data['ID'].to_i )
|
113
|
-
# when 'GFX' then m.where(:media_id => data['MEDIA_ID'] ).exists? ? m.where(:media_id => data['MEDIA_ID'] ).first : m.new(:media_id => data['MEDIA_ID'] )
|
114
|
-
# when 'COM' then m.where(:id => data['MLS'].to_i ).exists? ? m.where(:id => data['MLS'].to_i ).first : m.new(:id => data['MLS'].to_i )
|
115
|
-
# when 'LND' then m.where(:id => data['MLS'].to_i ).exists? ? m.where(:id => data['MLS'].to_i ).first : m.new(:id => data['MLS'].to_i )
|
116
|
-
# when 'MUL' then m.where(:id => data['MLS'].to_i ).exists? ? m.where(:id => data['MLS'].to_i ).first : m.new(:id => data['MLS'].to_i )
|
117
|
-
# when 'RES' then m.where(:id => data['MLS'].to_i ).exists? ? m.where(:id => data['MLS'].to_i ).first : m.new(:id => data['MLS'].to_i )
|
118
|
-
# when 'AGT' then m.where(:la_code => data['LA_LA_CODE'] ).exists? ? m.where(:la_code => data['LA_LA_CODE'] ).first : m.new(:la_code => data['LA_LA_CODE'] )
|
119
|
-
# when 'OFF' then m.where(:lo_code => data['LO_LO_CODE'] ).exists? ? m.where(:lo_code => data['LO_LO_CODE'] ).first : m.new(:lo_code => data['LO_LO_CODE'] )
|
120
|
-
# end
|
121
|
-
return obj
|
122
|
-
end
|
123
|
-
|
124
|
-
#=============================================================================
|
125
|
-
# Main updater
|
126
|
-
#=============================================================================
|
127
|
-
|
128
|
-
def self.update_after(date_modified, save_images = true)
|
129
|
-
self.delay(:priority => 10, :queue => 'rets').update_helper('PRO', date_modified, save_images)
|
130
|
-
# self.delay(:queue => 'rets').update_helper('RES', date_modified, save_images)
|
131
|
-
# self.delay(:queue => 'rets').update_helper('COM', date_modified, save_images)
|
132
|
-
# self.delay(:queue => 'rets').update_helper('LND', date_modified, save_images)
|
133
|
-
# self.delay(:queue => 'rets').update_helper('MUL', date_modified, save_images)
|
134
|
-
# self.delay(:queue => 'rets').update_helper('OFF', date_modified, save_images)
|
135
|
-
# self.delay(:queue => 'rets').update_helper('AGT', date_modified, save_images)
|
136
|
-
# self.delay(:queue => 'rets').update_helper('OPH', date_modified, save_images)
|
137
|
-
end
|
138
|
-
|
139
|
-
def self.update_helper(class_type, date_modified, save_images = true)
|
140
|
-
m = self.meta(class_type)
|
141
|
-
k = m.remote_key_field
|
142
|
-
d = date_modified.in_time_zone(CabooseRets::timezone).strftime("%FT%T")
|
143
|
-
params = {
|
144
|
-
:search_type => m.search_type,
|
145
|
-
:class => class_type,
|
146
|
-
:select => [m.remote_key_field],
|
147
|
-
:query => "(#{m.date_modified_field}=#{d}+)",
|
148
|
-
:standard_names_only => true,
|
149
|
-
:timeout => -1
|
150
|
-
}
|
151
|
-
self.client.search(params) do |data|
|
152
|
-
self.delay(:priority => 10, :queue => 'rets').import_properties( data[k], save_images)
|
153
|
-
# case class_type
|
154
|
-
# when 'RES' then self.delay(:priority => 10, :queue => 'rets').import_residential_property( data[k], save_images)
|
155
|
-
# when 'COM' then self.delay(:priority => 10, :queue => 'rets').import_commercial_property( data[k], save_images)
|
156
|
-
# when 'LND' then self.delay(:priority => 10, :queue => 'rets').import_land_property( data[k], save_images)
|
157
|
-
# when 'MUL' then self.delay(:priority => 10, :queue => 'rets').import_multi_family_property( data[k], save_images)
|
158
|
-
# when 'OFF' then self.delay(:priority => 10, :queue => 'rets').import_office( data[k], save_images)
|
159
|
-
# when 'AGT' then self.delay(:priority => 10, :queue => 'rets').import_agent( data[k], save_images)
|
160
|
-
# when 'OPH' then self.delay(:priority => 10, :queue => 'rets').import_open_house( data[k], save_images)
|
161
|
-
# end
|
162
|
-
end
|
163
|
-
end
|
164
|
-
|
165
|
-
#=============================================================================
|
166
|
-
# Single model import methods (called from a worker dyno)
|
167
|
-
#=============================================================================
|
168
|
-
|
169
|
-
def self.import_property(mls, save_images = true)
|
170
|
-
self.import('RES', "(MLS=*#{mls}*)")
|
171
|
-
p = CabooseRets::Property.where(:id => mls.to_i).first
|
172
|
-
# p = CabooseRets::ResidentialProperty.where(:id => mls.to_i).first
|
173
|
-
# if p.nil?
|
174
|
-
# self.import('COM', "(MLS=*#{mls}*)")
|
175
|
-
# p = CabooseRets::CommercialProperty.where(:id => mls.to_i).first
|
176
|
-
# if p.nil?
|
177
|
-
# self.import('LND', "(MLS=*#{mls}*)")
|
178
|
-
# p = CabooseRets::LandProperty.where(:id => mls.to_i).first
|
179
|
-
# if p.nil?
|
180
|
-
# self.import('MUL', "(MLS=*#{mls}*)")
|
181
|
-
# p = CabooseRets::MultiFamilyProperty.where(:id => mls.to_i).first
|
182
|
-
# return if p.nil?
|
183
|
-
# end
|
184
|
-
# end
|
185
|
-
# end
|
186
|
-
self.download_property_images(p, save_images)
|
187
|
-
end
|
188
|
-
|
189
|
-
def self.import_properties(mls, save_images = true)
|
190
|
-
self.import('PRO', "(MLS=*#{mls}*)")
|
191
|
-
p = CabooseRets::Property.where(:id => mls.to_i).first
|
192
|
-
self.download_property_images(p, save_images)
|
193
|
-
self.update_coords(p)
|
194
|
-
end
|
195
|
-
|
196
|
-
# def self.import_residential_property(mls, save_images = true)
|
197
|
-
# self.import('RES', "(MLS=*#{mls}*)")
|
198
|
-
# p = CabooseRets::ResidentialProperty.where(:id => mls.to_i).first
|
199
|
-
# self.download_property_images(p, save_images)
|
200
|
-
# self.update_coords(p)
|
201
|
-
# end
|
202
|
-
#
|
203
|
-
# def self.import_commercial_property(mls, save_images = true)
|
204
|
-
# self.import('COM', "(MLS=*#{mls}*)")
|
205
|
-
# p = CabooseRets::CommercialProperty.where(:id => mls.to_i).first
|
206
|
-
# self.download_property_images(p, save_images)
|
207
|
-
# self.update_coords(p)
|
208
|
-
# end
|
209
|
-
#
|
210
|
-
# def self.import_land_property(mls, save_images = true)
|
211
|
-
# self.import('LND', "(MLS=*#{mls}*)")
|
212
|
-
# p = CabooseRets::LandProperty.where(:id => mls.to_i).first
|
213
|
-
# self.download_property_images(p, save_images)
|
214
|
-
# self.update_coords(p)
|
215
|
-
# end
|
216
|
-
#
|
217
|
-
# def self.import_multi_family_property(mls, save_images = true)
|
218
|
-
# self.import('MUL', "(MLS=*#{mls}*)")
|
219
|
-
# p = CabooseRets::MultiFamilyProperty.where(:id => mls.to_i).first
|
220
|
-
# self.download_property_images(p, save_images)
|
221
|
-
# self.update_coords(p)
|
222
|
-
# end
|
223
|
-
#
|
224
|
-
# def self.import_office(lo_code, save_images = true)
|
225
|
-
# self.import('OFF', "(LO_LO_CODE=*#{lo_code}*)")
|
226
|
-
# office = CabooseRets::Office.where(:lo_code => lo_code.to_s).first
|
227
|
-
# self.download_office_image(office) if save_images == true
|
228
|
-
# end
|
229
|
-
#
|
230
|
-
# def self.import_agent(la_code, save_images = true)
|
231
|
-
# self.import('AGT', "(LA_LA_CODE=*#{la_code}*)")
|
232
|
-
# a = CabooseRets::Agent.where(:la_code => la_code.to_s).first
|
233
|
-
# self.download_agent_image(a) #if save_images == true
|
234
|
-
# end
|
235
|
-
#
|
236
|
-
# def self.import_open_house(id, save_images = true)
|
237
|
-
# self.import('OPH', "((ID=#{id}+),(ID=#{id}-))")
|
238
|
-
# end
|
239
|
-
|
240
|
-
def self.import_media(id, save_images = true)
|
241
|
-
self.import('GFX', "((MEDIA_ID=#{id}+),(MEDIA_ID=#{id}-))")
|
242
|
-
end
|
243
|
-
|
244
|
-
#=============================================================================
|
245
|
-
# Images
|
246
|
-
#=============================================================================
|
247
|
-
|
248
|
-
def self.download_property_images(p, save_images = true)
|
249
|
-
return if save_images == false
|
250
|
-
|
251
|
-
self.log("- Downloading GFX records for #{p.mls}...")
|
252
|
-
params = {
|
253
|
-
:search_type => 'Media',
|
254
|
-
:class => 'GFX',
|
255
|
-
:query => "(MLS=*#{p.mls}*)",
|
256
|
-
:timeout => -1
|
257
|
-
}
|
258
|
-
ids = []
|
259
|
-
self.client.search(params) do |data|
|
260
|
-
ids << data['MEDIA_ID']
|
261
|
-
m = CabooseRets::Media.where(:media_id => data['MEDIA_ID']).first
|
262
|
-
m = CabooseRets::Media.new if m.nil?
|
263
|
-
m.parse(data)
|
264
|
-
m.save
|
265
|
-
end
|
266
|
-
|
267
|
-
if ids.count > 0
|
268
|
-
# Delete any records in the local database that shouldn't be there
|
269
|
-
self.log("- Deleting GFX records for MLS ##{p.mls} in the local database that are not in the remote database...")
|
270
|
-
query = "select media_id from rets_media where mls = '#{p.mls}'"
|
271
|
-
rows = ActiveRecord::Base.connection.select_all(ActiveRecord::Base.send(:sanitize_sql_array, query))
|
272
|
-
local_ids = rows.collect{ |row| row['media_id'] }
|
273
|
-
ids_to_remove = local_ids - ids
|
274
|
-
if ids_to_remove && ids_to_remove.count > 0
|
275
|
-
query = ["delete from rets_media where media_id in (?)", ids_to_remove]
|
276
|
-
ActiveRecord::Base.connection.execute(ActiveRecord::Base.send(:sanitize_sql_array, query))
|
277
|
-
end
|
278
|
-
end
|
279
|
-
|
280
|
-
end
|
281
|
-
|
282
|
-
def self.download_agent_image(agent)
|
283
|
-
self.log "Saving image for #{agent.first_name} #{agent.last_name}..."
|
284
|
-
begin
|
285
|
-
self.client.get_object(:resource => :Agent, :type => :Photo, :location => true, :id => agent.la_code) do |headers, content|
|
286
|
-
agent.verify_meta_exists
|
287
|
-
agent.meta.image_location = headers['location']
|
288
|
-
agent.meta.save
|
289
|
-
end
|
290
|
-
rescue RETS::APIError => err
|
291
|
-
self.log "No image for #{agent.first_name} #{agent.last_name}."
|
292
|
-
self.log err
|
293
|
-
end
|
294
|
-
end
|
295
|
-
|
296
|
-
def self.download_office_image(office)
|
297
|
-
#self.log "Saving image for #{agent.first_name} #{agent.last_name}..."
|
298
|
-
#begin
|
299
|
-
# self.client.get_object(:resource => :Agent, :type => :Photo, :location => true, :id => agent.la_code) do |headers, content|
|
300
|
-
# agent.verify_meta_exists
|
301
|
-
# agent.meta.image_location = headers['location']
|
302
|
-
# agent.meta.save
|
303
|
-
# end
|
304
|
-
#rescue RETS::APIError => err
|
305
|
-
# self.log "No image for #{agent.first_name} #{agent.last_name}."
|
306
|
-
# self.log err
|
307
|
-
#end
|
308
|
-
end
|
309
|
-
|
310
|
-
#=============================================================================
|
311
|
-
# GPS
|
312
|
-
#=============================================================================
|
313
|
-
|
314
|
-
def self.update_coords(p = nil)
|
315
|
-
if p.nil?
|
316
|
-
models = CabooseRets::Property #[CabooseRets::CommercialProperty, CabooseRets::LandProperty, CabooseRets::MultiFamilyProperty, CabooseRets::ResidentialProperty]
|
317
|
-
names = ["commercial", "land", "multi-family", "residential"]
|
318
|
-
i = 0
|
319
|
-
models.each do |model|
|
320
|
-
self.log "Updating coords #{names[i]} properties..."
|
321
|
-
model.where(:latitude => nil).reorder(:mls).each do |p|
|
322
|
-
self.delay(:priority => 10, :queue => 'rets').update_coords(p)
|
323
|
-
end
|
324
|
-
i = i + 1
|
325
|
-
end
|
326
|
-
return
|
327
|
-
end
|
328
|
-
|
329
|
-
self.log "Getting coords for mls #{p.mls}..."
|
330
|
-
coords = self.coords_from_address(CGI::escape "#{p.street_num} #{p.street_name}, #{p.city}, #{p.state} #{p.zip}")
|
331
|
-
if coords.nil? || coords == false
|
332
|
-
self.log "Can't set coords for mls acct #{p.mls}..."
|
333
|
-
return
|
334
|
-
end
|
335
|
-
|
336
|
-
p.latitude = coords['lat']
|
337
|
-
p.longitude = coords['lng']
|
338
|
-
p.save
|
339
|
-
end
|
340
|
-
|
341
|
-
def self.coords_from_address(address)
|
342
|
-
#return false
|
343
|
-
begin
|
344
|
-
uri = "https://maps.googleapis.com/maps/api/geocode/json?address=#{address}&sensor=false"
|
345
|
-
uri.gsub!(" ", "+")
|
346
|
-
resp = HTTParty.get(uri)
|
347
|
-
json = JSON.parse(resp.body)
|
348
|
-
return json['results'][0]['geometry']['location']
|
349
|
-
rescue
|
350
|
-
self.log "Error: #{uri}"
|
351
|
-
sleep(2)
|
352
|
-
return false
|
353
|
-
end
|
354
|
-
end
|
355
|
-
|
356
|
-
#=============================================================================
|
357
|
-
# Purging
|
358
|
-
#=============================================================================
|
359
|
-
|
360
|
-
def self.purge
|
361
|
-
self.purge_properties
|
362
|
-
self.purge_residential
|
363
|
-
self.purge_commercial
|
364
|
-
self.purge_land
|
365
|
-
self.purge_multi_family
|
366
|
-
self.purge_offices
|
367
|
-
#self.purge_agents
|
368
|
-
self.purge_open_houses
|
369
|
-
self.purge_media
|
370
|
-
end
|
371
|
-
|
372
|
-
def self.purge_properties() self.delay(:priority => 10, :queue => 'rets').purge_helper('PRO', '2012-01-01') end
|
373
|
-
def self.purge_residential() self.delay(:priority => 10, :queue => 'rets').purge_helper('RES', '2012-01-01') end
|
374
|
-
def self.purge_commercial() self.delay(:priority => 10, :queue => 'rets').purge_helper('COM', '2012-01-01') end
|
375
|
-
def self.purge_land() self.delay(:priority => 10, :queue => 'rets').purge_helper('LND', '2012-01-01') end
|
376
|
-
def self.purge_multi_family() self.delay(:priority => 10, :queue => 'rets').purge_helper('MUL', '2012-01-01') end
|
377
|
-
def self.purge_offices() self.delay(:priority => 10, :queue => 'rets').purge_helper('OFF', '2012-01-01') end
|
378
|
-
def self.purge_agents() self.delay(:priority => 10, :queue => 'rets').purge_helper('AGT', '2012-01-01') end
|
379
|
-
def self.purge_open_houses() self.delay(:priority => 10, :queue => 'rets').purge_helper('OPH', '2012-01-01') end
|
380
|
-
def self.purge_media() self.delay(:priority => 10, :queue => 'rets').purge_helper('GFX', '2012-01-01') end
|
381
|
-
|
382
|
-
def self.purge_helper(class_type, date_modified)
|
383
|
-
m = self.meta(class_type)
|
384
|
-
|
385
|
-
self.log("Purging #{class_type}...")
|
386
|
-
|
387
|
-
# Get the total number of records
|
388
|
-
self.log("- Getting total number of records for #{class_type}...")
|
389
|
-
params = {
|
390
|
-
:search_type => m.search_type,
|
391
|
-
:class => class_type,
|
392
|
-
:query => "(#{m.date_modified_field}=#{date_modified}T00:00:01+)",
|
393
|
-
:standard_names_only => true,
|
394
|
-
:timeout => -1
|
395
|
-
}
|
396
|
-
self.client.search(params.merge({ :count_mode => :only }))
|
397
|
-
count = self.client.rets_data[:code] == "20201" ? 0 : self.client.rets_data[:count]
|
398
|
-
batch_count = (count.to_f/5000.0).ceil
|
399
|
-
|
400
|
-
ids = []
|
401
|
-
k = m.remote_key_field
|
402
|
-
(0...batch_count).each do |i|
|
403
|
-
self.log("- Getting ids for #{class_type} (batch #{i+1} of #{batch_count})...")
|
404
|
-
self.client.search(params.merge({ :select => [k], :limit => 5000, :offset => 5000*i })) do |data|
|
405
|
-
ids << (class_type == 'OPH' ? data[k].to_i : data[k])
|
406
|
-
end
|
407
|
-
end
|
408
|
-
|
409
|
-
# Only do stuff if we got a real response from the server
|
410
|
-
if ids.count > 0
|
411
|
-
|
412
|
-
# Delete any records in the local database that shouldn't be there
|
413
|
-
self.log("- Finding #{class_type} records in the local database that are not in the remote database...")
|
414
|
-
t = m.local_table
|
415
|
-
k = m.local_key_field
|
416
|
-
query = "select distinct #{k} from #{t}"
|
417
|
-
rows = ActiveRecord::Base.connection.select_all(ActiveRecord::Base.send(:sanitize_sql_array, query))
|
418
|
-
local_ids = rows.collect{ |row| row[k] }
|
419
|
-
ids_to_remove = local_ids - ids
|
420
|
-
self.log("- Found #{ids_to_remove.count} #{class_type} records in the local database that are not in the remote database.")
|
421
|
-
self.log("- Deleting #{class_type} records in the local database that shouldn't be there...")
|
422
|
-
query = ["delete from #{t} where #{k} in (?)", ids_to_remove]
|
423
|
-
ActiveRecord::Base.connection.execute(ActiveRecord::Base.send(:sanitize_sql_array, query))
|
424
|
-
|
425
|
-
# Find any ids in the remote database that should be in the local database
|
426
|
-
self.log("- Finding #{class_type} records in the remote database that should be in the local database...")
|
427
|
-
query = "select distinct #{k} from #{t}"
|
428
|
-
rows = ActiveRecord::Base.connection.select_all(ActiveRecord::Base.send(:sanitize_sql_array, query))
|
429
|
-
local_ids = rows.collect{ |row| row[k] }
|
430
|
-
ids_to_add = ids - local_ids
|
431
|
-
ids_to_add = ids_to_add.sort.reverse
|
432
|
-
self.log("- Found #{ids_to_add.count} #{class_type} records in the remote database that we need to add to the local database.")
|
433
|
-
ids_to_add.each do |id|
|
434
|
-
self.log("- Importing #{id}...")
|
435
|
-
self.delay(:priority => 10, :queue => 'rets').import_properties(id, false)
|
436
|
-
# case class_type
|
437
|
-
# when 'RES' then self.delay(:queue => 'rets').import_residential_property(id, false)
|
438
|
-
# when 'COM' then self.delay(:queue => 'rets').import_commercial_property(id, false)
|
439
|
-
# when 'LND' then self.delay(:queue => 'rets').import_land_property(id, false)
|
440
|
-
# when 'MUL' then self.delay(:queue => 'rets').import_multi_family_property(id, false)
|
441
|
-
# when 'OFF' then self.delay(:queue => 'rets').import_office(id, false)
|
442
|
-
# when 'AGT' then self.delay(:queue => 'rets').import_agent(id, false)
|
443
|
-
# when 'OPH' then self.delay(:queue => 'rets').import_open_house(id, false)
|
444
|
-
# when 'GFX' then self.delay(:queue => 'rets').import_media(id, false)
|
445
|
-
# end
|
446
|
-
end
|
447
|
-
|
448
|
-
end
|
449
|
-
|
450
|
-
end
|
451
|
-
|
452
|
-
def self.get_media_urls
|
453
|
-
m = self.meta(class_type)
|
454
|
-
|
455
|
-
# Get the total number of records
|
456
|
-
params = {
|
457
|
-
:search_type => m.search_type,
|
458
|
-
:class => class_type,
|
459
|
-
:query => "(#{m.date_modified_field}=#{date_modified}T00:00:01+)",
|
460
|
-
:standard_names_only => true,
|
461
|
-
:timeout => -1
|
462
|
-
}
|
463
|
-
self.client.search(params.merge({ :count_mode => :only }))
|
464
|
-
count = self.client.rets_data[:code] == "20201" ? 0 : self.client.rets_data[:count]
|
465
|
-
batch_count = (count.to_f/5000.0).ceil
|
466
|
-
|
467
|
-
ids = []
|
468
|
-
k = m.remote_key_field
|
469
|
-
(0...batch_count).each do |i|
|
470
|
-
self.client.search(params.merge({ :select => [k], :limit => 5000, :offset => 5000*i })) do |data|
|
471
|
-
ids << data[k]
|
472
|
-
# ids << case class_type
|
473
|
-
# when 'RES' then data[k]
|
474
|
-
# when 'COM' then data[k]
|
475
|
-
# when 'LND' then data[k]
|
476
|
-
# when 'MUL' then data[k]
|
477
|
-
# when 'OFF' then data[k]
|
478
|
-
# when 'AGT' then data[k]
|
479
|
-
# when 'OPH' then data[k].to_i
|
480
|
-
# when 'GFX' then data[k]
|
481
|
-
# end
|
482
|
-
end
|
483
|
-
end
|
484
|
-
|
485
|
-
if ids.count > 0
|
486
|
-
# Delete any records in the local database that shouldn't be there
|
487
|
-
t = m.local_table
|
488
|
-
k = m.local_key_field
|
489
|
-
query = ["delete from #{t} where #{k} not in (?)", ids]
|
490
|
-
ActiveRecord::Base.connection.execute(ActiveRecord::Base.send(:sanitize_sql_array, query))
|
491
|
-
|
492
|
-
# Find any ids in the remote database that should be in the local database
|
493
|
-
query = "select distinct #{k} from #{t}"
|
494
|
-
rows = ActiveRecord::Base.connection.select_all(ActiveRecord::Base.send(:sanitize_sql_array, query))
|
495
|
-
local_ids = rows.collect{ |row| row[k] }
|
496
|
-
ids_to_add = ids - local_ids
|
497
|
-
ids_to_add.each do |id|
|
498
|
-
self.log("Importing #{id}...")
|
499
|
-
self.delay(:priority => 10, :queue => 'rets').import_properties(id, false)
|
500
|
-
# case class_type
|
501
|
-
# when 'RES' then self.delay(:queue => 'rets').import_residential_property(id, false)
|
502
|
-
# when 'COM' then self.delay(:queue => 'rets').import_commercial_property(id, false)
|
503
|
-
# when 'LND' then self.delay(:queue => 'rets').import_land_property(id, false)
|
504
|
-
# when 'MUL' then self.delay(:queue => 'rets').import_multi_family_property(id, false)
|
505
|
-
# when 'OFF' then self.delay(:queue => 'rets').import_office(id, false)
|
506
|
-
# when 'AGT' then self.delay(:queue => 'rets').import_agent(id, false)
|
507
|
-
# when 'OPH' then self.delay(:queue => 'rets').import_open_house(id, false)
|
508
|
-
# when 'GFX' then self.delay(:queue => 'rets').import_media(id)
|
509
|
-
# end
|
510
|
-
end
|
511
|
-
end
|
512
|
-
|
513
|
-
end
|
514
|
-
|
515
|
-
#=============================================================================
|
516
|
-
# Logging
|
517
|
-
#=============================================================================
|
518
|
-
|
519
|
-
def self.log(msg)
|
520
|
-
puts "[rets_importer] #{msg}"
|
521
|
-
#Rails.logger.info("[rets_importer] #{msg}")
|
522
|
-
end
|
523
|
-
|
524
|
-
def self.log2(msg)
|
525
|
-
puts "======================================================================"
|
526
|
-
puts "[rets_importer] #{msg}"
|
527
|
-
puts "======================================================================"
|
528
|
-
#Rails.logger.info("[rets_importer] #{msg}")
|
529
|
-
end
|
530
|
-
|
531
|
-
#=============================================================================
|
532
|
-
# Locking update task
|
533
|
-
#=============================================================================
|
534
|
-
|
535
|
-
def self.update_rets
|
536
|
-
self.log2("Updating rets...")
|
537
|
-
if self.task_is_locked
|
538
|
-
self.log2("Task is locked, aborting.")
|
539
|
-
return
|
540
|
-
end
|
541
|
-
self.log2("Locking task...")
|
542
|
-
task_started = self.lock_task
|
543
|
-
|
544
|
-
begin
|
545
|
-
overlap = 30.seconds
|
546
|
-
if (DateTime.now - self.last_purged).to_i >= 1
|
547
|
-
self.purge
|
548
|
-
self.save_last_purged(task_started)
|
549
|
-
# Keep this in here to make sure all updates are caught
|
550
|
-
#overlap = 1.month
|
551
|
-
end
|
552
|
-
|
553
|
-
self.log2("Updating after #{self.last_updated.strftime("%FT%T%:z")}...")
|
554
|
-
self.update_after(self.last_updated - overlap)
|
555
|
-
|
556
|
-
self.log2("Saving the timestamp for when we updated...")
|
557
|
-
self.save_last_updated(task_started)
|
558
|
-
|
559
|
-
self.log2("Unlocking the task...")
|
560
|
-
self.unlock_task
|
561
|
-
rescue Exception => err
|
562
|
-
puts err
|
563
|
-
raise
|
564
|
-
ensure
|
565
|
-
self.log2("Unlocking task if last updated...")
|
566
|
-
self.unlock_task_if_last_updated(task_started)
|
567
|
-
end
|
568
|
-
|
569
|
-
# Start the same update process in five minutes
|
570
|
-
self.log2("Adding the update rets task for 5 minutes from now...")
|
571
|
-
q = "handler like '%update_rets%'"
|
572
|
-
count = Delayed::Job.where(q).count
|
573
|
-
if count == 0 || (count == 1 && Delayed::Job.where(q).first.locked_at)
|
574
|
-
self.delay(:run_at => 5.minutes.from_now, :priority => 10, :queue => 'rets').update_rets
|
575
|
-
end
|
576
|
-
end
|
577
|
-
|
578
|
-
def self.last_updated
|
579
|
-
if !Caboose::Setting.exists?(:name => 'rets_last_updated')
|
580
|
-
Caboose::Setting.create(:name => 'rets_last_updated', :value => '2013-08-06T00:00:01')
|
581
|
-
end
|
582
|
-
s = Caboose::Setting.where(:name => 'rets_last_updated').first
|
583
|
-
return DateTime.parse(s.value)
|
584
|
-
end
|
585
|
-
|
586
|
-
def self.last_purged
|
587
|
-
if !Caboose::Setting.exists?(:name => 'rets_last_purged')
|
588
|
-
Caboose::Setting.create(:name => 'rets_last_purged', :value => '2013-08-06T00:00:01')
|
589
|
-
end
|
590
|
-
s = Caboose::Setting.where(:name => 'rets_last_purged').first
|
591
|
-
return DateTime.parse(s.value)
|
592
|
-
end
|
593
|
-
|
594
|
-
def self.save_last_updated(d)
|
595
|
-
s = Caboose::Setting.where(:name => 'rets_last_updated').first
|
596
|
-
s.value = d.in_time_zone(CabooseRets::timezone).strftime("%FT%T%:z")
|
597
|
-
s.save
|
598
|
-
end
|
599
|
-
|
600
|
-
def self.save_last_purged(d)
|
601
|
-
s = Caboose::Setting.where(:name => 'rets_last_purged').first
|
602
|
-
s.value = d.in_time_zone(CabooseRets::timezone).strftime("%FT%T%:z")
|
603
|
-
s.save
|
604
|
-
end
|
605
|
-
|
606
|
-
def self.task_is_locked
|
607
|
-
return Caboose::Setting.exists?(:name => 'rets_update_running')
|
608
|
-
end
|
609
|
-
|
610
|
-
def self.lock_task
|
611
|
-
d = DateTime.now.utc.in_time_zone(CabooseRets::timezone)
|
612
|
-
Caboose::Setting.create(:name => 'rets_update_running', :value => d.strftime("%FT%T%:z"))
|
613
|
-
return d
|
614
|
-
end
|
615
|
-
|
616
|
-
def self.unlock_task
|
617
|
-
Caboose::Setting.where(:name => 'rets_update_running').first.destroy
|
618
|
-
end
|
619
|
-
|
620
|
-
def self.unlock_task_if_last_updated(d)
|
621
|
-
setting = Caboose::Setting.where(:name => 'rets_update_running').first
|
622
|
-
self.unlock_task if setting && d.in_time_zone.strftime("%FT%T%:z") == setting.value
|
623
|
-
end
|
624
|
-
|
625
|
-
end
|