caboose-rets 0.0.62 → 0.0.63

Sign up to get free protection for your applications and to get access to all the features.
Files changed (4) hide show
  1. checksums.yaml +8 -8
  2. data/app/models/caboose_rets/rets_importer.rb +217 -171
  3. data/lib/caboose_rets/version.rb +1 -1
  4. metadata +1 -1
checksums.yaml CHANGED
@@ -1,15 +1,15 @@
1
1
  ---
2
2
  !binary "U0hBMQ==":
3
3
  metadata.gz: !binary |-
4
- NWU0NDk2NmI3ZjJhNjA0M2RlYzdjMTZmNmMyZjkzN2Q2OTVlNzdlMA==
4
+ MDA2YWIxNjM3YWYyYmYwYjNjNGNkZDQ1YThmMGI3MDU4ZDFjYzYzZQ==
5
5
  data.tar.gz: !binary |-
6
- MDhjOWMwODBiNDg5OTUwOWUyNTg1NGY5ZmQ5NWVkNGQzMTk0ZmQxZQ==
6
+ NmMyNTFkYWE1NjBlYWZkNzNlNDk2ZDg0MGQyOGYxYmEwYmVlYTIzYg==
7
7
  SHA512:
8
8
  metadata.gz: !binary |-
9
- MmNjYzBiZWIxYzM3ZjlhMzBjNDAyYmZkOTA2NzA1MThhMDkxZWJkMjNlMjdi
10
- ZWYzNjM4ZjU4MjJmZDRlMTNmZjljODFmYWY4NjAxOTE4M2Y3OGVhMWQ4Nzg5
11
- MGRiY2Y0OTMyYjA0ZjI4OTdiY2U5NGQ3Yzk2NTJhMGVkODQ4NGY=
9
+ ODhkYWYyOWJhZmQyNzczNzQ1N2UzZTAwZTVkZGJmNDY4MjlmZGM3ODY0ZjI3
10
+ ZGMwNTkxODk5NWNiZWM5MWE2Y2VhYTc1OTBhYmUyN2Q1Y2IzYjVlYjNmYmI5
11
+ MWY5YTY0YmJlNWFjOTMwMjc3ZWY0MjIyYWE3MWViMGQxYTRjOGQ=
12
12
  data.tar.gz: !binary |-
13
- OTkxYzMyMmUxODZhNWYxZDk4NGM3ZDM0M2E2MDM5MWQ3NGRiNmE5NmY3NjJj
14
- ODQxMjRhMzllNmJiMmQzOWI2OWI0ZTk4NTMwYTU2ZTc0YWUxMzI4OTVkZjgz
15
- NDJhMjA0ZmQxY2NkZDRmYjQ1YzllNjJkOWMyMzY0YWRkNmM5MjM=
13
+ NmE1YTVkMWI2OTU2Nzg2MDhiMTM0ZDgyYTIwOTNmM2RlZDM0MmI1MTZmMDc4
14
+ ZTc1YWVmNDhiYWZkNmViNmI0MzQ1YzA5NmY0NWVjNzVkZjkyNmJkMTYxNWEy
15
+ NWQ0OTVmZGUxY2NlNGI4MzBkZWIwNGE2MDYzZGM2MjI0MmJlN2E=
data/app/models/caboose_rets/rets_importer.rb CHANGED
@@ -14,14 +14,14 @@ require 'json'
14
14
  # http://rets.solidearth.com/ClientHome.aspx
15
15
 
16
16
  class CabooseRets::RetsImporter # < ActiveRecord::Base
17
-
17
+
18
18
  @@rets_client = nil
19
19
  @@config = nil
20
-
20
+
21
21
  def self.config
22
22
  return @@config
23
23
  end
24
-
24
+
25
25
  def self.get_config
26
26
  @@config = {
27
27
  'url' => nil, # URL to the RETS login
@@ -35,11 +35,11 @@ class CabooseRets::RetsImporter # < ActiveRecord::Base
35
35
  config = config[Rails.env]
36
36
  config.each { |key,val| @@config[key] = val }
37
37
  end
38
-
38
+
39
39
  def self.client
40
40
  self.get_config if @@config.nil? || @@config['url'].nil?
41
-
42
- if (@@rets_client.nil?)
41
+
42
+ if @@rets_client.nil?
43
43
  @@rets_client = RETS::Client.login(
44
44
  :url => @@config['url'],
45
45
  :username => @@config['username'],
@@ -48,35 +48,49 @@ class CabooseRets::RetsImporter # < ActiveRecord::Base
48
48
  end
49
49
  return @@rets_client
50
50
  end
51
-
51
+
52
+ def self.meta(class_type)
53
+ case class_type
54
+ when 'RES' then Caboose::StdClass.new({ :search_type => 'Property' , :remote_key_field => 'MLS_ACCT' , :local_key_field => 'mls_acct' , :local_table => 'rets_residential' , :date_modified_field => 'DATE_MODIFIED' })
55
+ when 'COM' then Caboose::StdClass.new({ :search_type => 'Property' , :remote_key_field => 'MLS_ACCT' , :local_key_field => 'mls_acct' , :local_table => 'rets_commercial' , :date_modified_field => 'DATE_MODIFIED' })
56
+ when 'LND' then Caboose::StdClass.new({ :search_type => 'Property' , :remote_key_field => 'MLS_ACCT' , :local_key_field => 'mls_acct' , :local_table => 'rets_land' , :date_modified_field => 'DATE_MODIFIED' })
57
+ when 'MUL' then Caboose::StdClass.new({ :search_type => 'Property' , :remote_key_field => 'MLS_ACCT' , :local_key_field => 'mls_acct' , :local_table => 'rets_multi_family' , :date_modified_field => 'DATE_MODIFIED' })
58
+ when 'OFF' then Caboose::StdClass.new({ :search_type => 'Office' , :remote_key_field => 'LO_LO_CODE' , :local_key_field => 'lo_code' , :local_table => 'rets_offices' , :date_modified_field => 'LO_DATE_MODIFIED' })
59
+ when 'AGT' then Caboose::StdClass.new({ :search_type => 'Agent' , :remote_key_field => 'LA_LA_CODE' , :local_key_field => 'la_code' , :local_table => 'rets_agents' , :date_modified_field => 'LA_DATE_MODIFIED' })
60
+ when 'OPH' then Caboose::StdClass.new({ :search_type => 'OpenHouse' , :remote_key_field => 'ID' , :local_key_field => 'id' , :local_table => 'rets_open_houses' , :date_modified_field => 'DATE_MODIFIED' })
61
+ when 'GFX' then Caboose::StdClass.new({ :search_type => 'Media' , :remote_key_field => 'MEDIA_ID' , :local_key_field => 'media_id' , :local_table => 'rets_media' , :date_modified_field => 'DATE_MODIFIED' })
62
+ end
63
+ end
64
+
52
65
  #=============================================================================
53
66
  # Import method
54
67
  #=============================================================================
55
-
56
- def self.import(query, search_type, class_type)
57
- self.log("Importing #{search_type}:#{class_type} with query #{query}...")
68
+
69
+ def self.import(class_type, query)
70
+ m = self.meta(class_type)
71
+ self.log("Importing #{m.search_type}:#{class_type} with query #{query}...")
58
72
  self.get_config if @@config.nil? || @@config['url'].nil?
59
73
  params = {
60
- :search_type => search_type,
74
+ :search_type => m.search_type,
61
75
  :class => class_type,
62
76
  :query => query,
63
- :limit => -1,
77
+ :limit => -1,
64
78
  :timeout => -1
65
79
  }
66
80
  obj = nil
67
81
  self.client.search(params) do |data|
68
82
  obj = self.get_instance_with_id(class_type, data)
69
83
  if obj.nil?
70
- self.log("Error: object is nil")
84
+ self.log("Error: object is nil")
71
85
  self.log(data.inspect)
72
86
  next
73
87
  end
74
- obj.parse(data)
75
- obj.save
76
- end
88
+ obj.parse(data)
89
+ obj.save
90
+ end
77
91
  end
78
-
79
- def self.get_instance_with_id(class_type, data)
92
+
93
+ def self.get_instance_with_id(class_type, data)
80
94
  obj = nil
81
95
  m = case class_type
82
96
  when 'OPH' then CabooseRets::OpenHouse
@@ -87,183 +101,182 @@ class CabooseRets::RetsImporter # < ActiveRecord::Base
87
101
  when 'RES' then CabooseRets::ResidentialProperty
88
102
  when 'AGT' then CabooseRets::Agent
89
103
  when 'OFF' then CabooseRets::Office
90
- end
104
+ end
91
105
  obj = case class_type
92
106
  when 'OPH' then m.where(:id => data['ID'].to_i ).exists? ? m.where(:id => data['ID'].to_i ).first : m.new(:id => data['ID'].to_i )
93
107
  when 'GFX' then m.where(:media_id => data['MEDIA_ID'] ).exists? ? m.where(:media_id => data['MEDIA_ID'] ).first : m.new(:media_id => data['MEDIA_ID'] )
94
108
  when 'COM' then m.where(:id => data['MLS_ACCT'].to_i ).exists? ? m.where(:id => data['MLS_ACCT'].to_i ).first : m.new(:id => data['MLS_ACCT'].to_i )
95
- when 'LND' then m.where(:id => data['MLS_ACCT'].to_i ).exists? ? m.where(:id => data['MLS_ACCT'].to_i ).first : m.new(:id => data['MLS_ACCT'].to_i )
96
- when 'MUL' then m.where(:id => data['MLS_ACCT'].to_i ).exists? ? m.where(:id => data['MLS_ACCT'].to_i ).first : m.new(:id => data['MLS_ACCT'].to_i )
97
- when 'RES' then m.where(:id => data['MLS_ACCT'].to_i ).exists? ? m.where(:id => data['MLS_ACCT'].to_i ).first : m.new(:id => data['MLS_ACCT'].to_i )
109
+ when 'LND' then m.where(:id => data['MLS_ACCT'].to_i ).exists? ? m.where(:id => data['MLS_ACCT'].to_i ).first : m.new(:id => data['MLS_ACCT'].to_i )
110
+ when 'MUL' then m.where(:id => data['MLS_ACCT'].to_i ).exists? ? m.where(:id => data['MLS_ACCT'].to_i ).first : m.new(:id => data['MLS_ACCT'].to_i )
111
+ when 'RES' then m.where(:id => data['MLS_ACCT'].to_i ).exists? ? m.where(:id => data['MLS_ACCT'].to_i ).first : m.new(:id => data['MLS_ACCT'].to_i )
98
112
  when 'AGT' then m.where(:la_code => data['LA_LA_CODE'] ).exists? ? m.where(:la_code => data['LA_LA_CODE'] ).first : m.new(:la_code => data['LA_LA_CODE'] )
99
113
  when 'OFF' then m.where(:lo_code => data['LO_LO_CODE'] ).exists? ? m.where(:lo_code => data['LO_LO_CODE'] ).first : m.new(:lo_code => data['LO_LO_CODE'] )
100
114
  end
101
- return obj
102
- end
103
-
104
- #def self.delete_old_properties
105
- # self.log("Deleting old residential properties...")
106
- # self.get_config if @@config.nil? || @@config['url'].nil?
107
- #
108
- # d = DateTime.new(2000, 1, 1)
109
- # now = DateTime.now
110
- # while d < now
111
- # d2 = d + 3.months
112
- # puts "- Getting ids for #{d.strftime("%Y-%m-%d")} - #{d2.strftime("%Y-%m-%d")}..."
113
- # q = "((DATE_CREATED=#{d.strftime("%Y-%m-%d")}+),(DATE_CREATED=#{d2.strftime("%Y-%m-%d")}-))",
114
- # params = { :search_type => 'Property', :class => 'RES', :query => q, :limit => -1, :timeout => -1, :select => 'MLS_ACCT' }
115
- # ids = []
116
- # self.client.search(params) do |data|
117
- # ids << data['MLS_ACCT'].to_i
118
- # end
119
- # d = d + 3.months
120
- # end
121
- #
122
- # puts "Deleting properties..."
123
- # #CabooseRets::ResidentialProperty.where("id not in (?)", ids).destroy_all
124
- #end
125
-
115
+ return obj
116
+ end
117
+
126
118
  #=============================================================================
127
119
  # Main updater
128
120
  #=============================================================================
129
121
 
130
122
  def self.update_after(date_modified, save_images = true)
131
- self.update_residential_properties_modified_after(date_modified, save_images)
132
- self.update_commercial_properties_modified_after(date_modified, save_images)
133
- self.update_land_properties_modified_after(date_modified, save_images)
134
- self.update_multi_family_properties_modified_after(date_modified, save_images)
135
- self.update_offices_modified_after(date_modified, save_images)
136
- self.update_agents_modified_after(date_modified, save_images)
137
- self.update_open_houses_modified_after(date_modified, save_images)
138
- end
139
- def self.update_residential_properties_modified_after(date_modified, save_images = true) self.client.search({ :search_type => 'Property' , :class => 'RES', :select => ['MLS_ACCT'] , :query => "(DATE_MODIFIED=#{date_modified.strftime("%FT%T")}+)" , :standard_names_only => true, :timeout => -1 }) { |data| self.delay(:priority => 10).import_residential_property( data['MLS_ACCT' ], save_images) } end
140
- def self.update_commercial_properties_modified_after(date_modified, save_images = true) self.client.search({ :search_type => 'Property' , :class => 'COM', :select => ['MLS_ACCT'] , :query => "(DATE_MODIFIED=#{date_modified.strftime("%FT%T")}+)" , :standard_names_only => true, :timeout => -1 }) { |data| self.delay(:priority => 10).import_commercial_property( data['MLS_ACCT' ], save_images) } end
141
- def self.update_land_properties_modified_after(date_modified, save_images = true) self.client.search({ :search_type => 'Property' , :class => 'LND', :select => ['MLS_ACCT'] , :query => "(DATE_MODIFIED=#{date_modified.strftime("%FT%T")}+)" , :standard_names_only => true, :timeout => -1 }) { |data| self.delay(:priority => 10).import_land_property( data['MLS_ACCT' ], save_images) } end
142
- def self.update_multi_family_properties_modified_after(date_modified, save_images = true) self.client.search({ :search_type => 'Property' , :class => 'MUL', :select => ['MLS_ACCT'] , :query => "(DATE_MODIFIED=#{date_modified.strftime("%FT%T")}+)" , :standard_names_only => true, :timeout => -1 }) { |data| self.delay(:priority => 10).import_multi_family_property( data['MLS_ACCT' ], save_images) } end
143
- def self.update_offices_modified_after(date_modified, save_images = true) self.client.search({ :search_type => 'Office' , :class => 'OFF', :select => ['LO_LO_CODE'] , :query => "(LO_DATE_MODIFIED=#{date_modified.strftime("%FT%T")}+)", :standard_names_only => true, :timeout => -1 }) { |data| self.delay(:priority => 10).import_office( data['LO_LO_CODE'], save_images) } end
144
- def self.update_agents_modified_after(date_modified, save_images = true) self.client.search({ :search_type => 'Agent' , :class => 'AGT', :select => ['LA_LA_CODE'] , :query => "(LA_DATE_MODIFIED=#{date_modified.strftime("%FT%T")}+)", :standard_names_only => true, :timeout => -1 }) { |data| self.delay(:priority => 10).import_agent( data['LA_LA_CODE'], save_images) } end
145
- def self.update_open_houses_modified_after(date_modified, save_images = true) self.client.search({ :search_type => 'OpenHouse', :class => 'OPH', :select => ['ID'] , :query => "(DATE_MODIFIED=#{date_modified.strftime("%FT%T")}+)" , :standard_names_only => true, :timeout => -1 }) { |data| self.delay(:priority => 10).import_open_house( data['ID' ], save_images) } end
146
-
123
+ self.update_helper('RES', date_modified, save_images)
124
+ self.update_helper('COM', date_modified, save_images)
125
+ self.update_helper('LND', date_modified, save_images)
126
+ self.update_helper('MUL', date_modified, save_images)
127
+ self.update_helper('OFF', date_modified, save_images)
128
+ self.update_helper('AGT', date_modified, save_images)
129
+ self.update_helper('OPH', date_modified, save_images)
130
+ end
131
+
132
+ def self.update_helper(class_type, date_modified, save_images = true)
133
+ m = self.meta(class_type)
134
+ k = m.remote_key_field
135
+ params = {
136
+ :search_type => m.search_type,
137
+ :class => class_type,
138
+ :select => [m.remote_key_field],
139
+ :query => "(#{m.date_modified_field}=#{date_modified.strftime("%FT%T")}+)",
140
+ :standard_names_only => true,
141
+ :timeout => -1
142
+ }
143
+ self.client.search(params) do |data|
144
+ case class_type
145
+ when 'RES' then self.delay(:priority => 10).import_residential_property( data[k], save_images)
146
+ when 'COM' then self.delay(:priority => 10).import_commercial_property( data[k], save_images)
147
+ when 'LND' then self.delay(:priority => 10).import_land_property( data[k], save_images)
148
+ when 'MUL' then self.delay(:priority => 10).import_multi_family_property( data[k], save_images)
149
+ when 'OFF' then self.delay(:priority => 10).import_office( data[k], save_images)
150
+ when 'AGT' then self.delay(:priority => 10).import_agent( data[k], save_images)
151
+ when 'OPH' then self.delay(:priority => 10).import_open_house( data[k], save_images)
152
+ end
153
+ end
154
+ end
155
+
147
156
  #=============================================================================
148
157
  # Single model import methods (called from a worker dyno)
149
158
  #=============================================================================
150
-
151
- def self.import_property(mls_acct)
152
- self.import("(MLS_ACCT=*#{mls_acct}*)", 'Property', 'RES')
159
+
160
+ def self.import_property(mls_acct, save_images = true)
161
+ self.import('RES', "(MLS_ACCT=*#{mls_acct}*)")
153
162
  p = CabooseRets::ResidentialProperty.where(:id => mls_acct.to_i).first
154
163
  if p.nil?
155
- self.import("(MLS_ACCT=*#{mls_acct}*)", 'Property', 'COM')
156
- p = CabooseRets::CommercialProperty.where(:id => mls_acct.to_i).first
164
+ self.import('COM', "(MLS_ACCT=*#{mls_acct}*)")
165
+ p = CabooseRets::CommercialProperty.where(:id => mls_acct.to_i).first
157
166
  if p.nil?
158
- self.import("(MLS_ACCT=*#{mls_acct}*)", 'Property', 'LND')
167
+ self.import('LND', "(MLS_ACCT=*#{mls_acct}*)")
159
168
  p = CabooseRets::LandProperty.where(:id => mls_acct.to_i).first
160
169
  if p.nil?
161
- self.import("(MLS_ACCT=*#{mls_acct}*)", 'Property', 'MUL')
170
+ self.import('MUL', "(MLS_ACCT=*#{mls_acct}*)")
162
171
  p = CabooseRets::MultiFamilyProperty.where(:id => mls_acct.to_i).first
163
172
  return if p.nil?
164
173
  end
165
174
  end
166
175
  end
167
- self.download_property_images(p)
168
- end
169
-
170
- def self.import_residential_property(mls_acct, save_images = true)
171
- self.import("(MLS_ACCT=*#{mls_acct}*)", 'Property', 'RES')
172
- p = CabooseRets::ResidentialProperty.where(:id => mls_acct.to_i).first
173
- self.download_property_images(p, save_images)
174
- self.update_coords(p)
175
- end
176
-
177
- def self.import_commercial_property(mls_acct, save_images = true)
178
- self.import("(MLS_ACCT=*#{mls_acct}*)", 'Property', 'COM')
176
+ self.download_property_images(p, save_images)
177
+ end
178
+
179
+ def self.import_residential_property(mls_acct, save_images = true)
180
+ self.import('RES', "(MLS_ACCT=*#{mls_acct}*)")
181
+ p = CabooseRets::ResidentialProperty.where(:id => mls_acct.to_i).first
182
+ self.download_property_images(p, save_images)
183
+ self.update_coords(p)
184
+ end
185
+
186
+ def self.import_commercial_property(mls_acct, save_images = true)
187
+ self.import('COM', "(MLS_ACCT=*#{mls_acct}*)")
179
188
  p = CabooseRets::CommercialProperty.where(:id => mls_acct.to_i).first
180
189
  self.download_property_images(p, save_images)
181
190
  self.update_coords(p)
182
191
  end
183
-
184
- def self.import_land_property(mls_acct, save_images = true)
185
- self.import("(MLS_ACCT=*#{mls_acct}*)", 'Property', 'LND')
186
- p = CabooseRets::LandProperty.where(:id => mls_acct.to_i).first
192
+
193
+ def self.import_land_property(mls_acct, save_images = true)
194
+ self.import('LND', "(MLS_ACCT=*#{mls_acct}*)")
195
+ p = CabooseRets::LandProperty.where(:id => mls_acct.to_i).first
187
196
  self.download_property_images(p, save_images)
188
197
  self.update_coords(p)
189
198
  end
190
-
191
- def self.import_multi_family_property(mls_acct, save_images = true)
192
- self.import("(MLS_ACCT=*#{mls_acct}*)", 'Property', 'MUL')
199
+
200
+ def self.import_multi_family_property(mls_acct, save_images = true)
201
+ self.import('MUL', "(MLS_ACCT=*#{mls_acct}*)")
193
202
  p = CabooseRets::MultiFamilyProperty.where(:id => mls_acct.to_i).first
194
203
  self.download_property_images(p, save_images)
195
204
  self.update_coords(p)
196
205
  end
197
-
206
+
198
207
  def self.import_office(lo_code, save_images = true)
199
- self.import("(LO_LO_CODE=*#{lo_code}*)", 'Office', 'OFF')
208
+ self.import('OFF', "(LO_LO_CODE=*#{lo_code}*)")
200
209
  office = CabooseRets::Office.where(:lo_code => lo_code.to_s).first
201
210
  self.download_office_image(office) if save_images == true
202
211
  end
203
-
212
+
204
213
  def self.import_agent(la_code, save_images = true)
205
- self.import("(LA_LA_CODE=*#{la_code}*)", 'Agent', 'AGT')
214
+ self.import('AGT', "(LA_LA_CODE=*#{la_code}*)")
206
215
  a = CabooseRets::Agent.where(:la_code => la_code.to_s).first
207
216
  self.download_agent_image(a) if save_images == true
208
217
  end
209
-
218
+
210
219
  def self.import_open_house(id, save_images = true)
211
- self.import("(ID=*#{id}*)", 'OpenHouse', 'OPH')
220
+ self.import('OPH', "(ID=*#{id}*)")
221
+ end
222
+
223
+ def self.import_media(id, save_images = true)
224
+ self.import('GFX', "(MEDIA_ID=*#{id}*)")
212
225
  end
213
226
 
214
227
  #=============================================================================
215
228
  # Images
216
229
  #=============================================================================
217
-
218
- def self.download_property_images(p, save_images = true)
230
+
231
+ def self.download_property_images(p, save_images = true)
219
232
  self.refresh_property_media(p)
220
233
  return if save_images == false
221
-
234
+
222
235
  self.log("-- Downloading images and resizing for #{p.mls_acct}")
223
236
  media = []
224
237
  self.client.get_object(:resource => :Property, :type => :Photo, :location => true, :id => p.id) do |headers, content|
225
-
238
+
226
239
  # Find the associated media record for the image
227
240
  filename = File.basename(headers['location'])
228
241
  m = CabooseRets::Media.where(:mls_acct => p.mls_acct, :file_name => filename).first
229
-
242
+
230
243
  if m.nil?
231
244
  self.log("Can't find media record for #{p.mls_acct} #{filename}.")
232
- else
245
+ else
233
246
  m.image = URI.parse(headers['location'])
234
247
  media << m
235
248
  #m.save
236
- end
249
+ end
237
250
  end
238
-
251
+
239
252
  self.log("-- Uploading images to S3 for #{p.mls_acct}")
240
- media.each do |m|
253
+ media.each do |m|
241
254
  m.save
242
- end
255
+ end
243
256
  end
244
-
257
+
245
258
  def self.refresh_property_media(p)
246
- self.log("-- Deleting images and metadata for #{p.mls_acct}...")
259
+ self.log("-- Deleting images and metadata for #{p.mls_acct}...")
247
260
  #CabooseRets::Media.where(:mls_acct => p.mls_acct, :media_type => 'Photo').destroy_all
248
261
  CabooseRets::Media.where(:mls_acct => p.mls_acct).destroy_all
249
-
250
- self.log("-- Downloading image metadata for #{p.mls_acct}...")
262
+
263
+ self.log("-- Downloading image metadata for #{p.mls_acct}...")
251
264
  params = {
252
265
  :search_type => 'Media',
253
266
  :class => 'GFX',
254
267
  #:query => "(MLS_ACCT=*#{p.id}*),(MEDIA_TYPE=|I)",
255
268
  :query => "(MLS_ACCT=*#{p.id}*)",
256
269
  :timeout => -1
257
- }
258
- self.client.search(params) do |data|
270
+ }
271
+ self.client.search(params) do |data|
259
272
  m = CabooseRets::Media.new
260
273
  m.parse(data)
261
274
  #m.id = m.media_id
262
275
  m.save
263
276
  end
264
277
  end
265
-
266
- def self.download_agent_image(agent)
278
+
279
+ def self.download_agent_image(agent)
267
280
  self.log "Saving image for #{agent.first_name} #{agent.last_name}..."
268
281
  begin
269
282
  self.client.get_object(:resource => :Agent, :type => :Photo, :location => true, :id => agent.la_code) do |headers, content|
@@ -272,9 +285,9 @@ class CabooseRets::RetsImporter # < ActiveRecord::Base
272
285
  end
273
286
  rescue RETS::APIError => err
274
287
  self.log "No image for #{agent.first_name} #{agent.last_name}."
275
- end
288
+ end
276
289
  end
277
-
290
+
278
291
  def self.download_office_image(office)
279
292
  self.log "Saving image for #{office.lo_name}..."
280
293
  begin
@@ -282,58 +295,58 @@ class CabooseRets::RetsImporter # < ActiveRecord::Base
282
295
  office.image = URI.parse(headers['location'])
283
296
  office.save
284
297
  end
285
- rescue RETS::APIError => err
298
+ rescue RETS::APIError => err
286
299
  self.log "No image for #{office.lo_name}."
287
- end
300
+ end
288
301
  end
289
302
 
290
303
  #=============================================================================
291
304
  # GPS
292
305
  #=============================================================================
293
-
294
- def self.update_coords(p = nil)
306
+
307
+ def self.update_coords(p = nil)
295
308
  if p.nil?
296
309
  models = [CabooseRets::CommercialProperty, CabooseRets::LandProperty, CabooseRets::MultiFamilyProperty, CabooseRets::ResidentialProperty]
297
310
  names = ["commercial", "land", "multi-family", "residential"]
298
311
  i = 0
299
- models.each do |model|
312
+ models.each do |model|
300
313
  self.log "Updating coords #{names[i]} properties..."
301
314
  model.where(:latitude => nil).reorder(:mls_acct).each do |p|
302
- self.update_coords(p)
315
+ self.update_coords(p)
303
316
  end
304
317
  i = i + 1
305
318
  end
306
319
  return
307
320
  end
308
-
321
+
309
322
  self.log "Getting coords for mls_acct #{p.mls_acct}..."
310
323
  coords = self.coords_from_address(CGI::escape "#{p.street_num} #{p.street_name}, #{p.city}, #{p.state} #{p.zip}")
311
324
  return if coords.nil? || coords == false
312
-
325
+
313
326
  p.latitude = coords['lat']
314
327
  p.longitude = coords['lng']
315
- p.save
328
+ p.save
316
329
  end
317
-
318
- def self.coords_from_address(address)
330
+
331
+ def self.coords_from_address(address)
319
332
  begin
320
333
  uri = "https://maps.googleapis.com/maps/api/geocode/json?address=#{address}&sensor=false"
321
- uri.gsub!(" ", "+")
334
+ uri.gsub!(" ", "+")
322
335
  resp = HTTParty.get(uri)
323
336
  json = JSON.parse(resp.body)
324
- return json['results'][0]['geometry']['location']
337
+ return json['results'][0]['geometry']['location']
325
338
  rescue
326
339
  self.log "Error: #{uri}"
327
340
  sleep(2)
328
- return false
341
+ return false
329
342
  end
330
343
  end
331
-
344
+
332
345
  #=============================================================================
333
346
  # Purging
334
347
  #=============================================================================
335
-
336
- def self.purge
348
+
349
+ def self.purge
337
350
  self.purge_residential
338
351
  self.purge_commercial
339
352
  self.purge_land
@@ -343,56 +356,89 @@ class CabooseRets::RetsImporter # < ActiveRecord::Base
343
356
  self.purge_open_houses
344
357
  self.purge_media
345
358
  end
346
-
347
- def self.purge_residential() self.purge_helper('Property' , 'RES', 'MLS_ACCT' , 'DATE_MODIFIED' , '2012-01-01', "delete from rets_residential where mls_acct not in (?)") end
348
- def self.purge_commercial() self.purge_helper('Property' , 'COM', 'MLS_ACCT' , 'DATE_MODIFIED' , '2012-01-01', "delete from rets_commercial where mls_acct not in (?)") end
349
- def self.purge_land() self.purge_helper('Property' , 'LND', 'MLS_ACCT' , 'DATE_MODIFIED' , '2012-01-01', "delete from rets_land where mls_acct not in (?)") end
350
- def self.purge_multi_family() self.purge_helper('Property' , 'MUL', 'MLS_ACCT' , 'DATE_MODIFIED' , '2012-01-01', "delete from rets_multi_family where mls_acct not in (?)") end
351
- def self.purge_offices() self.purge_helper('Office' , 'OFF', 'LO_LO_CODE' , 'LO_DATE_MODIFIED' , '2012-01-01', "delete from rets_offices where lo_code not in (?)") end
352
- def self.purge_agents() self.purge_helper('Agent' , 'AGT', 'LA_LA_CODE' , 'LA_DATE_MODIFIED' , '2012-01-01', "delete from rets_agents where la_code not in (?)") end
353
- def self.purge_open_houses() self.purge_helper('OpenHouse' , 'OPH', 'ID' , 'DATE_MODIFIED' , '2012-01-01', "delete from rets_open_houses where id not in (?)") end
354
- def self.purge_media() self.purge_helper('Media' , 'GFX', 'MEDIA_ID' , 'DATE_MODIFIED' , '2012-01-01', "delete from rets_media where media_id not in (?)") end
355
-
356
- def self.purge_helper(search_type, class_type, key_field, date_modified_field, date_modified, delete_query)
357
-
359
+
360
+ def self.purge_residential() self.purge_helper('RES', '2012-01-01') end
361
+ def self.purge_commercial() self.purge_helper('COM', '2012-01-01') end
362
+ def self.purge_land() self.purge_helper('LND', '2012-01-01') end
363
+ def self.purge_multi_family() self.purge_helper('MUL', '2012-01-01') end
364
+ def self.purge_offices() self.purge_helper('OFF', '2012-01-01') end
365
+ def self.purge_agents() self.purge_helper('AGT', '2012-01-01') end
366
+ def self.purge_open_houses() self.purge_helper('OPH', '2012-01-01') end
367
+ def self.purge_media() self.purge_helper('GFX', '2012-01-01') end
368
+
369
+ def self.purge_helper(class_type, date_modified)
370
+ m = self.meta(class_type)
371
+
358
372
  # Get the total number of records
359
- params = { :search_type => search_type, :class => class_type, :query => "(#{date_modified_field}=#{date_modified}T00:00:01+)", :standard_names_only => true, :timeout => -1 }
373
+ params = {
374
+ :search_type => m.search_type,
375
+ :class => class_type,
376
+ :query => "(#{m.date_modified_field}=#{date_modified}T00:00:01+)",
377
+ :standard_names_only => true,
378
+ :timeout => -1
379
+ }
360
380
  self.client.search(params.merge({ :count_mode => :only }))
361
- count = self.client.rets_data[:code] == "20201" ? 0 : self.client.rets_data[:count]
381
+ count = self.client.rets_data[:code] == "20201" ? 0 : self.client.rets_data[:count]
362
382
  batch_count = (count.to_f/5000.0).ceil
363
383
 
364
384
  ids = []
365
- (0...batch_count).each do |i|
366
- self.client.search(params.merge({ :select => [key_field], :limit => 5000, :offset => 5000*i })){ |data| ids << data[key_field] }
367
- end
368
- ActiveRecord::Base.connection.execute(ActiveRecord::Base.send(:sanitize_sql_array, [delete_query, ids]))
385
+ k = m.remote_key_field
386
+ (0...batch_count).each do |i|
387
+ self.client.search(params.merge({ :select => [k], :limit => 5000, :offset => 5000*i })){ |data| ids << data[k] }
388
+ end
389
+
390
+ # Delete any records in the local database that shouldn't be there
391
+ t = m.local_table
392
+ k = m.local_key_field
393
+ query = ["delete from #{t} where #{k} not in (?)", ids]
394
+ ActiveRecord::Base.connection.execute(ActiveRecord::Base.send(:sanitize_sql_array, query))
395
+
396
+ # Find any ids in the remote database that should be in the local database
397
+ query = "select distinct #{k} from #{t}"
398
+ rows = ActiveRecord::Base.connection.select_all(ActiveRecord::Base.send(:sanitize_sql_array, query))
399
+ local_ids = rows.collect{ |row| row[k] }
400
+ ids_to_add = ids - local_ids
401
+ ids_to_add.each do |id|
402
+ puts "Importing #{id}..."
403
+ case class_type
404
+ when 'RES' then self.delay.import_residential_property(id)
405
+ when 'COM' then self.delay.import_commercial_property(id)
406
+ when 'LND' then self.delay.import_land_property(id)
407
+ when 'MUL' then self.delay.import_multi_family_property(id)
408
+ when 'OFF' then self.delay.import_office(id)
409
+ when 'AGT' then self.delay.import_agent(id)
410
+ when 'OPH' then self.delay.import_open_house(id)
411
+ when 'GFX' then self.delay.import_media(id)
412
+ end
413
+ end
414
+
369
415
  end
370
416
 
371
417
  #=============================================================================
372
418
  # Logging
373
419
  #=============================================================================
374
-
420
+
375
421
  def self.log(msg)
376
422
  #puts "[rets_importer] #{msg}"
377
423
  Rails.logger.info("[rets_importer] #{msg}")
378
- end
379
-
424
+ end
425
+
380
426
  #=============================================================================
381
427
  # Locking update task
382
428
  #=============================================================================
383
-
384
- def self.update_rets
429
+
430
+ def self.update_rets
385
431
  return if self.task_is_locked
386
432
  task_started = self.lock_task
387
-
433
+
388
434
  begin
389
- overlap = 30.seconds
390
- if (DateTime.now - self.last_purged).to_i > 1
435
+ overlap = 30.seconds
436
+ if (DateTime.now - self.last_purged).to_i > 1
391
437
  self.purge
392
438
  self.save_last_purged(task_started)
393
439
  overlap = 1.month
394
- end
395
- self.update_after(self.last_updated - overlap)
440
+ end
441
+ self.update_after(self.last_updated - overlap)
396
442
  self.save_last_updated(task_started)
397
443
  self.unlock_task
398
444
  rescue
@@ -400,11 +446,11 @@ class CabooseRets::RetsImporter # < ActiveRecord::Base
400
446
  ensure
401
447
  self.unlock_task_if_last_updated(task_started)
402
448
  end
403
-
449
+
404
450
  # Start the same update process in five minutes
405
- self.delay(:run_at => 1.minutes.from_now).update_rets
451
+ self.delay(:run_at => 1.minutes.from_now).update_rets
406
452
  end
407
-
453
+
408
454
  def self.last_updated
409
455
  if !Caboose::Setting.exists?(:name => 'rets_last_updated')
410
456
  Caboose::Setting.create(:name => 'rets_last_updated', :value => '2013-08-06T00:00:01')
@@ -412,7 +458,7 @@ class CabooseRets::RetsImporter # < ActiveRecord::Base
412
458
  s = Caboose::Setting.where(:name => 'rets_last_updated').first
413
459
  return DateTime.parse(s.value)
414
460
  end
415
-
461
+
416
462
  def self.last_purged
417
463
  if !Caboose::Setting.exists?(:name => 'rets_last_purged')
418
464
  Caboose::Setting.create(:name => 'rets_last_purged', :value => '2013-08-06T00:00:01')
@@ -420,36 +466,36 @@ class CabooseRets::RetsImporter # < ActiveRecord::Base
420
466
  s = Caboose::Setting.where(:name => 'rets_last_purged').first
421
467
  return DateTime.parse(s.value)
422
468
  end
423
-
469
+
424
470
  def self.save_last_updated(d)
425
471
  s = Caboose::Setting.where(:name => 'rets_last_updated').first
426
472
  s.value = d.strftime('%FT%T')
427
473
  s.save
428
474
  end
429
-
475
+
430
476
  def self.save_last_purged(d)
431
477
  s = Caboose::Setting.where(:name => 'rets_last_purged').first
432
478
  s.value = d.strftime('%FT%T')
433
479
  s.save
434
480
  end
435
-
481
+
436
482
  def self.task_is_locked
437
483
  return Caboose::Setting.exists?(:name => 'rets_update_running')
438
484
  end
439
-
485
+
440
486
  def self.lock_task
441
487
  d = DateTime.now.utc - 5.hours
442
488
  Caboose::Setting.create(:name => 'rets_update_running', :value => d.strftime('%F %T'))
443
489
  return d
444
490
  end
445
-
491
+
446
492
  def self.unlock_task
447
493
  Caboose::Setting.where(:name => 'rets_update_running').first.destroy
448
494
  end
449
-
495
+
450
496
  def self.unlock_task_if_last_updated(d)
451
497
  setting = Caboose::Setting.where(:name => 'rets_update_running').first
452
498
  self.unlock_task if setting && d.strftime('%F %T') == setting.value
453
499
  end
454
-
500
+
455
501
  end
data/lib/caboose_rets/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  module CabooseRets
2
- VERSION = '0.0.62'
2
+ VERSION = '0.0.63'
3
3
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: caboose-rets
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.62
4
+ version: 0.0.63
5
5
  platform: ruby
6
6
  authors:
7
7
  - William Barry