reso 0.1.0.0 → 0.1.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f40fc790012ddccc71df15ef4ded0e4e2da9c158
-  data.tar.gz: e83245209c33c892e5db60621b33409663610ca5
+  metadata.gz: ff149e91987ac675f1b345183c70e9a1e887cac9
+  data.tar.gz: e01f8ebe524c4f0a61cb25ea7f0a87e8d6856ba4
 SHA512:
-  metadata.gz: c9d4a8f7591292f3f68a4b1ec15fe90498250d76ea57214ec32cded2c661839910691491af8d877da43df6da7600d33bafccfcd2ca29c28e8e701c4ae22f885c
-  data.tar.gz: fdbacc52993e69b3260051e3e53bf05795ad0860f990bc17377b6ed62abc6496e6539a1aa18f4630b468c2a7e89d2823e33c6f4eaadb78fce6d4cd79dd9c5ee6
+  metadata.gz: 7fc1601255520d7219d6fa70c278a16f33d61ba1a1da68c645fe44456854b42790dceb863a26caa48d5f124288b0729b9ac4d856053533fa40e60873cc4b3d49
+  data.tar.gz: 2241c34f18b659df0c4f6184d91289332c72986121efd511208c3cea2d31c6bedde8923fd505b95c6f5c046c3cbce7d78ef4e0d580490a84116bf61d40b07fc6

data/app/models/queued_listing.rb

@@ -8,9 +8,14 @@ class QueuedListing < ActiveRecord::Base
   end
 
   def create_or_update_listing
-    listing = self.import.listings.eager_load(:address).eager_load(:photos).find_or_initialize_by(
-      listing_key: mapper.unique_identifier(self)
-    )
+    listing = self.import.listings.
+      eager_load(:address).
+      eager_load(:appliances).
+      eager_load(:participants).
+      eager_load(:photos).
+      find_or_initialize_by(
+        listing_key: mapper.unique_identifier(self)
+      )
     if (listing.modification_timestamp != mapper.modification_timestamp(self, listing))
       Mapper::RESO_LISTING_ATTRIBUTES.each do |attribute|
         listing.send("#{attribute}=", mapper.send(attribute, self, listing))

data/lib/class_extensions.rb

@@ -21,16 +21,14 @@ end
 Hash.class_eval do
   def drilldown drillees
     if (result = drillees.split(' ').inject(self){|res, el| res[el] ? res[el] : Hash.new })
-      result.present? ? (result.unwrap_attribute) : nil
+      result.present? ? result.unwrap_attribute : nil
     end
   end
 end
 
 Array.class_eval do
   def drilldown drillee
-    if (result = drillee.last)
-      result.present? ? (result.unwrap_attribute) : nil
-    end
+    drillee.last.unwrap_attribute
   end
 end
 

data/lib/generators/reso/templates/create_addresses.rb

@@ -2,7 +2,7 @@ class CreateAddresses < ActiveRecord::Migration
   def change
     create_table :addresses, options: 'DEFAULT CHARSET=utf8' do |t|
       t.string :addressable_type
-      t.references :addressable
+      t.references :addressable, index: true
 
       t.integer :preference_order, default: 1 
       t.integer :address_preference_order, default: 1
@@ -28,6 +28,6 @@ class CreateAddresses < ActiveRecord::Migration
     add_index :addresses, :state_or_province
     add_index :addresses, :postal_code
     add_index :addresses, :country
-    add_index :addresses, [:addressable_id, :addressable_type]
+    add_index :addresses, :addressable_type
   end
 end

data/lib/generators/reso/templates/create_businesses.rb

@@ -14,6 +14,5 @@ class CreateBusinesses < ActiveRecord::Migration
     end
     add_index :businesses, :name
     add_index :businesses, :type
-    add_index :businesses, [:name, :type]
   end
 end

data/lib/generators/reso/templates/create_enumerations.rb

@@ -1,11 +1,13 @@
 class CreateEnumerations < ActiveRecord::Migration
   def change
     create_table :enumerations, options: 'DEFAULT CHARSET=utf8' do |t|
-      t.string :name, null: false, index: true, limit: 128
-      t.string :type, null: false, index: true
+      t.string :name, null: false, limit: 128
+      t.string :type, null: false
 
       t.timestamps
     end
+    add_index :enumerations, :name
+    add_index :enumerations, :type
     add_index :enumerations, [:name, :type]
   end
 end

data/lib/generators/reso/templates/create_listing_media.rb

@@ -4,7 +4,7 @@ class CreateListingMedia < ActiveRecord::Migration
       t.integer :media_order_number
       t.string :media_url, null: false, limit: 512 
       t.string :media_caption
-      t.text :media_description
+      t.string :media_description
       t.string :media_modification_timestamp
       t.string :type, null: false
       t.references :listing, index: true, null: false, foreign_key: true 
@@ -13,7 +13,6 @@ class CreateListingMedia < ActiveRecord::Migration
     end
     add_index :listing_media, :media_order_number
     add_index :listing_media, :type
-    add_index :listing_media, [:listing_id, :media_order_number]
-    add_index :listing_media, [:media_url, :media_modification_timestamp], name: "on_url_and_timestamp"
+    add_index :listing_media, [:media_url, :media_modification_timestamp, :media_order_number, :media_caption, :media_description], name: "on_url_and_timestamp"
   end
 end

data/lib/generators/reso/templates/create_listing_providers.rb

@@ -7,6 +7,7 @@ class CreateListingProviders < ActiveRecord::Migration
 
       t.timestamps
     end
-    add_index :listing_providers, [:name, :source_provider_category_id, :url], name: "index_provider_on_name_and_category_and_url"
+    add_index :listing_providers, :name
+    add_index :listing_providers, :url
   end
 end

data/lib/generators/reso/templates/create_listings.rb

@@ -106,49 +106,6 @@ class CreateListings < ActiveRecord::Migration
     add_index :listings, :list_price
     add_index :listings, :listing_key
     add_index :listings, :mls_number
-    add_index :listings, :bedrooms
-    add_index :listings, :bathrooms
-    add_index :listings, :has_attic
-    add_index :listings, :has_barbecue_area
-    add_index :listings, :has_basement
-    add_index :listings, :has_ceiling_fan
-    add_index :listings, :has_deck
-    add_index :listings, :has_disabled_access
-    add_index :listings, :has_dock
-    add_index :listings, :has_doorman
-    add_index :listings, :has_double_pane_windows
-    add_index :listings, :has_elevator
-    add_index :listings, :has_fireplace
-    add_index :listings, :has_garden
-    add_index :listings, :has_gated_entry
-    add_index :listings, :has_greenhouse
-    add_index :listings, :has_hot_tub_spa
-    add_index :listings, :has_intercom
-    add_index :listings, :has_jetted_bath_tub
-    add_index :listings, :has_lawn
-    add_index :listings, :has_mother_in_law
-    add_index :listings, :has_patio
-    add_index :listings, :has_pond
-    add_index :listings, :has_pool
-    add_index :listings, :has_porch
-    add_index :listings, :has_rv_parking
-    add_index :listings, :has_sauna
-    add_index :listings, :has_security_system
-    add_index :listings, :has_skylight
-    add_index :listings, :has_sports_court
-    add_index :listings, :has_sprinkler_system
-    add_index :listings, :has_vaulted_ceiling
-    add_index :listings, :has_wet_bar
-    add_index :listings, :is_cable_ready
-    add_index :listings, :is_new_construction
-    add_index :listings, :is_waterfront
-    add_index :listings, :is_wired
-    add_index :listings, :living_area
-    add_index :listings, :lot_size
-    add_index :listings, :listing_date
-    add_index :listings, :latitude
-    add_index :listings, :longitude
     add_index :listings, :originating_system_key
-    add_index :listings, :year_built
   end
 end

data/lib/generators/reso/templates/create_participants.rb

@@ -6,7 +6,7 @@ class CreateParticipants < ActiveRecord::Migration
       t.string :participant_code
       t.string :first_name
       t.string :last_name
-      t.references :person, null: true, foreign_key: true
+      t.references :person, index:true, foreign_key: true
       t.references :participant_role, index: true
       t.string :primary_contact_phone
       t.string :office_phone
@@ -18,9 +18,9 @@ class CreateParticipants < ActiveRecord::Migration
 
       t.timestamps
     end
-    add_index :participants, :participant_key
+    add_index :participants, :email
     add_index :participants, :participant_identifier
-    add_index :participants, :person_id
+    add_index :participants, :participant_key
     add_index :participants, [:first_name, :last_name, :email]
   end
 end

data/lib/mapper.rb

@@ -111,4 +111,121 @@ module Mapper
     year_built
     year_updated
   )
+
+  def self.fetch_enumerations class_name
+    Rails.cache.fetch(class_name, expires_in: 1.hours) do
+      enums = {}
+      class_name.constantize.all.map{|item| enums[item.name] = item}
+      enums
+    end
+  end
+
+  def self.address_types
+    fetch_enumerations 'AddressType'
+  end
+  
+  def self.appliances
+    fetch_enumerations 'Appliance'
+  end
+
+  def self.architecture_styles
+    fetch_enumerations 'ArchitectureStyle'
+  end
+  
+  def self.area_units
+    fetch_enumerations 'AreaUnit'
+  end
+  
+  def self.cooling_systems
+    fetch_enumerations 'CoolingSystem'
+  end
+  
+  def self.currency_periods
+    fetch_enumerations 'CurrencyPeriod'
+  end
+  
+  def self.expense_categories
+    fetch_enumerations 'ExpenseCategory'
+  end
+  
+  def self.exterior_types
+    fetch_enumerations 'ExteriorType'
+  end
+  
+  def self.floor_coverings
+    fetch_enumerations 'FloorCovering'
+  end
+  
+  def self.foreclosure_statuses
+    fetch_enumerations 'ForeclosureStatus'
+  end
+  
+  def self.gender
+    fetch_enumerations 'Gender'
+  end
+  
+  def self.heating_fuels
+    fetch_enumerations 'HeatingFuel'
+  end
+  
+  def self.heating_systems
+    fetch_enumerations 'HeatingSystem'
+  end
+  
+  def self.home_features
+    fetch_enumerations 'HomeFeature'
+  end
+  
+  def self.import_formats
+    fetch_enumerations 'ImportFormat'
+  end
+  
+  def self.license_categories
+    fetch_enumerations 'LicenseCategory'
+  end
+  
+  def self.listing_categories
+    fetch_enumerations 'ListingCategory'
+  end
+  
+  def self.listing_statuses
+    fetch_enumerations 'ListingStatus'
+  end
+  
+  def self.parkings
+    fetch_enumerations 'Parking'
+  end
+  
+  def self.participant_roles
+    fetch_enumerations 'ParticipantRole'
+  end
+  
+  def self.property_sub_types
+    fetch_enumerations 'PropertySubType'
+  end
+  
+  def self.property_types
+    fetch_enumerations 'PropertyType'
+  end
+  
+  def self.roof_types
+    fetch_enumerations 'RoofType'
+  end
+  
+  def self.room_categories
+    fetch_enumerations 'RoomCategory'
+  end
+  
+  def self.school_categories
+    fetch_enumerations 'SchoolCategory'
+  end
+  
+  def self.source_provider_categories
+    fetch_enumerations 'SourceProviderCategory'
+  end
+  
+  def self.view_types
+    fetch_enumerations 'ViewType'
+  end
+
 end

data/lib/mapper/reso.rb

@@ -34,23 +34,23 @@ module Mapper
     end
 
     def self.appliances queued_listing, listing
-      (result = get_enums(queued_listing, %w(DetailedCharacteristics Appliances Appliance))) ? result : nil
+      get_enums(queued_listing, %w(DetailedCharacteristics Appliances Appliance))
     end
     
     def self.architecture_style queued_listing, listing
-      (result = unwrap_attribute(get_value(queued_listing, %w(DetailedCharacteristics ArchitectureStyle)))) ? ArchitectureStyle.find_by_name(result) : nil
+      (result = get_value(queued_listing, %w(DetailedCharacteristics ArchitectureStyle))) ? Mapper::architecture_styles[result] : nil
     end
     
     def self.architecture_style_description queued_listing, listing
-      (result = get_value(queued_listing, %w(DetailedCharacteristics ArchitectureStyle otherDescription))) ? result : nil
+      get_value(queued_listing, %w(DetailedCharacteristics ArchitectureStyle otherDescription))
     end
     
     def self.bathrooms queued_listing, listing
-      (result = get_value(queued_listing, %w(Bathrooms))) ? result.to_i : nil
+      get_value(queued_listing, %w(Bathrooms))
     end
 
     def self.bedrooms queued_listing, listing
-      (result = get_value(queued_listing, %w(Bedrooms))) ? result.to_i : nil
+      get_value(queued_listing, %w(Bedrooms))
     end
 
     def self.brokerage queued_listing, listing
@@ -62,20 +62,20 @@ module Mapper
     end
 
     def self.building_unit_count queued_listing, listing
-      (result = get_value(queued_listing, %w(DetailedCharacteristics BuildingUnitCount))) ? result.to_i : nil
+      get_value(queued_listing, %w(DetailedCharacteristics BuildingUnitCount))
     end
     
     def self.condo_floor_num queued_listing, listing
-      (result = get_value(queued_listing, %w(DetailedCharacteristics CondoFloorNum))) ? result.to_i : nil
+      get_value(queued_listing, %w(DetailedCharacteristics CondoFloorNum))
     end
     
     def self.cooling_systems queued_listing, listing
-      (result = get_enums(queued_listing, %w(DetailedCharacteristics CoolingSystems CoolingSystem))) ? result : nil
+      get_enums(queued_listing, %w(DetailedCharacteristics CoolingSystems CoolingSystem))
     end
     
     def self.county queued_listing, listing
       if (get_value(queued_listing, %w(Location County)))
-        County.find_or_initialize_by(
+        county = County.find_or_initialize_by(
           name: get_value(queued_listing, %w(Location County)),
           state_or_province: get_value(queued_listing, %w(Address StateOrProvince)),
           country: get_value(queued_listing, %w(Address Country))
@@ -86,45 +86,45 @@ module Mapper
     end
 
     def self.currency_code queued_listing, listing
-      (result = get_value(queued_listing, %w(ListPrice currencyCode))) ? result : nil
+      get_value(queued_listing, %w(ListPrice currencyCode))
     end
     
     def self.directions queued_listing, listing
-      (result = get_value(queued_listing, %w(Location Directions))) ? result : nil
+      get_value(queued_listing, %w(Location Directions))
     end
 
     def self.disclaimer queued_listing, listing
-      (result = get_value(queued_listing, %w(Disclaimer))) ? result : nil
+      get_value(queued_listing, %w(Disclaimer))
     end
 
     def self.disclose_address queued_listing, listing
-      (result = get_value(queued_listing, %w(DiscloseAddress))) ? result.to_s.to_bool : nil
+      get_boolean_value(queued_listing, %w(DiscloseAddress))
     end
     
     def self.elevation queued_listing, listing
-      (result = get_value(queued_listing, %w(Location Elevation))) ? result : nil
+      get_value(queued_listing, %w(Location Elevation))
     end
 
     def self.expenses queued_listing, listing
       if (result = get_repeaters(queued_listing, %w(Expenses Expense)))
         result.map{|item| listing.expenses.find_or_initialize_by(
-          expense_category: ExpenseCategory.find_by_name(item['ExpenseCategory']),
-          currency_period: CurrencyPeriod.find_by_name(item['ExpenseValue']['currencyPeriod']),
-          expense_value: unwrap_attribute(item['ExpenseValue'])
+          expense_category: Mapper::expense_categories[item['ExpenseCategory']],
+          currency_period: Mapper::expense_categories[item['ExpenseValue']['currencyPeriod']],
+          expense_value: item['ExpenseValue'].unwrap_attribute
         )}
       end
     end
 
     def self.exterior_types queued_listing, listing
-      (result = get_enums(queued_listing, %w(DetailedCharacteristics ExteriorTypes ExteriorType))) ? result : nil
+      get_enums(queued_listing, %w(DetailedCharacteristics ExteriorTypes ExteriorType))
     end
     
     def self.floor_coverings queued_listing, listing
-      (result = get_enums(queued_listing, %w(DetailedCharacteristics FloorCoverings FloorCovering))) ? result : nil
+      get_enums(queued_listing, %w(DetailedCharacteristics FloorCoverings FloorCovering))
     end
     
     def self.foreclosure_status queued_listing, listing
-      (result = get_value(queued_listing, %w(ForeclosureStatus))) ? ForeclosureStatus.find_by(name: result) : nil
+      (result = get_value(queued_listing, %w(ForeclosureStatus))) ? Mapper::foreclosure_statuses[result] : nil
     end
     
     def self.franchise queued_listing, listing
@@ -132,15 +132,15 @@ module Mapper
     end
 
     def self.full_bathrooms queued_listing, listing
-      (result = get_value(queued_listing, %w(FullBathrooms))) ? result.to_i : nil
+      get_value(queued_listing, %w(FullBathrooms))
     end
 
     def self.geocode_options queued_listing, listing
-      (result = get_value(queued_listing, %w(Location GeocodeOptions))) ? result : nil
+      get_value(queued_listing, %w(Location GeocodeOptions))
     end
 
     def self.half_bathrooms queued_listing, listing
-      (result = get_value(queued_listing, %w(HalfBathrooms))) ? result.to_i : nil
+      get_value(queued_listing, %w(HalfBathrooms))
     end
 
     def self.has_attic queued_listing, listing
@@ -268,11 +268,11 @@ module Mapper
     end
 
     def self.heating_fuels queued_listing, listing
-      (result = get_enums(queued_listing, %w(DetailedCharacteristics HeatingFuels HeatingFuel))) ? result : nil
+      get_enums(queued_listing, %w(DetailedCharacteristics HeatingFuels HeatingFuel)).uniq
     end
     
     def self.heating_systems queued_listing, listing
-      (result = get_enums(queued_listing, %w(DetailedCharacteristics HeatingSystems HeatingSystem))) ? result : nil
+      get_enums(queued_listing, %w(DetailedCharacteristics HeatingSystems HeatingSystem)).uniq
     end
 
     def self.is_cable_ready queued_listing, listing
@@ -292,27 +292,27 @@ module Mapper
     end
 
     def self.latitude queued_listing, listing
-      (result = get_value(queued_listing, %w(Location Latitude))) ? result.to_d : nil
+      get_value(queued_listing, %w(Location Latitude))
     end
 
     def self.lead_routing_email queued_listing, listing
-      (result = get_value(queued_listing, %w(LeadRoutingEmail))) ? result : nil
+      get_value(queued_listing, %w(LeadRoutingEmail))
     end
 
     def self.legal_description queued_listing, listing
-      (result = get_value(queued_listing, %w(DetailedCharacteristics LegalDescription))) ? result : nil
+      get_value(queued_listing, %w(DetailedCharacteristics LegalDescription))
     end
     
     def self.list_price queued_listing, listing
-      (result = get_value(queued_listing, %w(ListPrice)).unwrap_attribute) ? result : nil
+      get_value(queued_listing, %w(ListPrice))
     end
     
     def self.list_price_low queued_listing, listing
-      (result = get_value(queued_listing, %w(ListPriceLow)).unwrap_attribute) ? result : nil
+      get_value(queued_listing, %w(ListPriceLow))
     end
     
     def self.listing_category queued_listing, listing
-      (result = get_value(queued_listing, %w(ListingCategory))) ? ListingCategory.find_by_name(result) : nil
+      (result = get_value(queued_listing, %w(ListingCategory))) ? Mapper::listing_categories[result] : nil
     end
 
     def self.listing_date queued_listing, listing
@@ -320,70 +320,69 @@ module Mapper
     end
     
     def self.listing_description queued_listing, listing
-      (result = unwrap_attribute(get_value(queued_listing, %w(ListingDescription)))) ? result : nil
+      get_value(queued_listing, %w(ListingDescription))
     end
 
     def self.listing_media queued_listing, listing, elements
       if (result = get_repeaters(queued_listing, elements))
         result.map do |item|
-          media = listing.send(elements.last.tableize).find_or_initialize_by(
+          listing.send(elements.last.tableize).find_or_initialize_by(
             media_url: item.drilldown('MediaURL'),
-            media_modification_timestamp: item.drilldown('MediaModificationTimestamp')
-          )
-          media.assign_attributes({
+            media_modification_timestamp: item.drilldown('MediaModificationTimestamp'),
             media_order_number: item.drilldown('MediaOrderNumber'),
             media_caption: item.drilldown('MediaCaption'),
             media_description: item.drilldown('MediaDescription')
-          })
-          media
+          )
         end
       end
     end
 
     # TODO: Make ListingProvider and SourceProviderCategory Provider and ProviderCategory
     def self.listing_provider queued_listing, listing
-      (result = ListingProvider.find_or_initialize_by(
-        name: get_value(queued_listing, %w(ProviderName)),
-        url: get_value(queued_listing, %w(ProviderURL)),
-        source_provider_category: SourceProviderCategory.find_by_name(get_value(queued_listing, %w(ProviderCategory)))
-      )) ? result : nil
+      if get_value(queued_listing, %w(ProviderName)).present?
+        result = ListingProvider.find_or_initialize_by(
+          name: get_value(queued_listing, %w(ProviderName)),
+          url: get_value(queued_listing, %w(ProviderURL)),
+          source_provider_category: Mapper::source_provider_categories[get_value(queued_listing, %w(ProviderCategory))]
+        )
+      end
     end
 
     def self.listing_status queued_listing, listing
-      (result = get_value(queued_listing, %w(ListingStatus))) ? ListingStatus.find_by_name(result) : nil
+      (result = get_value(queued_listing, %w(ListingStatus))) ? Mapper::listing_statuses[result] : nil
     end
     
     def self.listing_title queued_listing, listing
-      (result = get_value(queued_listing, %w(ListingTitle))) ? result : nil
+      get_value(queued_listing, %w(ListingTitle))
     end
     
     def self.listing_url queued_listing, listing
-      (result = get_value(queued_listing, %w(ListingURL))) ? result : nil
+      get_value(queued_listing, %w(ListingURL))
     end
 
     def self.living_area queued_listing, listing
-      (result = get_value(queued_listing, %w(LivingArea))) ? result.to_i : nil
+      get_value(queued_listing, %w(LivingArea))
     end
     
     def self.living_area_unit queued_listing, listing
-      (result = get_value(queued_listing, %w(LivingArea areaUnits))) ? result : nil
+      get_value(queued_listing, %w(LivingArea areaUnits))
     end
 
     def self.longitude queued_listing, listing
-      (result = get_value(queued_listing, %w(Location Longitude))) ? result.to_d : nil
+      get_value(queued_listing, %w(Location Longitude))
     end
     
     def self.lot_size queued_listing, listing
-      (result = get_value(queued_listing, %w(LotSize))) ? result : nil
+      get_value(queued_listing, %w(LotSize))
     end
 
     def self.mls_number queued_listing, listing
-      (result = get_value(queued_listing, %w(MlsNumber))) ? result : nil
+      get_value(queued_listing, %w(MlsNumber))
     end
     
     def self.modification_timestamp queued_listing, listing
       # TODO: Change from string to datetime
-      (result = get_value(queued_listing, %w(ModificationTimestamp))) ? result : nil
+      get_value(queued_listing, %w(ModificationTimestamp))
     end
 
     def self.multiple_listing_service queued_listing, listing
@@ -407,15 +406,15 @@ module Mapper
           place
         end
       end
-      places ? places : []
+      places.present? ? places : []
     end
 
     def self.num_floors queued_listing, listing
-      (result = get_value(queued_listing, %w(DetailedCharacteristics NumFloors))) ? result.to_i : nil
+      get_value(queued_listing, %w(DetailedCharacteristics NumFloors))
     end
     
     def self.num_parking_spaces queued_listing, listing
-      (result = get_value(queued_listing, %w(DetailedCharacteristics NumParkingSpaces))) ? result.to_i : nil
+      get_value(queued_listing, %w(DetailedCharacteristics NumParkingSpaces))
     end
     
     def self.office queued_listing, listing
@@ -449,23 +448,26 @@ module Mapper
     end
 
     def self.one_quarter_bathrooms queued_listing, listing
-      (result = get_value(queued_listing, %w(OneQuarterBathrooms))) ? result.to_i : nil
+      get_value(queued_listing, %w(OneQuarterBathrooms))
     end
 
     # TODO: Figure out how to provide time zone for StartTime and EndTime 
     def self.open_houses queued_listing, listing
       if (result = get_repeaters(queued_listing, %w(OpenHouses OpenHouse)))
-        result.map{|open_house| listing.open_houses.find_or_initialize_by(
-          showing_date: Chronic::parse(open_house['Date']).to_date,
-          start_time: open_house['StartTime'],
-          end_time: open_house['EndTime'],
-          description: open_house['Description']
-        )}
+        result.map do |open_house| 
+          oh = listing.open_houses.find_or_initialize_by(
+            showing_date: Chronic::parse(open_house['Date']).to_date,
+            start_time: open_house['StartTime'],
+            end_time: open_house['EndTime']
+          )
+          oh.description = open_house['Description']
+          oh
+        end
       end
     end
 
     def self.parcel_id queued_listing, listing
-      (result = get_value(queued_listing, %w(Location ParcelId))) ? result : nil
+      get_value(queued_listing, %w(Location ParcelId))
     end
 
     # TODO: Change participant_identifier to participant_id
@@ -480,7 +482,7 @@ module Mapper
           participant.assign_attributes({
             participant_key: item['ParticipantKey'],
             participant_identifier: item['ParticipantId'],
-            participant_role: ParticipantRole.find_by(name: item['Role']),
+            participant_role: Mapper::participant_roles[item['Role']],
             primary_contact_phone: item['PrimaryContactPhone'],
             office_phone: item['OfficePhone'],
             fax: item['Fax'],
@@ -492,40 +494,40 @@ module Mapper
     end
 
     def self.permit_address_on_internet queued_listing, listing
-      (result = get_value(queued_listing, %w(MarketingInformation PermitAddressOnInternet))) ? result.to_s.to_bool : nil
+      get_boolean_value(queued_listing, %w(MarketingInformation PermitAddressOnInternet'))
     end
     
     def self.photos queued_listing, listing
-      (result = listing_media(queued_listing, listing, %w(Photos Photo))) ? result : nil
+      listing_media(queued_listing, listing, %w(Photos Photo))
     end
 
     def self.property_sub_type queued_listing, listing
-      (result = unwrap_attribute(get_value(queued_listing, %w(PropertySubType)))) ? PropertySubType.find_by_name(result) : nil
+      (result = get_value(queued_listing, %w(PropertySubType))) ? Mapper::property_sub_types[result] : nil
     end
     
     def self.property_sub_type_description queued_listing, listing
-      (result = get_value(queued_listing, %w(PropertySubType otherDescription))) ? result : nil
+      get_value(queued_listing, %w(PropertySubType otherDescription))
     end
     
     def self.property_type queued_listing, listing
-      (result = unwrap_attribute(get_value(queued_listing, %w(PropertyType)))) ? PropertyType.find_by_name(result) : nil
+      (result = get_value(queued_listing, %w(PropertyType))) ? Mapper::property_types[result] : nil
     end
 
     def self.property_type_description queued_listing, listing
-      (result = get_value(queued_listing, %w(PropertyType otherDescription))) ? result : nil
+      get_value(queued_listing, %w(PropertyType otherDescription))
     end
     
     def self.roof_types queued_listing, listing
-      (result = get_enums(queued_listing, %w(DetailedCharacteristics RoofTypes RoofType))) ? result : nil
+      get_enums(queued_listing, %w(DetailedCharacteristics RoofTypes RoofType))
     end
     
     def self.room_count queued_listing, listing
-      (result = get_value(queued_listing, %w(DetailedCharacteristics RoomCount))) ? result.to_i : nil
+      get_value(queued_listing, %w(DetailedCharacteristics RoomCount))
     end
     
     def self.rooms queued_listing, listing
       if (result = get_value(queued_listing, %w(DetailedCharacteristics Rooms Room)))
-        rooms = Array(result).map{|room_category| Room.new(listing: listing, room_category: RoomCategory.find_by(name: room_category))}
+        rooms = Array(result).map{|room_category| Room.new(listing: listing, room_category: Mapper::room_categories[room_category])}
       end
       rooms ? rooms : []
     end
@@ -541,19 +543,19 @@ module Mapper
     end
 
     def self.three_quarter_bathrooms queued_listing, listing
-      (result = get_value(queued_listing, %w(ThreeQuarterBathrooms))) ? result.to_i : nil
+      get_value(queued_listing, %w(ThreeQuarterBathrooms))
     end
 
     def self.videos queued_listing, listing
-      (result = listing_media(queued_listing, listing, %w(Videos Video))) ? result : nil
+      listing_media(queued_listing, listing, %w(Videos Video))
     end
 
     def self.view_types queued_listing, listing
-      (result = get_enums(queued_listing, %w(DetailedCharacteristics ViewTypes ViewType))) ? result : nil
+      get_enums(queued_listing, %w(DetailedCharacteristics ViewTypes ViewType))
     end
     
     def self.virtual_tours queued_listing, listing
-      (result = listing_media(queued_listing, listing, %w(VirtualTours VirtualTour))) ? result : nil
+      listing_media(queued_listing, listing, %w(VirtualTours VirtualTour))
     end
 
     def self.vow_address_display queued_listing, listing
@@ -569,11 +571,11 @@ module Mapper
     end
     
     def self.year_built queued_listing, listing
-      (result = get_value(queued_listing, %w(YearBuilt))) ? result.to_i : nil
+      get_value(queued_listing, %w(YearBuilt))
     end
 
     def self.year_updated queued_listing, listing
-      (result = get_value(queued_listing, %w(DetailedCharacteristics YearUpdated))) ? result.to_s.to_i : nil
+      get_value(queued_listing, %w(DetailedCharacteristics YearUpdated))
     end
     
     # Utilities
@@ -612,10 +614,6 @@ module Mapper
       (result = get_value(queued_listing, elements)) ? result.to_s.to_bool : nil
     end
     
-    def self.unwrap_attribute value
-      value.is_a?(Hash) ? (value['__content__'] ? value['__content__'] : value) : value
-    end
-    
     def self.get_repeaters queued_listing, elements
       if (value = get_value(queued_listing, elements[0..-2]))
         (result = value.drilldown(elements.last)) ? (result.is_a?(Array) ? result : [result]) : nil
@@ -624,12 +622,11 @@ module Mapper
       end
     end
     
-    # TODO: Should be find_by_name not find_or_create_by_name. Remove after before_save validation.
     def self.get_enums queued_listing, elements
       if (result = get_repeaters(queued_listing, elements))
-        enums = result.map{|name| elements.last.constantize.find_or_create_by(name: name)}
+        enums = result.map{|name| Mapper.send(elements.last.tableize)[name]}
       end
-      enums ? enums : nil
+      enums ? enums.compact : nil
     end
 
     def self.get_value queued_listing, elements

data/lib/reso/railties/reso.rake

@@ -31,17 +31,13 @@ namespace :reso do
     end
     nil # Just in cases
   end
-  
-  def process_item xml, xml_header, import
+
+  def create_queued_listing doc, import
     begin
-      doc = Nokogiri::XML([xml_header, xml].join).remove_namespaces!
       doc.css(import.repeating_element).each do |o|
-        listing_data = Hash.new
-        listing = Hash.from_xml(o.to_xml)
-        object = import.queued_listings.new
-        listing[import.repeating_element].each_pair{|key, value| listing_data[key] = value }
-        object.listing_data = listing_data
-        object.save
+        listing_data = {}
+        Hash.from_xml(o.to_xml)[import.repeating_element].each_pair{|key, value| listing_data[key] = value }
+        QueuedListing.create(import: import, listing_data: listing_data)
       end
     rescue Exception => e
       puts e.inspect
@@ -107,17 +103,19 @@ namespace :reso do
       # Grab the XML header to avoid namespace errors later 
       xml_header = get_xml_header filepath, import.repeating_element
 
-      start = Time.now
+      puts (start = Time.now)
       puts "Starting..." if Rails.env.development?
       File.foreach(filepath) do |line|
         stream += line
         while (from_here = stream.index(open_tag)) && (to_there = stream.index(close_tag))
           xml = stream[from_here..to_there + (close_tag.length-1)]
-          process_item xml, xml_header, import
+          doc = Nokogiri::XML([xml_header, xml].join).remove_namespaces!
+          create_queued_listing doc, import
           stream.gsub!(xml, '')
-          if (l += 1) % 1000 == 0
-            puts "#{l} - #{l/(Time.now - start)} listings/s" if Rails.env.development?
+          if ((l += 1) % 1000).zero?
+            puts "#{l}\t#{l/(Time.now - start)}" if Rails.env.development?
           end
+          GC.start if (l % 100).zero?
         end
       end
       puts "#{l} - #{l/(Time.now - start)} listings/s" if Rails.env.development?

data/lib/reso/version.rb

@@ -1,3 +1,3 @@
 module Reso
-  VERSION = "0.1.0.0"
+  VERSION = "0.1.0.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: reso
 version: !ruby/object:Gem::Version
-  version: 0.1.0.0
+  version: 0.1.0.1
 platform: ruby
 authors:
 - Michael Edlund
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-08-13 00:00:00.000000000 Z
+date: 2015-08-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake