reso 0.1.0.0

data/lib/mapper.rb

@@ -0,0 +1,114 @@
+module Mapper
+  RESO_LISTING_ATTRIBUTES = %w(
+    address
+    alternate_prices
+    appliances
+    architecture_style
+    architecture_style_description
+    bathrooms
+    bedrooms
+    brokerage
+    builder
+    building_unit_count
+    condo_floor_num
+    cooling_systems
+    county
+    currency_code
+    directions
+    disclaimer
+    disclose_address
+    elevation
+    expenses
+    exterior_types
+    floor_coverings
+    foreclosure_status
+    franchise
+    full_bathrooms
+    geocode_options
+    half_bathrooms
+    has_attic
+    has_barbecue_area
+    has_basement
+    has_ceiling_fan
+    has_deck
+    has_disabled_access
+    has_dock
+    has_doorman
+    has_double_pane_windows
+    has_elevator
+    has_fireplace
+    has_garden
+    has_gated_entry
+    has_greenhouse
+    has_hot_tub_spa
+    has_intercom
+    has_jetted_bath_tub
+    has_lawn
+    has_mother_in_law
+    has_patio
+    has_pond
+    has_pool
+    has_porch
+    has_rv_parking
+    has_sauna
+    has_security_system
+    has_skylight
+    has_sports_court
+    has_sprinkler_system
+    has_vaulted_ceiling
+    has_wet_bar
+    heating_fuels
+    heating_systems
+    import
+    is_cable_ready
+    is_new_construction
+    is_waterfront
+    is_wired
+    latitude
+    lead_routing_email
+    legal_description
+    list_price
+    list_price_low
+    listing_category
+    listing_date
+    listing_description
+    listing_provider
+    listing_status
+    listing_title
+    listing_url
+    living_area
+    living_area_unit
+    longitude
+    lot_size
+    mls_number
+    multiple_listing_service
+    modification_timestamp
+    neighborhoods
+    num_floors
+    num_parking_spaces
+    office
+    one_quarter_bathrooms
+    open_houses
+    parcel_info
+    participants
+    permit_address_on_internet
+    photos
+    property_sub_type 
+    property_sub_type_description
+    property_type
+    property_type_description
+    roof_types
+    room_count
+    rooms
+    taxes
+    three_quarter_bathrooms
+    videos
+    view_types
+    virtual_tours
+    vow_address_display
+    vow_automated_valuation_display
+    vow_consumer_comment
+    year_built
+    year_updated
+  )
+end

data/lib/reso/engine.rb

@@ -0,0 +1,13 @@
+require "reso"
+require "rails"
+
+module Reso
+  class Engine < ::Rails::Engine
+    engine_name :reso
+
+    rake_tasks do
+      load "reso/railties/reso.rake"
+    end
+
+  end
+end

data/lib/reso/railties/reso.rake

@@ -0,0 +1,128 @@
+namespace :reso do
+
+  require 'nokogiri'
+  require 'open-uri'
+  require 'open_uri_redirections'
+
+  def download_feed_to_import import
+    filename = import.source_url.split('/').last
+    filepath = Rails.root.join('tmp', filename).to_s
+    File.delete(filepath) if File.file? filepath
+    open(filepath, 'wb') do |file|
+      file << open(import.source_url, 
+                http_basic_authentication: [import.source_user, import.source_pass], 
+                allow_redirections: :all
+              ).read
+    end
+    filepath
+  end
+
+  def get_open_and_closing_tag_for repeating_element
+    ApplicationController.helpers.content_tag(repeating_element, "\n").split
+  end
+
+  def get_xml_header filepath, repeating_element
+    stream = ''
+    open_tag = get_open_and_closing_tag_for(repeating_element).first
+    File.foreach(filepath) do |line|
+      stream += line
+      pos = stream.index(open_tag)
+      return stream[0..pos-1] if pos
+    end
+    nil # Just in cases
+  end
+  
+  def process_item xml, xml_header, import
+    begin
+      doc = Nokogiri::XML([xml_header, xml].join).remove_namespaces!
+      doc.css(import.repeating_element).each do |o|
+        listing_data = Hash.new
+        listing = Hash.from_xml(o.to_xml)
+        object = import.queued_listings.new
+        listing[import.repeating_element].each_pair{|key, value| listing_data[key] = value }
+        object.listing_data = listing_data
+        object.save
+      end
+    rescue Exception => e
+      puts e.inspect
+      exit if Rails.env.development?
+    end
+  end
+  
+  def uncompress_and_return_new_filepath filepath
+    output_path = [filepath, '.xml'].join
+    File.delete(output_path) if File.file? output_path
+    Zlib::GzipReader.open(filepath) do |gz|
+      File.open(output_path, "w") do |g|
+        IO.copy_stream(gz, g)
+      end
+    end
+    File.delete(filepath)
+    output_path
+  end
+  
+  desc "Populate database with seed data."
+  task :seed => [:load_enumerations, :seed_imports] 
+
+  task :load_enumerations => [:environment] do
+    require "csv"
+    csv_text = File.read("#{Rails.root}/db/enumerations.csv")
+    csv = CSV.parse(csv_text, :headers => true)
+    csv.each do |row|
+      row = row.to_hash.with_indifferent_access
+      Enumeration.create!(row.to_hash.symbolize_keys)
+        puts row.inspect
+      end
+      puts
+  end
+
+  task :seed_imports => [:environment] do
+    imports = [{ name: "RESO Example", 
+                 token: "reso",
+                 import_format_id: ImportFormat.find_by(name: 'reso').id,
+                 source_url: "https://app.listhub.com/syndication-docs/example.xml"
+              }]
+  
+    imports.each do |import|
+      @import = Import.new
+      import.each_pair{|key, value| @import[key] = value }
+      @import.save
+    end
+  end
+
+  desc "Download and import data file for specified import."
+  task :import, [:import_token] => [:environment] do |t, args|
+
+    args.with_defaults(:import_token => "reso")
+    import = Import.find_by(token: args.import_token)
+
+    unless import.blank?
+      l, count, stream = 0, 0, ''
+      open_tag, close_tag = get_open_and_closing_tag_for import.repeating_element
+
+      # Grab a file to work with
+      filepath = download_feed_to_import import
+      filepath = uncompress_and_return_new_filepath(filepath) if filepath.split('.').last.downcase == 'gz'
+      
+      # Grab the XML header to avoid namespace errors later 
+      xml_header = get_xml_header filepath, import.repeating_element
+
+      start = Time.now
+      puts "Starting..." if Rails.env.development?
+      File.foreach(filepath) do |line|
+        stream += line
+        while (from_here = stream.index(open_tag)) && (to_there = stream.index(close_tag))
+          xml = stream[from_here..to_there + (close_tag.length-1)]
+          process_item xml, xml_header, import
+          stream.gsub!(xml, '')
+          if (l += 1) % 1000 == 0
+            puts "#{l} - #{l/(Time.now - start)} listings/s" if Rails.env.development?
+          end
+        end
+      end
+      puts "#{l} - #{l/(Time.now - start)} listings/s" if Rails.env.development?
+      File.delete(filepath)
+    end
+  end
+
+end

data/lib/reso/version.rb

@@ -0,0 +1,3 @@
+module Reso
+  VERSION = "0.1.0.0"
+end

data/lib/reso.rb

@@ -0,0 +1,10 @@
+require 'reso/version'
+require 'reso/engine' if defined?(Rails)
+require 'chronic'
+require 'class_extensions'
+require 'mapper'
+require 'mapper/reso'
+require 'street_address'
+
+module Reso
+end

data/reso.gemspec

@@ -0,0 +1,31 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'reso/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "reso"
+  spec.version       = Reso::VERSION
+  spec.authors       = ["Michael Edlund"]
+  spec.email         = ["medlund@mac.com"]
+  spec.description   = %q{RESO Data Dictionary data models and import.}
+  spec.summary       = %q{The RESO Data Dictionary gem provides models and data import based on the Real Estate Standards Organization Syndication Format for exchange of real estate listing data.}
+  spec.homepage      = "http://github.com/arcticleo/reso"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.required_ruby_version = '>= 1.9.3'
+
+  spec.add_development_dependency "rake", '~> 10'
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rspec", "~> 2.6"
+  spec.add_dependency "rails", '~> 4.0', '>= 4.0.0'
+  spec.add_dependency "nokogiri", '~> 1'
+  spec.add_dependency 'chronic', '~> 0'
+  spec.add_dependency 'StreetAddress', '~> 1'
+  spec.add_dependency 'open_uri_redirections', '~> 0'
+end

metadata

@@ -0,0 +1,261 @@
+--- !ruby/object:Gem::Specification
+name: reso
+version: !ruby/object:Gem::Version
+  version: 0.1.0.0
+platform: ruby
+authors:
+- Michael Edlund
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-08-13 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.6'
+- !ruby/object:Gem::Dependency
+  name: rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.0.0
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+- !ruby/object:Gem::Dependency
+  name: chronic
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: StreetAddress
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+- !ruby/object:Gem::Dependency
+  name: open_uri_redirections
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+description: RESO Data Dictionary data models and import.
+email:
+- medlund@mac.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- app/controllers/syndication_controller.rb
+- app/models/.keep
+- app/models/address.rb
+- app/models/address_type.rb
+- app/models/alternate_price.rb
+- app/models/appliance.rb
+- app/models/architecture_style.rb
+- app/models/area_unit.rb
+- app/models/brokerage.rb
+- app/models/builder.rb
+- app/models/business.rb
+- app/models/community.rb
+- app/models/cooling_system.rb
+- app/models/county.rb
+- app/models/currency_period.rb
+- app/models/enumeration.rb
+- app/models/expense.rb
+- app/models/expense_category.rb
+- app/models/exterior_type.rb
+- app/models/floor_covering.rb
+- app/models/foreclosure_status.rb
+- app/models/franchise.rb
+- app/models/gender.rb
+- app/models/heating_fuel.rb
+- app/models/heating_system.rb
+- app/models/home_feature.rb
+- app/models/import.rb
+- app/models/import_format.rb
+- app/models/license_category.rb
+- app/models/listing.rb
+- app/models/listing_category.rb
+- app/models/listing_media.rb
+- app/models/listing_provider.rb
+- app/models/listing_status.rb
+- app/models/multiple_listing_service.rb
+- app/models/neighborhood.rb
+- app/models/office.rb
+- app/models/open_house.rb
+- app/models/parking.rb
+- app/models/participant.rb
+- app/models/participant_license.rb
+- app/models/participant_role.rb
+- app/models/person.rb
+- app/models/photo.rb
+- app/models/place.rb
+- app/models/price.rb
+- app/models/property_sub_type.rb
+- app/models/property_type.rb
+- app/models/queued_listing.rb
+- app/models/roof_type.rb
+- app/models/room.rb
+- app/models/room_category.rb
+- app/models/school.rb
+- app/models/school_category.rb
+- app/models/source_provider_category.rb
+- app/models/tax.rb
+- app/models/video.rb
+- app/models/view_type.rb
+- app/models/virtual_tour.rb
+- app/views/syndication/rets.xml.erb
+- lib/class_extensions.rb
+- lib/generators/reso/install_generator.rb
+- lib/generators/reso/setup.rb
+- lib/generators/reso/templates/create_addresses.rb
+- lib/generators/reso/templates/create_businesses.rb
+- lib/generators/reso/templates/create_enumerations.rb
+- lib/generators/reso/templates/create_expenses.rb
+- lib/generators/reso/templates/create_imports.rb
+- lib/generators/reso/templates/create_join_tables.rb
+- lib/generators/reso/templates/create_listing_media.rb
+- lib/generators/reso/templates/create_listing_providers.rb
+- lib/generators/reso/templates/create_listings.rb
+- lib/generators/reso/templates/create_multiple_listing_services.rb
+- lib/generators/reso/templates/create_offices.rb
+- lib/generators/reso/templates/create_open_houses.rb
+- lib/generators/reso/templates/create_participant_licenses.rb
+- lib/generators/reso/templates/create_participants.rb
+- lib/generators/reso/templates/create_people.rb
+- lib/generators/reso/templates/create_places.rb
+- lib/generators/reso/templates/create_prices.rb
+- lib/generators/reso/templates/create_queued_listings.rb
+- lib/generators/reso/templates/create_rooms.rb
+- lib/generators/reso/templates/create_schools.rb
+- lib/generators/reso/templates/create_taxes.rb
+- lib/generators/reso/templates/enumerations.csv
+- lib/mapper.rb
+- lib/mapper/reso.rb
+- lib/reso.rb
+- lib/reso/engine.rb
+- lib/reso/railties/reso.rake
+- lib/reso/version.rb
+- reso.gemspec
+homepage: http://github.com/arcticleo/reso
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 1.9.3
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.6
+signing_key: 
+specification_version: 4
+summary: The RESO Data Dictionary gem provides models and data import based on the
+  Real Estate Standards Organization Syndication Format for exchange of real estate
+  listing data.
+test_files: []