reso 0.1.1.0 → 0.1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5e4cf4dfcb9e6c4b1ed0a73f10bc721b710762b7
-  data.tar.gz: c05035bf8cadb7a7044027357488f3e2ec65c709
+  metadata.gz: e761f8d043c586346c9cc9c319d279982e19d9f8
+  data.tar.gz: 4e6a3b16fc333523cc0da036e3f7cb932b07152e
 SHA512:
-  metadata.gz: c533cabc8a83c661b24d254bde866b56ba604107512e10217efbad93712753beb045348d7058ea3a3c833c8b893a490c32a69550ff253c048c2d00b6c0db5281
-  data.tar.gz: 9573993e629bb919d59776afc8ae4bdc75d1480cccf29c305154e4de4b57d6cbf621dbb85ba2a15a60a10970940c166fcdaf0775789d64fb952ff997fc94be03
+  metadata.gz: 621c9d180dd6a11a054484ef4fcecc19d45cc502c1ead96ff6a69c7e89d4023ab0ebd74f4cc33ff71f9609360c76ccb0c4cf9fdd311f238f451576ff78d7309a
+  data.tar.gz: d05f9c83ce99ba0c16f65720e893f1449001cf09cbd6c039ad9f5ec03826a5ed4cba3a499e871eeeb5790479a127f9b610d810893f62cda1658baaae4377c208

data/app/models/import.rb

@@ -1,15 +1,15 @@
 class Import < ActiveRecord::Base
 
-  belongs_to :import_format
+  require 'open-uri'
+  require 'open_uri_redirections'
 
+  before_save :set_import_format
+  belongs_to :import_format
   has_many :listings
   has_many :queued_listings
-  
   validates_uniqueness_of :token
 
-  before_save :set_import_format
-
-  def remove_listings_no_longer_present fresh_listing_keys
+  def remove_listings_not_present fresh_listing_keys
     existing_listing_keys = self.listings.all.pluck(:listing_key)
     stale_listing_keys = existing_listing_keys.delete_if{|key| fresh_listing_keys.include? key }
     stale_listing_keys.each do |listing_key|
@@ -22,4 +22,15 @@ class Import < ActiveRecord::Base
     self.import_format = ImportFormat.find_by(name: 'reso') unless self.import_format.present?
   end
 
+  def source_url_last_modified
+    open(self.source_url, 
+      http_basic_authentication: [self.source_user, self.source_pass], 
+      allow_redirections: :all
+    ){|f| return f.last_modified }
+  end
+  
+  def new_source_data_exists?
+    self.source_data_modified.eql? self.source_url_last_modified ? false : true
+  end
+
 end

data/lib/generators/reso/templates/create_imports.rb

@@ -9,6 +9,7 @@ class CreateImports < ActiveRecord::Migration
       t.string      :source_url
       t.string      :source_user
       t.string      :source_pass
+      t.datetime    :source_data_modified
 
       t.timestamps null: false
     end

data/lib/generators/reso/templates/reso.rake

@@ -96,36 +96,44 @@ namespace :reso do
     import = Import.find_by(token: args.import_token)
 
     unless import.blank?
-      l, count, incoming_listing_keys, stream = 0, 0, [], ''
-      open_tag, close_tag = get_open_and_closing_tag_for import.repeating_element
+      unless import.new_source_data_exists?
+        source_data_modified = import.source_url_last_modified
+        l, count, found_listing_keys, stream = 0, 0, [], ''
+        open_tag, close_tag = get_open_and_closing_tag_for import.repeating_element
 
-      # Grab a file to work with
-      filepath = download_feed_to_import import
-      filepath = uncompress_and_return_new_filepath(filepath) if filepath.split('.').last.downcase == 'gz'
+        # Grab a file to work with
+        filepath = download_feed_to_import import
+        filepath = uncompress_and_return_new_filepath(filepath) if filepath.split('.').last.downcase == 'gz'
       
-      # Grab the XML header to avoid namespace errors later 
-      xml_header = get_xml_header filepath, import.repeating_element
+        # Grab the XML header to avoid namespace errors later 
+        xml_header = get_xml_header filepath, import.repeating_element
 
-      puts (start = Time.now)
-      puts "Starting..." if Rails.env.development?
-      File.foreach(filepath) do |line|
-        stream += line
-        while (from_here = stream.index(open_tag)) && (to_there = stream.index(close_tag))
-          xml = stream[from_here..to_there + (close_tag.length-1)]
-          doc = Nokogiri::XML([xml_header, xml].join).remove_namespaces!
-          incoming_listing_keys << create_queued_listing_and_return_listing_key(doc, import)
-          stream.gsub!(xml, '')
-          if ((l += 1) % 1000).zero?
-            puts "#{l}\t#{l/(Time.now - start)}" if Rails.env.development?
+        start_time = Time.now
+        import_result = ImportResult.create(import: import, start_time: start_time)
+        File.foreach(filepath) do |line|
+          stream += line
+          while (from_here = stream.index(open_tag)) && (to_there = stream.index(close_tag))
+            xml = stream[from_here..to_there + (close_tag.length-1)]
+            doc = Nokogiri::XML([xml_header, xml].join).remove_namespaces!
+            found_listing_keys << create_queued_listing_and_return_listing_key(doc, import)
+            stream.gsub!(xml, '')
+            if ((l += 1) % 1000).zero?
+              puts "#{l}\t#{l/(Time.now - start)}" if Rails.env.development?
+            end
+            GC.start if (l % 100).zero?
           end
-          GC.start if (l % 100).zero?
         end
+        end_time = Time.now
+        removed_listing_keys = import.remove_listings_not_present(found_listing_keys)
+        import_result.assign_attributes({
+          end_time: end_time,
+          found_listing_keys: found_listing_keys,
+          removed_listing_keys: removed_listing_keys.inspect
+        })
+        import_result.save
+        import.update_attribute(:source_data_modified, source_data_modified)
+        File.delete(filepath)
       end
-      puts "Import speed: #{l/(Time.now - start)} listings/s" if Rails.env.development?
-      puts "Found #{l} new listings." if Rails.env.development?
-      stale_listing_keys = import.remove_listings_no_longer_present(incoming_listing_keys)
-      puts "Removed #{stale_listing_keys.count} old listings." if Rails.env.development?
-      File.delete(filepath)
     end
   end
 

data/lib/reso/version.rb

@@ -1,3 +1,3 @@
 module Reso
-  VERSION = "0.1.1.0"
+  VERSION = "0.1.2.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: reso
 version: !ruby/object:Gem::Version
-  version: 0.1.1.0
+  version: 0.1.2.0
 platform: ruby
 authors:
 - Michael Edlund
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-08-24 00:00:00.000000000 Z
+date: 2015-08-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake