reso 0.1.3.0 → 0.1.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 40d73be58419384a1c36b0cf68410db55bc95abb
-  data.tar.gz: d73d8c8395ce5c5829ad20d9f2f5b416a584107c
+  metadata.gz: b45456599ff2b14aec9e246637f1e2968a82d97f
+  data.tar.gz: f3ec499d115470a13d3e367c8cf28294cfd5c984
 SHA512:
-  metadata.gz: 1c98555ad280f777976132930c82db9bc6d7bbfcc8f917443c524d9130c8fd9961c44623c15c15c0a2abb6562935b59a4b6550bdd8330531d1370fe813b4be49
-  data.tar.gz: 3cfe39b4c852971963c92e76d9d34678e52a21a8022b93b7c3ee97f58bd53c50f4999819886ab33d17bbc85c7214aa084f00be0897d02998dd101811f3f2746f
+  metadata.gz: 75d3ff3bdb12d2948adb1506f6d4c46bfa5c7ba0c21e1f8524897e55d678174fe785c5d817fc02133a2cab2955a02a50c0fa1fe163dc081ae3bada94c781ad3d
+  data.tar.gz: abf3cb025a5d8fee6ce83b02fe8bd235cdfd7828fcaf7b150f3f8a5673433f8a169dc89c7b56bdd14c7c3697329857579e54a86e7308481fd1ca9217513b5530

data/app/models/import.rb

@@ -1,5 +1,6 @@
 class Import < ActiveRecord::Base
 
+  require 'nokogiri'
   require 'open-uri'
   require 'open_uri_redirections'
 
@@ -34,4 +35,101 @@ class Import < ActiveRecord::Base
     self.source_data_modified.eql? self.source_url_last_modified ? false : true
   end
 
+  def run_import
+    if self.status == 'active'
+      unless self.new_source_data_exists?
+        self.update_attribute(:status, :running)
+        source_data_modified = self.source_url_last_modified
+        l, count, found_listing_keys, stream = 0, 0, [], ''
+        open_tag, close_tag = get_open_and_closing_tag_for self.repeating_element
+
+        # Grab a file to work with
+        filepath = download_feed_to_import self
+        filepath = uncompress_and_return_new_filepath(filepath) if filepath.split('.').last.downcase == 'gz'
+      
+        # Grab the XML header to avoid namespace errors later 
+        xml_header = get_xml_header filepath, self.repeating_element
+
+        start_time = Time.now
+        import_result = ImportResult.create(import: self, start_time: start_time)
+        File.foreach(filepath) do |line|
+          stream += line
+          while (from_here = stream.index(open_tag)) && (to_there = stream.index(close_tag))
+            xml = stream[from_here..to_there + (close_tag.length-1)]
+            doc = Nokogiri::XML([xml_header, xml].join).remove_namespaces!
+            found_listing_keys << create_queued_listing_and_return_listing_key(doc, self)
+            stream.gsub!(xml, '')
+          end
+        end
+        end_time = Time.now
+        removed_listing_keys = self.remove_listings_not_present(found_listing_keys)
+        import_result.assign_attributes({
+          end_time: end_time,
+          found_listing_keys: found_listing_keys,
+          removed_listing_keys: removed_listing_keys.inspect
+        })
+        import_result.save
+        self.update_attribute(:status, :active)
+        self.update_attribute(:source_data_modified, source_data_modified)
+        File.delete(filepath)
+      end
+    end
+  end
+  
+  def download_feed_to_import import
+    filename = import.source_url.split('/').last
+    filepath = Rails.root.join('tmp', filename).to_s
+    File.delete(filepath) if File.file? filepath
+    open(filepath, 'wb') do |file|
+      file << open(import.source_url, 
+                http_basic_authentication: [import.source_user, import.source_pass], 
+                allow_redirections: :all
+              ).read
+    end
+    filepath
+  end
+
+  def get_open_and_closing_tag_for repeating_element
+    ApplicationController.helpers.content_tag(repeating_element, "\n").split
+  end
+
+  def get_xml_header filepath, repeating_element
+    stream = ''
+    open_tag = get_open_and_closing_tag_for(repeating_element).first
+    File.foreach(filepath) do |line|
+      stream += line
+      pos = stream.index(open_tag)
+      return stream[0..pos-1] if pos
+    end
+    nil # Just in cases
+  end
+
+  def create_queued_listing_and_return_listing_key doc, import
+    begin
+      doc.css(import.repeating_element).each do |o|
+        listing_data = {}
+        Hash.from_xml(o.to_xml)[import.repeating_element].each_pair{|key, value| listing_data[key] = value }
+        queued_listing = QueuedListing.new(import: import, listing_data: listing_data)
+        queued_listing.save
+        return Mapper::unique_identifier(queued_listing)
+      end
+    rescue Exception => e
+      puts e.inspect
+      exit if Rails.env.development?
+      return nil
+    end
+  end
+  
+  def uncompress_and_return_new_filepath filepath
+    output_path = [filepath, '.xml'].join
+    File.delete(output_path) if File.file? output_path
+    Zlib::GzipReader.open(filepath) do |gz|
+      File.open(output_path, "w") do |g|
+        IO.copy_stream(gz, g)
+      end
+    end
+    File.delete(filepath)
+    output_path
+  end
+  
 end

data/lib/reso/version.rb

@@ -1,3 +1,3 @@
 module Reso
-  VERSION = "0.1.3.0"
+  VERSION = "0.1.3.1"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: reso
 version: !ruby/object:Gem::Version
-  version: 0.1.3.0
+  version: 0.1.3.1
 platform: ruby
 authors:
 - Michael Edlund