mercator_icecat 0.0.1 → 0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4deffa8cd6c702eed77bac34ab3017b702f0f854
-  data.tar.gz: 867a87c65a178ec5f5967da1aaa15413364db81d
+  metadata.gz: 40286766b8ceba1e1c7b30b5212281e376266d76
+  data.tar.gz: 51c2ddba7064afe93b8d0cff1843966e0f5be042
 SHA512:
-  metadata.gz: aad16e7b3a246dfb64929f46ed9bab73bde106491dbb144a2ab12e667121ee1f36624f47799ed7093cf7151a4e75e070dfb8d2f0181ed6ba534cd37430528fb6
-  data.tar.gz: 5086e1e40dd3d7b8f0cf0ab89ae3b1f04368e95c93d11e6629ab6943ef1d616fc05ad63b97c52ffa5ce3b2c8c7223ec211805ef41a48cdcb76b8e45b169ab072
+  metadata.gz: 5305d1e84e0563e5f6a74948c62ecc00156381b3741a08663405fb5b3c2ffc61f5966f1a9a5276af2b08b9b188455dbc25fe61f6e788352dcd29099ddd179910
+  data.tar.gz: d8bcc01c56a69749d4aeb99d9ca5ea9dbae998ae7aae0e3357533f577c89bf75ede726d0bdb74e8bb0297cb138e824de6fe8851494f5ee34daf8bcd9decd2f23

data/README.rdoc

@@ -3,3 +3,14 @@
 This engines provides Icecat Import for the Mercator Guided Selling Application.
 
 This project uses the GPL V3 license. See file LICENSE in the same directory for details about the GPL V3.
+
+* Icecat Product: XY141ET corrupt Unicode Sequence
+
+* A typical seqence for updating data would include the tasks:
+  bundle exec rake icecat:catalog:download_daily RAILS_ENV=production
+  bundle exec rake icecat:metadata:import_daily RAILS_ENV=production
+  bundle exec rake icecat:metadata:assign_products RAILS_ENV=production
+  bundle exec rake icecat:metadata:download_xml RAILS_ENV=production
+  bundle exec rake icecat:metadata:update_products RAILS_ENV=production
+  bundle exec rake icecat:metadata:update_product_relations RAILS_ENV=production
+  bundle exec rake icecat:metadata:import_missing_images RAILS_ENV=production

data/app/controllers/admin/metadata_controller.rb

@@ -0,0 +1,14 @@
+class Admin::MetadataController < Admin::AdminSiteController
+
+  def self.model
+    MercatorIcecat::Metadatum
+  end
+
+  def self.model_name
+    mercator_icecat_metadata
+  end
+
+  include Hobo::Controller::Model
+  hobo_model_controller
+  auto_actions :all
+end

data/app/models/mercator_icecat/access.rb

@@ -0,0 +1,48 @@
+require 'open-uri'
+
+module MercatorIcecat
+  class Access
+
+    attr_accessor(:user, :password, :vendor, :lang, :typ, :base_uri, :full_index_url, :daily_index_url, :open_uri_options)
+
+    USER = "donrudl"
+    PASSWORD = "cpP42J"
+    VENDOR = "HP"
+    LANG = "int"
+    TYP = "productxml"
+    BASE_URL = "http://data.icecat.biz"
+    FULL_INDEX_URL = BASE_URL + "/export/freexml/files.index.xml"
+    DAILY_INDEX_URL = BASE_URL + "/export/freexml/daily.index.xml"
+
+    # --- Class Methods --- #
+
+    def self.open_uri_options
+      {:http_basic_authentication => [self::USER, self::PASSWORD]}
+    end
+
+    def self.download_index(full: false)
+      if full
+        file = File.new(Rails.root.join("vendor","catalogs","files.index.xml"), "w")
+        io = open( FULL_INDEX_URL, open_uri_options.merge({"Accept-Encoding" => "gzip"}) )
+      else
+        file = File.new(Rails.root.join("vendor","catalogs",Date.today.to_s + "-index.xml"), "w")
+        io = open( DAILY_INDEX_URL, open_uri_options.merge({"Accept-Encoding" => "gzip"}) )
+      end
+      unzipped_io = Zlib::GzipReader.new( io )
+      unzipped_io.each do |line|
+        file.write line
+      end
+      file.close
+      io.close
+    end
+
+    def self.product(product_id: nil, path: nil) # accepts product_id or path as parameter
+      return open(self.product_url(product_id), open_uri_options).read if product_id
+      return open(BASE_URL + "/" + path, open_uri_options).read if path
+    end
+
+    def self.product_url(product_id: nil)
+      BASE_URL + "/xml_s3/xml_server3.cgi?prod_id=" + product_id + ";vendor=" + VENDOR + ";lang=" + LANG + ";output=" + TYP
+    end
+  end
+end

data/app/models/mercator_icecat/metadatum.rb

@@ -0,0 +1,393 @@
+# encoding: utf-8
+
+require 'saxerator'
+require 'open-uri'
+require 'string_extensions'
+
+module MercatorIcecat
+  class Metadatum < ActiveRecord::Base
+
+    hobo_model # Don't put anything above this
+
+    fields do
+      path              :string
+      icecat_updated_at :datetime
+      quality           :string
+      supplier_id       :string
+      icecat_product_id :string, :index => true
+      prod_id           :string, :index => true
+      product_number    :string
+      cat_id            :string
+      on_market         :string
+      model_name        :string
+      product_view      :string
+      timestamps
+    end
+    attr_accessible :path, :cat_id, :product_id, :icecat_updated_at, :quality, :supplier_id,
+                    :prod_id, :on_market, :model_name, :product_view, :icecat_product_id
+
+    belongs_to :product, :class_name => "Product"
+
+    # --- Permissions --- #
+
+    def create_permitted?
+      acting_user.administrator?
+    end
+
+    def update_permitted?
+      acting_user.administrator?
+    end
+
+    def destroy_permitted?
+      acting_user.administrator?
+    end
+
+    def view_permitted?(field)
+      true
+    end
+
+    # --- Class Methods --- #
+
+    def self.import(full: false, date: Date.today)
+      if full
+        file = File.open(Rails.root.join("vendor","catalogs","files.index.xml"), "r")
+      else
+        file = File.open(Rails.root.join("vendor","catalogs",date.to_s + "-index.xml"), "r")
+      end
+
+      parser = Saxerator.parser(file) do |config|
+        config.put_attributes_in_hash!
+      end
+
+      # Hewlett Packard has Supplier_id = 1
+      parser.for_tag("file").with_attribute("Supplier_id", "1").each do |product|
+        metadatum = self.find_or_create_by_icecat_product_id(product["Product_ID"])
+        mode = Time.now - metadatum.created_at > 5 ? " updated." : " created."
+
+        model_name = product["Model_Name"].fix_utf8 if product["Model_Name"].present?
+
+        if metadatum.update(path:              product["path"],
+                            cat_id:            product["Catid"],
+                            icecat_product_id: product["Product_ID"],
+                            icecat_updated_at: product["Updated"],
+                            quality:           product["Quality"],
+                            supplier_id:       product["Supplier_id"],
+                            prod_id:           product["Prod_ID"],
+                            on_market:         product["On_Market"],
+                            model_name:        model_name,
+                            product_view:      product["Product_View"])
+          ::JobLogger.info("Metadatum " + product["Prod_ID"].to_s + mode)
+        else
+          ::JobLogger.error("Metadatum " + product["Prod_ID"].to_s + " could not be saved: " + metadatum.errors.first )
+        end
+      end
+      file.close
+    end
+
+    def self.assign_products(only_missing: true)
+      if only_missing
+        products = Product.without_icecat_metadata
+        ::JobLogger.warn(products.count.to_s + " products without metadata.")
+      else
+        products = Product.all
+      end
+
+      products.each do |product|
+        metadata = self.where(prod_id: product.icecat_article_number)
+        amount = metadata.count
+        metadata.each_with_index do |metadatum, index|
+          if metadatum.update(product_id: product.id)
+            ::JobLogger.info("Product " + product.number.to_s + " assigned to " + metadatum.id.to_s + " (" + index.to_s + "/" + amount.to_s + ")")
+          else
+            ::JobLogger.error("Product " + product.number.to_s + " assigned to " + metadatum.id.to_s)
+          end
+        end
+      end
+
+      products = Product.without_icecat_metadata
+      ::JobLogger.warn(products.count.to_s + " products without metadata.")
+    end
+
+    def self.download(overwrite: false, from_today: true)
+      if from_today
+        @twentyfivehours = Time.now - 25.hours
+        metadata = self.where{ (product_id != nil) & (updated_at > my{@twentyfivehours})}
+      else
+        metadata = self.where{ product_id != nil }
+      end
+
+      amount = metadata.count
+      metadata.each_with_index do |metadatum, index|
+        if metadatum.download(overwrite: overwrite)
+          ::JobLogger.info("XML Metadatum " + metadatum.prod_id.to_s + " downloaded. (" + index.to_s + "/" + amount.to_s + ")")
+        else
+          ::JobLogger.info("XML Metadatum " + metadatum.prod_id.to_s + " exists (no overwrite)!")
+        end
+      end
+    end
+
+    def self.update_products(from_today: true)
+      if from_today
+        @twentyfivehours = Time.now - 25.hours
+        metadata = self.where{ (product_id != nil) & (updated_at > my{@twentyfivehours})}.order(id: :asc)
+      else
+        metadata = self.where{ product_id != nil }.order(id: :asc)
+      end
+
+      amount = metadata.count
+      metadata.each_with_index do |metadatum, index|
+        # if-clause is handy for resume after dump
+        metadatum.update_product # if metadatum.id >= 109279
+        ::JobLogger.info("(" + index.to_s + "/" + amount.to_s + ")")
+      end
+    end
+
+    def self.update_product_relations(from_today: true)
+      if from_today
+        @twentyfivehours = Time.now - 25.hours
+        metadata = self.where{ (product_id != nil) & (updated_at > my{@twentyfivehours})}.order(id: :asc)
+      else
+        metadata = self.where{ product_id != nil }.order(id: :asc)
+      end
+
+      amount = metadata.count
+      metadata.each_with_index do |metadatum, index|
+        metadatum.update_product_relations
+        ::JobLogger.info("(" + index.to_s + "/" + amount.to_s + ")")
+      end
+    end
+
+    def self.import_missing_images
+      metadata = self.includes(:product).where{product.id != nil}
+                     .where{product.photo_file_name == nil}.references(:product).order(id: :asc)
+      metadata.each do |metadatum|
+        metadatum.import_missing_image
+      end
+    end
+
+
+    # --- Instance Methods --- #
+
+    def download(overwrite: false)
+      unless overwrite
+        return false if File.exist?(Rails.root.join("vendor","xml",icecat_product_id.to_s + ".xml"))
+      end
+
+      unless self.path
+        return false
+      end
+
+      # force_encoding fixes: Encoding::UndefinedConversionError: "\xC3" from ASCII-8BIT to UTF-8
+      begin
+        io = open(Access::BASE_URL + "/" + self.path, Access.open_uri_options).read.force_encoding('UTF-8')
+        file = File.new(Rails.root.join("vendor","xml",icecat_product_id.to_s + ".xml"), "w")
+        io.each_line do |line|
+          file.write line
+        end
+
+        file.close
+        return true
+      rescue
+        ::JobLogger.error("Download error: " + Access::BASE_URL + "/" + self.path)
+        return false
+      end
+    end
+
+    def update_product
+      # :en => lang_id = 1, :de => lang_id = 4
+      file = open(Rails.root.join("vendor","xml",icecat_product_id.to_s + ".xml")).read
+      product_nodeset = Nokogiri::XML(file).xpath("//ICECAT-interface/Product")[0]
+      product = self.product
+
+      description_de = try_to { product_nodeset.xpath("ProductDescription[@langid='4']")[0]["ShortDesc"].fix_utf8 }
+      description_en = try_to { product_nodeset.xpath("ProductDescription[@langid='1']")[0]["ShortDesc"].fix_utf8 }
+      long_description_de = try_to { product_nodeset.xpath("ProductDescription[@langid='4']")[0]["LongDesc"].fix_utf8 }
+      long_description_en = try_to { product_nodeset.xpath("ProductDescription[@langid='1']")[0]["LongDesc"].fix_utf8 }
+      warranty_de = try_to { product_nodeset.xpath("ProductDescription[@langid='4']")[0]["WarrantyInfo"].fix_utf8 }
+      warranty_en = try_to { product_nodeset.xpath("ProductDescription[@langid='1']")[0]["WarrantyInfo"].fix_utf8 }
+
+      product.update(# title_de: product_nodeset["Title"],
+                     # title_en: product_nodeset["Title"],
+                     description_de: description_de,
+                     description_en: description_en,
+                     long_description_de: long_description_de,
+                     long_description_en: long_description_en,
+                     warranty_de: warranty_de,
+                     warranty_en: warranty_en)
+
+      property_groups_nodeset = product_nodeset.xpath("CategoryFeatureGroup")
+      property_groups_nodeset.each do |property_group_nodeset|
+        icecat_id = property_group_nodeset["ID"]
+        name_en = try_to { property_group_nodeset.xpath("FeatureGroup/Name[@langid='1']")[0]["Value"].fix_utf8 }
+        name_de = try_to { property_group_nodeset.xpath("FeatureGroup/Name[@langid='4']")[0]["Value"].fix_utf8 }
+        name_de ||= name_en # English, if German not available
+        name_de ||= try_to { property_group_nodeset.xpath("FeatureGroup/Name")[0]["Value"].fix_utf8 }
+                    # anything if neither German nor English available
+
+        property_group = ::PropertyGroup.find_by_icecat_id(icecat_id)
+        unless property_group
+          property_group = ::PropertyGroup.new(icecat_id: icecat_id,
+                                               name_de: name_de,
+                                               name_en: name_en,
+                                               position: icecat_id) # no better idea ...
+          if property_group.save
+            ::JobLogger.info("PropertyGroup " + icecat_id.to_s + " created.")
+          else
+            ::JobLogger.error("PropertyGroup " + icecat_id.to_s + " could not be created: " + property_group.errors.first.to_s)
+          end
+        end
+      end
+
+      product.values.destroy_all
+
+      features_nodeset = product_nodeset.xpath("ProductFeature")
+      features_nodeset.each do |feature|
+        # icecat_presentation_value = feature.xpath("Presentation_Value") # not used here
+        icecat_feature_id = feature.xpath("Feature")[0]["ID"].to_i
+        icecat_value = feature["Value"]
+        icecat_feature_group_id = feature["CategoryFeatureGroup_ID"]
+
+        name_en = try_to { feature.xpath("Feature/Name[@langid='1']")[0]["Value"].fix_utf8 }
+        name_de = try_to { feature.xpath("Feature/Name[@langid='4']")[0]["Value"].fix_utf8 }
+        name_de ||= name_en # English, if German not available
+        name_de ||= try_to { feature.xpath("Feature/Name")[0]["Value"].fix_utf8 } # anything if neither German nor English available
+
+        unit_en = try_to { feature.xpath("Feature/Measure/Signs/Sign[@langid='1']")[0].content.fix_utf8 }
+        unit_de = try_to { feature.xpath("Feature/Measure/Signs/Sign[@langid='4']")[0].content.fix_utf8 }
+        unit_de ||= unit_en # English, if German not available
+        unit_de ||= try_to { feature.xpath("Feature/Measure/Signs/Sign")[0].content.fix_utf8 }
+                    # anything if neither German nor English available
+
+        property_group = PropertyGroup.find_by_icecat_id(icecat_feature_group_id)
+
+        property = Property.where(icecat_id: icecat_feature_id).first
+        unless property
+          property = Property.new(icecat_id: icecat_feature_id,
+                                  position: icecat_feature_id,
+                                  name_de: name_de,
+                                  name_en: name_en,
+                                  datatype: icecat_value.icecat_datatype)
+          if property.save
+            ::JobLogger.info("Property " + property.id.to_s + " saved.")
+          else
+            ::JobLogger.error("Property could not be saved:" + property.errors.first.to_s)
+          end
+        end
+
+        value = Value.where(property_group_id: property_group.id, property_id: property.id,
+                            product_id: product.id, state: icecat_value.icecat_datatype).first
+        unless value
+          value = Value.new(property_group_id: property_group.id, property_id: property.id, product_id: product.id)
+          value.state = icecat_value.icecat_datatype
+        end
+
+        if icecat_value.icecat_datatype == "flag"
+          value.flag = ( icecat_value == "Y" )
+        end
+
+        if icecat_value.icecat_datatype == "numeric"
+          value.amount = icecat_value.to_f
+          value.unit_de = try_to { unit_de.fix_utf8 }
+          value.unit_en = try_to { unit_en.fix_utf8 }
+        end
+
+        if icecat_value.icecat_datatype == "textual"
+          value.title_de = try_to { icecat_value.truncate(252).fix_utf8 }
+          value.title_en = try_to { icecat_value.truncate(252).fix_utf8 }
+          value.unit_de = try_to { unit_de.fix_utf8 }
+          value.unit_en = try_to { unit_en.fix_utf8 }
+        end
+
+        if value.save
+          ::JobLogger.info("Value " + value.id.to_s + " saved.")
+        else
+          ::JobLogger.error("Value could not be saved:" + value.errors.first)
+        end
+      end
+      ::JobLogger.info("=== Metadatum " + id.to_s + " updated Product " + product_id.to_s + " ===")
+    end
+
+    def delete_relations
+      product = self.product
+      relations_count = product.productrelations.count
+      product.productrelations.destroy_all
+      supplies_count = product.supplyrelations.count
+      product.supplyrelations.destroy_all
+      ::JobLogger.info(relations_count.to_s + " Productrel., " +
+                       supplies_count.to_s + " Supplyrel. deleted for Product " + product.id.to_s +
+                       " Metadatum " + self.id.to_s)
+    end
+
+    def update_product_relations
+      file = open(Rails.root.join("vendor","xml",icecat_product_id.to_s + ".xml")).read
+      product_nodeset = Nokogiri::XML(file).xpath("//ICECAT-interface/Product")[0]
+      product = self.product
+      cat_id = self.cat_id
+
+      self.delete_relations
+
+      unknown_products = 0
+      icecat_ids = []
+      product_nodeset.xpath("ProductRelated").each do |relation|
+        icecat_ids << try_to { relation.xpath("Product")[0]["ID"].to_i }
+      end
+
+      related_metadata = Metadatum.where(icecat_product_id: icecat_ids)
+      related_metadata.each do |related_metadatum|
+        related_product_id = try_to { related_metadatum.product_id.to_i }
+
+        if related_product_id > 0
+          if related_metadatum.cat_id == cat_id
+            product.productrelations.new(related_product_id: related_product_id)
+          else
+            product.supplyrelations.new(supply_id: related_product_id)
+          end
+        else
+          unknown_products += 1
+        end
+      end
+
+      if product.save(validate: false) # FIXME!: This time without validations ...
+        ::JobLogger.info("Product " + product.id.to_s + ": " +
+                         product.productrelations.count.to_s + " Productrel. " +
+                         product.supplyrelations.count.to_s + ", Supplyrel. created, " +
+                         unknown_products.to_s + " unknown.")
+      else
+        ::JobLogger.error("Product " + product.id.to_s + " could not be updated")
+      end
+    end
+
+    def import_missing_image
+      file = open(Rails.root.join("vendor","xml",icecat_product_id.to_s + ".xml")).read
+      product_nodeset = Nokogiri::XML(file).xpath("//ICECAT-interface/Product")[0]
+
+      product = self.product
+      return nil if product.photo_file_name # no overwriting intended
+
+      path = product_nodeset["HighPic"]
+
+      if path.empty?
+        ::JobLogger.warn("no Image available for Product " + product.id.to_s)
+        return nil # no image available
+      end
+
+      begin
+        io = StringIO.new(open(path, Access.open_uri_options).read)
+        io.class.class_eval { attr_accessor :original_filename }
+        io.original_filename = path.split("/").last
+
+        product.photo = io
+
+        if product.save(validate: false) # FIXME!: This time without validations ...
+          ::JobLogger.info("Image  " + path.split("/").last + " for Product " + product.id.to_s + " saved." )
+        else
+          ::JobLogger.error("Image  " + path.split("/").last + " for Product " + product.id.to_s + " could not be saved!" )
+        end
+      rescue Exception => e
+        ::JobLogger.warn("Image  " + path + " could not be loaded!" )
+        ::JobLogger.warn(e)
+      end
+    end
+
+  end
+end