jaleb 0.1.2

data/lib/jaleb.rb

@@ -0,0 +1,68 @@
+# = Jaleb
+# Author:: Hasan Basheer - 2017
+require "uri"
+require_relative "jaleb/version"
+require_relative "jaleb/data"
+require_relative "jaleb/model"
+require_relative "jaleb/string"
+require_relative "jaleb/nokogiri"
+
+
+module Jaleb  
+  LIBRARY_PATH       = File.join(File.dirname(__FILE__), 'jaleb')
+  MODEL_PATH         = File.join(LIBRARY_PATH, 'models')
+  
+  # Module Methods
+  class << self    
+    # Fetch information based on url
+    def fetch(url) 
+      # Look for model based on url
+      model_class = Jaleb::Model.identify(url)
+
+      # Use Base class for fallback
+      model_class = Jaleb::Model::Base unless model_class
+      
+      data = Jaleb::Data.read(url)
+
+      product = model_class.new
+      product.parse(data)
+
+      # Save url
+      product.url = url
+      
+      return product
+    end
+    
+
+    # get array of models syms
+    def models
+      models = Array.new
+      Dir[File.join(File.dirname(__FILE__), "jaleb", "models", "*.rb")].each do |f|
+        model = File.basename(f, ".rb").to_sym
+        models << model unless model == :base
+      end 
+      return models
+    end 
+
+    # get array of model classes
+    #   Jaleb.model_classes = [Amazon, Ebay, ThinkGeek]
+    def model_classes
+      models = Array.new
+      Dir[File.join(File.dirname(__FILE__), "jaleb", "models", "*.rb")].each do |f|
+        filename = File.basename(f, ".rb")
+        models << ["Jaleb", "Model", filename.camelize].join("::").constantize unless filename == "base"
+      end 
+      return models
+    end 
+  end
+
+
+
+  # Autoload Models
+  module Model
+    autoload :Base, File.join(MODEL_PATH, "base")
+    for model in Jaleb.models
+      autoload model.to_s.camelize.to_sym, File.join(MODEL_PATH, model.to_s)
+    end
+  end     
+end

data/lib/jaleb/data.rb

@@ -0,0 +1,22 @@
+require "nokogiri"
+require "open-uri"
+
+
+module Jaleb
+  # this class is responsible for fetching and parsing data 
+  class Data
+ 	# Get read url and get data object
+  	def self.read(url )
+      opt= {}
+      opt['User-Agent'] = 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.57 Safari/537.36'
+      response = open(url, opt)
+  	  doc = ::Nokogiri::HTML(response.read)
+  	   # Save contents of URL/Remote File for debugging
+      response.rewind
+      last_response_file = File.expand_path(File.join("..", "..", "last_response"), File.dirname(__FILE__))
+      File.new(last_response_file, "w+").write(response.read)
+  	  return doc	
+  	end
+  
+  end
+end

data/lib/jaleb/model.rb

@@ -0,0 +1,21 @@
+module Jaleb
+  module Model
+    class << self
+      #Detect model by url
+      def identify(url)
+        if url =~ ::URI.regexp
+          uri = ::URI::parse(url)
+          host = uri.host
+          match_model = nil 
+           for model in Jaleb.model_classes 
+             match_model = model if host =~ model.regexp
+           end
+           return match_model
+
+        else
+          raise AgrumentError , "not a url"
+        end
+      end
+    end
+  end
+end

data/lib/jaleb/models/amazon.rb

@@ -0,0 +1,51 @@
+module Jaleb module Model
+    class Amazon < Jaleb::Model::Base # A regular expression for determining if a url comes from a specific service/website
+      def self.regexp
+        /amazon\.com/
+      end
+        
+      # Parse data and look for object attributes to give to object    
+      def parse(data)
+        super(data)
+        
+        case doc
+        when Nokogiri::HTML::Document
+          # Get Name
+          self.name = doc.css("h1#title").first_string 
+          self.name = doc.xpath("string(//title)").split(" - ").first unless self.name
+           
+          # Get Description
+          self.description = doc.css(".productDescriptionWrapper").first_string    
+                
+          # Get description from meta title if not found
+          self.description = doc.xpath("//meta[@name='description']/@content").first_string if description.nil?
+          
+          # Get Price
+          parse_price(doc.css("#actualPriceValue").first_string) 
+          parse_price(doc.css("#priceblock_ourprice").first_string) unless self.price                  
+          parse_price(doc.css("#priceblock_saleprice").first_string) unless self.price
+          parse_price(doc.xpath("//span[contains(@id, 'price')]").first_string) unless self.price
+
+          # Get Unqualified Price 
+          parse_price(doc.xpath("//*[contains(@id, 'unqualifiedBuyBox')]//span").first_string) unless self.price
+
+          # Get Used Price 
+          parse_price(doc.xpath("//*[contains(@id, 'secondaryUsedAndNew')]//*[@class='price']").first_string) unless self.price
+
+
+          # Get Images
+          self.images = doc.xpath("//*[@data-action='main-image-click']//img").attribute_array
+          self.images = doc.xpath("//*[@id='imageBlock']//img").attribute_array unless self.images
+
+          # Get images for in-house products (kindle, etc.)
+          self.images = doc.xpath("//*[@id='kib-ma-container-0']//img").attribute_array if self.images.empty?
+
+          # Get images for third-party sellers
+          self.images = doc.xpath("//*[@id='prodImageContainer']//img").attribute_array if self.images.empty?
+
+          self.image = images.first
+        end            
+      end
+    end
+  end
+end

data/lib/jaleb/models/base.rb

@@ -0,0 +1,26 @@
+require "hashie"
+require "money"
+require "monetize"
+
+module Jaleb
+  module Model
+    class Base < ::Hashie::Mash         
+      # Parse data and set object attributes    
+      def parse(data)
+        self.doc = data # save data for if user wants to access it later      
+
+        # Get page title as name
+        self.name = doc.css("head > title").first_string
+      end      
+
+      # Parse a raw price string to get usable data
+      #   model.parse_price("$5.00") # => #<Money cents:500 currency:USD>
+      #   model.price.to_f # => 5.0
+      #   model.price.currency.symbol # => '$'
+      def parse_price(raw_price = nil)
+        return if raw_price.nil?
+        self.price = ::Monetize.parse(raw_price)
+      end
+    end # Baseasd
+  end # Model
+end # Jaleb

data/lib/jaleb/models/ebay.rb

@@ -0,0 +1,32 @@
+module Jaleb
+  module Model
+    class Ebay < Jaleb::Model::Base
+      # A regular expression for determining if a url comes from a specific service/website
+      def self.regexp
+        /ebay\.com/
+      end
+        
+      # Parse data and look for object attributes to give to object    
+      def parse(data)
+        super(data)
+
+        case doc
+        when Nokogiri::HTML::Document
+          # Get Name
+          self.name = doc.xpath("//h1[@itemprop='name']").first_string
+          
+          # Get Description
+          # OMITTED: This is tough to get because ebay item descriptions are custom html/content created by sellers
+
+          # Get Price
+          raw_price = doc.xpath("//span[@itemprop='price']").first_string
+          parse_price(raw_price.gsub(/US/, "")) if raw_price
+          
+          # Get Image
+          self.images = [{:src => doc.xpath("//span[@itemprop='image']/img").first_string}]
+          self.image = images.first
+        end            
+      end
+    end
+  end
+end

data/lib/jaleb/models/etsy.rb

@@ -0,0 +1,31 @@
+module Jaleb
+  module Model
+    class Etsy < Jaleb::Model::Base
+      # A regular expression for determining if a url comes from a specific service/website
+      def self.regexp
+        /etsy\.com/
+      end
+        
+      # Parse data and look for object attributes to give to object    
+      def parse(data)
+        super(data)
+        
+        case doc
+        when Nokogiri::HTML::Document
+          # Get Name
+          self.name = doc.xpath("//span[@itemprop='name']").first_string
+
+          # Get Description
+          self.description = doc.xpath("//div[@id='description-text']").first_string
+
+          # Get Price
+          parse_price(doc.css("span#listing-price").first_string)
+                          
+          # Get Images
+          self.images = doc.xpath("//div[@id='image-main']//img").attribute_array 
+          self.image = images.first
+        end            
+      end
+    end
+  end
+end

data/lib/jaleb/models/gamecouk.rb

@@ -0,0 +1,40 @@
+module Jaleb
+  module Model
+    class Gamecouk < Jaleb::Model::Base
+      # A regular expression for determining if a url comes from a specific service/website
+      def self.regexp
+        /game\.co\.uk/
+      end
+        
+      # Parse data and look for object attributes to give to object    
+      def parse(data)
+        super(data)
+
+        case doc
+        when Nokogiri::HTML::Document
+          require 'date'
+          # Get Name
+          self.name = doc.xpath("//meta[@property='og:title']/@content").first_string
+          
+          # Get Description
+          # OMITTED: This is tough to get because ebay item descriptions are custom html/content created by sellers
+          
+          self.description = doc.css('div#primary div#details.panel div.description').to_s
+          
+          self.release_date = doc.css('div#primary div#details.panel p.releaseDate').first_string
+          
+          self.release_date = Date.strptime(self.release_date.gsub('Released on ',''), '%d-%b-%Y') if self.release_date
+          
+          # Get Price
+          raw_price = doc.css("span.price").first_string
+          Money.default_currency = Money::Currency.new("GBP")
+          parse_price(raw_price.gsub(/Only /, "")) if raw_price
+          
+          # Get Image
+          self.images = [{:src => doc.xpath("//meta[@property='og:image']/@content").first_string}]
+          self.image = images.first
+        end            
+      end
+    end
+  end
+end

data/lib/jaleb/models/googleshopping.rb

@@ -0,0 +1,31 @@
+module Jaleb
+  module Model
+    class Googleshopping < Jaleb::Model::Base
+      # A regular expression for determining if a url comes from a specific service/website
+      def self.regexp
+        /google\.com/
+      end
+        
+      # Parse data and look for object attributes to give to object    
+      def parse(data)
+        super(data)
+        case doc
+        when Nokogiri::HTML::Document
+          # Get Name
+          name = doc.css('h1#product-name span.main-title').first_string
+          self.name = name if name
+
+          # Get Description
+          self.description = doc.css("#product-description-full").first_string   
+
+          # Get Price
+          parse_price(doc.css('#summary-prices .price').first_string)
+                          
+          # Get Images
+          self.images = doc.css('div#product-basic-info img').attribute_array
+          self.image = images.first
+        end            
+      end
+    end
+  end
+end

data/lib/jaleb/models/newegg.rb

@@ -0,0 +1,33 @@
+module Jaleb
+  module Model
+    class Newegg < Jaleb::Model::Base
+      # A regular expression for determining if a url comes from a specific service/website
+      def self.regexp
+        /newegg\.com/
+      end
+        
+      # Parse data and look for object attributes to give to object    
+      def parse(data)
+        super(data)
+        case doc
+        when Nokogiri::HTML::Document
+          # Get Name          
+          self.name = doc.css("#synopsis .grpDesc .wrapper h1 > span").first_string
+          #self.name = doc.css("#synopsis .grpDesc .wrapper > .span").first_string
+
+          # Description - Not always reliable.
+          self.description = doc.css(".itmDesc > p").first_string
+
+          # Get Price (currently disabled because price is displayed after page load)
+          #parse_price doc.css("li.price-current").attribute("content").content
+                          
+          # # Get Images
+          self.images = [{:src => doc.css('.mainSlide > img').attribute("src").content}]
+          self.image = images.first
+        end
+      rescue
+        return nil
+      end
+    end
+  end
+end

data/lib/jaleb/models/souq.rb

@@ -0,0 +1,32 @@
+module Jaleb
+  module Model
+    class Souq < Jaleb::Model::Base
+      # A regular expression for determining if a url comes from a specific service/website
+      def self.regexp
+        /souq\.com/
+      end
+        
+      # Parse data and look for object attributes to give to object    
+      def parse(data)
+        super(data)
+        
+        case doc
+        when Nokogiri::HTML::Document
+          # Get Name
+          self.name = doc.xpath("//meta[@property='og:title']/@content").first_string
+
+          # Get Description
+          self.description = doc.xpath("//meta[@property='og:description']/@content").first_string
+  
+          # Get Price
+          raw_price = doc.css('h3.price').first_string
+          parse_price(raw_price.gsub(/,/, "")) if raw_price
+          
+          # Get Images
+          self.images = [{:src => doc.xpath("//meta[@property='og:image']/@content").first_string}] 
+          self.image = images.first
+        end            
+      end
+    end
+  end
+end

data/lib/jaleb/models/thinkgeek.rb

@@ -0,0 +1,31 @@
+module Jaleb
+  module Model
+    class Thinkgeek < Jaleb::Model::Base
+      # A regular expression for determining if a url comes from a specific service/website
+      def self.regexp
+        /thinkgeek\.com/
+      end
+        
+      # Parse data and look for object attributes to give to object    
+      def parse(data)
+        super(data)
+        
+        case doc
+        when Nokogiri::HTML::Document
+          # Get Name
+          self.name = doc.xpath("//meta[@property='og:title']/@content").first_string
+
+          # Get Description
+          self.description = doc.xpath("//meta[@property='og:description']/@content").first_string
+  
+          # Get Price
+          parse_price(doc.xpath("//form[@id='buy']/h3").first_string) rescue nil
+          
+          # Get Images
+          self.images = [{:src => doc.xpath("//meta[@property='og:image']/@content").first_string}] 
+          self.image = images.first
+        end            
+      end
+    end
+  end
+end

data/lib/jaleb/nokogiri.rb

@@ -0,0 +1,53 @@
+require "cgi"
+
+module Jaleb
+  # This contains helper methods for Nokogiri interactions
+  module Nokogiri
+    module HTML
+      module Document
+         
+      end # Document
+    end # HTML
+    
+    module XML
+      module NodeSet
+        # get string from first nodeset model
+        def first_string
+          node = first
+          case node
+          # xml/html element?
+          when ::Nokogiri::XML::Element 
+            return node.content.sanitize
+          # xml/html attribute?
+          when ::Nokogiri::XML::Attr
+            return node.value.sanitize
+          end
+        end          
+        
+        # convert nodeset models to an array of hashes
+        #   @doc.xpath("//img")).attribute_array # => [{:element => "img", :src => ".../someimage.png"}]
+        def attribute_array
+          a = Array.new
+          each do |node|
+            temp_hash = Hash.new 
+            case node 
+            when ::Nokogiri::XML::Element
+              temp_hash[:element] = node.name
+              node.attributes.each do |key, value|
+                case value
+                when ::Nokogiri::XML::Attr
+                  temp_hash[key.to_sym] = value.value.sanitize
+                end 
+              end
+            end 
+            a << temp_hash            
+          end
+          return a          
+        end 
+      end # Nodeset
+    end # XML 
+  end # Nokogiri 
+end # Jaleb
+
+::Nokogiri::HTML::Document.send(:include, ::Jaleb::Nokogiri::HTML::Document)
+::Nokogiri::XML::NodeSet.send(:include, ::Jaleb::Nokogiri::XML::NodeSet)