joyceshop 0.0.2 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f8dbce4f8ea05d23d63f3d9c49bc5819afeb7bd7
-  data.tar.gz: ddc747fbfb484c3f4dc4f8d3e513153a243bd01c
+  metadata.gz: 2c4b143619c3fe55c48d39a9b419f39d6923fce0
+  data.tar.gz: 1b0fd41b0f9aff1067b680b43e2260a3f7f83cae
 SHA512:
-  metadata.gz: ed50d49f299ae7bf30e8f86f3e9284d4d9dd0eaebe325b0a9f7f3b01664e652b57faf20b57b05b17ab1b26ba6ac0149dd43693927e49890ab1b8db57e0ef1bf4
-  data.tar.gz: c7cd6da017475ef2f3b34d92774407f3fc7916f97c416ab9b0094488f605a302d38b554096f798dd6fda6f88aac53dc2ed82f53fffc0a546fa569caddfac6740
+  metadata.gz: 1b7810cbdc56d47096794d986417082dbe14dc84122e8d0539dbbc2e1f0e4c243f98ba451be45642fafb93fc0f053c0c258e8e4c6fcc6b288caded95c1b961c7
+  data.tar.gz: bd71e91f8adda7003cd73357e1eb2495c4e262d30e39169242bca461f4baaf0caffc5925849c0a5526f3a9fc3c438e2c05b3df9eb72170b1036e40ef063aa02f

data/bin/joyceshop

@@ -2,5 +2,6 @@
 # require 'joyceshop' # for production
 require_relative '../lib/joyceshop.rb' # for testing
 
-scraper = JoyceShop::Scraper.new()
+scraper = JoyceShop::Scraper.new
 puts scraper.search('紗針織衫', {price_boundary: [100, 443]})
+puts scraper.scrape(:tops, 1)

data/lib/joyceshop/scraper.rb

@@ -1,10 +1,14 @@
 #!/usr/bin/env ruby
 require 'oga'
+require 'uri'
 require 'open-uri'
 
 # scrape data
 module JoyceShop
   class Scraper
+    # Types
+    @@VALID_TYPES = [:tops, :popular, :pants, :pants, :accessories, :latest]
+
     # URI
     @@BASE_URI        = 'https://www.joyce-shop.com'
     @@LATEST_URI      = "#{@@BASE_URI}/PDList.asp?brand=01&item1=&item2=&ya19=&keyword=&recommand=1412170001&ob=F"
@@ -12,11 +16,12 @@ module JoyceShop
     @@TOPS_URI        = "#{@@BASE_URI}/PDList.asp?brand=01&item1=110&item2=111&ya19=&keyword=&recommand=&ob=F"
     @@PANTS_URI       = "#{@@BASE_URI}/PDList.asp?brand=01&item1=120&item2=121&ya19=&keyword=&recommand=&ob=F"
     @@ACCESSORIES_URI = "#{@@BASE_URI}/PDList.asp?brand=01&item1=140&item2=141&ya19=&keyword=&recommand=&ob=F"
+    @@SEARCH_URI      = "#{@@BASE_URI}/PDList.asp?"
 
     # Selectors
     @@ITEM_SELECTOR      = "//div[contains(@class, 'NEW_shop_list')]/ul/li/div[contains(@class, 'NEW_shop_list_pic')]"
-    @@LINK_SELECTOR      = 'a'
-    @@IMAGE_SELECTOR     = "a/img[contains(@class, 'lazyload')]"
+    @@LINK_SELECTOR      = 'a[1]/@href'
+    @@IMAGE_SELECTOR     = "a/img[contains(@class, 'lazyload')]/@src"
     @@ITEM_INFO_SELECTOR = "div[contains(@class, 'NEW_shop_list_info')]"
     @@TITLE_SELECTOR     = "#{@@ITEM_INFO_SELECTOR}/div[1]"
     @@PRICE_SELECTOR     = "#{@@ITEM_INFO_SELECTOR}/span"
@@ -58,6 +63,19 @@ module JoyceShop
       data = parse_html(body)
       filter(data, options)
     end
+
+    def search(keyword, options={})
+      uri  = uri_with_search(keyword)
+      body = fetch_data(uri)
+      data = parse_html(body)
+      filter(data, options)
+    end
+
+    def scrape(type, page, options = {})
+      abort "only supports #{@@VALID_TYPES}" unless @@VALID_TYPES.include?(type.to_sym)
+
+      method = self.method(type)
+      method.call(page, options)
     end
 
     private
@@ -65,6 +83,10 @@ module JoyceShop
       "#{uri}&pageno=#{page}"
     end
 
+    def uri_with_search(keyword)
+      "#{@@SEARCH_URI}keyword=#{URI.escape(keyword)}"
+    end
+
     def fetch_data(uri)
       open(uri) { |file| file.read }
     end
@@ -116,13 +138,13 @@ module JoyceShop
     end
 
     def extract_images(item)
-      image       = item.xpath(@@IMAGE_SELECTOR).attribute(:src).first.value
+      image       = item.xpath(@@IMAGE_SELECTOR).text
       image_hover = image.sub(/\.jpg/, '-h.jpg')
       ["#{@@BASE_URI}#{image}", "#{@@BASE_URI}#{image_hover}"]
     end
 
     def extract_link(item)
-      "#{@@BASE_URI}/#{item.xpath(@@LINK_SELECTOR).attribute(:href).first.value}"
+      "#{@@BASE_URI}/#{item.xpath(@@LINK_SELECTOR).text}"
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: joyceshop
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
 platform: ruby
 authors:
 - Even Chang
@@ -11,7 +11,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-01-03 00:00:00.000000000 Z
+date: 2016-01-04 00:00:00.000000000 Z
 dependencies: []
 description: This is a gem scraping joyceshop's website and returns the popular/latest
   items