joyceshop 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: a3acee65aadff3a04affd99a74cdf5504632c695
+  data.tar.gz: 2709281e4846fe17d1a70649554aa87ddbea2b81
+SHA512:
+  metadata.gz: 2f512e054122cfe33a784207d417de2539687646cf23127dbefe5719270c78c6e7f8740c7103c4242a1ab506b1af2f934a733389c48ba05624243e3cfc3d0d5f
+  data.tar.gz: af79d4953a8ad049e69f7074ca7bbc575ecc5e3df6518a9c6b53b08b0006f6b67ab46a94a625de4be3cc608f47999e7dc00bc9d2ec6f74912dd44dc249953632

data/bin/joyceshop

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+# require 'joyceshop' # for production
+require_relative '../lib/joyceshop.rb' # for testing
+
+scraper = JoyceShop::Scraper.new()
+puts scraper.accessories(1)

data/lib/joyceshop.rb

@@ -0,0 +1,2 @@
+#!/usr/bin/env ruby
+require_relative 'joyceshop/scraper'

data/lib/joyceshop/scraper.rb

@@ -0,0 +1,101 @@
+#!/usr/bin/env ruby
+require 'oga'
+require 'open-uri'
+
+# scrape data
+module JoyceShop
+  class Scraper
+    # URI
+    @@BASE_URI        = 'https://www.joyce-shop.com'
+    @@LATEST_URI      = "#{@@BASE_URI}/PDList.asp?brand=01&item1=&item2=&ya19=&keyword=&recommand=1412170001&ob=F"
+    @@POPULAR_URI     = "#{@@BASE_URI}/PDList.asp?brand=01&item1=&item2=&ya19=&keyword=&recommand=1305080002&ob=F"
+    @@TOPS_URI        = "#{@@BASE_URI}/PDList.asp?brand=01&item1=110&item2=111&ya19=&keyword=&recommand=&ob=F"
+    @@PANTS_URI       = "#{@@BASE_URI}/PDList.asp?brand=01&item1=120&item2=121&ya19=&keyword=&recommand=&ob=F"
+    @@ACCESSORIES_URI = "#{@@BASE_URI}/PDList.asp?brand=01&item1=140&item2=141&ya19=&keyword=&recommand=&ob=F"
+
+    # Selectors
+    @@ITEM_SELECTOR      = "//div[contains(@class, 'NEW_shop_list')]/ul/li/div[contains(@class, 'NEW_shop_list_pic')]"
+    @@LINK_SELECTOR      = 'a'
+    @@IMAGE_SELECTOR     = "a/img[contains(@class, 'lazyload')]"
+    @@ITEM_INFO_SELECTOR = "div[contains(@class, 'NEW_shop_list_info')]"
+    @@TITLE_SELECTOR     = "#{@@ITEM_INFO_SELECTOR}/div[1]"
+    @@PRICE_SELECTOR     = "#{@@ITEM_INFO_SELECTOR}/span"
+
+    # Regular
+    @@TITLE_REGEX = /([．\p{Han}[a-zA-Z]]+)/
+
+    def latest(page)
+      uri  = uri_with_page(@@LATEST_URI, page)
+      body = fetch_data(uri)
+      filter(body)
+    end
+
+    def popular(page)
+      uri  = uri_with_page(@@POPULAR_URI, page)
+      body = fetch_data(uri)
+      filter(body)
+    end
+
+    def tops(page)
+      uri  = uri_with_page(@@TOPS_URI, page)
+      body = fetch_data(uri)
+      filter(body)
+    end
+
+    def pants(page)
+      uri  = uri_with_page(@@PANTS_URI, page)
+      body = fetch_data(uri)
+      filter(body)
+    end
+
+    def accessories(page)
+      uri  = uri_with_page(@@ACCESSORIES_URI, page)
+      body = fetch_data(uri)
+      filter(body)
+    end
+
+    private
+    def uri_with_page(uri, page)
+      "#{uri}&pageno=#{page}"
+    end
+
+    def fetch_data(uri)
+      open(uri) {|file| file.read}
+    end
+
+    def filter(raw)
+      Oga.parse_html(raw)
+         .xpath(@@ITEM_SELECTOR)
+         .map { |item| parse(item) }
+    end
+
+    def parse(item)
+      {
+        title:  extract_title(item),
+        price:  extract_price(item),
+        images: extract_images(item),
+        link:   extract_link(item)
+      }
+    end
+
+    def extract_title(item)
+      item.xpath(@@TITLE_SELECTOR).text
+          .scan(@@TITLE_REGEX)
+          .flatten[0]
+    end
+
+    def extract_price(item)
+      item.xpath(@@PRICE_SELECTOR).text.to_i
+    end
+
+    def extract_images(item)
+      image       = item.xpath(@@IMAGE_SELECTOR).attribute(:src).first.value
+      image_hover = image.sub(/\.jpg/, '-h.jpg')
+      ["#{@@BASE_URI}#{image}", "#{@@BASE_URI}#{image_hover}"]
+    end
+
+    def extract_link(item)
+      "#{@@BASE_URI}/#{item.xpath(@@LINK_SELECTOR).attribute(:href).first.value}"
+    end
+  end
+end

metadata

@@ -0,0 +1,55 @@
+--- !ruby/object:Gem::Specification
+name: joyceshop
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Even Chang
+- Luis Herrera
+- Katy Lee
+- Frank Lee
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-12-14 00:00:00.000000000 Z
+dependencies: []
+description: This is a gem scraping joyceshop's website and returns the popular/latest
+  items
+email:
+- kiki44552002@gmail.com
+- lmherrera86@gmail.com
+- katylee41024@yahoo.com.tw
+- frank1234211@gmail.com
+executables:
+- joyceshop
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/joyceshop
+- lib/joyceshop.rb
+- lib/joyceshop/scraper.rb
+homepage: http://rubygems.org/gems/joyceshop
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.7
+signing_key: 
+specification_version: 4
+summary: Scraper for JoyceShop
+test_files: []