queenshop 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 4ccb94cb12167097b2f0d30b5602d911d37c4295
+  data.tar.gz: 68c743d55718026e871bf2b041adfe2e2b2329df
+SHA512:
+  metadata.gz: 721a7e7e251081cbdbe220481ffa5038b2bd4a80f6ae6e393207081b106730b1f91ee732dbaa779a9797c24f0ee3e025f0a2d0295aa8418d333c01694aa34df7
+  data.tar.gz: 5a925e85811e0f6eeba1848e8a63d0f6604d59a98aaf220e899f9b1b6cf3f9a0b99bef679037c3c591153a94f982c265919c7510a46c40266467d112e6f28567

data/bin/queenshop

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+# require 'queenshop' # for production
+require_relative '../lib/queenshop.rb' # for testing
+
+scraper = QueenShopScraper::Filter.new
+puts scraper.scrape

data/lib/queenshop.rb

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+require_relative 'queenshop/config'
+require_relative 'queenshop/scraper'

data/lib/queenshop/config.rb

@@ -0,0 +1,58 @@
+#!/usr/bin/env ruby
+
+# this class takes care of
+# parsing the parameters
+module Validate
+  attr_reader :parameters
+  attr_reader :pages
+
+  VALID_ARGS = [:item, :price, :pages]
+
+  def validate_args(args)
+    @parameters = {item: '', price: '', pages: '1..2'}
+    args.each do |arg|
+      begin
+        match = /(?<key>.*?)=(?<value>.*)/.match(arg)
+        fail unless VALID_ARGS.include?(match[:key].to_sym)
+        value = check(match)
+        @parameters[match[:key].to_sym] = value
+      rescue StandardError
+        abort "invalid usage...\n" << usage << "\n\n"
+      end
+    end
+  end # end validate_args
+
+  def check(match)
+    value = match[:value]
+    fail unless value =~ /^(>|<|>=|<=|==)\d*.\d*?$/ if match[:key].to_sym.eql?(:price)
+    # Float(value) if match[:key].to_sym.eql?(:price)
+    fail unless value =~ /^\d*([.]{2}\d*)?$/ if match[:key].to_sym.eql?(:pages)
+    value
+  rescue StandardError
+      abort "invalid parameters"
+  end
+
+  def pages
+    first_page = @parameters[:pages].scan(/\d+/).first.to_i
+    last_page = @parameters[:pages].scan(/\d+/).last.to_i
+    @pages = *(first_page..last_page)
+  end
+
+  def usage
+    'Usage: queenshop [options]
+      item=(string)
+      price=(float[,float])
+      examples:
+              queenshop item="blouse" price=300
+              queenshop price=0,100
+              queenshop item="skirt"'
+  end
+end
+
+class QConfig
+  include Validate
+  def initialize (args)
+    validate_args (args)
+    pages
+  end
+end

data/lib/queenshop/scraper.rb

@@ -0,0 +1,79 @@
+#!/usr/bin/env ruby
+require 'oga'
+require 'open-uri'
+require_relative './config'
+
+# scrape data
+module QueenShopScraper
+  # filter class basically uses xpath selectors to get attribs
+  class Filter
+    attr_reader :result
+    attr_writer :item_selector
+    attr_writer :title_selector
+    attr_writer :price_selector
+    attr_writer :site_url
+
+    private
+
+    def get_xmldata(url)
+      raw_html = open(url)
+      Oga.parse_html(raw_html)
+    rescue StandardError
+      'error'
+    end
+
+    def fetch_result(uri = '')
+      url = @site_url + uri
+      # try to open the url
+      document = get_xmldata(url)
+      # hard return on an error
+      return [] unless document != 'error'
+
+      items = document.xpath(@item_selector)
+      # loop through the items and get the title and price
+      items.map do |item|
+        title = item.xpath(@title_selector).text.force_encoding('UTF-8')
+        price = item.xpath(@price_selector).text
+        strip_filter(title, price)
+      end
+      @result
+    end
+
+    def strip_filter (title, price)
+      price = price.gsub!(/NT. /, '')
+      if !@price_filter.empty?
+        if eval("#{price} #{@price_filter}")
+          @result << { title: "#{title}", price: "#{price}" }
+        end
+      else
+        @result << { title: "#{title}", price: "#{price}" } unless title.empty?
+      end
+
+    end
+
+    public
+
+    def initialize
+      @result = []
+      # xml selectors that will be used to scrape data
+      @item_selector = "//div[@class=\'pditem\']/div[@class=\'pdicon\']"
+      @title_selector = "div[@class=\'pdicon_name\']/a"
+      @price_selector = "div[@class=\'pdicon_price\']/div[@style=\'font-weight:bold;\']"
+      @site_url = 'https://www.queenshop.com.tw/m/PDList2.asp?'
+      @price_filter = nil
+    end
+
+    def scrape (params=[])
+      params.concat(ARGV)
+      conf = QConfig.new(params)
+      @price_filter = conf.parameters[:price]
+      
+      conf.pages.map do |page|
+        paginated_uri = "&page=#{page}"
+        fetch_result (paginated_uri)
+      end
+      @result
+    end
+
+  end
+end

metadata

@@ -0,0 +1,56 @@
+--- !ruby/object:Gem::Specification
+name: queenshop
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Even Chang
+- Luis Herrera
+- Katy Lee
+- Frank Lee
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-10-23 00:00:00.000000000 Z
+dependencies: []
+description: This is a gem scraping queenshop's website and returns the items with
+  corresponding prices
+email:
+- kiki44552002@gmail.com
+- lmherrera86@gmail.com
+- katylee41024@yahoo.com.tw
+- frank1234211@gmail.com
+executables:
+- queenshop
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/queenshop
+- lib/queenshop.rb
+- lib/queenshop/config.rb
+- lib/queenshop/scraper.rb
+homepage: http://rubygems.org/gems/pinkoi
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.0
+signing_key: 
+specification_version: 4
+summary: Scraper for Queenshop
+test_files: []