shopee 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 8162d4eda0b852b61f80f873d566f046df929f5a
+  data.tar.gz: e8cfdea68b4163292dc62113e4ed92c3fc1e8622
+SHA512:
+  metadata.gz: 92fb9cd073ebc2dd6014e865afb70cf790106a2dd10c4c09d4e7ec79b35268eb53ea2345d8ad73174d2fb94037819b6f1fc4ee1faba1855b708115425cf5503e
+  data.tar.gz: e814c518173af4929411433132063c6b37b8e5c7844b8c89feff6b92a57e5af96d28b4030fc28ff37bf970205b03f1a056d96db8745460bed4c32c3814600d0b

data/.gitignore

@@ -0,0 +1 @@
+Gemfile.lock

data/.travis.yml

@@ -0,0 +1,4 @@
+language: ruby
+rvm:
+  - ruby-head
+  - jruby-head

data/Gemfile

@@ -0,0 +1,10 @@
+source 'https://rubygems.org'
+
+gem 'oga'
+gem 'json'
+gem 'minitest'
+gem 'vcr'
+gem 'webmock'
+group :test do
+  gem 'rake'
+end

data/README.md

@@ -0,0 +1,23 @@
+# Ideate-and-Scrape 
+
+[![Build Status](https://travis-ci.org/Smartibuy/ideate-and-scrape.svg?branch=master)](https://travis-ci.org/Smartibuy/ideate-and-scrape)
+
+## Description
+
+Parse Category from  [Shopee](http://shopee.tw/mobile/)
+
+### Installation
+- Install packages
+```
+$ bundle install
+```
+- Run
+```
+$ ./bin/shopee_titles
+```
+
+### Test
+```shell
+$ cd spec
+$ ruby shopee_spec.rb
+```

data/Rakefile

@@ -0,0 +1,10 @@
+require "rake/testtask"
+
+desc "Test Task"
+task :default => ["test"]
+desc "Test Task"
+task :test do
+    Rake::TestTask.new do |t|
+        t.pattern = "spec/*_spec.rb"
+    end
+end

data/bin/shopee_titles

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+require 'json'
+require './lib/shopee_scrap'
+
+begin
+  google_news = ShopeeTileScrape::ShopeeTile.new
+  news_arr = JSON.parse(google_news.titles)
+
+  news_arr.each do |hash|
+    puts "#{hash['title']} "
+  end
+
+rescue => e
+  puts "Error occured - see details: #{e}"
+end

data/lib/shopee_scrap.rb

@@ -0,0 +1,45 @@
+require 'oga'
+require 'open-uri'
+require 'json'
+
+
+module ShopeeTileScrape
+  class ShopeeTile
+
+    URL = 'http://mall.shopee.tw/?utm_source=OrganicA&utm_medium=OrganicA&utm_campaign=lp_home_mall'
+    XPATH_CARD = "//div[(@class='title')]"
+    CARD_TITLE_XPATH = "//div[(@class='list-wrapper')]"
+
+    def initialize
+      parse_html
+    end
+
+    def titles
+      @titles ||= extract_titles
+    end
+
+    private
+
+    def parse_html
+      url = "#{URL}"
+      @document = Oga.parse_html(open(url))
+    end
+
+    def extract_titles
+      result = []
+      @document.xpath(CARD_TITLE_XPATH).map do |card|
+        card.xpath(XPATH_CARD).map do |item|
+
+          stri = item.text
+          check =  stri.empty?
+          if check == false
+            element = {}
+            element['title'] = item.text
+            result << element
+          end
+        end
+      end
+      result.to_json
+    end
+  end
+end

data/spec/shopee_spec.rb

@@ -0,0 +1,25 @@
+require 'minitest/autorun'
+require 'webmock/minitest'
+require 'json'
+require_relative '../lib/shopee_scrap'
+require_relative '../vcr/vcr.rb'
+
+CATEGORY_LIST = ['女生衣著', '男生衣著', '女生包包與配件', '男生包包與配件', '女鞋', '男鞋','3C相關', '手機平板與周邊', '居家生活',
+'娛樂、收藏', '代買代購', '服務、票券', '化妝、保養品', '嬰幼童與母親', '美食、伴手禮', '寵物', '戶外、運動',
+'家電影音', '電玩遊戲相關', '其他類別']
+
+TESTAMENT_ARR = []
+    
+CATEGORY_LIST.each do |e|
+  TESTAMENT_ARR.push({'title' => e})
+end
+
+VCR.use_cassette('shopee', :record=>:once) do
+  scraper = ShopeeTileScrape::ShopeeTile.new
+  describe 'Shopee testament using vcr' do
+    it 'should return an array of string and name of categories' do  
+      category_arr = JSON.parse(scraper.titles)
+      category_arr.must_equal TESTAMENT_ARR
+    end
+  end
+end

data/vcr/vcr.rb

@@ -0,0 +1,7 @@
+require 'rubygems'
+require 'vcr'
+
+VCR.configure do |config|
+  config.cassette_library_dir = "fixtures/vcr_cassettes"
+  config.hook_into :webmock # or :fakeweb
+end

metadata

@@ -0,0 +1,127 @@
+--- !ruby/object:Gem::Specification
+name: shopee
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Sheng Jung Wu
+- Calvin Jeng
+- Henry Chang
+- Yi Wei Huang
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-10-17 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: vcr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: oga
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Scrape categories of shopee!
+email: 
+executables:
+- shopee_titles
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- Gemfile
+- README.md
+- Rakefile
+- bin/shopee_titles
+- lib/shopee_scrap.rb
+- spec/shopee_spec.rb
+- vcr/vcr.rb
+homepage: https://github.com/Smartibuy/ideate-and-scrape
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.6
+signing_key: 
+specification_version: 4
+summary: Scrape categories of shopee!
+test_files:
+- spec/shopee_spec.rb